from raid_night_summarizer import *
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
from textwrap import wrap
import re
import pandas as pd

antorus_ordered_handles = {"Heroic Garothi Worldbreaker": 1,
            "Heroic Felhounds of Sargeras": 2,
            "Heroic The Defense of Eonar": 3,
            "Heroic Portal Keeper Hasabel": 4,
            "Heroic Antoran High Command": 5,
            "Heroic Imonar the Soulhunter": 6,
            "Heroic Kin'garoth": 7,
            "Heroic Varimathras": 8,
            "Heroic The Coven of Shivarra": 9,
            "Heroic Aggramar": 10,
            "Heroic Argus the Unmaker": 11}

uldir_ordered_handles = {"Heroic Taloc": 1,
            "Heroic MOTHER": 2,
            "Heroic Fetid Devourer": 3,
            "Heroic Vectis": 4,
            "Heroic Zek'voz": 5,
            "Heroic Zul": 6,
            "Heroic Mythrax": 7,
            "Heroic G'huun": 8}

def get_parse_color(parse_number):
    color_key = [(20, '#9d9d9d'), #common-gray
                (50, '#1eff00'), #uncommon-green
                (75, '#0070dd'), #rare-blue
                (94, '#a335ee'), #epic-purple
                (100, '#ff8000')] #legendary-orange

    for colorcode in color_key:
        if parse_number <= colorcode[0]:
            return colorcode[1]

    return 'gray'

def make_avg_ilvl_parse_bar_plot(raidnight_object):
    average_parses = []
    for boss_name in raidnight_object.parse_scrapes.keys():
        iparselist = [raidnight_object.parse_scrapes[boss_name][name]['ilvl-performance'] for name in raidnight_object.parse_scrapes[boss_name].keys()]
        average_parse = np.mean(iparselist)
        average_parses.append((average_parse,boss_name))

    fig = plt.figure(1)
    ax = fig.add_subplot(111)
    barplot = ax.bar(['\n'.join(wrap(x[1],16)) for x in average_parses], [x[0] for x in average_parses], color=[get_parse_color(x[0]) for x in average_parses], edgecolor='black')
    #plt.xticks(np.arange(len(average_parses)), [x[1] for x in average_parses])
    plt.xticks(rotation=70)
    plt.yticks(np.arange(0,110,10))
    plt.xlabel('Boss')
    plt.tight_layout()
    for a,b in enumerate(x[0] for x in average_parses):
        ax.text(a, b +1, f'{b:{4}.{3}}', color=get_parse_color(b), fontweight='bold', horizontalalignment='center')
    ax.set_facecolor('#A9A9A9')
    fig.set_facecolor('gray')
    plt.show()

def make_heroic_raid_avg_ilvl_parse_scatter_plot(raid_folder):
    average_parses = []

    for filename in [f for f in listdir(raid_folder) if isfile(join(raid_folder, f))]:
        if 'Ant' in filename:
            continue
        raidnight = RaidnightData(filename, 'MyDudes')
        date = pd.to_datetime(datetime.date.fromtimestamp(raidnight.raidnight_date))

        for boss in raidnight.parse_scrapes.keys():
            if not re.search('Heroic', boss):
                continue
            iparselist = [raidnight.parse_scrapes[boss][name]['overall-performance'] for name in raidnight.parse_scrapes[boss].keys()]
            iparse_average = np.mean(iparselist)
            average_parses.append((boss, iparse_average, date))

    average_parses = sorted(average_parses, key = lambda x: x[0])

    parse_df = pd.DataFrame(average_parses, columns=["Boss", "Parse", "Date"])
    parse_df.set_index("Date", inplace=True)

    fig = plt.figure(figsize=(15,8))
    ax = fig.add_subplot(111)

    parse_df.groupby('Boss')['Parse'].plot(legend=True, marker='o', linestyle='')

    ax.set_title("Raid Overall Parses by Date")
    ax.set_ylabel("Parse Percentile")
    ax.set_xlabel("Date")

    handles, labels = ax.get_legend_handles_labels()
    handles, labels = zip(*sorted(zip(handles, labels), key=lambda x: uldir_ordered_handles[x[1]]))

    ax.legend(handles, labels, bbox_to_anchor=(1.02, 1), loc=2, borderaxespad=0.)
    plt.tight_layout()

    ax.set_yticks(np.arange(0,110,10))

    plt.show()

def make_ilvl_chart(raid_folder, playername=None):
    '''
    defaults to raid average ilvl
    '''
    title_string = "Raid Average Equipped ilvl"
    playerregex = ".*"
    if playername:
        playerregex = playername
        title_string = playername + " Best Equipped ilvl"

    average_parses = []

    for filename in [join(raid_folder, f) for f in listdir(raid_folder) if isfile(join(raid_folder, f))]:
        if 'Ant' in filename:
            continue
        raidnight = RaidnightData(filename, 'MyDudes')
        date = pd.to_datetime(datetime.date.fromtimestamp(raidnight.raidnight_date))
        top_average_ilevel = 0

        for boss in raidnight.damage_done.keys():
            try:
                player_ilevels = [x['itemLevel'] for x in raidnight.damage_done[boss]['entries'] if re.match(playerregex,x['name'])]
            except KeyError:
                continue
            average_ilevel = np.mean(player_ilevels)
            if average_ilevel > top_average_ilevel:
                top_average_ilevel = average_ilevel

        if top_average_ilevel != 0:
            average_parses.append((date, top_average_ilevel))

    parse_df = pd.DataFrame(average_parses, columns=["Date", "ilevel"])
    parse_df.set_index("Date", inplace=True)

    fig = plt.figure(figsize=(15,8))
    ax = fig.add_subplot(111)
    parse_df["ilevel"].plot(legend=None)

    ax.set_ylabel("ilevel")
    ax.set_title(title_string)

    plt.show()

def make_raidstats_chart(raid_folder):
    #TODO: raid duration, cumulative bosses down (heroic only AND normal only)
    raidstats_data_columns = ["Date", "Lockout Number", "Duration"] + list(uldir_ordered_handles.keys())

    raidstats_dictionary = dict()
    for column_header in raidstats_data_columns:
        raidstats_dictionary[column_header] = []

    for filename in [f for f in listdir(raid_folder) if isfile(join(raid_folder, f))]:
        if filename[:3] == "Ant":
            continue

        raidnight = RaidnightData(filename, raid_folder)

        raidstats_dictionary["Lockout Number"].append(raidnight.get_raid_lockout_period())

        date = pd.to_datetime(datetime.date.fromtimestamp(raidnight.raidnight_date))
        raidstats_dictionary["Date"].append(date)

        raidnight_duration = pd.to_timedelta(datetime.datetime.fromtimestamp(raidnight.fights['end']//1000) - datetime.datetime.fromtimestamp(raidnight.fights['start']//1000))
        raidstats_dictionary["Duration"].append(raidnight_duration)

        for boss in raidstats_data_columns[3:]:
            if boss in raidnight.parse_scrapes.keys():
                raidstats_dictionary[boss].append(uldir_ordered_handles[boss])
            else:
                raidstats_dictionary[boss].append(None)

    raidstats_df = pd.DataFrame(raidstats_dictionary, columns=raidstats_data_columns)
    raidstats_df.set_index("Date", inplace=True)
    print(raidstats_df.head())
    def timeTicks(nanoseconds, pos):
        seconds = nanoseconds//1000000000
        hours = str(int(seconds//3600))
        minutes = str(int((seconds%3600)//60))
        seconds = str(int(seconds%60))
        return ':'.join([hours, minutes.zfill(2),seconds.zfill(2)])
    formatter = mpl.ticker.FuncFormatter(timeTicks)
    fig = plt.figure(1)
    ax = fig.add_subplot(111)

    '''for column in raidstats_df.keys():
        if column == "Duration":
            continue
        plt.scatter(raidstats_df.index, raidstats_df[column])

    ax.set_yticks(np.arange(12))'''

    plt.bar(raidstats_df.index, raidstats_df["Duration"])
    ax.set_yticks(np.arange(0,3600*1000000000*4, 1800*1000000000))
    ax.yaxis.set_major_formatter(formatter)
    plt.show()

make_heroic_raid_avg_ilvl_parse_scatter_plot('MyDudes')


"""game time ps state

Revision ID: 603e4def37ae
Revises: be58e854b159
Create Date: 2019-10-26 11:45:00.046144

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '603e4def37ae'
down_revision = 'be58e854b159'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('game', schema=None) as batch_op:
        batch_op.add_column(sa.Column('last_time', sa.DateTime(), nullable=True))
        batch_op.add_column(sa.Column('ps', sa.String(length=140), nullable=True))
        batch_op.add_column(sa.Column('start_time', sa.DateTime(), nullable=True))
        batch_op.add_column(sa.Column('state', sa.Integer(), nullable=True))

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('game', schema=None) as batch_op:
        batch_op.drop_column('state')
        batch_op.drop_column('start_time')
        batch_op.drop_column('ps')
        batch_op.drop_column('last_time')

    # ### end Alembic commands ###


# -*- coding: utf-8 -*-
"""

followed by:

"""
import numpy as np
import pandas as pd
import os
from pyproj import Proj, transform
import xarray as xr
from scipy.spatial import cKDTree
import matplotlib.pyplot as plt
# import cfgrib
from datetime import datetime
from datetime import timedelta

if os.getlogin() == 'jason':
    base_path = '/Users/jason/Dropbox/CARRA/CARRA_ERA5_events'
    heavy_data_path='/Users/jason/0_dat/ERA5/events/'

os.chdir(base_path)

# %% CARRA coordinates

def lon360_to_lon180(lon360):

    # reduce the angle
    lon180 =  lon360 % 360

    # force it to be the positive remainder, so that 0 <= angle < 360
    lon180 = (lon180 + 360) % 360;

    # force into the minimum absolute value residue class, so that -180 < angle <= 180
    lon180[lon180 > 180] -= 360

    return lon180

# CARRA West grid dims
ni = 1269 ; nj = 1069

# read lat lon arrays
fn = './meta/CARRA/2.5km_CARRA_west_lat_1269x1069.npy'
lat = np.fromfile(fn, dtype=np.float32)
clat_mat = lat.reshape(ni, nj)

fn = './meta/CARRA/2.5km_CARRA_west_lon_1269x1069.npy'
lon = np.fromfile(fn, dtype=np.float32)
lon_pn = lon360_to_lon180(lon)
clon_mat = lon_pn.reshape(ni, nj)

# fn='./meta/CARRA/CARRA_W_domain_ice_mask.nc'
# ds=xr.open_dataset(fn)
# mask = np.array(ds.z)

# %% reproject 4326 (lat/lon) CARRA coordinates to 3413 (orth polar projection in meters)

#from lat/lon to meters
inProj = Proj(init='epsg:4326')
outProj = Proj(init='epsg:3413')

x1, y1 = clon_mat.flatten(), clat_mat.flatten()
cx, cy = transform(inProj, outProj, x1, y1)
cx_mat = cx.reshape(ni, nj)
cy_mat = cy.reshape(ni, nj)

cols, rows = np.meshgrid(np.arange(np.shape(clat_mat)[1]),
                         np.arange(np.shape(clat_mat)[0]))

CARRA_positions = pd.DataFrame({'rowc': rows.ravel(),
                                'colc': cols.ravel(),
                                'xc': cx_mat.ravel(),
                                'yc': cy_mat.ravel()})

# ,
# 'maskc': mask.flatten()}
#import CARRA datset
# ds = xr.open_dataset(raw_path+'tp_2012.nc')
# CARRA_data = np.array(ds.tp[0, :, :]).flatten()

# %% load ERA5 coordinates

#ERA 5 elevation data
fn='./meta/ERA5/ERA5_mask_and_terrain.nc'
ds=xr.open_dataset(fn)

#from lat/lon to meters
inProj = Proj(init='epsg:4326')
outProj = Proj(init='epsg:3413')

niE=np.shape(ds.latitude.values)[0]
njE=np.shape(ds.longitude.values)[0]

lat_mesh, lon_mesh = np.meshgrid(ds.latitude.values, ds.longitude.values)
x1, y1 = lon360_to_lon180(lon_mesh.flatten()), lat_mesh.flatten()
ex, ey = transform(inProj, outProj, x1, y1)
ex_mat = ex.reshape(niE, njE)
ey_mat = ey.reshape(niE, njE)

cols2, rows2 = np.meshgrid(np.arange(np.shape(ds.latitude.values)[0]), np.arange(np.shape(ds.longitude.values)[0]))
lat_e, lon_e = np.meshgrid(ds.latitude.values, ds.longitude.values)
ERA5_positions = pd.DataFrame({
"row_e": rows2.ravel(),
"col_e": cols2.ravel(),
"lon_e": lon_e.ravel(),
"lat_e": lat_e.ravel(),})

if os.getlogin() == 'jason':
    base_path = '/Users/jason/Dropbox/CARRA/CARRA_ERA5_events/'
    heavy_data_path='/Users/jason/0_dat/ERA5/events/'

os.chdir(base_path)

fn='/Users/jason/Dropbox/CARRA/CARRA_rainfall_study/stats/rf_extremes.csv'
os.system('ls -lF '+fn)
os.system('head '+fn)
os.system('tail '+fn)
df=pd.read_csv(fn)

print(df.columns)
#%%
# minval=300 ;scaling_factor=3000
# v=np.where(df.maxlocalrate>minval)

v=np.where(df.Gt_overall>2.2)
# v=np.where(df.Gt_overall>2.7)
# v=v[0]
# print(v[0])

# print()
minval=np.min(df.maxlocalrate[v[0]]) ; scaling_factor=1500
listx=[v[0][3]] # 2021
# listx=[v[0][-4]] # 2017
# listx=v[0]
# listx=[v[0][0]] # 2012a
listx=[v[0][12]] # 2017

model='CARRA'
model='ERA5'

if model=='CARRA':
    choices=['tpt2m']
else:
    choices=['tpsf']

for ii,i in enumerate(listx):
    print(model,df.date[i],df.Gt_overall[i],ii)
    #%%
    ni=1269 ; nj=1069

    # fn='./meta/CARRA/2.5km_CARRA_west_elev_1269x1069.npy'
    # elevx=np.fromfile(fn, dtype=np.float32)
    # elevx=elevx.reshape(ni, nj)
    # elevx=np.rot90(elevx.T)

    # global plot settings
    th=1
    font_size=18
    plt.rcParams['axes.facecolor'] = 'k'
    plt.rcParams['axes.edgecolor'] = 'k'
    plt.rcParams["font.size"] = font_size
    plt.rcParams['axes.facecolor']='w'
    plt.rcParams['savefig.facecolor']='w'

    timex=pd.to_datetime(df.date[i])

    casex=timex.strftime('%Y-%m-%d')
    casex2=timex.strftime('%d %B, %Y')

    timex=pd.to_datetime(df.date[i])

    ddx=int(timex.strftime('%j'))
    ddx0=ddx-1 ; ddx1=ddx+1
    ymd=timex.strftime('%Y-%m-%d')
    year=timex.strftime('%Y')
    month=timex.strftime('%m')
    day=timex.strftime('%d')

    timex0=pd.to_datetime(df.date[i])- timedelta(days=1)
    ymd0=timex0.strftime('%Y-%m-%d')
    day_before=timex0.strftime('%d')

    timex1=pd.to_datetime(df.date[i])+ timedelta(days=1)
    ymd1=timex1.strftime('%Y-%m-%d')

    print(i,ymd0,ymd,ymd1,ddx,df.lon[i],df.lat[i],df.Gt_overall[i],df.maxlocalrate[i])

    day_before=str(int(day)-1).zfill(2)
    day_three=str(int(day)+1).zfill(2)
    # day_eight=str(int(day)+6).zfill(2)

    choice='tpsf'
    opath='/Users/jason/0_dat/ERA5/events/tpsf/'
    ofile=opath+str(year)+str(month).zfill(2)+str(day_before).zfill(2)+'-'+day_three+'_3hourly_'+choice+'.grib'
    print(ofile)
    #%%
    # casex='2017-09-14'; casex2='14 September, 2017'
    # casex='2022-06-17'; casex2='17 June, 2022'
    #load ERA5 dataset

    # fn='/Users/jason/0_dat/ERA5/events/tpsf/20210814-17_3hourly_tpsf.grib'
    # fn='/Users/jason/0_dat/ERA5/events/tzuv/20210814-17_3hourly_tzuv.grib'
    ds=xr.open_dataset(ofile,engine='cfgrib')
    # ds = cfgrib.open_dataset(fn)

    # print(ds.variables)
    # print(np.shape(ds.tp))
    #%%

    steps=['-21','-00','-03','-06']
    steps=['00','01','02','03']
    times=ds.variables['time']
    # for time in times:
    #     # print(str(np.array(time)))
    #     dates.append(str(np.array(time)))
    #     temp=str(np.array(time))[0:16]
    #     date_strings.append(temp)
    #     dates.append(datetime.strptime(temp, "%Y-%m-%dT%H:%M"))
    # print(dates)

    # str_dates = [i.strftime("%Y-%m-%dT%H:%M") for i in time]
    cc=0
    sumx=np.zeros((1269, 1069))
    for i,time in enumerate(times):
        for step in range(4):
            if ((cc>0) & (cc<25)):
                temp=str(np.array(time))[0:16]
                x=datetime.strptime(temp, "%Y-%m-%dT%H:%M")
                timex0=pd.to_datetime(times[i].to_numpy()) + timedelta(hours=step*3-6)
                if x.strftime('%H')=='18':
                    timex0=pd.to_datetime(times[i].to_numpy()) + timedelta(hours=(step*3)-6)
                timex0=timex0 + timedelta(hours=9)

                date_stringx=timex0.strftime('%Y-%m-%d-%H')
                print(cc,date_stringx)

                var_choice='tp'
                var_choice='rf'

                if var_choice=='t2m':
                    ERA_data = np.array(ds.t2m[i, :, :])-273.15
                if var_choice=='tp':
                    ERA_data = np.array(ds.tp[i,step, :, :])*1000*3
                if var_choice=='rf':
                    ERA_data = np.array(ds.tp[i,step, :, :])*1000*3-np.array(ds.sf[i,step, :, :])*1000*3
                # print(np.shape(ERA_data))

                #  nearest neighbours

                #dataset to be upscaled -> ERA5
                nA = np.column_stack((ex_mat.ravel(), ey_mat.ravel()) )
                #dataset to provide the desired grid -> CARRA
                nB = np.column_stack((cx_mat.ravel(), cy_mat.ravel()))

                btree = cKDTree(nA)  #train dataset
                dist, idx = btree.query(nB, k=1)  #apply on grid

                #collocate matching cells
                CARRA_cells_for_ERA5 = ERA5_positions.iloc[idx]

                # Output resampling key ERA5 data in CARRA grid
                path_tools='/tmp/'
                CARRA_cells_for_ERA5.to_pickle(path_tools+'resampling_key_ERA5_to_CARRA.pkl')

                outpath='/Users/jason/0_dat/ERA5/events/resampled/'+var_choice+'/'
                os.system('mkdir -p '+outpath)

                #  visualisation

                new_grid= ERA_data[CARRA_cells_for_ERA5.col_e, CARRA_cells_for_ERA5.row_e]
                new_grid = np.rot90(new_grid.reshape(ni, nj).T)
                plt.close()
                plt.imshow(new_grid,vmin=0,vmax=12)
                plt.axis('off')
                plt.title(var_choice+' '+date_stringx)
                plt.colorbar()
                DPI=200
                ly='p'
                if ly=='x':
                    plt.show()
                if ly == 'p':
                    figpath=outpath
                    # figpath='/Users/jason/Dropbox/CARRA/CARRA_ERA5_events/Figs/ERA5/'
                    # os.system('mkdir -p '+figpath)
                    plt.savefig(figpath+date_stringx+'.png', bbox_inches='tight', pad_inches=0.04, dpi=DPI, facecolor='w', edgecolor='k')

                new_grid.astype(dtype=np.float16).tofile(outpath+'/'+date_stringx+'_1269x1069.npy',)
                sumx+=new_grid
                if ((cc==8)or(cc==16)or(cc==24)):
                    sumx.astype(dtype=np.float16).tofile(outpath+'/'+timex0.strftime('%Y-%m-%d')+'_1269x1069.npy',)

                    plt.close()
                    plt.imshow(sumx,vmin=0,vmax=48)
                    plt.axis('off')
                    plt.title(var_choice+' '+timex0.strftime('%Y-%m-%d'))
                    plt.colorbar()
                    DPI=200
                    ly='p'
                    if ly=='x':
                        plt.show()
                    if ly == 'p':
                        figpath=outpath
                        # figpath='/Users/jason/Dropbox/CARRA/CARRA_ERA5_events/Figs/ERA5/'
                        # os.system('mkdir -p '+figpath)
                        plt.savefig(figpath+timex0.strftime('%Y-%m-%d')+'.png', bbox_inches='tight', pad_inches=0.04, dpi=DPI, facecolor='w', edgecolor='k')
                    sumx=np.zeros((1269, 1069))

            cc+=1


"""
This bot listens to port 5002 for incoming connections from Facebook. It takes
in any messages that the bot receives and echos it back.
"""
from flask import Flask, request
from pymessenger.bot import Bot
import requests
import ipdb
app = Flask(__name__)
TOKEN = "<access_token>"
bot = Bot(TOKEN)

@app.route("/webhook", methods = ['GET', 'POST'])
def hello():
    if request.method == 'GET':
        if (request.args.get("hub.verify_token") == "<token you define during"\
                "the verification phase>"):
                return request.args.get("hub.challenge")
    if request.method == 'POST':
        output = request.json
        event = output['entry'][0]['messaging']
        for x in event:
            if (x.get('message') and x['message'].get('text')):
                message = x['message']['text']
                recipient_id = x['sender']['id']
                bot.send_text_message(recipient_id, message)
            else:
                pass
        return "success"

if __name__ == "__main__":
    app.run(port=5002, debug=True)


from .. import subparsers
from . import load_mixin, save_mixin, weights_mixin
from .utils import OptionStringHelpFormatter, formatter_factory

compile_parser = subparsers.add_parser(
    "compile",
    description="configure the model for training",
    formatter_class=formatter_factory(
        OptionStringHelpFormatter,
        max_help_position=30
    )
)

compile_mixins = {
    weights_mixin: {},
    load_mixin: {},
    save_mixin: {
        "required": True
    }
}
for mixin, kwargs in compile_mixins.items():
    mixin(compile_parser, **kwargs)

compile_group = compile_parser.add_argument_group(
    "compilation parameters"
)
compile_group.add_argument(
    "--optimizer",
    help="name of optimizer",
    metavar="<optimizer>",
),
compile_group.add_argument(
    "--loss",
    help="name of objective function",
    metavar="<loss>",
),
compile_group.add_argument(
    "--metric",
    action="append",
    help="metric to be evaluated by the model during training and testing",
    metavar="<metric>",
)


#!/usr/bin/env python
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import sklearn.datasets
import pandas as pd
from tensorflow import keras
tf.random.set_seed(0)

def main():
    test_data=np.load('test_data.npy')
    test_label=np.load('test_label.npy')
    training_data=np.load('training_data.npy')
    training_label=np.load('training_label.npy')

    model=keras.Sequential()
    model.add(keras.layers.LSTM(30, input_shape=(training_data.shape[1], training_data.shape[2])))
    model.add(keras.layers.Dense(1,))

    model.compile(
        optimizer=keras.optimizers.Adam(learning_rate=0.001),
        loss='mean_squared_error',
        metrics=['accuracy']
        )

    model.fit(training_data, training_label, epochs=25, batch_size=32, verbose=1)

    plt.figure(figsize=(10,8))

    plt.subplot(3,1,1)
    plt.plot(range(training_label.shape[0]),training_label, label='Training data')
    plt.plot(range(training_label.shape[0], training_label.shape[0]+test_label.shape[0]) ,test_label, label='Test data')
    plt.title('Training data')
    plt.xlabel('Time')
    plt.ylabel('Data')
    plt.legend()

    plt.subplot(3,2,3)
    plt.plot(range(test_label.shape[0]) ,test_label, label='true')
    plt.plot(range(test_label.shape[0]) ,model.predict(test_data), label='pred')
    plt.title('Test prediction')
    plt.xlabel('Time')
    plt.ylabel('Data')
    plt.legend()

    plt.subplot(3,2,4)
    plt.plot(range(100) ,test_label[0:100], label='true')
    plt.plot(range(100) ,model.predict(test_data)[0:100], label='pred')
    plt.title('First 100 days')
    plt.xlabel('Time')
    plt.ylabel('Data')
    plt.legend()

    plt.subplot(3,2,5)
    plt.plot(range(test_label.shape[0]) ,test_label-model.predict(test_data), label='true')
    plt.title('True - prediction')
    plt.xlabel('Time')
    plt.ylabel('Difference')
    plt.legend()

    plt.subplot(3,2,6)
    plt.scatter(model.predict(test_data), test_label, s=2, color='black')
    plt.title('Scatter plot')
    plt.xlabel('Prediction')
    plt.ylabel('True')
    plt.legend()

    plt.subplots_adjust(hspace = 0.5, wspace=0.3)
    plt.show()

    return 0

if __name__ == '__main__':
    main()


import cv2
import matplotlib.pyplot as plt
import numpy as np
import time
import thrusters

hsv = None
img = None
bounding_rect = None
empty = None
cam_dex = 0
baud_rate = 115200
thrust_manager = thrusters.ThrustManager()
error_queue = None
target_orientation = 0
current_orientation = None  #link controls orientation list here
# BECAUSE PARAMETERS ARE CRINGE

def detect_rope():
    global hsv
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    lower_red = np.array([0,50,50])
    upper_red = np.array([10,255,255])
    mask0 = cv2.inRange(hsv, lower_red, upper_red)

    lower_red = np.array([170,50,50])
    upper_red = np.array([180,255,255])
    mask1 = cv2.inRange(hsv, lower_red, upper_red)

    mask_red = mask0+mask1
    hsv[np.where(mask_red==0)] = 0

def find_defect():
    global bounding_rect, empty
    rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
    gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)

    _, thresh = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)
    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    if len(contours) == 0:
        empty = True
        return (0, 0)

    cnt = contours[0]
    hull = cv2.convexHull(cnt, returnPoints = False)
    defects = cv2.convexityDefects(cnt, hull)

    if not defects:
        return (-1, -1)

    defect = defects[0, 0]
    bounding_rect = cv2.boundingRect(cnt)
    defect_pt = tuple(cnt[defect[2]][0])
    return defect_pt

def read_frame():
    global img, hsv
    vid = cv2.VideoCapture(cam_dex)
    _, img = vid.read()
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

def turn(direction):
    left = False
    right = True

    if direction == left:
        #packet =
        pass
    elif direction == right:
        #packet =
        pass

    send_packet()
    time.sleep((len(packet)+2)/baud_rate)
    get_orientation_data()

    # Get angular acceleration data, then do 0.5at2 and figure out t

def adjust_orientation(orientation):
    orientation %= 360
    if (orientation < 0):
        orientation += 360
    return orientation

def get_error(current_orientation, target_orientation):
    error = adjust_orientation(target_orientation - current_orientation[0])
    if (error > 180):
        error = 180 - error
    return error

def run():
    finished = False
    thrust_manager.multiplier = 0.1
    thrust_manager.getTSpeeds((0, 1, 0), (0, 0, 0))    # Move forward

    while(not finished):
        read_frame()
        detect_rope()
        defect_pt = find_defect()

        if defect_pt == (0, 0):

            finished = True
        elif defect_pt != (-1, -1):
            x, y, w, h = bounding_rect
            thrust_manager.multiplier = 0.1 #stop moving forward
            thrust_manager.getTSpeeds((0, 0, 0), (0, 0, 0)) # Stop moving forward
            left = False
            right = True

            if ((defect_pt[0] - x) > (w / 2.0)):
                target_orientation = current_orientation[0] + 90 #turn left
                target_orientation = adjust_orientation(target_orientation)
                #START ERROR READING IN QUEUE, DON'T READ HERE PROBABLY
                error = adjust_orientation(target_orientation - current_orientation[0])
            else:
                turn(right);    #turn right


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import warnings
import sys
import argparse

from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Flatten
from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras.applications import vgg16

class Runner(object):

    def __init__(self):

        self.logDirectory, self.configs = self.parseArgs()
        self.currentDir = os.path.dirname(os.path.abspath(__file__))
        self.pathToAdvertismentDetectionSourceCode = self.currentDir + '/src'
        self.pathToDataPathesFile = self.currentDir + '/src/pathVariables.txt'
        self.pathToWeights = str(self.logDirectory) + '/model.h5'
        self.pathToSaveHistory = str(self.logDirectory) + '/history'

        sys.path.insert(0, self.pathToAdvertismentDetectionSourceCode)
        self.data = self.readInData()
        self.testIndices = self.data.testIndices
        self.model = self.getModel(input_shape = self.data.imageShape)

    def start(self):
        from advertisementDetection import RecognitionSystem
        recogSystem = RecognitionSystem(data = self.data, pathToWeights = self.pathToWeights, pathToSaveHistory = self.pathToSaveHistory, configs = self.configs, model = self.model, testIndices=self.testIndices)
        recogSystem.data.printInformationAboutData()
        recogSystem.printModelSummary()
#        recogSystem.trainModel()
        recogSystem.evaluateModel()

    def readInData(self):
        with open(self.pathToDataPathesFile) as pathVariables:
            pathes = pathVariables.read().splitlines()
            imagesPath = pathes[0]
            labelsPath = pathes[1]
        normalizeData = self.configs['normalizeData'] if 'normalizeData' in self.configs else False
        from advertisementDetection import Preprocessor
        preprocessedData = Preprocessor(imagesPath, labelsPath, normalizeData = normalizeData)
        return preprocessedData

    def getModel(self, input_shape, classes=1):
        vgg16_model = vgg16.VGG16(include_top = False, weights = 'imagenet', input_shape = input_shape)

        model = Sequential()
        for layer in vgg16_model.layers:
            model.add(layer)
        for layer in model.layers:
            layer.trainable = False

        model.add(Flatten(name='flatten'))
        model.add(Dense(1000, activation='relu'))
        model.add(Dense(classes, activation = 'sigmoid'))

        return model

    def parseArgs(self):
        configs = {
            'learningRate':1e-4,
            'decay':1e-6,
            'momentum':0.9,
    #        'normalizeData':True,
            'nesterov':True,
            'batchSize':20,
            'epochs':15,
            'loss':'binary_crossentropy',
            'metrics':['accuracy']
        }

        parser = argparse.ArgumentParser(description='Description of your program')
        parser.add_argument('--logDir', help='The folder to save everything', required=True)
        parser.add_argument('--learningRate')
        parser.add_argument('--decay')
        parser.add_argument('--momentum')
        parser.add_argument('--batchSize')
        parser.add_argument('--epochs')

        args = vars(parser.parse_args())

        logDir = args['logDir']

        for arg in configs:
            if arg in args and args[arg] is not None:
                if(arg in ['learningRate','decay', 'momentum']):
                        configs[arg] = float(args[arg])
                else:
                        configs[arg] = int(args[arg])

        print(configs)
        return logDir, configs

if __name__ == "__main__":
    runner = Runner()
    runner.start()

{'RecognitionSystem': 'advertisementDetection.RecognitionSystem', 'Preprocessor': 'advertisementDetection.Preprocessor'}


import numpy as np
import pytest

from kalm_benchmark.evaluation.scanner.kubeaudit import Scanner
from kalm_benchmark.evaluation.scanner.scanner_evaluator import CheckCategory

class TestCheckCategorization:
    @pytest.mark.parametrize(
        "check_id",
        [
            "AppArmorAnnotationMissing",
            "AppArmorDisabled",
            "AppArmorInvalidAnnotation",
        ],
    )
    def test_apparmor_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "AutomountServiceAccountTokenDeprecated",
            "AutomountServiceAccountTokenTrueAndDefaultSA",
        ],
    )
    def test_automount_sa_token_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        ["CapabilityAdded", "CapabilityShouldDropAll", "CapabilityOrSecurityContextMissing"],
    )
    def test_capabilities_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "NamespaceHostNetworkTrue",
            "NamespaceHostIPCTrue",
            "NamespaceHostPIDTrue",
        ],
    )
    def test_hostns_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "ImageTagMissing",
            "ImageTagIncorrect",
            "ImageCorrect",
        ],
    )
    def test_image_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "LimitsNotSet",
            "LimitsCPUNotSet",
            "LimitsMemoryNotSet",
            "LimitsCPUExceeded",
            "LimitsMemoryExceeded",
        ],
    )
    def test_limits_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.Workload

    @pytest.mark.parametrize(
        "check_id",
        [
            "SensitivePathsMounted",
        ],
    )
    def test_mounts_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "MissingDefaultDenyIngressAndEgressNetworkPolicy",
            "MissingDefaultDenyIngressNetworkPolicy",
            "MissingDefaultDenyEgressNetworkPolicy",
            "AllowAllIngressNetworkPolicyExists",
            "AllowAllEgressNetworkPolicyExists",
        ],
    )
    def test_netpol_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.Network

    @pytest.mark.parametrize(
        "check_id",
        [
            "RunAsUserCSCRoot",
            "RunAsUserPSCRoot",
            "RunAsNonRootCSCFalse",
            "RunAsNonRootPSCNilCSCNil",
            "RunAsNonRootPSCFalseCSCNil",
        ],
    )
    def test_nonroot_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "AllowPrivilegeEscalationNil",
            "AllowPrivilegeEscalationTrue",
        ],
    )
    def test_privilege_escalation_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "PrivilegedTrue",
            "PrivilegedNil",
        ],
    )
    def test_privileged_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "ReadOnlyRootFilesystemFalse",
            "ReadOnlyRootFilesystemNil",
        ],
    )
    def test_root_fs_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "SeccompAnnotationMissing",
            "SeccompDeprecatedPod",
            "SeccompDisabledPod",
            "SeccompDeprecatedContainer",
            "SeccompDisabledContainer",
        ],
    )
    def test_seccomp_checks(self, check_id: str):
        cat = Scanner.categorize_check(check_id)
        assert cat == CheckCategory.PodSecurity

    @pytest.mark.parametrize(
        "check_id",
        [
            "NotAValidCheck",
            "",
            None,
            np.nan,
        ],
    )
    def test_invalid_or_unknown_ids(self, check_id: str | None):
        cat = Scanner.categorize_check(check_id)
        assert cat is None  # no category assigned


"""Code is based on
https://github.com/rwightman/pytorch-image-models/blob/master/timm/data/mixup.py
"""
import numpy as np
import torch

def to_one_hot(x, num_classes, on_value=1., off_value=0.):
    if len(x.size()) > 1 and x.size(-1) == num_classes:
        # already one-hot form
        return x
    x = x.long().view(-1, 1)
    return torch.full((x.size()[0], num_classes), off_value, device=x.device
                      ).scatter_(dim=1, index=x, value=on_value)

def squeeze_one_hot(x):
    return x.topk(1)[1].long()

def smooth_target(target, smoothing=0.1, num_classes=1000):
    target *= (1. - smoothing)
    target += (smoothing / num_classes)
    return target

def mix_target(target_a, target_b, num_classes, lam=1., smoothing=0.0):
    y1 = smooth_target(target_a, smoothing, num_classes)
    y2 = smooth_target(target_b, smoothing, num_classes)
    return lam * y1 + (1. - lam) * y2

def rand_bbox(size, lam):
    W = size[2]
    H = size[3]
    cut_rat = np.sqrt(1. - lam)
    cut_w = np.int(W * cut_rat)
    cut_h = np.int(H * cut_rat)

    # uniform
    cx = np.random.randint(W)
    cy = np.random.randint(H)

    bbx1 = np.clip(cx - cut_w // 2, 0, W)
    bby1 = np.clip(cy - cut_h // 2, 0, H)
    bbx2 = np.clip(cx + cut_w // 2, 0, W)
    bby2 = np.clip(cy + cut_h // 2, 0, H)

    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (size[2] * size[3]))

    return bbx1, bby1, bbx2, bby2, lam

def cutmix_batch(image, target, mix_args):
    if not mix_args.disable and np.random.rand(1) < mix_args.prob:
        lam = np.random.beta(mix_args.beta, mix_args.beta)
    else:
        target = smooth_target(
            target=target,
            smoothing=mix_args.smoothing,
            num_classes=mix_args.num_classes)
        return image, target

    rand_index = torch.randperm(image.size()[0], device=image.device)
    target_a = target
    target_b = target[rand_index]
    bbx1, bby1, bbx2, bby2, lam = rand_bbox(image.size(), lam)
    image[:, :, bbx1:bbx2, bby1:bby2] = image[rand_index, :, bbx1:bbx2,
                                              bby1:bby2]

    target = mix_target(target_a=target_a,
                        target_b=target_b,
                        num_classes=mix_args.num_classes,
                        lam=lam,
                        smoothing=mix_args.smoothing)
    return image, target


import codecs
import sys
import os

from lib import NaiveBayesClassifier

def deserialize(filename):
    source = codecs.open(filename, 'r', 'utf-8')
    data = source.readlines()
    source.close()

    return data

def run(args):
    source = args[1]
    os.chdir(source)

    labels = os.listdir('.')
    data = {label: deserialize(label) for label in labels}

    nb = NaiveBayesClassifier()
    nb.train_from_data(data)

    code = """
#!/usr/bin/env python
def normalize_space(s):
    \"\"\"Return s stripped of leading/trailing whitespace
    and with internal runs of whitespace replaced by a single SPACE\"\"\"
    # This should be a str method :-(
    return ' '.join(s.split())

    replacement = [normalize_space(i) for i in hello]}return
    """

    results = []
    for language in labels:
        results.append((language, nb.probability(code, language)))

    results = results[0:10]

    sorted_results = sorted(results, key = lambda tup: -tup[1])
    print(sorted_results)

if __name__ == '__main__':
    run(sys.argv)


from sys import float_repr_style
from typing import Optional, List, Dict
from pydantic import BaseModel, ValidationError, root_validator, validator
from datetime import date, datetime
from dateutil.relativedelta import relativedelta

class stock(BaseModel):
    stock_code: str
    stock_name: str
    sector_code: str

    @root_validator()
    def check_stock(cls, values):
        assert (
            len(values.get("stock_code")) >= 3 and len(values.get("stock_code")) <= 5
        ), "stock_code must be between 3 and 5 characters long"
        assert (
            len(values.get("stock_name")) <= 30
        ), "stock_name must be less than 30 characters long"
        assert (
            len(values.get("sector_code")) == 3
        ), "sector_code must be exactly 3 characters long"
        return values

class stock_collection:
    stockCollection = []

    def __init__(self, stocks: List):
        # make sure there aren't any duplicate stock codes passed in
        if self.checkUnique(stocks):
            [
                self.stockCollection.append(
                    stock(
                        stock_code=x.stock_code,
                        stock_name=x.stock_name,
                        sector_code=x.sector_code,
                    )
                )
                for x in stocks
            ]
        else:
            return False

    def diffCollection(self, queryStocks: List):
        # make sure there aren't any duplicate stocks being queried
        assert self.checkUnique(queryStocks), (
            "Duplicate stock_codes in list: [ "
            + ", ".join([str(elem) for elem in queryStocks])
            + " ]"
        )
        notInCollection = []

        if all(elem in self.stockCollection for elem in queryStocks):
            return []
        else:
            notFound = []
            for qStock in queryStocks:
                found = False
                for cStock in self.stockCollection:
                    if qStock == cStock.stock_code:
                        found = True
                        break
                if not found:
                    notFound.append(qStock)
            return notFound

    def checkUnique(self, someList: List):
        return len(someList) == len(set(someList))

    def getDuplicates(self, someList: List):
        seen = set()
        uniq = [x for x in someList if x in seen or seen.add(x)]
        return seen

class stock_filter(BaseModel):
    stock_code: Optional[List] = None

    @root_validator()
    def check_stock_filter(cls, values: Dict[str, List[str]]) -> Dict[str, List[str]]:
        # optional, so none is okay
        if values.get("stock_code") == None:
            # but just make sure there's no other attributes set
            assert len(values) == 0, "invalid attribute: " + str(values.keys())
            return values

        assert len(values) == 1, "multiple attributes given, expecting just stock_code"
        assert "stock_code" in values, "unknown attribute given in " + str(
            values.keys()
        )
        for x in values.get("stock_code"):
            assert len(x) >= 2 and len(x) <= 6, (
                "stock_codes must be between 3 and 5 characters long.  Failed on '"
                + x
                + "'"
            )

        return values

class sector_filter(BaseModel):
    sector_code: Optional[List] = None

    @root_validator()
    def check_sector_filter(cls, values: Dict[str, List[str]]) -> Dict[str, List[str]]:
        # optional, so none is okay
        if values.get("sector_code") == None:
            # but just make sure there's no other attributes set
            assert len(values) == 0, "invalid attribute: " + str(values.keys())
            return values

        assert len(values) == 1, "multiple attributes given, expecting just sector_code"
        assert "sector_code" in values, "unknown attribute given in " + str(
            values.keys()
        )
        for x in values.get("sector_code"):
            assert len(x) == 3, (
                "sector_codes must be between 3 and 5 characters long.  Failed on " + x
            )

        return values

class date_filter(BaseModel):
    # inputs
    start_date: Optional[str] = None
    end_date: Optional[str] = None
    period: Optional[str] = None

    # date values that we'll search with
    start: Optional[date] = None
    end: Optional[date] = None
    periodSymbols = ["d", "w", "m", "y"]

    @validator("start_date", "end_date", pre=True)
    def check_date_type(cls, v):
        try:
            datetime.strptime(v, "%Y-%m-%d")
            return v
        except:
            assert False, "dates must be formatted as yyyy-mm-dd"

    @root_validator()
    def check_attributes(cls, values):
        # can't set period and start/end
        if values.get("period") != None:
            # make sure they haven't also set a start or end date
            assert (
                values.get("start_date") == None
            ), "Cannot specify 'period' and 'start_date'"
            assert (
                values.get("end_date") == None
            ), "Cannot specify 'period' and 'end_date'"

            assert (
                len(values.get("period")) > 1
            ), "Must specify an integer and a period symbol.  Examples: 1d, 4w, 7y"
            periodSymbol = values.get("period")[-1]
            assert periodSymbol in values.get("periodSymbols"), (
                "Unrecognised period symbol '" + periodSymbol + "'"
            )
        else:
            # can't set end to be before start
            # don't need to worry about typing for start_date or end_date since its validated already
            if values.get("start_date") != None and values.get("end_date") != None:
                start = datetime.strptime(values.get("start_date"), "%Y-%m-%d")
                end = datetime.strptime(values.get("end_date"), "%Y-%m-%d")
                assert start < end, "Cannot specify 'end_date' before 'start_date'"
        return values

    def processSearchDates(self):
        # they didn't give any input at all
        if self.period == None and self.start_date == None and self.end_date == None:
            # search without a date range
            # this technically should never run, given the driver has to check for this anyway when issuing the query
            return True
        elif self.period != None:
            # use period
            return self.setPeriod()
        elif self.start_date == None and self.end_date != None:
            # no start, but end specified. go from the beginning of records
            self.start = datetime.strptime("2001-01-01", "%Y-%m-%d")
            self.end = self.end_date
            return True
        elif self.start_date != None and self.end_date == None:
            # specified start, but no end. go from start until today
            self.start = datetime.strptime(self.start_date, "%Y-%m-%d")
            self.end = date.today()
            return True
        elif self.start_date != None and self.end_date != None:
            # specified a start and an end date
            self.start = datetime.strptime(self.start_date, "%Y-%m-%d")
            self.end = datetime.strptime(self.end_date, "%Y-%m-%d")
            return True
        else:
            # logically this can't happen.... famous last words
            return False

    def setPeriod(self):
        # try catches any issues with date and relativedelta
        try:
            self.end = date.today()
            if self.period[-1] == "d":
                self.start = date.today() + relativedelta(days=-int(self.period[:-1]))
            elif self.period[-1] == "w":
                self.start = date.today() + relativedelta(weeks=-int(self.period[:-1]))
            elif self.period[-1] == "m":
                self.start = date.today() + relativedelta(months=-int(self.period[:-1]))
            elif self.period[-1] == "y":
                self.start = date.today() + relativedelta(years=-int(self.period[:-1]))
            else:
                # default to days
                self.start = date.today() + relativedelta(days=-int(self.period[:-1]))
            return True
        except:
            return False


#

# writing from zie's json format into the csr

import json
import logging
import math
from collections import Counter

# helper function for add ef
def add_ef(csr, arg_ef, check_type=False):
    extra_info = arg_ef.get("extra_info", {})
    span_info = extra_info.get("posi")
    if span_info is not None:
        ef_type = arg_ef["type"]
        if check_type and ef_type.split(":")[0] not in ["ldcOnt", "aida"]:
            ef_type = None
        ef_score = arg_ef.get("score")
        if ef_score is not None and ef_score < 0.:
            ef_score = math.exp(ef_score)
        ent = csr.add_entity_mention(
            span_info["head_span"], span_info["span"], span_info["text"], ef_type,
            entity_form=extra_info.get("entity_form"), component=extra_info.get("component"), score=ef_score,
        )
        return ent
    else:
        return None

# read file and add to csr
def add_zie_event(zie_event_file, csr):
    with open(zie_event_file) as fd:
        # read json
        doc = json.load(fd)
        evt_posi_mapping = Counter()
        # --
        # add (certain) efs
        ef_all_dict_mappings = {}  # ef-id -> ef (all)
        ef_dict_mappings = {}  # ef-id -> ef
        ef_csr_mappings = {}  # ef-id -> csr_ef
        for ef in doc["entity_mentions"] + doc["fillers"]:
            ef_all_dict_mappings[ef["id"]] = ef
            if ef["type"].startswith("ldcOnt:") or ef["type"].startswith("aida:"):
                ef_dict_mappings[ef["id"]] = ef  # store for later adding
        # --
        # add events
        evt_csr_mappings = {}  # event-id -> csr_evm
        for evt in doc["event_mentions"]:
            if evt["type"].startswith("rel:"):
                continue  # these will be added later
            if not (evt["type"].startswith("ldcOnt:") or evt["type"].startswith("aida:")):
                logging.info(f"Skipping evt for {evt['type']}({evt['extra_info']['posi']['text']})")
                continue
            extra_info = evt["extra_info"]
            span_info = extra_info["posi"]
            # add one
            csr_evm = csr.add_event_mention(
                span_info["head_span"], span_info["span"], span_info["text"], evt["type"],
                realis=evt["realis"], component=extra_info["component"], score=math.exp(evt["score"]),
                extent_text=span_info.get('extra_text', None), extent_span=span_info.get('extra_span', None)
            )
            # span_key = tuple(span_info["span"])
            # evt_posi_mapping[span_key] += 1
            # if evt_posi_mapping[span_key] > 1:
            #     logging.warning("Same posi-span evt inside zie!!")
            # add args
            if csr_evm is not None:
                evt_csr_mappings[evt["id"]] = csr_evm
            else:
                logging.warning(f"Adding evt failed for {evt['type']}({span_info['text']})")
        # --
        # add args
        for evt in doc["event_mentions"]:
            csr_evm = evt_csr_mappings.get(evt["id"])
            if csr_evm is None:
                continue  # adding failed
            for arg in evt["em_arg"]:
                if not (arg["role"].startswith("ldcOnt:") or arg["role"].startswith("aida:")):
                    logging.info(f"Skipping evt arg for {arg['role']}({arg['extra_info']['posi']['text']})")
                    continue
                # do we want to add things that might not be in ontology?
                # if not arg["role"].startswith("ldcOnt:"):
                #     continue
                arg_extra_info = arg["extra_info"]
                arg_span_info = arg_extra_info["posi"]
                if arg_extra_info.get("is_evt2evt_link", False):
                    # directly use event's adding method to add another event as arg
                    arg_evm = evt_csr_mappings.get(arg["aid"])
                    if arg_evm is not None:
                        arg_onto, slot_type = arg["role"].split(":", 1)
                        arg_id = csr.get_id('arg')
                        csr_arg = csr_evm.add_arg(arg_onto, slot_type, arg_evm, arg_id,
                                                  component=arg_extra_info["component"], score=math.exp(arg["score"]))
                    else:
                        csr_arg = None
                else:
                    # add ent only if used as arg
                    arg_ef = ef_dict_mappings.get(arg["aid"])
                    if arg_ef is not None:
                        if arg["aid"] not in ef_csr_mappings:  # no repeated adding
                            ent = add_ef(csr, arg_ef)
                            if ent is not None:  # set for later use
                                ef_csr_mappings[arg["aid"]] = ent
                    # add arg
                    csr_arg = csr.add_event_arg_by_span(
                        csr_evm, arg_span_info["head_span"], arg_span_info["span"], arg_span_info["text"], arg["role"],
                        component=arg_extra_info["component"], score=math.exp(arg["score"])
                    )
                if csr_arg is None:
                    logging.warning(f"Adding evt arg failed for {arg['role']}({arg_span_info['text']})")
        # --
        # add special rels
        for rel in doc["event_mentions"]:
            if not rel["type"].startswith("rel:"):
                continue
            try:  # to be safe ...
                extra_info = rel["extra_info"]
                span_info = extra_info["posi"]
                rel_type = "ldcOnt:" + rel["type"].split(":")[-1]
                rel_arg_ids, rel_arg_roles = [], []
                for arg in rel["em_arg"]:
                    if not (arg["role"].startswith("rel:") or arg["role"].startswith("aida:")):
                        continue
                    # find the arg
                    arg_extra_info = arg["extra_info"]
                    if arg_extra_info.get("is_evt2evt_link", False):
                        one_arg = evt_csr_mappings.get(arg["aid"])
                    else:
                        arg_ef = ef_all_dict_mappings.get(arg["aid"])
                        one_arg = ef_csr_mappings.get(arg["aid"])
                        if one_arg is None and arg_ef is not None:
                            one_arg = add_ef(csr, arg_ef, check_type=True)
                            if one_arg is not None:
                                ef_csr_mappings[arg["aid"]] = one_arg
                    # skip if not added
                    if one_arg is None:
                        logging.warning(f"Adding rel_arg failed for {arg})")
                        continue
                    # put it
                    arg_role = arg['role'] if arg["role"].startswith("aida:") else f"{rel_type}_{arg['role'].split(':')[-1]}"
                    rel_arg_ids.append(one_arg.id)
                    rel_arg_roles.append(arg_role)
                # finally checking
                if len(rel_arg_ids) >= 2:  # allow extra ones for time
                    # finally adding
                    csr_rel = csr.add_relation(
                        arguments=rel_arg_ids, arg_names=rel_arg_roles,
                        component=extra_info.get("component"), span=span_info["span"],
                        score=math.exp(rel["score"]),
                    )
                    if csr_rel:
                        csr_rel.add_type(rel_type)
                else:
                    logging.warning(f"Adding rel failed (invalid num of args) for {rel})")
            except:
                logging.warning(f"Adding rel failed (some except?) for {rel})")
        # --

# =====
# possible ways to modify "combine_runner.py"
"""
# adding a config option for the dir of zie_event: config.zie_event
# after "if config.conllu_folder:" and before "if config.rich_event:", insert these
# =====
        if config.zie_event:
            if os.path.exists(config.zie_event):
                zie_event_file = find_by_id(config.zie_event, docid)
                if zie_event_file:
                    logging.info("Adding events with zie output: {}".format(zie_event_file))
                    zie_integration_utils.add_zie_event(zie_event_file, csr)
                else:
                    logging.error(f"Cannot find zie output for {docid}")
            else:
                logging.info("No zie event output.")
# =====
"""

# =====
# how to move these to _export?
# msp/ -> msp/; tasks/zie/{common,main,models2,models3,opera,tools} -> tasks/zie/; tasks/zie/csr/* -> tasks/zie/csr/(setup)
# fix previous eval bug
# sed -i 's/ldcOnt:PER:/ldcOnt:PER./g' *
# run combine_runner: at csr dir
# output_base=../hector_out/english/ csr_resources=../csr_resources/ PYTHONPATH=./ python event/mention/combine_runner.py conf/aida/combine.py  --CombineParams.csr_output=../zout_csr_norich/ --CombineParams.rich_event=NOPE
# output_base=../hector_out/english/ csr_resources=../csr_resources/ PYTHONPATH=./ python event/mention/combine_runner.py conf/aida/combine.py  --CombineParams.csr_output=../zout_csr_rich/
# visualization
# python -m event.io.csr2stuff ../zout_csr_norich/ ../zout_visual_norich/ ../csr_resources/LDCOntologyM36.jsonld


import torch.nn as nn
import torch.nn.functional as F

def get_feature_extractor(backbone_type: str, input_dim):
    if backbone_type == 'MLP':
        return nn.ModuleList([
            nn.Dropout(0.1),
            nn.Linear(input_dim, 1000),   # hidden = [1000, 500, 100]
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(1000, 500),
            nn.ReLU(),
            # nn.Dropout(0.5),
            # nn.Linear(500, 100),
            # nn.ReLU()
        ])
    elif backbone_type == 'LSTM':
        return nn.ModuleList([
            nn.LSTM(input_size=input_dim, hidden_size=1000, batch_first=True),
            nn.Dropout(0.3),
            nn.LSTM(input_size=1000, hidden_size=500, batch_first=True),
            nn.Dropout(0.3),
            nn.LSTM(input_size=500, hidden_size=100, batch_first=True)
        ])
    else:
        raise NotImplementedError

def get_predictor(num_classes):
    return nn.ModuleList([
        nn.Dropout(0.1),
        nn.Linear(500, 100),
        nn.ReLU(),
        nn.Dropout(0.1),
        nn.Linear(100, num_classes),
    ])


import argparse

def get_arguments():
    parser = argparse.ArgumentParser(description=("MIDI music generator is a script for generating "
                                                  "simple MIDI music with different opions"))
    parser.add_argument('--narcossity', '-n', type=int, choices=range(1, 5), dest='narcossity_level', default=1,
                        help="Change the level of narcossity.")
    parser.add_argument('-p', "--path", type=str, dest="fileName", default="myMidi.mid",
                        help=("Sets the path of the generated midi file. Note, that the name "
                              "should end witch .mid. If only a name is given the file is created in"
                              "working directory"))
    parser.add_argument('-k', '--key', type=int, dest='key', default=7,
                        help="Set the base key for whole midi file. The KEY should be"
                             "between 0 and 12.")
    parser.add_argument('-b', '--beats', type=int, dest='beats_per_bar', default=16,
                        help="Set the number of beats per bar. BEATS_PER_BAR should be between 4 and 64")
    parser.add_argument('-B', '--bars', type=int, dest='number_of_bars', default=16,
                        help="Set the number of bars = length of the music")
    parser.add_argument('-j', '--jazz', dest='genre', action='store_const', const=1,
                        help='Sets genre type to jazz. Setting flag -j is similar to setting: '
                             '-n 2 -b 20')
    parser.add_argument('-s', '--samba', dest='genre', action='store_const', const=2,
                        help='Sets genre type to samba. Setting flag -s is similar to setting: '
                             '-n 4 -b 16')

    arguments = parser.parse_args()

    if arguments.key not in range(0, 12) or arguments.beats_per_bar not in range(4, 65):
        print("Bad arguments, type -h or --help to see help message")
        exit(1)

    if arguments.genre == 1:
        arguments.narcossity_level = 2
        arguments.beats_per_bar = 20
    elif arguments.genre == 2:
        arguments.narcossity_level = 4
        arguments.beats_per_bar = 16

    return arguments


from collections import deque

FILENAME = "input.txt"
LOOK_BACK = 3

def filter_empty(item: str) -> bool:
    return len(item) == 0

if __name__ == "__main__":

    previous = None
    increase = 0
    window = deque()

    with open(FILENAME, "r") as fid:
        while True:
            text = fid.readline().strip()
            try:
                # empty string at last position
                line = int(text)
            except ValueError:
                break
            window.append(line)
            if len(window) > LOOK_BACK:
                previous = window.popleft()
            try:
                if line > previous:
                    increase += 1
            except TypeError:
                pass

    print("Total count", increase)


import logging
import os
import shutil

class PyTestLogger:

    session_log_dir_name = "Pytest_logs"
    workspace = "C:\\dev"
    log_source_dir = "pytest_project"
    log_destination_root_path = "C:\\dev\\Log"
    log_level_name_list = ["debug", "info", "warning", "error", "critical"]

    def __init__(self, log_file_name, log_level_list):
        """ Initialize self.  See help(type(self)) for accurate signature. """

        self.logger = logging.getLogger(log_file_name)
        self.logger.setLevel(logging.DEBUG)
        self.log_file_name = log_file_name
        self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s: %(message)s')
        self.log_level_list = log_level_list
        self.in_progress_flag = False

        if self.log_level_list.count(logging.DEBUG):
            self.file_handler_debug = logging.FileHandler(self.log_file_name + "_" +
                                                          PyTestLogger.log_level_name_list[0] + ".log", mode="w")
            self.file_handler_debug.setLevel(logging.DEBUG)

        if self.log_level_list.count(logging.INFO):
            self.file_handler_info = logging.FileHandler(self.log_file_name + "_" +
                                                         PyTestLogger.log_level_name_list[1] + ".log", mode="w")
            self.file_handler_info.setLevel(logging.INFO)

        if self.log_level_list.count(logging.WARNING):
            self.file_handler_warning = logging.FileHandler(self.log_file_name + "_" +
                                                            PyTestLogger.log_level_name_list[2] + ".log", mode="w")
            self.file_handler_warning.setLevel(logging.WARNING)

        if self.log_level_list.count(logging.ERROR):
            self.file_handler_error = logging.FileHandler(self.log_file_name + "_" +
                                                          PyTestLogger.log_level_name_list[3] + ".log", mode="w")
            self.file_handler_error.setLevel(logging.ERROR)

        if self.log_level_list.count(logging.CRITICAL):
            self.file_handler_critical = logging.FileHandler(self.log_file_name + "_" +
                                                             PyTestLogger.log_level_name_list[4] + ".log", mode="w")
            self.file_handler_critical.setLevel(logging.CRITICAL)

        self.update_formatter()

    @classmethod
    def set_session_log_dir_name(cls, dir_name):  # real signature unknown
        cls.session_log_dir_name = dir_name

    def set_formatter(self, formatter):
        if not self.in_progress_flag:
            self.formatter = logging.Formatter(formatter)
            self.update_formatter()
        else:
            raise Exception("Failed to set formatter! Logging is in progress! Please call this function before "
                            "start_logger() or after stop_logger()!")

    def update_formatter(self):
        if self.log_level_list.count(logging.DEBUG):
            self.file_handler_debug.setFormatter(self.formatter)
        if self.log_level_list.count(logging.INFO):
            self.file_handler_info.setFormatter(self.formatter)
        if self.log_level_list.count(logging.WARNING):
            self.file_handler_warning.setFormatter(self.formatter)
        if self.log_level_list.count(logging.ERROR):
            self.file_handler_error.setFormatter(self.formatter)
        if self.log_level_list.count(logging.CRITICAL):
            self.file_handler_critical.setFormatter(self.formatter)

    def start_logger(self):
        self.in_progress_flag = True
        if self.log_level_list.count(logging.DEBUG):
            self.logger.addHandler(self.file_handler_debug)
        if self.log_level_list.count(logging.INFO):
            self.logger.addHandler(self.file_handler_info)
        if self.log_level_list.count(logging.WARNING):
            self.logger.addHandler(self.file_handler_warning)
        if self.log_level_list.count(logging.ERROR):
            self.logger.addHandler(self.file_handler_error)
        if self.log_level_list.count(logging.CRITICAL):
            self.logger.addHandler(self.file_handler_critical)

    def stop_logger(self):

        if self.log_level_list.count(logging.DEBUG):
            self.file_handler_debug.close()
            self.logger.removeHandler(self.file_handler_debug)
        if self.log_level_list.count(logging.INFO):
            self.file_handler_info.close()
            self.logger.removeHandler(self.file_handler_info)
        if self.log_level_list.count(logging.WARNING):
            self.file_handler_warning.close()
            self.logger.removeHandler(self.file_handler_warning)
        if self.log_level_list.count(logging.ERROR):
            self.file_handler_error.close()
            self.logger.removeHandler(self.file_handler_error)
        if self.log_level_list.count(logging.CRITICAL):
            self.file_handler_critical.close()
            self.logger.removeHandler(self.file_handler_critical)

        self.in_progress_flag = False

    def save_log_files(self, test_name, file_name):

        test_suite_dir = file_name
        session_dir = PyTestLogger.session_log_dir_name

        log_source_path = os.path.join(PyTestLogger.workspace, PyTestLogger.log_source_dir)
        log_save_path = os.path.join(PyTestLogger.log_destination_root_path, session_dir, test_suite_dir, test_name)

        log_files_list = list()

        for root, dirs, files in os.walk(log_source_path):
            for file in files:
                for log_level_name in self.log_level_name_list:
                    if ((self.log_file_name + "_" + log_level_name + ".log") == file) or ("witef.log" == file):
                        log_files_list.append(os.path.join(root, file))

        if not os.path.exists(log_save_path):
            os.makedirs(log_save_path)

        for path in log_files_list:
            print(path)
            shutil.copy(path, log_save_path)


import time
import numpy as np
import pandas as pd
from itertools import combinations, product
from sklearn.metrics import accuracy_score
from collections import defaultdict

class RULES:
    def __init__(self, contains_header=True, discretize_mode='equal', number_bins=7, discretize_ints=False):
        self.contains_header = contains_header
        self.discretize_mode = discretize_mode
        self.number_bins = number_bins
        self.discretize_ints = discretize_ints
        self.bins = []

        self.attribute_names = None
        self.preproc_dict = None
        self.labels_dict = None

        self.rules = []
        self.most_probable_class = None
        self.n_attributes = 0

    # ######################### F I T #########################
    def fit(self, x, y, method='RRULES', show_rules=True, show_metrics=True, show_time=True):
        since = time.time()
        x, y = self.__preproc_train_data(x, y)
        preproc_time = time.time() - since

        since = time.time()
        if method == 'RRULES':
            print('Training with RRULES...')
            self.__fit_RRULES(x, y)
        elif method == 'Original':
            print('Training with Original RULES...')
            self.__fit_original_RULES(x, y)
        fit_time = time.time() - since

        since = time.time()
        if show_rules:
            metrics = None
            if show_metrics:
                metrics = self.__compute_metrics(x, y)
                metrics_time = time.time() - since
                since = time.time()
            print('\nInduced rules:')
            print(self.__print_rules(metrics))
            print()
        print_time = time.time() - since

        if show_time:
            print(f"Time to preprocess data = {preproc_time:.2f}s")
            print(f"Time to fit data = {fit_time:.2f}s")
            if show_rules:
                if show_metrics:
                    print(f"Time to comput metrics = {metrics_time:.2f}s")
                print(f"Time to print rules = {print_time:.2f}s")
            print()

    def __fit_RRULES(self, x, y):
        # We calculate the most probable class to create a default rule for unseen combinations of attributes
        classes, counts = np.unique(y, return_counts=True)
        self.most_probable_class = classes[np.argmax(counts)]

        # ##### RRULES #####
        n_examples, n_attributes = x.shape
        self.n_attributes = n_attributes
        # Track non-classified by index
        indices_not_classified = np.arange(n_examples)

        # For each n_conditions = 1, ..., n_attributes
        for n_conditions in range(1, n_attributes + 1):
            # Generate all possible combinations of attributes (without repetition and without order)
            # of length n_conditions
            attribute_combinations_n = combinations(range(n_attributes), n_conditions)
            # For each combination of attributes (columns)
            for attribute_group in attribute_combinations_n:
                lists_of_values = []
                # Calculate the unique values of the chosen attributes from the non-classified instances,
                # and generate all combinations of selectors <attribute, value> given the chosen attributes
                # These combination of selectors form conditions
                for attribute in attribute_group:
                    lists_of_values.append(np.unique(x[indices_not_classified, attribute]))
                value_combinations = product(*lists_of_values)
                # For each condition <att1, val1>, <att2, val2>, ...
                for value_group in value_combinations:
                    # Find indices of ALL INSTANCES that match the condition
                    indices_match = np.where((x[:, list(attribute_group)] == value_group).all(axis=1))[0]
                    # Find indices of NON-CLASSIFIED INSTANCES that match the condition
                    indices_match_not_classified = \
                        np.where((x[np.ix_(indices_not_classified, attribute_group)] == value_group).all(axis=1))[0]
                    if len(indices_match) == 0:
                        # This condition is not present in the training set of examples
                        continue
                    if len(indices_match_not_classified) == 0:
                        # Although this condition is present in the set of examples,
                        # it does not match any non-classified instance
                        # It is the case of a condition that could end generating an IRRELEVANT RULE
                        continue
                    # Take the ground truth of the matched instances and look if they belong to a single class
                    classes = y[indices_match]
                    unique_classes = np.unique(classes)
                    if len(unique_classes) == 1:
                        # Generate the rule and add it to the set of rules
                        # The rule is encoded as the set of attributes to match, their values and the class
                        self.rules.append((attribute_group, value_group, unique_classes[0]))
                        # Remove the classified instances from the set of non-classified ones
                        indices_not_classified = np.setdiff1d(indices_not_classified, indices_match, assume_unique=True)
                        # If there aren't any more instances to classify, return
                        if len(indices_not_classified) == 0:
                            return
                    # If we are in the last iteration, we are checking for the full antecedent
                    # If there is more than a single class, we have a contradiction in the data
                    # Let's choose the most probable class (or random if tie)
                    elif n_conditions == n_attributes:
                        print("WARNING: There are contradictions in the training set")
                        unique_classes, counts = np.unique(classes, return_counts=True)
                        self.rules.append((attribute_group, value_group, unique_classes[np.argmax(counts)]))
                        # Remove the classified instances from the set of non-classified ones
                        indices_not_classified = np.setdiff1d(indices_not_classified, indices_match, assume_unique=True)
                        # If there aren't any more instances to classify, return
                        if len(indices_not_classified) == 0:
                            return

    def __fit_original_RULES(self, x, y):
        # We calculate the most probable class to create a default rule for unseen combinations of attributes
        classes, counts = np.unique(y, return_counts=True)
        self.most_probable_class = classes[np.argmax(counts)]

        # ##### RULES #####
        n_examples, n_attributes = x.shape
        self.n_attributes = n_attributes
        # Track non-classified by index
        indices_not_classified = np.arange(n_examples)

        # For each n_conditions = 1, ..., n_attributes
        for n_conditions in range(1, n_attributes + 1):
            # Check stopping condition only at the beginning of every outer iteration
            if len(indices_not_classified) == 0:
                return
            # Generate all combinations of selectors of length n_conditions
            conditions = []
            # Generate all possible combinations of attributes (without repetition and without order)
            # of length n_conditions
            attribute_combinations_n = combinations(range(n_attributes), n_conditions)
            # For each combination of attributes (columns)
            for attribute_group in attribute_combinations_n:
                lists_of_values = []
                # Calculate the unique values of the chosen attributes, and generate
                # all combinations of selectors <attribute, value> given the chosen attributes
                # These combination of selectors form conditions
                for attribute in attribute_group:
                    lists_of_values.append(np.unique(x[indices_not_classified, attribute]))
                value_combinations = product(*lists_of_values)
                for value_group in value_combinations:
                    conditions.append((attribute_group, value_group))

            # For each condition <att1, val1>, <att2, val2>, ...
            for attribute_group, value_group in conditions:
                # Find indices of ALL INSTANCES that match the condition
                indices_match = np.where((x[:, list(attribute_group)] == value_group).all(axis=1))[0]
                if len(indices_match) == 0:
                    # This condition is not present in the training set of examples
                    continue
                # Take the ground truth of the matched instances and look if they belong to a single class
                classes = y[indices_match]
                unique_classes = np.unique(classes)
                if len(unique_classes) == 1:
                    # Check for irrelevant conditions
                    is_irrelevant = False
                    if n_conditions > 1:
                        for rule in self.rules:
                            # If there is a previous rule (with less conditions) that includes all the selectors for
                            # the new rule, this becomes irrelevant as it does not classify any new instance
                            if all(selector in zip(attribute_group, value_group) for selector in zip(rule[0], rule[1])):
                                is_irrelevant = True
                                break
                    if not is_irrelevant:
                        # Generate the rule and add it to the set of rules
                        # The rule is encoded as the set of attributes to match, their values and the class
                        self.rules.append((attribute_group, value_group, unique_classes[0]))
                        # Remove the classified instances from the set of non-classified ones
                        indices_not_classified = np.setdiff1d(indices_not_classified, indices_match, assume_unique=True)

                # If we are in the last iteration, we are checking for the full antecedent
                # If there is more than a single class, we have a contradiction in the data
                # Let's choose the most probable class (or random if tie)
                # There won't be irrelevant conditions here because it is the last iteration and we have
                # non-classified instances!
                elif n_conditions == n_attributes:
                    print("WARNING: There are contradictions in the training set")
                    unique_classes, counts = np.unique(classes, return_counts=True)
                    self.rules.append((attribute_group, value_group, unique_classes[np.argmax(counts)]))
                    # Remove the classified instances from the set of non-classified ones
                    indices_not_classified = np.setdiff1d(indices_not_classified, indices_match, assume_unique=True)

    # ######################### P R E D I C T #########################
    def predict(self, x):
        # Predict
        y_pred = self.__predict(x)
        # Predictions are integers, convert back to original values
        y_pred = np.vectorize(self.labels_dict[-1].get)(y_pred)
        return y_pred

    def score(self, x, y):
        y_pred = self.__predict(x)
        # Convert true class to integers (predictions already are)
        y = np.vectorize(self.preproc_dict[-1].get)(y)
        return accuracy_score(y, y_pred)

    def __predict(self, x):
        print('Predicting...')
        x = self.__preproc_test_data(x)
        y_pred = []
        # For each instance
        for instance in x:
            classified = False
            # For each rule
            for attributes, values, tag in self.rules:
                # Check if antecedent matches
                if np.array_equal(instance[list(attributes)], values):
                    y_pred.append(tag)
                    classified = True
                    break
            # No rule matched, we apply the default rule --> Most probable class
            if not classified:
                print("WARNING: There are unseen combinations")
                y_pred.append(self.most_probable_class)
        return np.array(y_pred)

    # ################ M E T R I C S   A N D   R U L E S ################
    def compute_metrics(self, x, y):
        # Use the already computed (training) bins and integer conversions
        x = self.__preproc_test_data(x)
        y = np.vectorize(self.preproc_dict[-1].get)(y)
        return self.__compute_metrics(x, y)

    def __compute_metrics(self, x, y):
        n_examples = x.shape[0]

        # Store (Coverage, Precision) for each rule
        metrics = []
        overall_coverage = []
        overall_precision = []
        for attributes, values, tag in self.rules:
            indices_match_condition = np.where((x[:, list(attributes)] == values).all(axis=1))[0]
            coverage = len(indices_match_condition) / n_examples
            indices_match_rule = np.where(y[indices_match_condition] == tag)[0]
            precision = len(indices_match_rule) / len(indices_match_condition)
            overall_precision.append(precision)
            overall_coverage.append(coverage)
            metrics.append((coverage, precision))
        # Add overall metrics
        metrics.append((sum(overall_coverage), sum(overall_precision) / len(overall_precision)))

        return metrics

    def __print_rules(self, metrics=None):
        # Set default attribute names if not present in dataset
        if self.attribute_names is None:
            attribute_names = [f"Attribute_{i + 1}" for i in range(self.n_attributes)]
            attribute_names.append("Class")
            self.attribute_names = attribute_names

        rule_strings = []
        for i in range(len(self.rules)):
            attributes, values, tag = self.rules[i]
            rule = f"Rule {i + 1:3}.  IF " \
                   f"{' AND '.join([f'{self.attribute_names[attributes[j]]} IS {self.labels_dict[attributes[j]][values[j]]}' for j in range(len(attributes))])}" \
                   f" THEN {self.attribute_names[-1]} IS {self.labels_dict[-1][tag]}"

            if metrics:
                rule = f"{rule:100}    Coverage = {100 * metrics[i][0]:5.2f}%    Precision = " \
                       f"{100 * metrics[i][1]:5.2f}%"
            rule_strings.append(rule)
        if metrics:
            overall = f"Overall Coverage = {100 * metrics[-1][0]:5.2f}%     Overall Precision = {100 * metrics[-1][1]:5.2f}%"
            rule_strings.append(overall)
        return '\n'.join(rule_strings)

    # ############### P R E P R O C E S S I N G ###############
    def __preproc_train_data(self, x, y):
        # Set attribute names for pretty printing
        if self.contains_header:
            names_x = x.columns.values.tolist()
            name_y = y.columns.values.tolist()
            self.attribute_names = names_x + name_y

        column_types = x.dtypes
        # Missing Values
        for attribute, dtype in zip(x, column_types):
            # We take the mean for floats
            if np.issubdtype(dtype, np.floating):
                x.loc[:, attribute].fillna(x[attribute].mean(), inplace=True)
            # We take the mode for categoricals (including integers)
            else:
                # Intermediate conversion into '?' (to join different representations for mv)
                x.loc[:, attribute].fillna('?', inplace=True)
                uniques, counts = np.unique(x[attribute], return_counts=True)
                mode = uniques[np.argmax(counts)]
                if mode == '?':
                    mode = uniques[np.argsort(counts)[-2]]
                x.loc[x[attribute] == '?', attribute] = mode

        # Discretize Numeric attributes
        # Store exact discretization bins for test data
        if self.number_bins != 0:
            to_discretize = np.number if self.discretize_ints else np.floating
            for attribute, dtype in zip(x, column_types):
                if np.issubdtype(dtype, to_discretize):
                    if self.discretize_mode == 'equal':
                        x[attribute], bins = pd.cut(x[attribute], bins=self.number_bins, retbins=True)
                        self.bins.append((attribute, bins, np.unique(x[attribute])))
                    elif self.discretize_mode == 'freq':
                        x[attribute], bins = pd.qcut(x[attribute], q=self.number_bins, retbins=True)
                        self.bins.append((attribute, bins, np.unique(x[attribute])))
                    else:
                        raise ValueError("Wrong discretize_mode")

        # Move everything to integer, so numpy works faster
        x = x.to_numpy()
        y = y.to_numpy()
        data = np.concatenate((x, y), axis=1)
        _, n_cols = data.shape

        # Store conversion from original values to integers for pretty printing
        inv_conversions = []
        conversions = []
        for i in range(n_cols):
            col = data[:, i]
            uniques = np.unique(col).tolist()
            d = defaultdict(lambda: -1, zip(uniques, range(len(uniques))))
            d_inv = dict(zip(range(len(uniques)), uniques))
            data[:, i] = np.vectorize(d.get)(col)
            conversions.append(d)
            inv_conversions.append(d_inv)

        # Preprocessing Dictionary
        # Contains all the conversions from original values to integers --> To be used when preprocessing test data
        self.preproc_dict = conversions
        # Labels Dictionary
        # Contains all the conversions from integers to original values --> To be used when pretty printing
        self.labels_dict = inv_conversions

        return data[:, :-1].astype(np.uint8), data[:, -1].astype(np.uint8)

    def __preproc_test_data(self, x):
        # Preprocess only attributes (not class)
        # Use the same exact steps than in training
        #   MV
        #   Discretization using same bins
        #   Conversion to integers using same mapping

        column_types = x.dtypes
        # Missing Values
        for attribute, dtype in zip(x, column_types):
            # We take the mean for floats
            if np.issubdtype(dtype, np.floating):
                x.loc[:, attribute].fillna(x[attribute].mean(), inplace=True)
            # We take the mode for categoricals (including integers)
            else:
                # Intermediate conversion into '?' (to join different representations for mv)
                x.loc[:, attribute].fillna('?', inplace=True)
                uniques, counts = np.unique(x[attribute], return_counts=True)
                mode = uniques[np.argmax(counts)]
                if mode == '?':
                    mode = uniques[np.argsort(counts)[-2]]
                x.loc[x[attribute] == '?', attribute] = mode

        # Discretize Numeric attributes using training bins
        for attribute, bins, labels in self.bins:
            if len(labels) + 1 == len(bins):
                x[attribute] = pd.cut(x[attribute], bins=bins, labels=labels)
            else:
                x[attribute] = pd.cut(x[attribute], bins=bins)

        # Move everything to integer, so numpy works faster
        data = x.to_numpy()
        _, n_cols = data.shape

        # Use original-integer training mapping
        for i in range(n_cols):
            col = data[:, i]
            data[:, i] = np.vectorize(self.__my_vectorized_mapping)(i, col)

        return data.astype(np.uint8)

    def __my_vectorized_mapping(self, i, x):
        return self.preproc_dict[i][x]


import os
import html
import time
import asyncio

from amiyabot import QQGuildBotInstance
from amiyabot.builtin.message import MessageStructure

from core.database.group import GroupSetting
from core.database.messages import *
from core.util import TimeRecorder, AttrDict
from core import send_to_console_channel, Message, Chain, AmiyaBotPluginInstance, bot as main_bot

from .helper import WeiboUser

curr_dir = os.path.dirname(__file__)

class WeiboPluginInstance(AmiyaBotPluginInstance):
    ...

bot = WeiboPluginInstance(
    name='明日方舟微博推送',
    version='2.8',
    plugin_id='amiyabot-weibo',
    plugin_type='official',
    description='在明日方舟相关官微更新时自动推送到群',
    document=f'{curr_dir}/README.md',
    instruction=f'{curr_dir}/README_USE.md',
    global_config_schema=f'{curr_dir}/config_schema.json',
    global_config_default=f'{curr_dir}/config_default.yaml',
)

@table
class WeiboRecord(MessageBaseModel):
    user_id: int = IntegerField()
    blog_id: str = CharField()
    record_time: int = IntegerField()

async def send_by_index(index: int, weibo: WeiboUser, data: MessageStructure):
    result = await weibo.get_weibo_content(index - 1)

    if not result:
        return Chain(data).text('博士...暂时无法获取微博呢...请稍后再试吧~')
    else:
        chain = (
            Chain(data)
            .text(result.user_name + '\n')
            .text(html.unescape(result.html_text) + '\n')
            .image(result.pics_list)
        )

        if not isinstance(data.instance, QQGuildBotInstance):
            chain.text(f'\n\n{result.detail_url}')

        return chain

def get_index_from_text(text: str, array: list):
    r = re.search(r'(\d+)', text)
    if r:
        index = abs(int(r.group(1))) - 1
        if index >= len(array):
            index = len(array) - 1

        return index

@bot.on_message(group_id='weibo', keywords=['开启微博推送'])
async def _(data: Message):
    if not data.is_admin:
        return Chain(data).text('抱歉，微博推送只能由管理员设置')

    channel: GroupSetting = GroupSetting.get_or_none(group_id=data.channel_id, bot_id=data.instance.appid)
    if channel:
        GroupSetting.update(send_weibo=1).where(
            GroupSetting.group_id == data.channel_id,
            GroupSetting.bot_id == data.instance.appid,
        ).execute()
    else:
        if GroupSetting.get_or_none(group_id=data.channel_id):
            GroupSetting.update(bot_id=data.instance.appid, send_weibo=1).where(
                GroupSetting.group_id == data.channel_id
            ).execute()
        else:
            GroupSetting.create(group_id=data.channel_id, bot_id=data.instance.appid, send_weibo=1)

    return Chain(data).text('已在本群开启微博推送')

@bot.on_message(group_id='weibo', keywords=['关闭微博推送'])
async def _(data: Message):
    if not data.is_admin:
        return Chain(data).text('抱歉，微博推送只能由管理员设置')

    GroupSetting.update(send_weibo=0).where(
        GroupSetting.group_id == data.channel_id, GroupSetting.bot_id == data.instance.appid
    ).execute()

    return Chain(data).text('已在本群关闭微博推送')

@bot.on_message(group_id='weibo', keywords=['微博'])
async def _(data: Message):
    listens: list = bot.get_config('listen')
    setting = AttrDict(bot.get_config('setting'))

    if len(listens) == 1:
        weibo = WeiboUser(listens[0]['uid'], setting)
    else:
        md = '回复序号选择已关注的微博：\n\n|序号|微博ID|备注|\n|----|----|----|\n'
        for index, item in enumerate(listens):
            md += '|{index}|{uid}|{name}|\n'.format(index=index + 1, **item)

        wait = await data.wait(Chain(data).markdown(md))
        if not wait:
            return None

        index = get_index_from_text(wait.text_digits, listens)
        if index is None:
            return None

        weibo = WeiboUser(listens[index]['uid'], setting)

    message = data.text_digits
    index = 0

    r = re.search(r'(\d+)', message)
    if r:
        index = abs(int(r.group(1)))

    if '最新' in message:
        index = 1

    if index:
        return await send_by_index(index, weibo, data)
    else:
        blog_list = await weibo.get_blog_list()
        user_name = await weibo.get_user_name()

        if not blog_list:
            return Chain(data).text('博士...暂时无法获取微博列表呢...请稍后再试吧~')

        md = f'博士，这是【{user_name}】的微博列表，回复【序号】来获取详情吧\n\n|序号|日期|内容|\n|----|----|----|\n'
        for item in blog_list:
            md += '|{index}|{date}|{content}|\n'.format(**item)

        reply = Chain(data).markdown(md)

        wait = await data.wait(reply)
        if wait:
            r = re.search(r'(\d+)', wait.text_digits)
            if r:
                index = abs(int(r.group(1)))
                return await send_by_index(index, weibo, wait)

@bot.timed_task(each=30)
async def _(_):
    listens: list = bot.get_config('listen')
    for listen in listens:
        user = listen['uid']
        weibo = WeiboUser(user, AttrDict(bot.get_config('setting')))
        new_id = await weibo.get_weibo_id(0)
        if not new_id:
            continue

        record = WeiboRecord.get_or_none(blog_id=new_id)
        if record:
            continue

        WeiboRecord.create(user_id=user, blog_id=new_id, record_time=int(time.time()))

        target: List[GroupSetting] = GroupSetting.select().where(GroupSetting.send_weibo == 1)

        if not target:
            continue

        time_rec = TimeRecorder()
        async_send_tasks = []

        result = await weibo.get_weibo_content(0)

        if not result:
            await send_to_console_channel(Chain().text(f'微博获取失败\nUSER: {user}\nID: {new_id}'))
            return

        send = True
        for regex in bot.get_config("block"):
            if re.match(regex, html.unescape(result.html_text)):
                await send_to_console_channel(Chain().text(f'微博正文触发正则屏蔽，跳过推送\nUSER: {user}\nID: {new_id}'))
                send = False
                break
            if re.search(regex, html.unescape(result.html_text)):
                await send_to_console_channel(Chain().text(f'微博正文触发搜索屏蔽，跳过推送\nUSER: {user}\nID: {new_id}'))
                send = False
                break

        if not send:
            continue

        await send_to_console_channel(
            Chain().text(f'开始推送微博\nUSER: {result.user_name}\nID: {new_id}\n目标数: {len(target)}')
        )

        for item in target:
            data = Chain()

            instance = main_bot[item.bot_id]

            if not instance:
                continue

            data.text(f'来自 {result.user_name} 的最新微博\n\n{html.unescape(result.html_text)}')

            if isinstance(instance.instance, QQGuildBotInstance):
                if not instance.private:
                    for url in result.pics_urls:
                        data.image(url=url)
                else:
                    data.image(result.pics_list)
            else:
                data.image(result.pics_list).text(f'\n\n{result.detail_url}')

            if bot.get_config('sendAsync'):
                async_send_tasks.append(instance.send_message(data, channel_id=item.group_id))
            else:
                await instance.send_message(data, channel_id=item.group_id)
                await asyncio.sleep(bot.get_config('sendInterval'))

        if async_send_tasks:
            await asyncio.wait(async_send_tasks)

        await send_to_console_channel(Chain().text(f'微博推送结束:\n{new_id}\n耗时{time_rec.total()}'))


import cv2
import numpy as np
import csv

import sys, os
from tqdm import *

from os import listdir
from os.path import isfile, join

import argparse
import logging

import cPickle as pickle

AGES = ['child', 'young', 'adult', 'elder']
GENDERS = ['male', 'female']

IMG_DIM = (224, 224)

class Dataset():
    def load_labeled_data(self, raw_data_path):
        X = []
        T = []
        for g_i in range(len(GENDERS)):
            for a_i in range(len(AGES)):
                age = AGES[a_i]
                gender = GENDERS[g_i]
                label = g_i
                label = np.array([label], dtype=np.int32)[np.newaxis, :]

                path =  join(raw_data_path, age, gender)
                onlyfiles = [f for f in listdir(path) if isfile(join(path, f))]
                onlyfiles.sort(key=lambda f: int(filter(str.isdigit, f)))

                for i in tqdm(range(len(onlyfiles))):
                    img = (cv2.imread(path + '/' + onlyfiles[i]))
                    img = cv2.resize(img, IMG_DIM)
                    X.append(img[np.newaxis, :])
                    T.append(label)
        X = np.asarray(np.concatenate(X))
        T = np.asarray(np.concatenate(T))
        return X, T

    def load_unlabeled_data(self, raw_data_path):
        X = []
        F = []
        for g_i in range(len(GENDERS)):
            for a_i in range(len(AGES)):
                age = AGES[a_i]
                gender = GENDERS[g_i]
                label = g_i
                label = np.array([label], dtype=np.int32)[np.newaxis, :]

                path =  join(raw_data_path, age, gender)
                onlyfiles = [f for f in listdir(path) if isfile(join(path, f))]
                onlyfiles.sort(key=lambda f: int(filter(str.isdigit, f)))

                for i in tqdm(range(len(onlyfiles))):
                    img = cv2.cvtColor(cv2.imread(join(path, onlyfiles[i]), cv2.COLOR_RGB2GRAY))
                    img = cv2.resize(img, IMG_DIM)
                    X.append(img[np.newaxis, :])
                    F.append(onlyfiles[i])
        X = np.asarray(np.concatenate(X))
        return X, F

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--type', help='load labeled/unlabeled data',
            choices=['label', 'unlabel'], type=str, default='label')
    parser.add_argument('--input', help='input', type=str, default='data')
    parser.add_argument('--output', help='output', type=str, default='data')

    args = parser.parse_args()
    dataset = Dataset()

    if args.type == 'label':
        X, T = dataset.load_labeled_data(args.input)
        np.save(join(args.output, 'X'), X)
        np.save(join(args.output, 'T'), T)
    elif args.type == 'unlabel':
        X, F = dataset.load_unlabeled_data(args.input)
        np.save(join(args.output, 'X'), X)
        pickle.dump(F, open(join(args.input, 'X.index'), "wb"), True)