import pandas as pd
import numpy as np

import sys

from sklearn.metrics import mean_absolute_error
sys.path.append(sys.path[0][0:-5])
PATH = sys.path[-1]
# Estimate
X_train_estimated_a: pd.DataFrame = pd.read_parquet(
    PATH+'/A/X_train_estimated.parquet')
X_train_estimated_b: pd.DataFrame = pd.read_parquet(
    PATH+"/B/X_train_estimated.parquet")
X_train_estimated_c: pd.DataFrame = pd.read_parquet(
    PATH+"/C/X_train_estimated.parquet")

# Test estimates
X_test_estimated_a: pd.DataFrame = pd.read_parquet(
    PATH+"/A/X_test_estimated.parquet")
X_test_estimated_b: pd.DataFrame = pd.read_parquet(
    PATH+"/B/X_test_estimated.parquet")
X_test_estimated_c: pd.DataFrame = pd.read_parquet(
    PATH+"/C/X_test_estimated.parquet")

# Observations
X_train_observed_a: pd.DataFrame = pd.read_parquet(
    PATH+"/A/X_train_observed.parquet")
X_train_observed_b: pd.DataFrame = pd.read_parquet(
    PATH+"/B/X_train_observed.parquet")
X_train_observed_c: pd.DataFrame = pd.read_parquet(
    PATH+"/C/X_train_observed.parquet")

# Targets
Y_train_observed_a: pd.DataFrame = pd.read_parquet(
    PATH+"/A/train_targets.parquet")
Y_train_observed_b: pd.DataFrame = pd.read_parquet(
    PATH+"/B/train_targets.parquet")
Y_train_observed_c: pd.DataFrame = pd.read_parquet(
    PATH+"/C/train_targets.parquet")

test_df_example = pd.read_csv(PATH+"/test.csv")

best_submission: pd.DataFrame = pd.read_csv(
    PATH+"/mikael/submissions/fourth_submission.csv")

class Pipin:

    def __init__(self):
        pass

    def get_combined_datasets(self, timeSeries=False, randomize=True, data_sets: set = {"A", "B", "C"}):
        # get for location A, B and C and concatinate them
        if not data_sets.issubset({"A", "B", "C"}):
            raise Exception("set must contain A, B or C")

        df_a = self.get_data(
            "A", timeSeries) if "A" in data_sets else pd.DataFrame()
        df_b = self.get_data(
            "B", timeSeries) if "B" in data_sets else pd.DataFrame()
        df_c = self.get_data(
            "C", timeSeries) if "C" in data_sets else pd.DataFrame()

        dataSets = [df_a, df_b, df_c]
        location = ["A", "B", "C"]
        for location, dataset in zip(location, dataSets):
            if dataset.empty:
                continue
            dataset["location:idx"] = location

        if timeSeries:
            # hypotese: The year of the data is not important, and the data can be used from all locations to make a better model
            df_b.index = df_b.index + pd.DateOffset(years=5)
            df_c.index = df_c.index + pd.DateOffset(years=10)
            all_dfs = pd.concat([df_a, df_b, df_c])
            all_dfs["time"] = all_dfs.index
            groupe_for_ts = self.grouped_by_hour_ts(all_dfs)
            return groupe_for_ts
        else:
            df = pd.concat([df_a, df_b, df_c]).reset_index(drop=True)
            if randomize:
                df = df.sample(frac=1).reset_index(drop=True)
            return df

    def add_location_to_datasets(self, dfs: list, locations: list):
        if not locations.issubset({"A", "B", "C"}) or len(dfs) != len(locations):
            raise Exception("set must contain A, B or C")
        for location, dataset in zip(location, dfs):
            if dataset.empty:
                continue
            dataset["location:idx"] = location
        return dfs

    def get_data(self, location: str, timeSeries: bool = False):
        if location == "A":
            Y_train_x = Y_train_observed_a
            X_train_observed_x = X_train_observed_a
            X_train_estimated_x = X_train_estimated_a
        if location == "B":
            X_train_observed_x = X_train_observed_b
            Y_train_x = Y_train_observed_b
            X_train_estimated_x = X_train_estimated_b
        if location == "C":
            X_train_observed_x = X_train_observed_c
            Y_train_x = Y_train_observed_c
            X_train_estimated_x = X_train_estimated_c

        # concatinate the estimated and observed data

        X_train_total = pd.concat(
            [X_train_observed_x, X_train_estimated_x]).reset_index().drop(columns=["index"])
        X_train_total["date_forecast"] = pd.to_datetime(
            X_train_total["date_forecast"])

        # adjust feature data

        X_train_group = self.grouped_by_hour(X_train_total)
        if not timeSeries:
            X_train_group = self.unzip_date_feature(X_train_group)
        # rename the date_forecast column to time to merge with the target data
        X_train_group.rename(columns={"date_forecast": "time"}, inplace=True)

        # clean the traget data
        # Y_train_x = self.remove_consecutive_measurments(
        #     Y_train_x, consecutive_threshold=24)

        # Merge the targets and features and remove bad targets
        merged = pd.merge(X_train_group, Y_train_x, on="time", how="inner")
        mask = merged["pv_measurement"].notna()
        merged = merged.loc[mask].reset_index().drop(columns=["index"])

        # add the location to the data
        # merged["location:idx"] = location # do this when getting combined data

        if timeSeries:
            # make sure all houres are present. Even empty ones
            merged = self.grouped_by_hour_ts(merged)
        return merged

    def get_irrelevant_features(self):
        # "location:idx" # this is relevant when getting combined data and not single location.
        return ["date_calc", "time", "consecutive_count"]

    def get_combined_test_data(self,   time_series: bool = False, data_sets: set = {"A", "B", "C"}):
        if not data_sets.issubset({"A", "B", "C"}):
            raise Exception("set must contain A, B or C")

        df_a = self.get_test_data(
            "A", time_series) if "A" in data_sets else pd.DataFrame()
        df_b = self.get_test_data(
            "B", time_series) if "B" in data_sets else pd.DataFrame()
        df_c = self.get_test_data(
            "C", time_series) if "C" in data_sets else pd.DataFrame()

        dataSets = [df_a, df_b, df_c]
        location = ["A", "B", "C"]
        for location, dataset in zip(location, dataSets):
            if dataset.empty:
                continue
            dataset["location:idx"] = location

        df = pd.concat([df_a, df_b, df_c])

        if not time_series:
            df.reset_index().drop(
                columns=["index"])

        return df

    def get_test_data(self, location: str = None, time_series=False) -> pd.DataFrame:
        if location == "A":
            df = X_test_estimated_a
        elif location == "B":
            df = X_test_estimated_b
        elif location == "C":
            df = X_test_estimated_c
        else:
            raise Exception("location must be A, B or C")

        df = self.grouped_by_hour(df)
        df = self.unzip_date_feature(df)
        df.rename(columns={"date_forecast": "time"}, inplace=True)
        # df = self._shorten_dataset_to_prediction_scale(df, location)
        df = self.scale_after_test_example_csv(df, location)
        df.reset_index().drop(columns=["index"])
        # add the location to the data
        # df["location:idx"] = location

        if time_series:
            df = self.grouped_by_hour_ts(df)
            # df["time"] = df.index
            # df = self.scale_after_test_example_csv(df, location)
            # df.index = df["time"]
            # df.drop(columns=["time"], inplace=True)

        return df

    def scale_after_test_example_csv(self, df: pd.DataFrame, location: str = None):
        test_df = test_df_example
        if location == "A" or location == "B" or location == "C":
            # location is not marked at a idx in the test_df
            test_df = test_df.loc[test_df["location"] == location]

        test_df = test_df[["time"]]
        test_df["time"] = pd.to_datetime(test_df["time"])

        filter_on_time = df.merge(test_df, on="time", how="right")
        return filter_on_time

    def get_categorical_features(self, df: pd.DataFrame):
        categorical_columns = [c for c in df.columns if ":idx" in c]
        return categorical_columns

    def grouped_by_hour(self, df: pd.DataFrame, date_column: str = "date_forecast"):
        def custom_agg_categorical(x):
            if x.empty:
                return np.nan
            else:
                mode_result = x.mode()
                if not mode_result.empty:
                    return mode_result.iloc[0]
                else:
                    return np.nan

        categorical_columns = self.get_categorical_features(df)
        numeric_columns = list(
            set(df.columns) - set(categorical_columns) - {date_column})
        X_train_group = df.groupby(pd.Grouper(key=date_column, freq="1H", )).agg(
            {**{col: 'mean' for col in numeric_columns}, **{col: custom_agg_categorical for col in categorical_columns}}).reset_index()
        return X_train_group

    def grouped_by_hour_ts(self, df: pd.DataFrame):
        # print("df", df.columns)
        categorical_columns = self.get_categorical_features(df)
        numeric_columns = list(
            set(df.columns) - set(categorical_columns) - {"time"})

        df["index"] = pd.to_datetime(df["time"])
        df.drop(columns=["time"], inplace=True)
        df.set_index('index', inplace=True)

        def custom_agg_categorical(x):
            if x.empty:
                return np.nan
            else:
                mode_result = x.mode()
                if not mode_result.empty:
                    return mode_result.iloc[0]
                else:
                    return np.nan

        numeric_df = df[numeric_columns]
        numeric_df.ffill(inplace=True)
        numeric_df = numeric_df.resample("H").mean()

        categorical_df = df[categorical_columns]
        categorical_df = categorical_df.resample(
            "H").apply(custom_agg_categorical)

        merged_df = pd.merge(categorical_df, numeric_df,
                             left_index=True, right_index=True)
        return merged_df

    def unzip_date_feature(self, df: pd.DataFrame, date_column: str = "date_forecast"):
        df["date_forecast"] = pd.to_datetime(df["date_forecast"])
        df["day_of_year"] = df["date_forecast"].dt.day_of_year
        df["hour"] = df["date_forecast"].dt.hour
        df["month"] = df["date_forecast"].dt.month
        return df

    def remove_outliers(self, df: pd.DataFrame, column: str, threshold: float = 3):
        df = df[np.abs(df[column]-df[column].mean())
                <= (threshold*df[column].std())]
        return df

    def remove_consecutive_measurments(self, df: pd.DataFrame, consecutive_threshold: int = 48):
        column_to_check = 'pv_measurement'
        mask = (df[column_to_check] != df[column_to_check].shift(2)).cumsum()

        df['consecutive_count'] = df.groupby(
            mask).transform('count')[column_to_check]
        mask = (df['consecutive_count'] > consecutive_threshold)
        df.drop(columns=["consecutive_count"], inplace=True)
        return df.loc[~mask]

    def find_consecutive_measurements(self, df: pd.DataFrame):
        column_to_check = 'pv_measurement'
        mask = (df[column_to_check] != df[column_to_check].shift()).cumsum()

        df['consecutive_count'] = df.groupby(
            mask).transform('count')[column_to_check]
        start_date = '2020-07-12 15:00:00'
        end_date = '2020-08-26'
        # mask = (df['time'] > start_date) & (df['time'] <= end_date)
        too_hig = 48
        mask = (df['consecutive_count'] > too_hig) & (
            df["pv_measurement"].notna()) & (df["pv_measurement"] > 0)
        df_sub = df.loc[mask]
        df_sub['time'] = pd.to_datetime(df_sub['time'])
        print("sub", df_sub)
        return df_sub

    def compare_mae(self, df: pd.DataFrame):
        best_submission: pd.DataFrame = pd.read_csv(
            PATH+"/mats/submissions/lightgbm.csv")
        best_submission = best_submission[["prediction"]]

        if best_submission.shape != df.shape:
            print("best_submission", best_submission.shape)
            print("df", df.shape)
            raise Exception("Dataframe shape must be the same")

        # calculate mae of the predicitons
        mae = mean_absolute_error(
            best_submission["prediction"], df["prediction"])
        return mae

# pipin = Pipin()
# x = pipin.get_combined_test_data(time_series=True)
# print(x)

# data0 = pipin.get_data("A", timeSeries=True)

# pipin.compare_mae(pd.DataFrame({"prediction": [1,2,3,4,5]}))

# print("df", big_data.head())
# pipin = Pipin()
# test = pipin.get_combined_test_data()
# pipin.get_data("B")


import collections
import itertools
import queue
import threading
import weakref

import numpy as np
import pygame as pg

from .constants import COLORMAP
from .stuff import maze_build, maze_solver
from .util import pim_to_clipboard, pygame_to_pim

class Eventman:
    wsdic = collections.defaultdict(weakref.WeakSet)

    @classmethod
    def broadcast(cls, channel, *args, **kwargs):
        if channel in cls.wsdic:
            for func in cls.wsdic[channel]:
                func(*args, **kwargs)

    @classmethod
    def addlistener(cls, channel, func):
        cls.wsdic[channel].add(func)
        return func

    @classmethod
    def listener(cls, channel):
        def decorator(func):
            cls.wsdic[channel].add(func)
            return func
        return decorator

class Taskman:
    def __init__(self):
        self.tasks = set()

    def add(self, cor):
        self.tasks.add(cor)
        return cor

    def update(self):
        for cor in self.tasks:
            try:
                next(cor)
            except StopIteration:
                self.tasks.remove(cor)

class World:
    keep_running = True

    def __init__(self, cellsize: int, cellshape: tuple[int]):
        self.CS = cellsize
        self.shape = cellshape
        surface = pg.display.get_surface()
        self.cellsurf = surface.copy()
        self.q = queue.Queue(2)
        self.th = threading.Thread(target=self.maze_produce, daemon=True)
        self.th.start()
        self.__listener = [
            Eventman.addlistener(pg.KEYDOWN, self.keydown),
            Eventman.addlistener(pg.QUIT, self.quit),
            Eventman.addlistener(pg.MOUSEMOTION, self.mousemotion),
        ]

    def maze_produce(self):
        while self.keep_running:
            field = np.ones(self.shape, 'uint8')
            start, goal = maze_build(field)
            route1 = list(maze_solver(
                field, start, goal, depthfirst=True))
            route2 = list(maze_solver(field, start, goal))
            if len(route1) != len(route2):
                self.q.put((field, route1, route2))

    def set_field(self, field):
        self.field = field
        surf = self.cellsurf
        surf.fill((0, 0, 0))
        for pos in itertools.product(*map(range, field.shape)):
            v = field[pos]
            self.drawcell(surf, pos, COLORMAP[v])
        self.prev_field = field.copy()

    def update_cellsurf(self):
        field = self.field
        surf = self.cellsurf
        for pos in zip(*np.where(field != self.prev_field)):
            v = field[pos]
            self.drawcell(surf, pos, COLORMAP[v])
        self.prev_field = field.copy()

    def toscreen(self, cellpos):
        CS = self.CS
        CH, CW = self.shape
        y, x = cellpos
        y %= CH
        x %= CW
        return x * CS + CS / 2, y * CS + CS / 2

    def drawcell(self, surface, pos, col, width=0):
        y, x = pos
        CS = self.CS
        x *= CS
        y *= CS
        pg.draw.rect(surface, col, [x, y, CS, CS], width)

    def drawcell2(self, surface, pos, col):
        pg.draw.circle(
            surface, col,
            self.toscreen(pos), self.CS / 3)

    def quit(self, *args):
        self.keep_running = False

    def screenshot(self, ev):
        # pdb.set_trace()
        surface = pg.display.get_surface()
        pim = pygame_to_pim(surface)
        pim_to_clipboard(pim)

    def keydown(self, ev):
        handler = self.keydown.handlerz.get(ev.key)
        if handler is not None:
            handler(self, ev)

    keydown.handlerz = {
        pg.K_ESCAPE: quit,
        pg.K_q: quit,
        pg.K_PRINTSCREEN: screenshot,
    }

    def mousemotion(self, ev):
        return
        # print(ev)
        # ipos = np.array([ev.pos[0], ev.pos[1], 1]) @ self.inv
        # print(ipos)


class Node:
    def __init__(self, data, next = None):
        self.data = data
        self.next = next

def cycleDetection(head):

    current = head
    s = set()

    while current:

        if current in s:
            return True

        s.add(current)

        current = current.next

    return False

if __name__ == '__main__':

    head = None
    for i in reversed(range(5)):
        head = Node(i+1, head)

    head.next.next.next.next.next = head.next.next

    if cycleDetection(head):
        print("Cycle Found")
    else:
        print("Cycle Not Found")


def is_touching(head_pos, tail_pos):
    # overlapping
    if head_pos == tail_pos:
        return True
    # adjacent
    if head_pos[0] == tail_pos[0]:
        return abs(head_pos[1] - tail_pos[1]) == 1
    if head_pos[1] == tail_pos[1]:
        return abs(head_pos[0] - tail_pos[0]) == 1
    # diagonally adjacent
    if abs(head_pos[0] - tail_pos[0]) == 1 and abs(head_pos[1] - tail_pos[1]) == 1:
        return True
    return False

def update_tail(first_part, second_part):
    # head is two steps directly L or R
    if first_part[0] == second_part[0]:
        diff = first_part[1] - second_part[1]
        second_part[1] += 1 if diff > 0 else -1

    # head is two steps directly U or D
    elif first_part[1] == second_part[1]:
        diff = first_part[0] - second_part[0]
        second_part[0] += 1 if diff > 0 else -1
    else:
        tmp = second_part.copy()
        if is_touching(first_part, [second_part[0] + 1, second_part[1] + 1]):
            second_part = [tmp[0] + 1, tmp[1] + 1]
        elif is_touching(first_part, [second_part[0] - 1, second_part[1] - 1]):
            second_part = [tmp[0] - 1, tmp[1] - 1]
        elif is_touching(first_part, [second_part[0] + 1, second_part[1] - 1]):
            second_part = [tmp[0] + 1, tmp[1] - 1]
        elif is_touching(first_part, [second_part[0] - 1, second_part[1] + 1]):
            second_part = [tmp[0] - 1, tmp[1] + 1]

    return second_part

def parse():
    instructions = []
    inp = input()
    while inp:
        instructions.append(inp.split())
        inp = input()
    return instructions

def day9(moves, knots):
    visited = set()
    visited.add((0, 0))

    # this list would contain positions of all parts of the rope
    POSITIONS = [[0,0] for _ in range(knots)]

    for move in moves:
        direction, step = move
        step = int(step)
        for s in range(step):
            if direction == "R":
                POSITIONS[0][1] += 1
            if direction == "L":
                POSITIONS[0][1] -= 1
            if direction == "U":
                POSITIONS[0][0] += 1
            if direction == "D":
                POSITIONS[0][0] -= 1

            for first_index in range(knots - 1):
                first_part = POSITIONS[first_index]
                second_part = POSITIONS[first_index + 1]
                if not is_touching(first_part, second_part):
                    POSITIONS[first_index + 1] = update_tail(first_part, second_part)

            visited.add(tuple(POSITIONS[-1]))
    return len(visited)

moves = parse()
print(day9(moves, 2))
print(day9(moves, 10))


import os
import sys
import pickle
#from nltk.corpus.reader.wordnet import Lemma
import codecs
import networkx as nx
import re
from pprint import pprint
import matplotlib.pyplot as plt
import pandas as pd
import copy
import operator
import collections
import itertools
from colorama import Fore, Back, Style
from sentences import  getSentenceDFTagList, getResultsDFTagList, TXConvert

from nltk.stem.wordnet import WordNetLemmatizer
lmtzr=WordNetLemmatizer()#create a lemmatizer object

import nltk.data
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

def getSourceDestKindForRelation(source, dest, rel, parentConnectionExceptions={}):
    if not parentConnectionExceptions:
        parentConnectionExceptions["txBeComparedTo txNonPred narg source"] = 1.00
        parentConnectionExceptions["txBeFrom txNamed narg source"] = 1.00
        parentConnectionExceptions["txBeLocatedAt txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txBeLocatedAt txNamed narg source"] = 1.00
        parentConnectionExceptions["txBeLocatedAt txNonPred narg source"] = 0.72
        parentConnectionExceptions["txBeLocatedAt txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txBeLocatedAt txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txBeLocatedAt txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txBeTemporallyAt txNonPred narg source"] = 0.60
        parentConnectionExceptions["txBeTemporallyAt txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txBeTemporallyAt txPred narg source"] = 1.00
        parentConnectionExceptions["txFrequentDouble txNamed narg source"] = 0.86
        parentConnectionExceptions["txFrequentDouble txNonPred narg source"] = 0.98
        parentConnectionExceptions["txHaveConcession txDateEntity narg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txHaveConcession txFrequentDouble arg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txHaveCondition arg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txNamed arg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txNamed narg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txNonPred arg source"] = 0.90
        parentConnectionExceptions["txHaveConcession txNonPred narg source"] = 0.93
        parentConnectionExceptions["txHaveConcession txPerson0 narg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txPerson1 arg source"] = 0.67
        parentConnectionExceptions["txHaveConcession txPerson1 narg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txPred narg source"] = 1.00
        parentConnectionExceptions["txHaveConcession txThing1 arg dest"] = 0.75
        parentConnectionExceptions["txHaveCondition txNamed arg source"] = 1.00
        parentConnectionExceptions["txHaveCondition txNamed narg source"] = 1.00
        parentConnectionExceptions["txHaveCondition txNonPred arg source"] = 0.88
        parentConnectionExceptions["txHaveCondition txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txHaveCondition txPerson0 arg source"] = 1.00
        parentConnectionExceptions["txHaveCondition txPerson2 arg dest"] = 1.00
        parentConnectionExceptions["txHaveCondition txPerson2 arg source"] = 1.00
        parentConnectionExceptions["txHaveCondition txPred narg source"] = 1.00
        parentConnectionExceptions["txHaveCondition txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txHaveCondition txThing1 arg source"] = 1.00
        parentConnectionExceptions["txHaveManner txNonPred arg source"] = 1.00
        parentConnectionExceptions["txHaveManner txNonPred narg source"] = 1.00
        parentConnectionExceptions["txHaveOrg txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txHaveOrg txHaveRel arg dest"] = 1.00
        parentConnectionExceptions["txHaveOrg txNamed narg source"] = 0.94
        parentConnectionExceptions["txHaveOrg txNonPred narg source"] = 0.89
        parentConnectionExceptions["txHaveOrg txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txHaveOrg txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txHaveOrg txPerson2 arg dest"] = 1.00
        parentConnectionExceptions["txHaveOrg txPred narg source"] = 0.93
        parentConnectionExceptions["txHaveOrg txTemporalQuantity narg source"] = 1.00
        parentConnectionExceptions["txHavePart txNamed narg source"] = 1.00
        parentConnectionExceptions["txHavePart txNonPred narg source"] = 0.83
        parentConnectionExceptions["txHavePart txPred narg source"] = 1.00
        parentConnectionExceptions["txHavePurpose txHaveConcession arg dest"] = 1.00
        parentConnectionExceptions["txHavePurpose txHaveConcession arg source"] = 1.00
        parentConnectionExceptions["txHavePurpose txNamed arg source"] = 0.62
        parentConnectionExceptions["txHavePurpose txNonPred arg source"] = 0.61
        parentConnectionExceptions["txHavePurpose txNonPred narg source"] = 0.55
        parentConnectionExceptions["txHavePurpose txPred narg source"] = 1.00
        parentConnectionExceptions["txHavePurpose txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txHavePurpose txThing1 arg source"] = 1.00
        parentConnectionExceptions["txHaveQuant txNonPred narg source"] = 1.00
        parentConnectionExceptions["txHaveRel txMost narg source"] = 1.00
        parentConnectionExceptions["txHaveRel txNonPred narg source"] = 0.55
        parentConnectionExceptions["txHaveRel txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txHaveRel txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txHaveRel txPerson2 arg dest"] = 1.00
        parentConnectionExceptions["txHaveSubevent txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txHaveSubevent txFrequentDouble arg source"] = 1.00
        parentConnectionExceptions["txHaveSubevent txNamed arg source"] = 1.00
        parentConnectionExceptions["txHaveSubevent txNonPred arg source"] = 1.00
        parentConnectionExceptions["txHaveSubevent txNonPred narg source"] = 1.00
        parentConnectionExceptions["txInclude txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txInclude txFrequentDouble arg source"] = 1.00
        parentConnectionExceptions["txInclude txFrequentDouble narg dest"] = 1.00
        parentConnectionExceptions["txInclude txFrequentDouble narg source"] = 1.00
        parentConnectionExceptions["txInclude txInclude arg dest"] = 1.00
        parentConnectionExceptions["txInclude txMonetaryQuantity narg source"] = 1.00
        parentConnectionExceptions["txInclude txMore narg source"] = 1.00
        parentConnectionExceptions["txInclude txMost narg source"] = 1.00
        parentConnectionExceptions["txInclude txNamed narg source"] = 1.00
        parentConnectionExceptions["txInclude txNonPred narg source"] = 0.94
        parentConnectionExceptions["txInclude txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txInclude txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txInclude txPerson2 arg dest"] = 1.00
        parentConnectionExceptions["txInclude txPred narg source"] = 0.91
        parentConnectionExceptions["txInclude txThing0 arg dest"] = 1.00
        parentConnectionExceptions["txInclude txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txInclude txThing1 narg dest"] = 1.00
        parentConnectionExceptions["txInclude txThing1 narg source"] = 1.00
        parentConnectionExceptions["txInsteadOf txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txInsteadOf txFrequentDouble arg source"] = 1.00
        parentConnectionExceptions["txInsteadOf txNamed arg source"] = 1.00
        parentConnectionExceptions["txInsteadOf txNamed narg source"] = 1.00
        parentConnectionExceptions["txInsteadOf txNonPred arg source"] = 0.82
        parentConnectionExceptions["txInsteadOf txNonPred narg source"] = 1.00
        parentConnectionExceptions["txInsteadOf txRateEntity arg source"] = 1.00
        parentConnectionExceptions["txMonetaryQuantity txNamed narg source"] = 1.00
        parentConnectionExceptions["txMonetaryQuantity txPerson0 narg dest"] = 1.00
        parentConnectionExceptions["txMonetaryQuantity txPerson2 narg dest"] = 1.00
        parentConnectionExceptions["txMore txBeLocatedAt arg dest"] = 1.00
        parentConnectionExceptions["txMore txMonetaryQuantity narg source"] = 1.00
        parentConnectionExceptions["txMore txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txMore txThing0 arg dest"] = 1.00
        parentConnectionExceptions["txMore txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txMore txThing1 narg dest"] = 1.00
        parentConnectionExceptions["txMore txThing2 arg dest"] = 1.00
        parentConnectionExceptions["txMost txInclude arg dest"] = 1.00
        parentConnectionExceptions["txMost txPerson1 narg dest"] = 1.00
        parentConnectionExceptions["txMost txThing1 narg dest"] = 1.00
        parentConnectionExceptions["txNamed txBeLocatedAt narg dest"] = 1.00
        parentConnectionExceptions["txNamed txFrequentDouble narg dest"] = 1.00
        parentConnectionExceptions["txNamed txPerson0 narg dest"] = 1.00
        parentConnectionExceptions["txNonPred txFrequentDouble narg dest"] = 1.00
        parentConnectionExceptions["txNonPred txHaveConcession narg dest"] = 0.60
        parentConnectionExceptions["txNonPred txHaveOrg narg dest"] = 0.55
        parentConnectionExceptions["txNonPred txInclude narg dest"] = 1.00
        parentConnectionExceptions["txNonPred txPerson0 narg dest"] = 0.72
        parentConnectionExceptions["txNonPred txThing0 narg dest"] = 1.00
        parentConnectionExceptions["txNonPred txThing1 narg dest"] = 0.80
        parentConnectionExceptions["txNonPred txThing2 narg dest"] = 0.93
        parentConnectionExceptions["txPerson0 txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txPerson0 txHaveOrg arg dest"] = 1.00
        parentConnectionExceptions["txPerson0 txMore narg source"] = 1.00
        parentConnectionExceptions["txPerson0 txNamed narg source"] = 0.88
        parentConnectionExceptions["txPerson0 txNonPred narg source"] = 0.71
        parentConnectionExceptions["txPerson0 txPerson0 narg dest"] = 1.00
        parentConnectionExceptions["txPerson0 txPerson0 narg source"] = 1.00
        parentConnectionExceptions["txPerson0 txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txPerson0 txThing2 arg dest"] = 1.00
        parentConnectionExceptions["txPerson1 txBeLocatedAt narg source"] = 1.00
        parentConnectionExceptions["txPerson1 txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txPerson1 txMost narg source"] = 1.00
        parentConnectionExceptions["txPerson1 txNamed narg source"] = 0.76
        parentConnectionExceptions["txPerson1 txNonPred narg source"] = 0.58
        parentConnectionExceptions["txPerson1 txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txPerson2 txHaveCondition arg dest"] = 1.00
        parentConnectionExceptions["txPerson2 txNamed narg source"] = 0.92
        parentConnectionExceptions["txPerson2 txNonPred narg source"] = 0.62
        parentConnectionExceptions["txPerson2 txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txPerson2 txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txPerson2 txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txPred txBeTemporallyAt arg dest"] = 0.60
        parentConnectionExceptions["txPred txBeTemporallyAt narg dest"] = 1.00
        parentConnectionExceptions["txPred txFrequentDouble arg dest"] = 0.99
        parentConnectionExceptions["txPred txFrequentDouble narg dest"] = 1.00
        parentConnectionExceptions["txPred txHaveConcession arg dest"] = 0.77
        parentConnectionExceptions["txPred txHaveOrg arg dest"] = 0.80
        parentConnectionExceptions["txPred txHavePart arg dest"] = 0.70
        parentConnectionExceptions["txPred txHaveRel arg dest"] = 0.58
        parentConnectionExceptions["txPred txInclude arg dest"] = 0.89
        parentConnectionExceptions["txPred txInclude narg dest"] = 1.00
        parentConnectionExceptions["txPred txPerson0 arg dest"] = 0.80
        parentConnectionExceptions["txPred txPerson1 arg dest"] = 0.63
        parentConnectionExceptions["txPred txPerson2 arg dest"] = 0.69
        parentConnectionExceptions["txPred txRateEntity arg dest"] = 0.67
        parentConnectionExceptions["txPred txThing0 arg dest"] = 0.84
        parentConnectionExceptions["txPred txThing0 narg dest"] = 1.00
        parentConnectionExceptions["txPred txThing1 arg dest"] = 0.79
        parentConnectionExceptions["txPred txThing1 narg dest"] = 0.79
        parentConnectionExceptions["txPred txThing2 arg dest"] = 0.95
        parentConnectionExceptions["txPred txThing2 narg dest"] = 1.00
        parentConnectionExceptions["txPred txWhy arg dest"] = 0.56
        parentConnectionExceptions["txRateEntity txNamed narg source"] = 1.00
        parentConnectionExceptions["txThing0 txNamed narg source"] = 1.00
        parentConnectionExceptions["txThing0 txNonPred narg source"] = 0.57
        parentConnectionExceptions["txThing0 txPerson1 narg dest"] = 1.00
        parentConnectionExceptions["txThing0 txPerson1 narg source"] = 1.00
        parentConnectionExceptions["txThing0 txPred narg source"] = 0.75
        parentConnectionExceptions["txThing1 txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txThing1 txMore narg source"] = 1.00
        parentConnectionExceptions["txThing1 txMost narg source"] = 1.00
        parentConnectionExceptions["txThing1 txNonPred narg source"] = 0.73
        parentConnectionExceptions["txThing1 txPerson0 arg dest"] = 1.00
        parentConnectionExceptions["txThing1 txPerson0 narg source"] = 1.00
        parentConnectionExceptions["txThing1 txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txThing1 txPred narg source"] = 0.77
        parentConnectionExceptions["txThing1 txThing1 arg dest"] = 1.00
        parentConnectionExceptions["txThing2 txFrequentDouble arg dest"] = 1.00
        parentConnectionExceptions["txThing2 txNamed narg source"] = 1.00
        parentConnectionExceptions["txThing2 txNonPred narg source"] = 0.81
        parentConnectionExceptions["txThing2 txPerson1 arg dest"] = 1.00
        parentConnectionExceptions["txThing2 txPred narg source"] = 0.86
        parentConnectionExceptions["txWhy txDateEntity narg source"] = 1.00
        parentConnectionExceptions["txWhy txHaveConcession arg dest"] = 1.00
        parentConnectionExceptions["txWhy txNonPred narg source"] = 0.75

    sourceType = 'parent'
    destType = 'parent'
    if rel[:3]=='ARG':
        if '%s %s %s %s' % (source,dest,'arg','source') in parentConnectionExceptions:
            sourceType = 'child'
        if '%s %s %s %s' % (source,dest,'arg','dest') in parentConnectionExceptions:
            destType = 'child'
    else:
        if '%s %s %s %s' % (source,dest,'narg','source') in parentConnectionExceptions:
            sourceType = 'child'
        if '%s %s %s %s' % (source,dest,'narg','dest') in parentConnectionExceptions:
            destType = 'child'
    return sourceType, destType

def getConceptDetailedTranslationHashes(wdir='./AMR have-rel-role_etc'):

    fn = 'morph-verbalization-v1.01.txt'
    fn = 'verbalization-list-v1.01.txt'
    fn = 'have-rel-role-91-roles-v1.01.txt'
    fn = 'have-org-role-91-roles-v1.01.txt'

    allDirs = {}

    fn = 'morph-verbalization-v1.01.txt'
    dtype = 'morphVerbalization'
    allDirs[dtype] = {}
    for line in open(wdir + '/' + fn ):
        zz= line.split()
        t = [re.sub(r'\"', '', z ) for z in zz]
        allDirs[dtype][ t[3] ] = t

    # VERBALIZE abusive TO abuse-01
    fn = 'verbalization-list-v1.01.txt'
    dtype = 'verbalization'
    allDirs[dtype] = {}
    for line in open(wdir + '/' + fn ):
        zz= line.split()
        if zz[0]=='VERBALIZE':
            t = [re.sub(r'\"', '', z ) for z in zz]
            allDirs[dtype][ t[1] ] = t[2:]   # changed

    # USE-HAVE-REL-ROLE-91-ARG2 enemy
    fn = 'have-rel-role-91-roles-v1.01.txt'
    dtype = 'relRole'
    allDirs[dtype] = {}
    for line in open(wdir + '/' + fn ):
        zz= line.split()
        if len(zz)==2:
            t = [re.sub(r'\"', '', z ) for z in zz]
            allDirs[dtype][ t[1] ] = t[0]

    # USE-HAVE-ORG-ROLE-91-ARG2 undersecretary-general
    fn = 'have-org-role-91-roles-v1.01.txt'
    dtype = 'orgRole'
    allDirs[dtype] = {}
    for line in open(wdir + '/' + fn ):
        zz= line.split()
        if zz[0]=='USE-HAVE-ORG-ROLE-91-ARG2':
            t = [re.sub(r'\"', '', z ) for z in zz]
            allDirs[dtype][ t[1] ] = t[0]

    return (allDirs)

def getTxFuncs(nameWordIndices, tag):
    txFuncs=[]
    for i,ix in enumerate(nameWordIndices):
        if len(nameWordIndices)==1:
            txFuncs.append( (ix, 'S_'+tag)  )
        elif i==0:
            txFuncs.append( (ix, 'B_'+tag)  )
        elif i==(len(nameWordIndices)-1):
            txFuncs.append( (ix, 'E_'+tag)  )
        else:
            txFuncs.append( (ix, 'I_'+tag)  )
    return txFuncs

def nodesUnassigned(conceptList, nodes):
    for n in nodes:
        if not n in conceptList:
            return False
    return True

def getNEWSentenceDFTagList():
    #  _p and _c for parent, child:
    # ar0_arg  ar0_ix ar1_arg  ar1_ix  ar2_arg  ar2_ix  ar3_arg  ar3_ix
    # attr0_lbl attr0_val attr1_lbl attr1_val  attr2_lbl  attr2_val  attr3_lbl  attr3_val
    # nar0_ix nar0_lbl  nar1_ix nar1_lbl  nar2_ix  nar2_lbl  nar3_ix  nar3_lbl
    # kind sense
    #  add this
    # edge_p_c

    """
    sentIX
    wordIX
    words
    txBIOES

    # _p and _c for parent, child:
    ar0_arg  ar0_ix ar1_arg  ar1_ix  ar2_arg  ar2_ix  ar3_arg  ar3_ix
    attr0_lbl attr0_val attr1_lbl attr1_val  attr2_lbl  attr2_val  attr3_lbl  attr3_val
    nar0_ix nar0_lbl  nar1_ix nar1_lbl  nar2_ix  nar2_lbl  nar3_ix  nar3_lbl
    kind sense

    # add this
    edge_p_c

    NcatProb pVectorL0  pVectorL0Args  pVectorL0Attr  pVectorL0Nargs txBIOESProb

    # maybe the rest of these could go away
    nameCategory
    wiki
    nnChild nnParent
    txFunc
    top
    txBIOES_2 txBIOES_3 txBIOES_4
    wordIX0  wordIX1  wordIX2  wordIX3  wordIX4  wordIX5  wordIX6  wordIX7
    """

    tagList = [ 'sentIX', 'wordIX', 'words',  'txBIOES', 'txBIOESProb', 'NcatProb', 'sense',   ] # NEW 6/29/16 put sense back in

    for nodePostfix in ['_p', '_c']:
        for i in range(4): # max of four args per predicate
            tagList += [  'ar%d_arg%s' % (i,nodePostfix), 'ar%d_ix%s' % (i,nodePostfix), 'ar%d_destPC%s' % (i,nodePostfix)  ]
        for i in range(4): # max of four nonarg relations per nonPred
            tagList += [  'nar%d_lbl%s' % (i,nodePostfix), 'nar%d_ix%s' % (i,nodePostfix), 'nar%d_destPC%s' % (i,nodePostfix)  ]
        for i in range(4): # max of four attributes per concept
            tagList += [  'attr%d_lbl%s' % (i,nodePostfix), 'attr%d_val%s' % (i,nodePostfix)  ]
        tagList += ['kind'+nodePostfix ]

    tagList += ['edge_p_c']
    tagList += ['pVectorL0',  'pVectorL0Args',  'pVectorL0Nargs',  'pVectorL0Attr']

    # obsolete legacy, delete when safety is confirmed
    tagList += ['txFunc', 'nameCategory', 'wiki', 'top', 'nnParent', 'nnChild']
    for i in range(8): # store all wordIX's
        tagList += [  'wordIX%d' % i   ]
    return tagList

def compareDictionaries( ref, sys ):
    compared = 0
    refTotal = len(ref.keys())
    sysTotal = len(sys.keys())

    for key, val in ref.iteritems():
        if key in sys:
            if sys[key].lower() == val.lower():
                compared += 1

    return refTotal, sysTotal, compared

def getDateEntityAttributes(words):
    allWords = ' '.join(words)
    attribDict = {}
    year=0
    month=0
    day=0
    if len(words) == 1 and words[0].isdigit():
        m = re.match(r'^(\d\d\d\d)$', allWords)
        if m:
            #print m.group(1), allWords
            attribDict['year'] = m.group(1)

        m = re.match(r'^(\d\d)(\d\d)(\d\d)$', allWords)
        if m:
            year  = m.group(1)
            month = m.group(2)
            day   = m.group(3)
            if int(year) < 40:
                year = '20'+year
            else:
                year = '19'+year
        m = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', allWords)
        if m:
            year  = m.group(1)
            month = m.group(2)
            day   = m.group(3)
            #print year, month, day, allWords
        m = re.match(r'^(\d\d)$', allWords)
        if m:
            day   = m.group(1)
            #print year, month, day, allWords
        if year or month or day:
            if int(year)>0:
                attribDict['year'] = year
            if int(month)>0:
                attribDict['month'] = str(int(month))
            if int(day)>0:
                attribDict['day'] = str(int(day))

    if attribDict == {}: # 2008 - 09 - 26

        m = re.search(r'(\s+|^)(\d\d\d\d)\s+\D*\s+(\d\d)\s+\D*\s+(\d\d)(\s+|$)', allWords);
        if m:
            year  = m.group(2)
            month = m.group(3)
            day   = m.group(4)
        if year or month or day:
            if int(year)>0:
                attribDict['year'] = year
            if int(month)>0:
                attribDict['month'] = str(int(month))
            if int(day)>0:
                attribDict['day'] = str(int(day))

    if attribDict == {}: # Named Month
        months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
        for mn in months:
            m = re.search(r'(?:\s+|^)(%s)' % mn, allWords, flags=re.IGNORECASE)
            if m:
                month = months.index(m.group(1).lower()) + 1
        m = re.search(r'(?:\s+|^)(\d\d\d\d)(?:\s+|$)', allWords)
        if m:
           year = m.group(1)
        m = re.search(r'(?:\s+|^)(\d\d)(?:\s+|$)', allWords)
        if m:
           day = m.group(1)
        m = re.search(r'(?:\s+|^)(\d)(?:\s+|$)', allWords)
        if m:
           day = m.group(1)
        if year or month or day:
            if year and int(year)>0:
                attribDict['year'] = year
            if month and int(month)>0:
                attribDict['month'] = str(int(month))
            if day and int(day)>0:
                attribDict['day'] = str(int(day))

    return attribDict

def text2int(textnum, numwords={}):
    # from http://stackoverflow.com/questions/493174/is-there-a-way-to-convert-number-words-to-integers
    found=False
    textnum = textnum.lower()
    if not numwords:
      units = [
        "zero", "one", "two", "three", "four", "five", "six", "seven", "eight",
        "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
        "sixteen", "seventeen", "eighteen", "nineteen",
      ]

      tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]

      scales = ["hundred", "thousand", "million", "billion", "trillion"]

      numwords["and"] = (1, 0)
      for idx, word in enumerate(units):    numwords[word] = (1, idx)
      for idx, word in enumerate(tens):     numwords[word] = (1, idx * 10)
      for idx, word in enumerate(scales):   numwords[word] = (10 ** (idx * 3 or 2), 0)

    current = result = 0
    for word in textnum.split():
        if word in numwords:
            scale, increment = numwords[word]
            if (scale > 100) and not found:
                current = 1
            current = current * scale + increment
            if scale > 100:
                result += current
                current = 0
            found=True
    if not found:
        if 'hundreds'  in textnum : return 1e2
        if 'thousands' in textnum : return 1e3
        if 'millions'  in textnum : return 1e6
        if 'billions'  in textnum : return 1e9
        if 'trillions' in textnum : return 1e12

        return None
    else:
        return result + current

def getTemporalQuantityQuantUnit(words):
    """
    from training data:
year        392  56.402878      392   56.402878
day         115  16.546763      507   72.949640
month        70  10.071942      577   83.021583
week         36   5.179856      613   88.201439
hour         35   5.035971      648   93.237410
decade       22   3.165468      670   96.402878
minute       16   2.302158      686   98.705036
century       4   0.575540      690   99.280576
second        4   0.575540      694   99.856115
    """
    allWords = ' '.join(words)

    unitKind = 'year'
    quant = None
    units = ['year', 'day', 'month', 'week', 'hour', 'decade', 'minute', 'century', 'second']
    for u in units:
        if u in allWords : unitKind=u; break
    if 'centuries' in allWords : unitKind='century'
    if 'daily' in allWords : unitKind='day'
    m = re.search(r'([0-9,\.]+)', allWords.lower())
    if m:
        quant = m.group(1)
        quant = quant.replace(",", "")
        if quant=='.':
            quant=None
    if not quant:
        q = text2int(allWords)
        if q:
            quant = str(q)
    if not quant:
        quant = '1'

    return   quant, unitKind

def getMonetaryQuantityQuantUnit(words):
    """
    from training data:
          count       perc  cum_sum    cum_perc
kind_c
dollar      133  60.454545      133   60.454545
yuan         36  16.363636      169   76.818182
euro         23  10.454545      192   87.272727
pound         7   3.181818      199   90.454545
cent          5   2.272727      204   92.727273
rupee         3   1.363636      207   94.090909
renminbi      2   0.909091      209   95.000000
diram         1   0.454545      210   95.454545
figure        1   0.454545      211   95.909091
franc         1   0.454545      212   96.363636
mark          1   0.454545      213   96.818182
baht          1   0.454545      214   97.272727
quid          1   0.454545      215   97.727273
ruble         1   0.454545      216   98.181818
somoni        1   0.454545      217   98.636364
won           1   0.454545      218   99.090909
yen           1   0.454545      219   99.545455
RMB           1   0.454545      220  100.000000
    """
    allWords = ' '.join(words)

    unitKind = 'dollar'
    quant = None
    units = ['dollar', 'yuan', 'euro', 'pound', 'cent', 'rupee', 'franc', 'mark', 'yen']
    for u in units:
        if u in allWords : unitKind=u; break
    if 'centuries' in allWords : unitKind='century'
    if 'daily' in allWords : unitKind='day'
    m = re.search(r'([0-9,\.]+)', allWords.lower())
    if m:
        quant = m.group(1)
        quant = quant.replace(",", "")
        quant = float(quant)
        if quant=='.':
            quant=None
    if not quant:
        q = text2int(allWords)
        if q:
            quant = q
    else:
        m = text2int(allWords)
        if m:
            quant *= m
    if not quant:
        quant = 1
    quant = ('%f' % quant).rstrip('0').rstrip('.')

    return   quant, unitKind

def getMassQuantityQuantUnit(words):
    """
    from training data:
          count       perc  cum_sum    cum_perc
kind_c
kilogram     66  43.421053       66   43.421053
ton          45  29.605263      111   73.026316
gram         24  15.789474      135   88.815789
pound        11   7.236842      146   96.052632
kilo          3   1.973684      149   98.026316
ounce         1   0.657895      150   98.684211
picogram      1   0.657895      151   99.342105
tonne         1   0.657895      152  100.000000
    """
    allWords = ' '.join(words)

    unitKind = 'kilogram'
    quant = None
    units = ['kilogram', 'tonne', 'ton', 'picogram', 'gram', 'pound',  'ounce']
    for u in units:
        if u in allWords : unitKind=u; break
    if 'kilo' in allWords : unitKind='kilogram'
    if 'daily' in allWords : unitKind='day'
    m = re.search(r'([0-9,\.]+)', allWords.lower())
    if m:
        quant = m.group(1)
        quant = quant.replace(",", "")
        quant = float(quant)
        if quant=='.':
            quant=None
    if not quant:
        q = text2int(allWords)
        if q:
            quant = q
    else:
        m = text2int(allWords)
        if m:
            quant *= m
    if not quant:
        quant = 1
    quant = ('%f' % quant).rstrip('0').rstrip('.')

    return   quant, unitKind

def getDistanceQuantityQuantUnit(words):
    """
    from training data:
kind_c
kilometer      31  37.349398       31   37.349398
mile           23  27.710843       54   65.060241
meter           9  10.843373       63   75.903614
foot            8   9.638554       71   85.542169
inch            7   8.433735       78   93.975904
millimeter      2   2.409639       80   96.385542
nanometer       1   1.204819       81   97.590361
step            1   1.204819       82   98.795181
street          1   1.204819       83  100.000000
    """
    allWords = ' '.join(words)

    unitKind = 'kilometer'
    quant = None
    units = ['kilometer', 'mile', 'millimeter', 'nanometer', 'meter', 'foot', 'inch', 'step', 'street']
    for u in units:
        if u in allWords : unitKind=u; break
    if 'feet' in allWords : unitKind='foot'
    m = re.search(r'([0-9,\.]+)', allWords.lower())
    if m:
        quant = m.group(1)
        quant = quant.replace(",", "")
        quant = float(quant)
        if quant=='.':
            quant=None
    if not quant:
        q = text2int(allWords)
        if q:
            quant = q
    else:
        m = text2int(allWords)
        if m:
            quant *= m
    if not quant:
        quant = 1
    quant = ('%f' % quant).rstrip('0').rstrip('.')

    return   quant, unitKind

def getAreaQuantityQuantUnit(words):
    """
    from training data:
              count       perc  cum_sum    cum_perc
kind_c
hectare           7  58.333333        7   58.333333
acre              2  16.666667        9   75.000000
meter             1   8.333333       10   83.333333
square-foot       1   8.333333       11   91.666667
square-meter      1   8.333333       12  100.000000
    """
    allWords = ' '.join(words)

    unitKind = 'hectare'
    quant = None
    units = ['hectare', 'acre', 'euro', 'meter', 'square-foot', 'square-meter' ]
    for u in units:
        if u in allWords : unitKind=u; break
    if 'square foot' in allWords : unitKind='square-foot'
    if 'square feet' in allWords : unitKind='square-foot'
    if 'square meter' in allWords : unitKind='square-meter'
    m = re.search(r'([0-9,\.]+)', allWords.lower())
    if m:
        quant = m.group(1)
        quant = quant.replace(",", "")
        quant = float(quant)
        if quant=='.':
            quant=None
    if not quant:
        q = text2int(allWords)
        if q:
            quant = q
    else:
        m = text2int(allWords)
        if m:
            quant *= m
    if not quant:
        quant = 1
    quant = ('%f' % quant).rstrip('0').rstrip('.')

    return   quant, unitKind

        #subGraph['attrDict_p'] =  attrDict_p

def updateSubGraphStats(tx, words, refSubGraph, subGraph, subGraphStats):
    if not tx in subGraphStats:
        subGraphStats[tx] = {}
    # subGraphStats[tx][words][subGraph['kind_p']] += 1
    # print '%10s %30s: %s' % (tx, words, refSubGraph)
    # print '%10s %30s: %s' % ('', '', subGraph)

[senseDict, allWordStats, singleWordStats] = pickle.load(  open( 'conceptDicts.pcl', "rb" ) )
cdtHash = getConceptDetailedTranslationHashes()  # do this lazily, like other hash list loading

def getMostCommonTranslation(tx, words, k):
    if tx in allWordStats:
        if k in allWordStats[tx]:
            allWords = ' '.join(words).lower()
            if allWords in allWordStats[tx][k]:
                if len(allWordStats[tx][k][allWords]) > 1:
                    mxCount=0
                    mxItem=''
                    for item in allWordStats[tx][k][allWords]:
                        if allWordStats[tx][k][allWords][item] > mxCount:
                            mxCount = allWordStats[tx][k][allWords][item]
                            mxItem  = item
                    return mxItem
                else:
                    return allWordStats[tx][k][allWords].keys()[0]
            if words[0] in allWordStats[tx][k]:
                if len(allWordStats[tx][k][words[0]]) > 1:
                    mxCount=0
                    mxItem=''
                    for item in allWordStats[tx][k][words[0]]:
                        if allWordStats[tx][k][words[0]][item] > mxCount:
                            mxCount = allWordStats[tx][k][words[0]][item]
                            mxItem  = item
                    return mxItem
                else:
                    return allWordStats[tx][k][words[0]].keys()[0]
        lemma=lmtzr.lemmatize(words[0].lower())
        if tx == 'txPred':
            if words[0] in senseDict:
                sense = senseDict[words[0]]
            else:
                sense = '01'
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
            return predWithSense
        elif tx == 'txNamed':
            return 'person'
        else:
            return TXConvert.convert(lemma)
    return None

def getSubGraph(txBIOES, words, wkLink='-'):

    subGraph = { 'kind_p':None, 'attrDict_p':{},  'kind_c':None, 'attrDict_c':{}, 'edge_p_c':None }

    if txBIOES=='txNonPred':
        #lemma=lmtzr.lemmatize(words[0].lower())
        #subGraph['kind_p'] = TXConvert.convert(lemma)
        #print 'txNonPred DEBUG', words[0].lower(), lemma, TXConvert.convert(lemma)
        subGraph['kind_p'] = getMostCommonTranslation(txBIOES, words, 'kind_p')

    if (txBIOES=='txPred-01'):
        if words[0] in senseDict:
            sense = senseDict[words[0]]
        else:
            sense = '01'
        lemma=lmtzr.lemmatize(words[0].lower(), 'v')
        predWithSense = lemma
        if (sense):
            predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)  # dominance-01 => dominate-01  (sense affects lemma)
        alternate = getMostCommonTranslation(txBIOES, words, 'kind_p')
        if alternate:
            subGraph['kind_p'] = alternate

    if (txBIOES=='txPred-02'):
        sense='02'
        #if words[0] in senseDict:
        #    sense = senseDict[words[0]]
        #else:
        #    sense = '01'
        lemma=lmtzr.lemmatize(words[0].lower(), 'v')
        predWithSense = lemma
        if (sense):
            predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)  # dominance-01 => dominate-01  (sense affects lemma)
        alternate = getMostCommonTranslation(txBIOES, words, 'kind_p')
        if alternate:
            subGraph['kind_p'] = alternate

    if (txBIOES=='txPred'):
        if words[0] in senseDict:
            sense = senseDict[words[0]]
        else:
            sense = '01'
        lemma=lmtzr.lemmatize(words[0].lower(), 'v')
        predWithSense = lemma
        if (sense):
            predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)  # dominance-01 => dominate-01  (sense affects lemma)
        alternate = getMostCommonTranslation(txBIOES, words, 'kind_p')
        if alternate:
            subGraph['kind_p'] = alternate

    if (txBIOES=='txNamed'):
        # Need words, op, category, wiki for each wikification
        subGraph['kind_p'] = getMostCommonTranslation(txBIOES, words, 'kind_p')
        subGraph['kind_c'] =  'name'
        subGraph['edge_p_c'] =  'name'
        attrDict_p={}
        attrDict_p['wiki'] = wkLink # '"' + wkLink + '"'
        attrDict_c={}
        for ai in range(len(words)):
            lbl = 'op%d' % (ai+1)
            val = words[ai].lower()
            attrDict_c[lbl] = val

        subGraph['attrDict_p'] =  attrDict_p
        subGraph['attrDict_c'] =  attrDict_c

    if (txBIOES=='txDateEntity'):  # get this from the stanford parser
        #dt = dateparser.parse(words.join())
        #print words, dt
        # date-entity         nan         nan                                        {}  {u'year': u'2008', u'day': u'24', u'month': u'8'}
        subGraph['kind_p']      =  'date-entity'
        subGraph['attrDict_p']  = getDateEntityAttributes(words)

    if (txBIOES=='txHaveOrg'):
        # txHaveOrg                     have-org-role-91        ARG2    official                                        {}  {}

        """

        some txHaveOrg don't have three elements?
        lots of child kind miscompares (lower case, singularization)
                txFunc  count  refCount  sysCount  goodCount  precision    recall        f1  kind_p_acc  kind_c_acc  edge_p_c_acc  attrDict_p_acc  attrDict_c_acc  sense_acc  lemma_acc
4            txHaveOrg   1742      5187      5226       2358   0.451206  0.454598  0.452895    1.000000    0.363794             1        0.000000             NaN        NaN        NaN
4            txHaveOrg   1742      5187      1742       1729   0.992537  0.333333  0.499062    1.000000           0             0        0.000000             NaN        NaN        NaN
        """

        subGraph['kind_p'] = 'have-org-role-91'
        #subGraph['kind_c'] = words[0]
        subGraph['kind_c'] = lmtzr.lemmatize(words[0].lower(), 'n').lower()
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txPerson0'):
        # txPerson0                     abuse-02        ARG0      person                                        {}  {}
        # morph-verbalization-v1.01.txt:::DERIV-VERB "decide" ::DERIV-NOUN "decision"

        k = 'verbalization'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'person'
        subGraph['edge_p_c'] = 'ARG0'

    if (txBIOES=='txFrequentDouble'):
        # txFrequentDouble              govern-01        ARG0  government-organization                                        {}  {}
        subGraph['kind_p'] = 'govern-01'
        subGraph['kind_c'] = 'government-organization'
        subGraph['edge_p_c'] = 'ARG0'

    if (txBIOES=='txTemporalQuantity'):
        # txTemporalQuantity            temporal-quantity        unit         day                                        {}  {u'quant': u'1'}
        subGraph['kind_p']   = 'temporal-quantity'
        subGraph['edge_p_c'] = 'unit'
        quant, unitKind    = getTemporalQuantityQuantUnit(words)
        subGraph['attrDict_p']  = {'quant': quant}
        subGraph['kind_c']      =  unitKind

    if (txBIOES=='txThing1'):
        # txThing1                      find-01        ARG1       thing                                        {}  {}
        k = 'verbalization'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'thing'
        subGraph['edge_p_c'] = 'ARG1'

    if (txBIOES=='txPerson1'):
        # txPerson1                     vote-01        ARG1      person                                        {}  {}

        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'person'
        subGraph['edge_p_c'] = 'ARG1'

    if (txBIOES=='txMore'):
        # txMore                        late      degree        more                                        {}  {}
        subGraph['kind_p'] = getMostCommonTranslation(txBIOES, words, 'kind_p')
        subGraph['kind_c'] = 'more'
        subGraph['edge_p_c'] = 'degree'

    if (txBIOES=='txInclude'):
        # txInclude                     include-91        ARG2       power                                        {}  {}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'include-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txThing2'):
        # txThing2                      use-01        ARG2       thing                                        {}  {}
        # txThing1                      find-01        ARG1       thing                                        {}  {}
        k = 'verbalization'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'thing'
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txHaveRel'):
        # txHaveRel                     have-rel-role-91        ARG2       enemy                                        {}  {}
        subGraph['kind_p'] = 'have-rel-role-91'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'have-rel-role-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txMost'):
        # txMost                        long-03      degree        most                                        {}  {}
        subGraph['kind_p'] = getMostCommonTranslation(txBIOES, words, 'kind_p')
        subGraph['kind_c']   = 'most'
        subGraph['edge_p_c'] = 'degree'

    if (txBIOES=='txPerson2'):
        # txPerson1                     vote-01        ARG1      person                                        {}  {}

        k = 'verbalization'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'person'
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txThing0'):
        # txThing0                      evidence-01        ARG0       thing                                        {}  {}
        # txThing1                      find-01        ARG1       thing                                        {}  {}
        k = 'verbalization'
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        if w in cdtHash['morphVerbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('morphVerbalization', w, cdtHash['morphVerbalization'][w])
            w = cdtHash['morphVerbalization'][w][1]
        if w in cdtHash['verbalization']:
            #print ' ========> %24s  %20s -> %20s' % ('verbalization', w, cdtHash['verbalization'][w])
            predWithSense = cdtHash['verbalization'][w][-1]
        else:
            #print '  NOT FOUND', w
            lemma=lmtzr.lemmatize(words[0].lower(), 'v')
            sense = '01'
            predWithSense = lemma
            if (sense):
                predWithSense = lemma + '-' + sense
        subGraph['kind_p'] = TXConvert.convert(predWithSense)
        subGraph['kind_c'] = 'thing'
        subGraph['edge_p_c'] = 'ARG0'

    if (txBIOES=='txHaveConcession'):
        # txHaveConcession              have-concession-91        ARG2          or                                        {}  {'TOP': u'have-concession-91'}
        subGraph['kind_p'] = 'have-concession-91'
        subGraph['kind_c'] = getMostCommonTranslation(txBIOES, words, 'kind_c')
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txMonetaryQuantity'):
        # txMonetaryQuantity            monetary-quantity        unit      dollar                                        {}  {u'quant': u'295000000'}
        subGraph['kind_p'] = 'monetary-quantity'
        subGraph['edge_p_c'] = 'unit'
        quant, unitKind    = getMonetaryQuantityQuantUnit(words)
        subGraph['attrDict_p']  = {'quant': quant}
        subGraph['kind_c']      =  unitKind

    if (txBIOES=='txMassQuantity'):
        # txMassQuantity                mass-quantity        unit         ton                                        {}  {u'quant': u'1.6'}
        subGraph['kind_p'] = 'mass-quantity'
        subGraph['edge_p_c'] = 'unit'
        quant, unitKind    = getMassQuantityQuantUnit(words)
        subGraph['attrDict_p']  = {'quant': quant}
        subGraph['kind_c']      =  unitKind

    if (txBIOES=='txAgo'):
        # txAgo                         before         op1         now                                        {}  {}
        subGraph['kind_p'] = 'before'
        subGraph['kind_c'] = 'now'
        subGraph['edge_p_c'] = 'op1'

    if (txBIOES=='txDistanceQuantity'):
        # txDistanceQuantity            distance-quantity        unit   kilometer                                        {}  {u'quant': u'76'}
        subGraph['kind_p'] = 'distance-quantity'
        subGraph['edge_p_c'] = 'unit'
        quant, unitKind    = getDistanceQuantityQuantUnit(words)
        subGraph['attrDict_p']  = {'quant': quant}
        subGraph['kind_c']      =  unitKind

    if (txBIOES=='txAreaQuantity'):
        # txDistanceQuantity            distance-quantity        unit   kilometer                                        {}  {u'quant': u'76'}
        subGraph['kind_p'] = 'area-quantity'
        subGraph['edge_p_c'] = 'unit'
        quant, unitKind    = getAreaQuantityQuantUnit(words)
        subGraph['attrDict_p']  = {'quant': quant}
        subGraph['kind_c']      =  unitKind

    if (txBIOES=='txWhy'):
        # txWhy                         cause-01        ARG0  amr-unknown                                        {}  {}
        subGraph['kind_p'] = 'cause-01'
        subGraph['kind_c'] = 'amr-unknown'
        subGraph['edge_p_c'] = 'ARG0'

    if (txBIOES=='txHaveCondition'):
        # txHaveCondition               have-condition-91        ARG2       go-06                                        {}  {}
        subGraph['kind_p'] = 'have-condition-91'
        subGraph['kind_c'] = getMostCommonTranslation(txBIOES, words, 'kind_c')
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txBeLocatedAt'):
        # txBeLocatedAt                 be-located-at-91        ARG2        edge                                        {}  {'TOP': u'be-located-at-91'}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'be-located-at-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txInsteadOf'):
        # txInsteadOf                   instead-of-91        ARG2     missile                                        {}  {}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'instead-of-91'
        subGraph['kind_c'] = getMostCommonTranslation(txBIOES, words, 'kind_c')
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txHavePart'):
        # txHaveParthave-part-91        ARG2        that                                        {}  {'TOP': u'have-part-91'}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'have-part-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txBeTemporallyAt'):
        # txBeTemporallyAt              be-temporally-at-91        ARG2        week                                        {}  {'TOP': u'be-temporally-at-91'}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'be-temporally-at-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txHavePurpose'):
        # txHavePurpose                 have-purpose-91        ARG2         and                                        {}  {'TOP': u'have-purpose-91'}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'have-purpose-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txRateEntity'):
        # txRateEntity                  nan         nan         nan                                        {}  {}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'rate-entity-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txVolumeQuantity'):
        # txVolumeQuantity              volume-quantity        unit         ton                                        {}  {u'quant': u'200'}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'volume-quantity'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'unit'

    if (txBIOES=='txHaveSubevent'):
        # txHaveSubevent                have-subevent-91        ARG2    violence                                        {}  {}
        w = words[0].lower()
        w=lmtzr.lemmatize(w.lower())
        subGraph['kind_p'] = 'have-subevent-91'
        subGraph['kind_c'] = w
        subGraph['edge_p_c'] = 'ARG2'

    if (txBIOES=='txAnd'):
        # txHaveSubevent                have-subevent-91        ARG2    violence                                        {}  {}
        subGraph['kind_p'] = 'and'

    if not subGraph['kind_p']:
        subGraph['kind_p'] = txBIOES   # at least assign something to this sub graph.

    return subGraph

if __name__ == '__main__':

    exit(1)


#Biggie Size - Given a list, write a function that changes all positive numbers in the list to "big".
def biggie_size(lst):
    for i in range(0,len(lst)):
        if(lst[i] > 0):
            lst[i] = "big"
    return lst
print(biggie_size([-1, 3, 5, -5]))

#Count Positives - Given a list of numbers, create a function to replace the last value with the number of positive values.
def count_positives(lst):
    count = 0
    for val in lst:
        if( val > 0):
            count+= 1
    lst[len(lst) - 1] = count
    return lst
print(count_positives([-1,1,1,1]))

#Sum Total - Create a function that takes a list and returns the sum of all the values in the list.
def sum_total(lst):
    sum = 0
    for val in lst:
        sum+= val
    return sum
print(sum_total([1,2,3,4]))

#Create a function that takes a list and returns the average of all the values
def avarage(lst):
    sum = 0
    for val in lst:
        sum+= val
    return sum/len(lst)
print(avarage([1,2,3,4]))

#Length - Create a function that takes a list and returns the length of the list.
def length(lst):
    return len(lst)
print(length([]))

#Minimum - Create a function that takes a list of numbers and returns the minimum value in the list.
#  If the list is empty, have the function return False.
def minimum(lst):
    if(len(lst) == 0):
        return False
    min = lst[0]
    for val in lst:
        if val < min:
            min = val
    return min
print(minimum([]))

#Maximum - Create a function that takes a list and returns the maximum value in the list. If the list is empty, have the function return False.
def maximum(lst):
    if(len(lst) == 0):
        return False
    max = lst[0]
    for val in lst:
        if val > max:
            max = val
    return max
print(maximum([37,2,1,-9]))

#Ultimate Analysis - Create a function that takes a list and returns a dictionary that has
# the sumTotal, average, minimum, maximum and length of the list.
def ultimate_analysis(lst):
    sumTotal = 0
    minimum = lst[0]
    maximum = lst[0]
    for val in lst:
        if( val > maximum):
            maximum = val
        if(val < minimum):
            minimum = val
        sumTotal+= val
    dict_lst = {
        'sumTotal': sumTotal,
        'average': sumTotal/len(lst),
        'minimum': minimum,
        'maximum': maximum,
        'length': len(lst)
    }
    return dict_lst
print(ultimate_analysis([37,2,1,-9]))

#Reverse List - Create a function that takes a list and return that list with values reversed. Do this without creating a second list.
def reverse_list(lst):
    for i in range(0, (len(lst)//2)):
        temp = lst[i]
        lst[i] = lst[len(lst) - i - 1]
        lst[len(lst) - i - 1] = temp
    return lst
print(reverse_list([37,2,1,-9]))


# coding: utf-8

__author__ = 'Yemilice_lau'

"""
给出一个字符串数组words组成的一本英语词典。

从中找出最长的一个单词，该单词是由words词典中其他单词逐步添加一个字母组成。

若其中有多个可行的答案，则返回答案中字典序最小的单词。

若无答案，则返回空字符串。

示例 1：

输入：
words = ["w","wo","wor","worl", "world"]
输出："world"
解释：
单词"world"可由"w", "wo", "wor", 和 "worl"添加一个字母组成。
示例 2：

输入：
words = ["a", "banana", "app", "appl", "ap", "apply", "apple"]
输出："apple"
解释：
"apply"和"apple"都能由词典中的单词组成。但是"apple"的字典序小于"apply"。

"""

class Solution(object):
    def longestWord(self, words):
        ans = ""
        wordset = set(words)
        for word in words:
            if len(word) > len(ans) or len(word) == len(ans) and word < ans:
                if all(word[:k] in wordset for k in xrange(1, len(word))):
                    ans = word

        return ans


def user_name():
    pass

def loop_array():
    arr = [1,2,3,4,5]
    for item in arr:
        print(item)

# you can use boolean operators: | or, & and, not. &&, || determine if something is 'truthy or falsy'

# lists are mutable, tuples are immutable

# you can't add elements to tuples as they are immutable

# you can have multiple types of data in a tuple

# data type is a dictionary

class DevOps:
    pass
object_ = DevOps()

def append_(arr= [1,2,3,4]):
    arr.append(5)
    return arr

def arguments(arg):
    if arg == "devops":
        return True
    return False

class ClassZ:
    pass

class ClassY(ClassZ):
    def __init__(self):
        super().__init__()

def odd_even(lst, value):
    for i in range(len(lst)):
        if lst[i] == value:
            if (i + 1) % 2 == 0:
                return True
            else:
                return False
        else:
            return None

def percentage(value1,value2):
    if value1 | value2 == 0:
        v_1 = int(input("Enter a new number. "))
        v_2 = int(input("Enter a new number. "))
        percentage(v_1,v_2)
    return 100 * value1 / value2

def divide(value1,value2):
    if value1 | value2 == 0:
        v_1 = int(input("Enter a new number. "))
        v_2 = int(input("Enter a new number. "))
        divide(v_1,v_2)
    return value1/value2

def dob(day, month):
    if day < 1:
        v_1 = int(input("Enter a day. "))
        v_2 = int(input("Enter a month. "))
        day, month = dob(v_1, v_2)
    return day, month

print(dob(0,12))

def multiply(value1, value2):
    if value1 | value2 == 0:
        v_1 = int(input("Enter a new number. "))
        v_2 = int(input("Enter a new number. "))
        multiply(v_1,v_2)

    return value1 * value2

def find_odd_even(lst):
    """finds locations of odd and even numbers in a list"""
    even_lst = []
    odd_lst = []
    for i in range(len(lst)):
        if lst[i] % 2 == 0:
            even_lst.append(i)
        else:
            odd_lst.append(i)
    return even_lst, odd_lst

def bill():
    shopping_lst = {
        "milk": 3.4,
        "potato": 5.5,
        "bread": 6,
        "butter": 7,
        "cookies": 5.4
    }
    total = 0
    for key, value in shopping_lst.items():
        total += value
    return total


from collections import deque

n = int(input())
dq = deque()
for i in range(1, n+1):
    dq.append(i)

while len(dq) != 1:
    dq.popleft()
    dq.append(dq.popleft())

print(dq.pop())

# n = int(input())
# s = 1
# while s<n:
#     s *= 2
# print(s if s == n else 2*n-s)


from setuptools import setup, find_packages
import os

def read(*rnames):
    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()

long_description = read('README.txt') + '\n' +\
    read('CHANGES.txt')

setup(name='hodgepodge',
      version = '0.1',
      description="Distributed component publish/subscribe.",
      long_description=long_description,
      long_description_content_type="text",
      classifiers=['Development Status :: 5 - Alpha',
                   'Environment :: Network Environment',
                   'Intended Audience :: Developers',
                   'License :: OSI Approved :: Lesser General Public License v2.0',
                   'Operating System :: OS Independent',
                   'Programming Language :: Python',
                   'Topic :: Internet :: TCP/IP',
                   'Topic :: Software Development :: Libraries',
                   'Topic :: Component Software',
                   ],
      keywords='publish subscribe library event zope interface zeromq 0mq',
      author='Paul Sephton',
      author_email='prsephton@gmail.com',
      license='LGPL',
      packages=find_packages('src'),
      package_dir = {'': 'src'},
      namespace_packages=[],
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'setuptools',
          'zope.site',
          'zope.interface',
          'zope.schema',
          'zope.component',
          'zope.event',
          'zope.location',
          'zope.container',
          'pyzmq',
          'dill',
         ],
      entry_points="""
      # -*- Entry points: -*-
      [console_scripts]
         hp_python = hodgepodge.scripts.interpreter:main
      """,
      )


from django.conf.urls import patterns, include, url

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'vrautotest.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    #--------------SuperUser------------#

    url(r'^superuser', 'vrapp.views.superuser', name='superuser'),
    url(r'^addContest', 'vrapp.views.addContest', name='addContest'),
    url(r'^deleteContest', 'vrapp.views.deleteContest', name='deleteContest'),
    url(r'^checkContestName', 'vrapp.views.checkContestName', name='checkContestName'),

	#------------Home---------------#

    url(r'^$', 'vrapp.views.home', name='home'),
    url(r'^registration', 'vrapp.views.registration', name='registration'),
    url(r'^checkUserName', 'vrapp.views.checkUserName', name='checkUserName'),
    url(r'^regisuccess', 'vrapp.views.regisuccess', name='regisuccess'),

	#---------Login----------#

    url(r'^loginform', 'vrapp.views.loginform', name='loginform'),
    url(r'^loginvalidate', 'vrapp.views.loginvalidate', name='loginvalidate'),
    url(r'^logout', 'vrapp.views.logout', name='logout'),

	#------------Contestant Home------------#

    url(r'^contestanthome', 'vrapp.views.contestanthome', name='contestanthome'),
    #url(r'^submissions', 'vrapp.views.submissions', name='submissions'),

	#------------TestAdmin Home------------#

    url(r'^testadminhome', 'vrapp.views.testadminhome', name='testadminhome'),
    url(r'^puppetrun', 'vrapp.views.puppetrun', name='puppetrun'),
    url(r'^puppetstop', 'vrapp.views.puppetstop', name='puppetstop'),
    url(r'^deactivateuser', 'vrapp.views.deactivateuser', name='deactivateuser'),

	#------------TestCreator Home------------#

    url(r'^testcreatorhome$', 'vrapp.views.testcreatorhome', name='testcreatorhome'),
    url(r'^createquestionpaper', 'vrapp.views.createquestionpaper', name='createquestionpaper'),
    url(r'^addquestion$','vrapp.views.addquestion', name='addquestion'),

	#------------ParticipantApprover Home------------#

    url(r'^participantapproverhome', 'vrapp.views.participantapproverhome', name='participantapproverhome'),
    url(r'^approve', 'vrapp.views.approve', name='approve'),
)


from loggers.loggers import logger2, logger

class FlatIterator:
    def __init__(self, list_of_list):
        self.flat_list = []
        self.set_flat(list_of_list)
        self.counter = -1

    def __iter__(self):
        return self

    def __next__(self):
        self.counter += 1
        if self.counter >= len(self.flat_list):
            raise StopIteration
        return self.flat_list[self.counter]

    def set_flat(self, list_of_lists):
        for pos, element in enumerate(list_of_lists):
            if type(element) is list:
                self.set_flat(list_of_lists[pos])
            else:
                self.flat_list.append(element)

@logger
@logger2("log_4.log")
def project_run():

    list_of_lists_1 = [
        ['a', 'b', 'c'],
        ['d', 'e', 'f', 'h', False],
        [1, 2, None]
    ]
    return FlatIterator(list_of_lists_1)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Nov  3 00:57:59 2019

@author: saeem
"""

x = [1,2,3,4,5]
print(x)
str_arr = input().split(',')
arr = [int(num) for num in str_arr]
print(arr)


from math import dist
import matplotlib.pyplot as plt
from shapely.geometry import LineString,Polygon,Point

class Node:
    def __init__(self,x,y):
        self.x = x;
        self.y = y;
        self.parent = None;
        self.neighbors = [];
        self.open = True;
        self.pathDist2Start = None;
        self.straightDist2End = None;

    def resetVisibility(self):
        self.neighbors = [];

    def resetAncestory(self):
        self.parent = None;
        self.pathDist2Start = None;
        self.straightDist2End = None;

    def add_neighbor(self,neighbor_node):
        self.neighbors.append(neighbor_node);

    def get_xy(self):
        return (self.x,self.y)

    def dist_to_node(self,n2):
        x2,y2 = n2.get_xy();
        return dist([self.x,self.y],[x2,y2]);

    def dist_to_coords(self,x2,y2):
        return dist([self.x,self.y],[x2,y2]);

    def evaluate_neighbors(self):
        #print('Me:', self.get_xy(),self.pathDist2Start,self.parent.get_xy())
        for n in self.neighbors:
            if n is self.parent:
                continue
            dn = self.dist_to_node(n);
            d = self.pathDist2Start + dn;
            if n.pathDist2Start is not None:
                if (self.pathDist2Start + dn)<n.pathDist2Start:
                    n.parent = self;
                    n.pathDist2Start = d;
                elif(n.pathDist2Start+dn)<self.pathDist2Start:
                    self.parent = n;
                    self.pathDist2Start = n.pathDist2Start+dn;

            else:
                n.parent = self;
                n.pathDist2Start = d;

            #print(n.get_xy(),n.pathDist2Start,n.parent.get_xy())
        return;

class Obstacle:
    def __init__ (self,nodes):
        self.nodes = nodes;
        self.poly = None;
        self.line = None;

        self.shape = self.create_shape();

    def create_shape(self):
        pts = [];
        for n in self.nodes:
            pts.append(n.get_xy());

        pts.append(self.nodes[0].get_xy())#close the loop
        return Polygon(pts);

    def contains(self,node):
        if self.shape.contains(Point(*node.get_xy())):
            return True;
        else:
            return False;

class Graph:
    def __init__(self):
        self.nodes = [];
        self.obstacles = [];

    def create_node(self,x,y):
        n = Node(x,y);
        self.append_node(n);
        return n;

    def create_obstacle(self,nodes):
        o = Obstacle(nodes);
        self.append_obstacle(o);
        return o;

    def append_node(self,n):
        self.nodes.append(n);

    def append_obstacle(self,o):
        self.obstacles.append(o);

    def closest_node_to(self,x,y):

        closest_node = None;
        closest_dist = float('inf');

        for n in self.nodes:
            d = n.dist_to_coords(x,y);
            if (d < closest_dist):
                closest_node = n;
                closest_dist = d;

        return closest_node;

    def build_visibility(self):

        for n in self.nodes:
            if not n.open:
                continue
            for m in self.nodes:
                if not m.open:
                    continue
                if n is m:
                    continue

                if self.is_visible(n,m):
                    n.add_neighbor(m);
                    m.add_neighbor(n);

            n.open = False;

    def is_visible(self,n1,n2):

        x1,y1 = n1.get_xy();
        x2,y2 = n2.get_xy();
        ls = LineString([(x1,y1), (x2,y2)]);

        for o in self.obstacles:
            if o.contains(n1):
                n1.open = False; #node is contained within obstacle and is therefore not visible by anyone
                return False;
            elif o.contains(n2):
                n2.open = False;
                return False;
            if ls.crosses(o.shape):
                return False;
            if ls.within(o.shape):
                return False;

        return True;

    def plot_graph(self,path = None):
        #Obstacles
        for o in self.obstacles:
            plt.plot(*o.shape.exterior.xy,c='m',ls='-');

        #Nodes
        for n in self.nodes:
            x,y = n.get_xy();
            plt.scatter(x,y,c='red');

            #visiblility lines
            for m in n.neighbors:
                x2,y2 = m.get_xy();
                plt.plot([x,x2],[y,y2],c='y',ls=':');

        #Ancestory
        for n in self.nodes:
            if n.parent is not None:
                x1,y1 = n.get_xy();
                x2,y2 = n.parent.get_xy();
                plt.plot([x1,x2],[y1,y2],c='b',ls='-');
                if n.parent is n:
                    plt.scatter(x1,y1,100,c='m')

        #path
        if path is not None:
            for i in range(len(path)-1):
                x1,y1 = path[i].get_xy();
                x2,y2 = path[i+1].get_xy();
                plt.plot([x1,x2],[y1,y2],c='y',ls='--');

        axes = plt.gca();
        axes.set_aspect(1);

        plt.show();

    def reset_parents(self):
        for n in self.nodes:
            n.resetAncestory();

    def aStar(self,n1,n2):
        self.reset_parents();

        #See if nodes already exist in graph or if they need to be added
        i1 = None;
        i2 = None;
        for i in range(len(self.nodes)):
            if n1 is self.nodes[i]:
                i1 = i;
            if n2 is self.nodes[i]:
                i2 = i;

        rebuild = False;

        if i1 is None:
            self.append_node(n1);
            i1 = len(self.nodes)-1;
            rebuild = True;

        if i2 is None:
            self.append_node(n2);
            i2 = len(self.nodes)-1;
            rebuild = True;

        if rebuild:
            self.build_visibility();

        #if n1 or n2 have no neighbors, then no path can connect the two, so need to try a path between different nodes
        if not n1.neighbors or not n2.neighbors:
            return [];

        #A* search algorithm
        n1.parent = n1;
        n1.pathDist2Start = 0;
        n1.straightDist2End = n1.dist_to_node(n2);

        openSet = [n1];

        while openSet: #while there are still nodes in the openSet to evaluate
            curr = self.getLowestCostNode(openSet);

            if curr is n2: #reached the end
                path = [n2];
                while(path[-1] is not n1):
                    path.append(path[-1].parent)
                return path;

            for n in curr.neighbors:
                tent_score = curr.pathDist2Start + curr.dist_to_node(n);
                if n.pathDist2Start is None or tent_score < n.pathDist2Start:
                    n.parent = curr;
                    n.pathDist2Start = tent_score;
                    if n.straightDist2End is None:
                        n.straightDist2End = n.dist_to_node(n2);
                    if n not in openSet:
                        openSet.append(n);

        return [];

    def getLowestCostNode(self,nodes):
        id = None;
        low_cost = float('inf');

        for i in range(len(nodes)):
            cost = nodes[i].pathDist2Start + nodes[i].straightDist2End;
            if cost<low_cost:
                id = i;
                low_cost = cost;

        return nodes.pop(id);

'''
    def shortest_path(self,n1,n2):
        self.reset_parents();
        self.evaluated = [False for i in range(len(self.nodes))];

        #See if nodes already exist in graph or if they need to be added
        i1 = None;
        i2 = None;
        for i in range(len(self.nodes)):
            if n1 is self.nodes[i]:
                i1 = i;
            if n2 is self.nodes[i]:
                i2 = i;

        rebuild = False;

        if i1 is None:
            self.append_node(n1);
            i1 = len(self.nodes)-1;
            rebuild = True;

        if i2 is None:
            self.append_node(n2);
            i2 = len(self.nodes)-1;
            rebuild = True;

        if rebuild:
            self.build_visibility();

        if not n1.neighbors or not n2.neighbors:
            return [];

        n1.parent = n1;
        n1.pathDist2Start = 0;
        while not all(self.evaluated): #REVISIT: Nodes that are within obstacles aren't visible by anyone. Don't evaluate nodes that have no neighbors. Return error code if there is no path to node
            for i in range(len(self.nodes)):
                if not self.nodes[i].neighbors: #has no neighbors. Effectively cannot be seen by anyone
                    self.evaluated[i] = True;
                if self.nodes[i].parent is not None and not self.evaluated[i]:
                    self.nodes[i].evaluate_neighbors();
                    self.evaluated[i] = True;

        path = [n2];

        while(path[-1] is not n1):
            path.append(path[-1].parent)

        return path;
'''


from socket import socket, AF_INET, SOCK_STREAM
import logging
from threading import Thread
import sys

class ChatClient:
    def __init__(self, host, port) -> None:
        self.logger = self._setup_logger()
        self.socket = self._setup_socket(host, port)

        # starts a thread to wait for user input.
        thread = Thread(target=self.send_messages)
        thread.daemon = True
        thread.start()

        # recv is a blocking call.
        # receives data from server.
        # upon server termination client shuts down.
        try:
            while True:
                data = self.socket.recv(4096)
                self.logger.info(data.decode(errors='replace'))
        except ConnectionResetError:
            self.logger.warning('Server terminated. Exiting.')
            sys.exit()

    def send_messages(self):
        # tries to receive input from client user.
        # upon termination closes the socket and shuts down the client.
        try:
            while True:
                user_message = input()
                self.socket.send(user_message.encode('utf-8', errors='replace'))
        except (KeyboardInterrupt, EOFError):
            self.socket.close()
            sys.exit()

    def _setup_socket(self, host, port):
        sock = socket(AF_INET, SOCK_STREAM)
        try:
            sock.connect((host, port))
        except ConnectionRefusedError:
            self.logger.warning('No server available. Shutting down.')
            sys.exit()
        return sock

    def _setup_logger(self):
        logger = logging.getLogger('chat_client')
        logger.addHandler(logging.StreamHandler())
        logger.setLevel(logging.DEBUG)
        return logger

if __name__ == "__main__":
    client = ChatClient('192.168.49.2', 31515)