from ctypes import WinError
from pysc.DTC import DTCProtocol_protobuf as DTC_protobuf
from pysc.DTC import DTCProtocol_binary as DTC_binary
from pysc.DTC import DTCProtocol_binaryVLS as DTC_binaryVLS
from google.protobuf.json_format import MessageToDict
import datetime as dt
import socket
import threading
import struct
import selectors
import collections
from threading import Thread
from queue import Queue
import time
import logging
import pandas as pd

logger = logging.getLogger(__name__)

class ApiNotAuthorised(Exception):
    '''Raised when the API rejects a historical data request becuase the user is not authorised'''
    pass

class ApiNoHistoricalData(Exception):
    '''Raised when the API rejects a historical data request becuase the user is not authorised'''
    pass

class ApiUnexpectedError(Exception):
    '''Raised when there is an error we really did't expect'''
    pass

class ApiInvalidSymbol(Exception):
    '''Raised when the API rejects a historical data request becuase the user is not authorised'''
    pass

class ApiResponse:
    def __init__(self, type, msg):
        self.type = type
        self.msg = msg

class DtcClientService():
    """ DTC connection and data retrieval

    """

    def __init__(self, dtc_server, encoding='protobuf', heartbeat=True, ClientName='', GeneralTextData='', HardwareIdentifier='', UserName=''):
        """
        """

        self.heartbeat = heartbeat
        self.ClientName = ClientName
        self.GeneralTextData = GeneralTextData
        self.HardwareIdentifier = HardwareIdentifier
        self.UserName = UserName
        self.dtc_server = dtc_server
        self.heartbeat_interval = 20
        self.messages = Queue()
        self.receiver_running = False
        self.logged_on = False

        self.request_id = 0  # Counter for ID used by HistoricalPriceDataRequest
        self.timeout = 30

        # Make the message library a class attribute so we can swap it easily
        if encoding=='protobuf':
            self.DTC = DTC_protobuf
            self.encoding_wanted = self.DTC.PROTOCOL_BUFFERS
        elif encoding=='binary':
            self.DTC = DTC_binary
            self.encoding_wanted = self.DTC.BINARY_ENCODING
        elif encoding=='binaryVLS':
            self.DTC = DTC_binaryVLS
            self.encoding_wanted = self.DTC.BINARY_WITH_VARIABLE_LENGTH_STRINGS
        else:
            raise RuntimeError('Encoding is invalid')

        return

    def login(self):
        self.logged_on = False

        while self.receiver_running:
            self.receiver_run = False
            self.sock.close()
            time.sleep(0.1)

        self.encoding = DTC_binary.BINARY_ENCODING  # Always start with binary encoding, switch after encoding request
        try:
            self.sock = socket.create_connection(self.dtc_server)
            self.selector = selectors.DefaultSelector()
            self.selector.register(self.sock, selectors.EVENT_READ)
        except ConnectionRefusedError:
            logger.exception(
                "Connection refused. Check that DTC server is running and listening at {}:{}".format(
                    dtc_server[0], dtc_server[1]
                ), exc_info=False)
            raise ConnectionError

        self.receiver_thread = receiver_thread(self, self.DTC)
        self.receiver_thread.start()

        logger.info(f'Requesting encoding: {self.encoding}')

        message = self.dtc_binary_encoding_request_message(self.encoding_wanted)

        self.send_message(message, 6, self.sock)

        response = self.messages.get(timeout=self.timeout)

        self.encoding = response.msg.Encoding
        logger.info(f'Agreed encoding: {self.encoding}')

        # Construct and send Logon Request
        logon_req = self.DTC.LogonRequest()
        logon_req.ProtocolVersion = self.DTC.CURRENT_VERSION
        logon_req.Username = self.UserName

        if self.heartbeat:
            logon_req.HeartbeatIntervalInSeconds = self.heartbeat_interval
        else:
            logon_req.HeartbeatIntervalInSeconds = -1

        logon_req.ClientName = self.ClientName
        logon_req.GeneralTextData = self.GeneralTextData
        logon_req.HardwareIdentifier = self.HardwareIdentifier

        message = logon_req.SerializeToString()
        self.send_message(message, self.DTC.LOGON_REQUEST, self.sock)

        logon_response = self.messages.get(timeout=self.timeout)

        logger.info(logon_response.msg.ResultText)

        if logon_response.msg.Result != self.DTC.LOGON_SUCCESS:
            logger.error(logon_response.msg.ResultText)
            self.logout()
        else:
            self.logged_on = True

        if self.heartbeat:
            # Begin sending heartbeats
            self.heartbeat_thread = Thread(target=self.heartbeat_thread)
            self.heartbeat_thread.start()

        return

    def heartbeat_thread(self):
        message = self.DTC.Heartbeat().SerializeToString()
        next_heartbeat_time = time.time()
        while self.logged_on:
            if time.time() > next_heartbeat_time:
                try:
                    self.send_message(message, self.DTC.HEARTBEAT, self.sock)
                    next_heartbeat_time = time.time() + self.heartbeat_interval
                except OSError:
                    # The socket closed, exit becuase we will need to log back on
                    break
            time.sleep(0.1)

    def send_message(self, m, m_type, sock, no_retry = False):
        """ Prepend header to the message and send it to the specified socket
        """
        total_len = 4 + len(m)  # 2 bytes Size + 2 bytes Type
        header = struct.pack('<HH', total_len, m_type)  # Prepare 4-byte little-endian header

        try:
            sock.send(header + m)  # Send message
        except OSError as exc:
            # we got disconnected, happens when debugging, try again
            if no_retry==False:
                self.login()
                sock.send(header + m)  # Send message
            else:
                # if logoff fails, we dont actually care
                # Can happen after a big history download due
                # to being slow handling a big DF
                if m_type != self.DTC.LOGOFF:
                    raise exc
        logger.debug(f"Sent {self.DTC.DtcMsgNameFromType(m_type)}")

    def dtc_binary_encoding_request_message(self, encoding_enum):
        msg = DTC_binary.EncodingRequest()
        msg.ProtocolVersion  = self.DTC.CURRENT_VERSION
        msg.Encoding = encoding_enum
        msg.ProtocolType = 'DTC'
        return msg.SerializeToString()

    def logout(self):
        """ Gracefully logoff and close the connection """
        logger.info("Disconnecting from DTC server")
        logoff = self.DTC.Logoff()
        logoff.Reason = "Client terminating"
        message = logoff.SerializeToString()
        self.send_message(message, self.DTC.LOGOFF, self.sock, no_retry=True)
        self.logged_on = False
        #self.selector.unregister(self.sock)
        self.sock.close()
        return

    def historical_data_request(self, symbol, exchange, interval, start=0, end=0):
        """ Request historical data from the server
        """
        logger.info('Starting HistoricalPriceDataRequest')
        self.request_id += 1
        print(f'REQUEST ID {self.request_id}**********************************')
        if interval =='TICK':
            rec_interval = self.DTC.INTERVAL_TICK
        elif interval =='M':
            rec_interval = self.DTC.INTERVAL_1_MINUTE
        elif interval =='H':
            rec_interval = self.DTC.INTERVAL_1_HOUR
        elif interval =='D':
            rec_interval = self.DTC.INTERVAL_1_DAY
        else:
            raise RuntimeError('Interval must be one of TICK, M, H or D')

        if start is None:
            start=0

        if end is None:
            end = 0

        self.hist_rec_interval = rec_interval
        self.hist_start_tstamp = start
        data_req = self.DTC.HistoricalPriceDataRequest()
        data_req.RequestID = self.request_id
        data_req.Symbol = symbol
        data_req.Exchange = exchange
        data_req.RecordInterval = rec_interval
        data_req.StartDateTime = int(pd.Timestamp(start).timestamp())
        data_req.EndDateTime = int(pd.Timestamp(end).timestamp())
        data_req.MaxDaysToReturn = 0
        message = data_req.SerializeToString()

        self.send_message(message, self.DTC.HISTORICAL_PRICE_DATA_REQUEST, self.sock)

        complete = False
        df = pd.DataFrame(columns = ['Open', 'High', 'Low', 'Close', 'TotalVolume', 'NumTrades', 'BidVol', 'AskVol'])
        recs=0
        while not complete:
            response = self.messages.get(timeout=self.timeout)

            if response.msg.RequestID != self.request_id:
                logger.error('Unexpected error, RequestID in response does not match request')
                raise ApiUnexpectedError
            if response.type == self.DTC.HISTORICAL_PRICE_DATA_REJECT:
                # If we got reject - display reject text and logout
                logger.error(response.msg.RejectText)
                raise ApiNotAuthorised
            elif response.type == self.DTC.HISTORICAL_PRICE_DATA_RESPONSE_HEADER:
                if bool(response.msg.NoRecordsToReturn):
                    # If no records available - give warning and logout
                    logger.info("No historical data records available. Check symbol name.")
                    raise ApiNoHistoricalData
            elif response.type == self.DTC.HISTORICAL_PRICE_DATA_RECORD_RESPONSE:
                    msg = response.msg
                    time = dt.datetime.fromtimestamp(msg.StartDateTime)
                    if msg.StartDateTime > 0:
                        df.loc[time] = [msg.OpenPrice, msg.HighPrice, msg.LowPrice, msg.LastPrice, msg.Volume, msg.NumTrades, msg.BidVolume, msg.AskVolume]
                    recs+=1
                    #print(f'Rec {recs} final {msg.IsFinalRecord}')
                    if msg.IsFinalRecord > 0:
                        complete = True
            else:
                logger.error('Received unexpected message waiting for response')
                raise ApiUnexpectedError

        logger.info(f'Received {recs} records and created DataFrame with {len(df)} records')
        return df

    def securityDefinitionForSymbol(self, symbol, exchange):
        """
        """
        assert self.messages.qsize() ==0
        logger.info(f"Starting SecurityDefinitionForSymbolRequest Symbol:'{symbol}' Exchange:'{exchange}'")
        self.request_id += 1
        print(f'REQUEST ID {self.request_id}**********************************\n')
        request = self.DTC.SecurityDefinitionForSymbolRequest()
        request.RequestID = self.request_id
        request.Symbol = symbol
        request.Exchange = exchange
        message = request.SerializeToString()
        self.send_message(message, self.DTC.SECURITY_DEFINITION_FOR_SYMBOL_REQUEST , self.sock)

        complete = False
        result = []
        while not complete:
            response = self.messages.get(timeout=self.timeout)

            if response.type == self.DTC.SECURITY_DEFINITION_REJECT :
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError
            elif response.type == self.DTC.SECURITY_DEFINITION_RESPONSE  :
                result.append(MessageToDict(response.msg))
                if response.msg.IsFinalMessage > 0:
                    complete = True
            else:
                logger.error('Received {self.DTC.DtcMsgNameFromType(m_type)}, was expecting {SecurityDefinitionForSymbolRequest}')
                raise ApiUnexpectedError

            if response.msg.ExchangeSymbol == '':
                logger.error(f'Symbol {symbol} does not appear to have a security definition')
                raise ApiInvalidSymbol

            if response.msg.RequestID != self.request_id:
                logger.error(f'Unexpected error, RequestID {response.msg.RequestID} in response does not match RequestID {self.request_id} in request')
                print('')
                raise ApiUnexpectedError

        logger.info(f"Received security definition for Symbol:'{symbol}' Exchange:'{exchange}'")
        return result

    def securityTypes(self):
        """
        """
        st ={}
        st['unset'] = self.DTC.SECURITY_TYPE_UNSET
        st['futures'] = self.DTC.SECURITY_TYPE_FUTURE
        st['stock'] = self.DTC.SECURITY_TYPE_STOCK
        st['forex'] = self.DTC.SECURITY_TYPE_FOREX
        st['index'] = self.DTC.SECURITY_TYPE_INDEX
        st['futures_strategy'] = self.DTC.SECURITY_TYPE_FUTURES_STRATEGY
        st['futures_option'] = self.DTC.SECURITY_TYPE_FUTURES_OPTION
        st['stock_option'] = self.DTC.SECURITY_TYPE_STOCK_OPTION
        st['index_option'] = self.DTC.SECURITY_TYPE_INDEX_OPTION
        st['bond'] = self.DTC.SECURITY_TYPE_BOND
        st['mutual_fund'] = self.DTC.SECURITY_TYPE_MUTUAL_FUND
        return st

    def SymbolsForUnderlyingRequest(self, symbol, exchange, securityType='futures'):
        """
        """
        logger.info('Starting SymbolsForUnderlyingRequest')
        SECURITY_TYPE = self.securityTypes()[securityType.lower()]
        self.request_id += 1
        request = self.DTC.SymbolsForUnderlyingRequest()
        request.RequestID = self.request_id
        request.UnderlyingSymbol = symbol
        request.Exchange = exchange
        request.SecurityType = SECURITY_TYPE
        message = request.SerializeToString()
        self.send_message(message, self.DTC.SYMBOLS_FOR_UNDERLYING_REQUEST , self.sock)

        complete = False
        result = []

        while not complete:
            response = self.messages.get(timeout=self.timeout)

            if response.msg.RequestID != self.request_id:
                logger.error('Unexpected error, RequestID in response does not match request')
                raise ApiUnexpectedError

            if response.type == self.DTC.SECURITY_DEFINITION_REJECT :
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError
            elif response.type == self.DTC.SECURITY_DEFINITION_RESPONSE  :
                underlying = MessageToDict(response.msg)
                print(underlying)
                result.append(underlying)
                if response.msg.IsFinalMessage > 0:
                    complete = True
            else:
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError

        logger.info(f'Received {len(result)} security definitions for synbol:"{symbol}" exchange:"{exchange}"')
        return result

    def SymbolSearchRequest(self, searchText, exchange, securityType='futures', searchType = 'by_symbol'):
        """
        """
        logger.info('Starting SymbolSearchRequest')
        SEARCH_TYPE_DICT={}
        SEARCH_TYPE_DICT['by_symbol'] = self.DTC.SEARCH_TYPE_BY_SYMBOL
        SEARCH_TYPE_DICT['by_description'] = self.DTC.SEARCH_TYPE_BY_DESCRIPTION
        SEARCH_TYPE = SEARCH_TYPE_DICT[searchType]
        SECURITY_TYPE = self.securityTypes()[securityType.lower()]

        self.request_id += 1
        request = self.DTC.SymbolSearchRequest()
        request.RequestID = self.request_id
        request.SearchText = searchText
        request.Exchange = exchange
        request.SecurityType = SECURITY_TYPE
        request.SearchType = SEARCH_TYPE
        message = request.SerializeToString()
        self.send_message(message, self.DTC.SYMBOL_SEARCH_REQUEST , self.sock)

        complete = False
        result = []
        while not complete:
            response = self.messages.get(timeout=self.timeout)

            if response.msg.RequestID != self.request_id:
                logger.error('Unexpected error, RequestID in response does not match request')
                raise ApiUnexpectedError

            if response.type == self.DTC.SECURITY_DEFINITION_REJECT :
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError
            elif response.type == self.DTC.SECURITY_DEFINITION_RESPONSE  :
                underlying = MessageToDict(response.msg)
                result.append(underlying)
                if response.msg.IsFinalMessage > 0:
                    complete = True
            else:
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError

        logger.info(f'Received {len(result)} security definitions for synbol:"{symbol}" exchange:"{exchange}"')
        return result

    def UnderlyingSymbolsForExchangeRequest(self, exchange, securityType='futures'):
        """
        """
        logger.info('Starting SymbolSearchRequest')
        SECURITY_TYPE = self.securityTypes()[securityType.lower()]

        self.request_id += 1
        request = self.DTC.UnderlyingSymbolsForExchangeRequest()
        request.RequestID = self.request_id
        request.Exchange = exchange
        request.SecurityType = SECURITY_TYPE
        message = request.SerializeToString()
        self.send_message(message, self.DTC.UNDERLYING_SYMBOLS_FOR_EXCHANGE_REQUEST  , self.sock)

        complete = False
        result = []
        while not complete:
            response = self.messages.get(timeout=self.timeout)

            if response.msg.RequestID != self.request_id:
                logger.error('Unexpected error, RequestID in response does not match request')
                raise ApiUnexpectedError

            if response.type == self.DTC.SECURITY_DEFINITION_REJECT :
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError
            elif response.type == self.DTC.SECURITY_DEFINITION_RESPONSE  :
                underlying = MessageToDict(response.msg)
                result.append(underlying)
                if response.msg.IsFinalMessage > 0:
                    complete = True
            else:
                logger.error(response.msg.RejectText)
                raise ApiUnexpectedError

        logger.info(f'Received {len(result)} security definitions for exchange:"{exchange}"')
        return result

class receiver_thread(Thread):
    """
    """
    def __init__(self, parent, DTC):
        """
        """
        self.parent = parent
        self.DTC = DTC
        super().__init__()

    def run(self):
        """
        """
        logger.info(f'Message receiver thread starting')
        self.parent.receiver_run = True
        timeout_secs = 30
        self.parent.error = None
        while self.parent.receiver_run:
            self.parent.receiver_running = True
            events = self.parent.selector.select()#timeout=timeout_secs)
            # Either we received data or the socket closed

            try:
                msg_header = self.parent.sock.recv(4)
            except (OSError, WindowsError) as exc:
                # The socket was closed
                # Happens if we call logout on the DTC server
                # Exit the thread loop
                logger.info(f'Socket Closed, receiver thread exiting')
                break

            if msg_header == b'':
                self.parent.error = 'timeout'
                break

            msg_size = struct.unpack_from('<H', msg_header)[0]
            msg_type = struct.unpack_from('<H', msg_header, 2)[0]

            msg_body = self.parent.sock.recv(msg_size - 4)

            #logger.info(f'Received {msg_type} message')

            try:
                if self.parent.encoding == self.parent.DTC.BINARY_ENCODING:
                    dtc_message = DTC_binary.DtcMsgObjectFromType(msg_type);
                elif self.parent.encoding == self.parent.DTC.PROTOCOL_BUFFERS:
                    dtc_message = DTC_protobuf.DtcMsgObjectFromType(msg_type);
                elif self.parent.encoding == self.parent.DTC.BINARY_WITH_VARIABLE_LENGTH_STRINGS:
                    dtc_message = DTC_binaryVLS.DtcMsgObjectFromType(msg_type);
                else:
                    raise RuntimeError('Invalid encoding type')
            except KeyError:

                logger.info("Received unknown message type: {0}".format(msg_type))
                self.parent.error = 'key error'
                break

            dtc_message.ParseFromString(msg_body)

            if msg_type == self.parent.DTC.GENERAL_LOG_MESSAGE:
                logger.info("Got general message from server: '{}'".format(dtc_message.MessageText))
                continue
            elif msg_type == self.parent.DTC.HEARTBEAT:
                logger.debug("Recv {0}".format(self.DTC.DtcMsgNameFromType(msg_type)))
                continue # Do not store heartbeat messages

            self.parent.messages.put( ApiResponse(msg_type,dtc_message) )
        self.parent.receiver_running = False
        return


import sqlite3
from sqlite3 import Error

from ReferenceStructure import *

# Checked
def createDB(db_path):
    conn = createConnectionToDB(db_path)
    if conn is not None:
        # create projects table
        initTables(conn)
    else:
        print("Error! cannot create the database connection.")

def initTables(dbConnection):
    # Create Article table
    # sql_create_article_table = """ CREATE TABLE IF NOT EXISTS "Article" (
    #                                     `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
    #                                     `Title` TEXT NOT NULL,
    #                                     `Author` TEXT,
    #                                     `Type` TEXT NOT NULL,
    #                                     `Journal` TEXT,
    #                                     `Year` INTEGER NOT NULL,
    #                                     `Labels` TEXT,
    #                                     `AddedTime` TEXT NOT NULL
    #                                 ); """
    # print(sql_create_article_table)
    for type in BibTeXTypes:
        tablename = type.capitalize()
        sql_head = " CREATE TABLE IF NOT EXISTS " + "\"" + tablename + "\" ("
        DB_BaseStr = "`ID` INTEGER NOT NULL PRIMARY KEY UNIQUE," \
                   + "`RefAbsID` INTEGER NOT NULL," \
                   + "`Labels` TEXT," \
                   + "`AddedTime` TEXT NOT NULL," \
                   + "`Citekey` TEXT," \
                   + "`Flags` TEXT," \
                   + "`Attachments` TEXT," \
                   + "`Links` TEXT,"
        DB_FieldsStrList = []
        DB_ExtendFieldsStrList = []
        for field in DatabaseStandardStructure[tablename]:
            tempStr = "`" + field + "` TEXT"
            DB_FieldsStrList.append(tempStr)
        for field in DB_ExtendFields:
            tempStr = "`" + field + "` TEXT"
            DB_ExtendFieldsStrList.append(tempStr)
        DB_FieldsStr = ",".join(DB_FieldsStrList + DB_ExtendFieldsStrList)
        sql_tail = "); "
        sql_create_table_expr =  sql_head + DB_BaseStr + DB_FieldsStr + sql_tail
        createTable(dbConnection, sql_create_table_expr)

    # Create Citation table
    sql_create_citation_table = """ CREATE TABLE IF NOT EXISTS "Citation" (
                                        `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
                                        `Self` TEXT,
                                        `Cited` TEXT,
                                        `CitedBy` TEXT
                                    ); """
    createTable(dbConnection, sql_create_citation_table)
    # Create Labels table
    sql_create_labels_table = """ CREATE TABLE IF NOT EXISTS "Labels" (
                                        `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
                                        `Name` TEXT NOT NULL UNIQUE
                                    ); """
    createTable(dbConnection, sql_create_labels_table)
    # Create PubIn table
    sql_create_pubin_table = """ CREATE TABLE IF NOT EXISTS "PubIn" (
                                        `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
                                        `Name` TEXT NOT NULL UNIQUE
                                    ); """
    createTable(dbConnection, sql_create_pubin_table)
    # Create Trash table
    sql_create_trash_table = """ CREATE TABLE IF NOT EXISTS "Trash" (
                                        `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
                                        `Title` TEXT NOT NULL UNIQUE
                                    ); """
    createTable(dbConnection, sql_create_trash_table)
    # Create Years table
    sql_create_years_table = """ CREATE TABLE IF NOT EXISTS "Years" (
                                        `ID` INTEGER NOT NULL PRIMARY KEY UNIQUE,
                                        `Year` TEXT NOT NULL UNIQUE
                                    ); """
    createTable(dbConnection, sql_create_years_table)

# Checked
def createTable(dbConnection, create_table_sql):
    """ create a table from the create_table_sql statement
    :param conn: Connection object
    :param create_table_sql: a CREATE TABLE statement
    :return:
    """
    try:
        c = dbConnection.cursor()
        c.execute(create_table_sql)
    except Error as e:
        print(e)

# Checked
def createConnectionToDB(db_file):
    """ create a database connection to the SQLite database
        specified by the db_file
    :param db_file: database file
    :return: Connection object or None
    """
    try:
        conn = sqlite3.connect(db_file)
        return conn
    except Error as e:
        print(e)
        return None

def countAllRefsInDB(dbConnection):
    allRefNum = 0
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempNum = countRefsInTable(dbConnection, tablename)
        allRefNum = allRefNum + tempNum
    return allRefNum

def countRefsInTable(dbConnection, tablename):
    sql = "SELECT Count(*) FROM " + tablename
    cur = dbConnection.cursor()
    cur.execute(sql)
    refNum = cur.fetchone()
    return refNum[0]

# Checked
def countRefs(dbConnection):
    sql = "SELECT Count(*) FROM ReferencesData"
    cur = dbConnection.cursor()
    cur.execute(sql)
    refNum = cur.fetchone()
    return refNum[0]

# Checked
def createTempCitationTable(dbConnection):
    """
    Create a table in the database
    """
    sql = ''' CREATE TABLE TempCitation
              (ID INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, Self TEXT, Cited TEXT, CitedBy TEXT)'''
    cur = dbConnection.cursor()
    cur.execute(sql)
    dbConnection.commit()

# Checked
def deleteTempCitationTable(dbConnection):
    """
    Delete temp citation table from the database
    """
    sql = ''' DROP TABLE TempCitation'''
    cur = dbConnection.cursor()
    cur.execute(sql)
    dbConnection.commit()

def readRefFromDBByDict(dbConnection, refItem1):
    """
    Read one piece of reference from database with reference dictionary
    :param dbConnection: the Connection object
    :param refItem: reference dictionary
    :return:
    """
    tablename = refItem1['MType']
    sql = "SELECT * FROM " + tablename + " WHERE title=? AND author=? AND year=?"
    cur = dbConnection.cursor()
    cur.execute(sql, (refItem1['title'], refItem1['author'], refItem1['year']))
    rows = cur.fetchall()
    refItem2 = {}
    tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[tablename] + DB_ExtendFields
    if len(rows) >= 1:
        if len(rows[0]) <= len(tempDBFieldsList):
            for i in range(len(rows[0])):
                refItem2[tempDBFieldsList[i]] = rows[0][i]

    return refItem2

# Checked
def writeRefToDB(dbConnection, refDict):
    # Check item existance first
    tempItem = readRefFromDBByDict(dbConnection, refDict)
    if len(tempItem):
        tempItem1 = {}
        tempItem1['title'] = tempItem['title']
        tempItem1['author'] = tempItem['author']
        tempItem1['year'] = tempItem['year']
        tempItem2 = {}
        tempItem2['title'] = refDict['title']
        tempItem2['author'] = refDict['author']
        tempItem2['year'] = refDict['year']
        if tempItem1 == tempItem2:
            pass
        else:
            # Not exact the same, wait to check by users
            tablename = refDict['MType']
            tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[tablename] + DB_ExtendFields
            sql = "INSERT INTO " + tablename + "(" + ",".join(tempDBFieldsList[1:]) + ") VALUES(" + ",".join(["?"]*len(tempDBFieldsList[1:])) + ")"
            task = ()
            for field in tempDBFieldsList[1:]:
                task = task + (refDict[field],)
            cur = dbConnection.cursor()
            cur.execute(sql, task)
            dbConnection.commit()
    else:
        # Not exist, add to database
        tablename = refDict['MType']
        tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[tablename] + DB_ExtendFields
        sql = "INSERT INTO " + tablename + "(" + ",".join(tempDBFieldsList[1:]) + ") VALUES(" + ",".join(["?"]*len(tempDBFieldsList[1:])) + ")"
        task = ()
        for field in tempDBFieldsList[1:]:
            task = task + (refDict[field],)
        cur = dbConnection.cursor()
        cur.execute(sql, task)
        dbConnection.commit()

# Checked
def writeRefsToDB(dbConnection, refDictList):
    if len(refDictList):
        for refDict in refDictList:
            writeRefToDB(dbConnection, refDict)
            updateRefAbsID(dbConnection, refDict)

def updateRefAbsID(dbConnection, refDict):
    tempItem = readRefFromDBByDict(dbConnection, refDict)
    tablename = refDict['MType']
    sql = "UPDATE " + tablename + " SET  RefAbsID= ? WHERE id = ?"
    refAbsID = int(str(DB_TypeCode[tablename])+str(tempItem['ID']).zfill(8))
    task = (refAbsID, tempItem['ID'])
    cur = dbConnection.cursor()
    cur.execute(sql, task)
    dbConnection.commit()

def readRefInDBTableByID(dbConnection, refType, refAbsID):
    tablename = refType
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM " + tablename +" WHERE RefAbsID=?", (refAbsID,))
    rows = cur.fetchall()
    refItem = {}
    tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[refType] + DB_ExtendFields
    row = rows[0]
    refItem = DB2Dict(rows, tablename)[0]
    return refItem

# Checked
def readRefFromDBByID(dbConnection, refAbsID):
    """
    Read one piece of reference from database with reference id
    :param dbConnection: the Connection object
    :param refAbsID: reference absolution id
    :return:
    """
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM ReferencesData WHERE id=?", (refAbsID,))
    rows = cur.fetchall()
    refItem = {}
    if len(rows[0]) <= len(DatabaseReferenceStructure):
        for i in range(len(rows[0])):
            refItem[DatabaseReferenceStructure[i]] = rows[0][i]
    return refItem

# Checked
def readRefsFromDBByIDs(dbConnection, refAbsIDList):
    refDictList = []
    for refID in refAbsIDList:
        refItem = readRefFromDBByID(dbConnection, refID)
        refDictList.append(refItem)
    return refDictList

def readAllRecentInDB(dbConnection, timeStr):
    allRefItemList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempList = readRecentInTable(dbConnection, tablename, timeStr)
        allRefItemList = allRefItemList + tempList
    return allRefItemList

def readRecentInTable(dbConnection, tablename, timeStr):
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM " + tablename + " WHERE AddedTime>?", (timeStr,))
    rows = cur.fetchall()
    refItemList = DB2Dict(rows, tablename)
    return refItemList

def DB2Dict(dbRows, tablename):
    refItemList = []
    tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[tablename] + DB_ExtendFields
    if len(dbRows):
        for row in dbRows:
            if len(row) <= len(tempDBFieldsList):
                refItem = {}
                refItem['MType'] = tablename
                refItem['PubIn'] = ""
                baseLength = len(DB_BaseFields)
                for i in range(baseLength):
                    refItem[DB_BaseFields[i]] = row[i]

                for i in range(baseLength, len(row)):
                    tempFieldName = tempDBFieldsList[i].capitalize()
                    if tempFieldName == 'Journal':
                        tempFieldName = 'PubIn'
                    elif tempFieldName == 'Booktitle':
                        tempFieldName = 'PubIn'
                    refItem[tempFieldName] = row[i]
                    if row[i] is None:
                        refItem[tempFieldName] = ""
                refItemList.append(refItem)
    return refItemList

def readAllRefsInDBByField(dbConnection, fieldList, keywordList):
    allRefItemList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempList = readAllRefsInTableByField(dbConnection, tablename, fieldList, keywordList)
        allRefItemList = allRefItemList + tempList
    return allRefItemList

def readAllRefsInTableByField(dbConnection, tablename, fieldList, keywordList):
    checkFlag = False
    tempFieldList = list(fieldList)
    if 'PubIn' not in fieldList:
        checkFlag = True
    else:
        tempInd = tempFieldList.index('PubIn')
        if tablename == 'Article':
            tempFieldList[tempInd] = 'journal'
            checkFlag = True
        elif tablename == 'Conference':
            tempFieldList[tempInd] = 'booktitle'
            checkFlag = True

    refItemList = []
    if checkFlag:
        sql = "SELECT * FROM " + tablename + " WHERE "
        if len(fieldList) == 1:
            sql = sql + tempFieldList[0] + "=?"
            cur = dbConnection.cursor()
            cur.execute(sql, (keywordList[0],))
        elif len(fieldList) == 2:
            sql = sql + tempFieldList[0] + "=? AND " + tempFieldList[1] + "=?"
            cur = dbConnection.cursor()
            cur.execute(sql, (keywordList[0],keywordList[1]))

        rows = cur.fetchall()
        refItemList = DB2Dict(rows, tablename)
    return refItemList

def readAllRefsInDBByLabelPartialMatch(dbConnection, keyword):
    allRefItemList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempList = readAllRefsInTableByLabelPartialMatch(dbConnection, tablename, keyword)
        allRefItemList = allRefItemList + tempList
    return allRefItemList

def readAllRefsInTableByLabelPartialMatch(dbConnection, tablename, keyword):
    refItemList = []
    sql = '''SELECT * FROM ''' + tablename + ''' WHERE Labels LIKE "%''' + keyword + '''%"'''
    cur = dbConnection.cursor()
    cur.execute(sql)
    rows = cur.fetchall()
    if len(rows) > 0:
        refItemList = DB2Dict(rows, tablename)
    return refItemList

def searchRefInTable(dbConnection, tablename, searchTarget):
    refItemList = []
    # sql = '''SELECT * FROM ''' + tablename + ''' WHERE Year=?'''
    # values = (searchTarget[0][1], )
    sql_base = '''SELECT * FROM ''' + tablename + ''' WHERE '''
    query_str = ""
    pubinField = ""
    for tarItem in searchTarget:
        tarKey = tarItem[0]
        tarValue = tarItem[1]
        if tarKey == 'Year':
            if len(query_str) == 0:
                query_str = query_str + "Year=" + tarValue
            else:
                query_str = query_str + " AND Year=" + tarValue
        elif tarKey == 'Title':
            if len(query_str) == 0:
                query_str = query_str + '''title LIKE "%''' + tarValue + '''%"'''
            else:
                query_str = query_str + ''' AND title LIKE "%''' + tarValue + '''%"'''
        elif tarKey == 'Author':
            if len(query_str) == 0:
                query_str = query_str + '''author LIKE "%''' + tarValue + '''%"'''
            else:
                query_str = query_str + ''' AND author LIKE "%''' + tarValue + '''%"'''
        elif tarKey == 'PubIn':
            if tablename == 'Article':
                pubinField = 'journal'
            elif tablename == 'Inproceedings':
                pubinField = 'booktitle'
            else:
                pass
            if len(pubinField) > 0:
                if len(query_str) == 0:
                    query_str = query_str + pubinField + ''' LIKE "%''' + tarValue + '''%"'''
                else:
                    query_str = query_str + " AND " + pubinField + ''' LIKE "%''' + tarValue + '''%"'''
    sql_str = sql_base + query_str
    if len(pubinField)==0 and len(query_str) == 0:
        pass
    else:
        cur = dbConnection.cursor()
        cur.execute(sql_str)
        rows = cur.fetchall()
        if len(rows) > 0:
            refItemList = DB2Dict(rows, tablename)
    return refItemList

def searchRefInDB(dbConnection, searchTarget):
    allRefItemList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempList = searchRefInTable(dbConnection, tablename, searchTarget)
        allRefItemList = allRefItemList + tempList
    return allRefItemList

def readAllRefsFromDB(dbConnection):
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM ReferencesData")
    rows = cur.fetchall()
    refItemList = []
    if len(rows):
        for row in rows:
            if len(row) <= len(DatabaseReferenceStructure):
                refItem = {}
                for i in range(len(row)):
                    refItem[DatabaseReferenceStructure[i]] = row[i]
                refItemList.append(refItem)
    return refItemList

def readAllRefsInDB(dbConnection):
    allRefItemList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempList = readAllRefsInTable(dbConnection, tablename)
        allRefItemList = allRefItemList + tempList
    return allRefItemList

def readAllRefsInTable(dbConnection, tablename):
    sql = "SELECT * FROM " + tablename
    cur = dbConnection.cursor()
    cur.execute(sql)
    rows = cur.fetchall()
    refItemList = []
    # tempDBFieldsList = DB_BaseFields + DatabaseStandardStructure[tablename] + DB_ExtendFields
    if len(rows):
        refItemList = DB2Dict(rows, tablename)
    return refItemList

def updateRefToDBByID(dbConnection, refAbsID, value):
    sql = ''' UPDATE ReferencesData
              SET Labels = ?
              WHERE id = ?'''
    task = (value, refAbsID)
    cur = dbConnection.cursor()
    cur.execute(sql, task)
    dbConnection.commit()

# Checked
def updateRefFieldToDBByID(dbConnection, refAbsID, field, value):
    sql = ''' UPDATE ReferencesData
              SET ''' + field + ''' = ?
              WHERE id = ?'''
    task = (value, refAbsID)
    cur = dbConnection.cursor()
    cur.execute(sql, task)
    dbConnection.commit()

# New
def updateRefLabelByID(dbConnection, tablename, refAbsID, value):
    sql = ''' UPDATE ''' + tablename + '''
              SET Labels = ?
              WHERE RefAbsID = ?'''
    task = (value, refAbsID)
    cur = dbConnection.cursor()
    cur.execute(sql, task)
    dbConnection.commit()

# Checked
def getLabelsFromDB(dbConnection):
    """
    Get labels from database
    This function will concentrate the labels to a list
    """
    labelRows = readLabelsFromDB(dbConnection)
    labels = list(map(lambda x: x[1], labelRows))
    return labels

# Checked
def readLabelsFromDB(dbConnection):
    """
    Read labels from Labels table of the database
    """
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM Labels")
    rows = cur.fetchall()
    return rows

def addLabelToLabelsTable(dbConnection, labelText):
    tempLabelList = getLabelsFromDB(dbConnection)
    if labelText not in tempLabelList:
        sql = ''' INSERT INTO Labels(Name)
                  VALUES(?) '''
        task = (labelText,)
        cur = dbConnection.cursor()
        cur.execute(sql, task)
        dbConnection.commit()

# Checked
def getCitationsFromDB(dbConnection, refAbsID):
    citationRows = readCitationsFromDB(dbConnection, refAbsID)
    citationListStr = None
    if len(citationRows):
        citations = citationRows[0][2]
        if len(citations):
            try:
                citationListStr = citations.split(",")
            except:
                pass
    return citationListStr

# Checked
def readCitationsFromDB(dbConnection, refAbsID):
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM Citation WHERE Self=?", (refAbsID,))
    rows = cur.fetchall()
    return rows

# Checked
def copyCitationData(dbConnection, refAbsID):
    citationRows = readCitationsFromDB(dbConnection, refAbsID)
    if len(citationRows) > 0:
        cur = dbConnection.cursor()
        cur.execute("SELECT * FROM TempCitation WHERE Self=?", (refAbsID,))
        rows = cur.fetchall()
        if len(rows) == 0:
            sql = ''' INSERT INTO TempCitation(Self, Cited, CitedBy)
                      VALUES(?,?,?) '''
            task = (citationRows[0][1], citationRows[0][2], citationRows[0][3])
            cur = dbConnection.cursor()
            cur.execute(sql, task)
            dbConnection.commit()
        else:
            pass

# Checked
def getTempCitationsFromDB(dbConnection):
    citationRows = readTempCitationsFromDB(dbConnection)
    return citationRows

# Checked
def readTempCitationsFromDB(dbConnection):
    cur = dbConnection.cursor()
    cur.execute("SELECT * FROM TempCitation")
    rows = cur.fetchall()
    return rows

def UpdateDatabase(dbConnection):
    indexYearsInDB(dbConnection)
    indexPubInDB(dbConnection)

def indexYearsInDB(dbConnection):
    allYearList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempYearList = indexYearsInTable(dbConnection, tablename)
        allYearList = allYearList + tempYearList
    allYearList = list(set(allYearList))
    allYearList.sort()
    updateYearsTable(dbConnection, allYearList)

def updateYearsTable(dbConnection, yearList):
    cur = dbConnection.cursor()
    cur.execute("DELETE FROM Years")
    for year in yearList:
        sql = "INSERT INTO Years (Year) VALUES(?)"
        cur = dbConnection.cursor()
        cur.execute(sql, (year,))
        dbConnection.commit()

def indexYearsInTable(dbConnection, tablename):
    cur = dbConnection.cursor()
    cur.execute("SELECT GROUP_CONCAT(Year, ',') FROM " + tablename)
    tempValue = cur.fetchone()
    yearList = []
    if len(tempValue):
        if tempValue[0] is None:
            pass
        else:
            yearList = tempValue[0].split(',')
    return yearList

def indexPubInDB(dbConnection):
    allPubInList = []
    for type in BibTeXTypes:
        tablename = type.capitalize()
        tempPubInList = indexPubInTable(dbConnection, tablename)
        allPubInList = allPubInList + tempPubInList
    allPubInList = list(set(allPubInList))
    allPubInList.sort()
    updatePubInTable(dbConnection, allPubInList)

def updatePubInTable(dbConnection, pubInList):
    cur = dbConnection.cursor()
    cur.execute("DELETE FROM PubIn")
    for pubin in pubInList:
        sql = "INSERT INTO PubIn (Name) VALUES(?)"
        cur = dbConnection.cursor()
        cur.execute(sql, (pubin,))
        dbConnection.commit()

def indexPubInTable(dbConnection, tablename):
    fieldname = ""
    if tablename == 'Article':
        fieldname = 'journal'
    elif tablename == 'Conference':
        fieldname = 'booktitle'
    elif tablename == 'Inproceedings':
        fieldname = 'booktitle'
    pubInList = []
    if len(fieldname)>0:
        sql = "SELECT GROUP_CONCAT(" + fieldname + ", ';') FROM " + tablename
        cur = dbConnection.cursor()
        cur.execute(sql)
        tempValue = cur.fetchone()
        if len(tempValue):
            if tempValue[0] is None:
                pass
            else:
                pubInList = tempValue[0].split(';')
    return pubInList

if __name__ == "__main__":
    createDB("Test.db")


# coding=utf-8

""" Batch module """

import ee
from . import alerts, utils
import requests
import os
from geetools import batch as gbatch

FUNCTIONS = {
    'probable': alerts.get_probable,
    'confirmed': alerts.get_confirmed,
    'both': alerts.oneday,
    'period': alerts.period
}

def mask(image, vector, raster):
    """ Mask out a vector mask or a raster mask """
    if vector:
        return image.clip(vector)
    elif raster:
        return image.updateMask(raster)
    else:
        return image

def downloadFile(url, name, ext, path=None):
    """ Download a file from a given url

    :param url: full url
    :type url: str
    :param name: name for the file (can contain a path)
    :type name: str
    :param ext: extension for the file
    :type ext: str
    :return: the created file (closed)
    :rtype: file
    """
    response = requests.get(url, stream=True)
    code = response.status_code

    if path is None:
        path = os.getcwd()

    path = os.path.join(path, name)

    while code != 200:
        if code == 400:
            return None
        response = requests.get(url, stream=True)
        code = response.status_code
        size = response.headers.get('content-length', 0)
        if size: print('size:', size)

    filename = '{}.{}'.format(path, ext)

    with open(filename, "wb") as handle:
        for data in response.iter_content():
            handle.write(data)

    return handle

def _download(vector, name, extension='JSON', path=None, verbose=True,
              logger=None):
    if extension in ['JSON', 'json', 'geojson', 'geoJSON']:
        try:
            gbatch.Download.table.toGeoJSON(vector, name, path)
        except Exception as e:
            msg = 'Download method failed: {} \n\ntrying another method...'.format(e)
            if verbose:
                print(msg)
            if logger:
                logger.log(msg)
            try:
                gbatch.Download.table.toLocal(vector, name, 'geojson',
                                              path=path)
            except Exception as e:
                msg = "Download failed: {}".format(e)
                if verbose:
                    print(msg)
                if logger:
                    logger.log(msg)
    else:
        print('Format {} not supported'.format(extension))

def _toDrive(vector, filename, folder, extension, **kwargs):
    verbose = kwargs.get('verbose', False)
    logger = kwargs.get('logger', None)

    try:
        task = ee.batch.Export.table.toDrive(vector, filename,
                                             folder, filename,
                                             extension)
        task.start()
        msg = 'uploading {} to {} in GDrive'.format(filename, folder)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)

    except Exception as e:
        msg = 'ERROR writing {} - {}'.format(filename, e)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)

def _toAsset(vector, filename, folder, **kwargs):
    verbose = kwargs.get('verbose', False)
    logger = kwargs.get('logger', None)

    user = ee.data.getAssetRoots()[0]['id']
    path = '{}/{}'.format(user, folder)

    assetId = '{}/{}'.format(path, filename)

    try:
        # task = ee.batch.Export.table.toAsset(vector, filename, assetId)
        # task.start()
        gbatch.Export.table.toAsset(vector, path, filename)
        msg = 'uploading {} to {} in Assets'.format(filename, path)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)

    except Exception as e:
        msg = 'ERROR in {} to {} in Assets - {}'.format(filename, path, e)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)

def _toLocal(vector, filename, folder=None, extension='geojson',
             subfolders=True, subname=None, **kwargs):

    verbose = kwargs.get('verbose', True)
    logger = kwargs.get('logger', None)

    # MANAGE ALERTS PATH
    if folder is None:
        folder = os.path.join(os.getcwd(), 'alerts')

    # make path if not present
    if not os.path.isdir(folder):
        if verbose:
            print('creating {} folder'.format(folder))
        os.mkdir(folder)

    if subfolders:
        subpath = os.path.join(folder, subname)
        if not os.path.isdir(subpath):
            if verbose:
                print('creating {}'.format(subpath))
            os.mkdir(subpath)
    else:
        subpath = folder

    msg = '{}: Downloading "{}" to "{}"'.format(subname, filename, subpath)
    if verbose:
        print(msg)
    if logger:
        logger.log(msg)

    try:
        _download(vector, filename, extension, subpath)
    except Exception as e:
        msg = '{}: ERROR writing {}'.format(subname, filename)
        if logger:
            logger.log(msg)
    else:
        msg = '{}: "{}" downloaded to "{}"'.format(subname, filename, subpath)
        if logger:
            logger.log(msg)

def _are_alerts(alert, name, date, clas, region, verbose, logger):
    try:
        count = utils.histogram(alert, clas, region).getInfo()
    except Exception as e:
        msg = '{}: ERROR getting histogram - {}'.format(name, e)
        if logger:
            logger.log(msg)
        return False

    if count == 0:
        msg = '{}: no alerts for {}'.format(name, date)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)
        return False
    else:
        return True

def _process_period(start, end, geometry, limit, year=None,
                    eightConnected=False, useProxy=False, mask=None,
                    destination='local', name=None, folder=None, **kwargs):
    verbose = kwargs.get('verbose', True)
    logger = kwargs.get('logger', None)

    try:
        alert = FUNCTIONS['period'](start, end, geometry, limit, year,
                                    eightConnected, useProxy, mask)
    except Exception as e:
        msg = 'ERROR while getting period alert {} to {}'.format(start, end)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)
        raise e

    date_str = '{} to {}'.format(start, end)

    are_alerts = _are_alerts(alert, name, date_str, 'both', geometry, **kwargs)
    if not are_alerts:
        return None

    filename = '{}_{}_to_{}'.format(name, start, end)

    vector = utils.make_alerts_vector(alert, geometry)
    # LOCAL
    if destination == 'local':
        subfolders = kwargs.get('subfolders', True)
        ext = kwargs.get('extension', 'geojson')
        _toLocal(vector, filename, folder, ext, subfolders,
                 name, **kwargs)

    elif destination == 'drive':
        filename = filename.encode().decode('ascii', errors='ignore')
        ext = kwargs.get('extension', 'geojson')
        _toDrive(vector, filename, folder, ext, **kwargs)

    elif destination == 'asset':
        _toAsset(vector, filename, folder, **kwargs)

def _process(geometry, date, clas, limit, folder, raster_mask, destination,
             filename,  name, **kwargs):
    verbose = kwargs.get('verbose', True)
    logger = kwargs.get('logger', None)

    try:
        alert = FUNCTIONS[clas](geometry, date, limit, mask=raster_mask)
    except Exception as e:
        msg = 'ERROR while getting alert for {}'.format(date)
        if verbose:
            print(msg)
        if logger:
            logger.log(msg)
        raise e

    # SKIP IF EMPTY ALERT
    are_alerts = _are_alerts(alert, name, date, clas, geometry, **kwargs)
    if not are_alerts:
        return None

    vector = utils.make_alerts_vector(alert, geometry)
    # LOCAL
    if destination == 'local':
        subfolders = kwargs.get('subfolders', True)
        ext = kwargs.get('extension', 'geojson')
        _toLocal(vector, filename, folder, ext, subfolders,
                 name, **kwargs)

    elif destination == 'drive':
        filename = filename.encode().decode('ascii', errors='ignore')
        ext = kwargs.get('extension', 'geojson')
        _toDrive(vector, filename, folder, ext, **kwargs)

    elif destination == 'asset':
        _toAsset(vector, filename, folder, **kwargs)

def period(site, start, end, limit, year=None, proxy=False, eightConnected=False,
           folder=None, property_name=None, raster_mask=None,
           destination='local', verbose=True, logger=None):
    """ General download function for a period """

    args = dict(verbose=verbose, logger=logger)

    # START PROCESS
    # If it is a FeatureCollection and there is a property name
    if isinstance(site, ee.FeatureCollection) and property_name:
        names = utils.get_options(site, property_name)
        names_cli = names.getInfo()
        for name in names_cli:

            geom = site.filterMetadata(
                property_name, 'equals', name).first().geometry()

            _process_period(start, end, geom, limit, year, eightConnected,
                            proxy, raster_mask, destination, name, folder,
                            **args)
    else:
        if isinstance(site, ee.Feature) and property_name:
            name = ee.String(site.get(property_name)).getInfo()
        else:
            name = 'N/A'

        # GET GEOMETRY
        if isinstance(site, (ee.FeatureCollection, ee.Feature)):
            geom = site.geometry()
        else:
            geom = site

        _process_period(start, end, geom, limit, year, eightConnected,
                        proxy, raster_mask, destination, name, folder,
                        **args)

def download(site, date, clas, limit, folder=None, property_name=None,
             raster_mask=None, destination='local', verbose=True, logger=None):
    """ General download function """
    if not utils.has_image(date, alerts.ALERTS).getInfo():
        msg = 'GLAD alerts not available for date {}'.format(date)
        if logger:
            logger.log(msg)
        return None

    if clas not in ['probable', 'confirmed', 'both']:
        clas = 'both'

    # BASE NAME FOR OUTPUT FILE
    if clas == 'both':
        basename = 'alerts_for'
    else:
        basename = '{}_alerts_for'.format(clas)

    args = dict(verbose=verbose, logger=logger)

    # START PROCESS
    # If it is a FeatureCollection and there is a property name
    if isinstance(site, ee.FeatureCollection) and property_name:
        names = utils.get_options(site, property_name)
        names_cli = names.getInfo()
        for name in names_cli:
            filename = '{}_{}_{}'.format(basename, date, name)

            geom = site.filterMetadata(
                property_name, 'equals', name).first().geometry()

            _process(geom, date, clas, limit, folder, raster_mask, destination,
                     filename,  name, **args)
    else:
        if isinstance(site, ee.Feature) and property_name:
            name = ee.String(site.get(property_name)).getInfo()
            filename = '{}_{}_{}'.format(basename, date, name)
        else:
            name = 'N/A'
            filename = '{}_{}'.format(basename, date)

        # GET GEOMETRY
        if isinstance(site, (ee.FeatureCollection, ee.Feature)):
            geom = site.geometry()
        else:
            geom = site

        _process(geom, date, clas, limit, folder, raster_mask, destination,
                 filename,  name, **args)


import numpy as np
from settings import loader_settings
import medpy.io
import os, pathlib

class Seg():
    def __init__(self):
        # super().__init__(
        #     validators=dict(
        #         input_image=(
        #             UniqueImagesValidator(),
        #             UniquePathIndicesValidator(),
        #         )
        #     ),
        # )
        return

    def process(self):
        inp_path = loader_settings['InputPath']  # Path for the input
        out_path = loader_settings['OutputPath']  # Path for the output
        file_list = os.listdir(inp_path)  # List of files in the input
        file_list = [os.path.join(inp_path, f) for f in file_list]
        for fil in file_list:
            dat, hdr = medpy.io.load(fil)  # dat is a numpy array
            im_shape = dat.shape
            dat = dat.reshape(1, 1, *im_shape)  # reshape to Pytorch standard
            # Convert 'dat' to Tensor, or as appropriate for your model.
            ###########
            ### Replace this section with the call to your code.
            mean_dat = np.mean(dat)
            dat[dat > mean_dat] = 1
            dat[dat <= mean_dat] = 0
            ###
            ###########
            dat = dat.reshape(*im_shape)
            out_name = os.path.basename(fil)
            out_filepath = os.path.join(out_path, out_name)
            print(f'=== saving {out_filepath} from {fil} ===')
            medpy.io.save(dat, out_filepath, hdr=hdr)
        return

if __name__ == "__main__":
    pathlib.Path("/output/images/stroke-lesion-segmentation/").mkdir(parents=True, exist_ok=True)
    Seg().process()


"""
A Python implementation of the FIFO, LRU and OPT page replacement algorithms.

Author: Jaron Cohen
Student Number: CHNJAR003

Sources:
[CSC3002F, 2020] CSC3002F (2020). A Comparison of Three Page Replacement Algorithms:
FIFO, LRU and Optimal. Department of Computer Science, University of Cape Town,
South Africa.
[Silberschatz et al., 2018] Silberschatz, A., Galvin, P., and Gagne, G. (2018). Operating Sys-
tem Concepts - 10th Edition. John Wiley & Sons, New York, USA.

"""

import sys
import random
from collections import deque

"""
Performs the FIFO algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def FIFO(size, pages):
    memory = []
    queue = deque([])
    numPageFaults = 0

    for page in pages:
        if len(memory) < size:
            memory.append(page)
            queue.append(page)
            numPageFaults += 1
        else:
            if page not in memory:
                pageToRemove = queue.popleft()
                indexOfPageToRemove = memory.index(pageToRemove)
                memory.remove(pageToRemove)
                memory.insert(indexOfPageToRemove, page)
                queue.append(page)
                numPageFaults += 1
    return numPageFaults

"""
Performs the LRU algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def LRU(size, pages):
    memory = []
    stack = deque()
    numPageFaults = 0

    for page in pages:
        if len(memory) < size:
            memory.append(page)
            stack.appendleft(page)
            numPageFaults += 1

        else:
            if page in memory:  # If page is referenced, remove and place on top of stack

                pageStackIndex = stack.index(page)

                del stack[pageStackIndex]
                stack.appendleft(page)

            else:

                minPage = stack.pop()

                indexOfPageToRemove = memory.index(minPage)

                memory.remove(minPage)
                memory.insert(indexOfPageToRemove, page)
                stack.appendleft(page)
                numPageFaults += 1

    return numPageFaults

"""
Performs the OPT algorithm on an array of integer page references in the range of 0 to 9 for a given memory size.
parameters:
size - integer number in range 1 to 7 of number of frames in memory
pages - array of integer page references with each element in the range of 0 to 9
"""

def OPT(size, pages):
    memory = []
    numPageFaults = 0
    for i in range(len(pages)):

        if len(memory) < size:
            memory.append(pages[i])
            numPageFaults += 1

        else:

            if pages[i] not in memory:

                lastToBeUsed = -1
                # Find page in memory that is never referenced in the future
                for frame in memory:
                    # If there is a page in a frame in memory that doesn't appear in the rest of the pages sequence
                    if frame not in pages[i+1:]:
                        lastToBeUsed = frame
                        break

                # If found page not referenced in future, replace it
                if lastToBeUsed != -1:

                    indexOfPageToRemove = memory.index(lastToBeUsed)

                    memory.remove(lastToBeUsed)
                    memory.insert(indexOfPageToRemove, pages[i])
                    numPageFaults += 1
                else:  # If all pages in memory are referenced in future, find farthest in future

                    farthestIndex = pages[i+1:].index(memory[0])
                    pageWithFarthestIndex = memory[0]
                    # Iterate through memory frames and find farthest one in sequence
                    for frameIndex in range(1, len(memory)):
                        newIndex = pages[i+1:].index(memory[frameIndex])
                        if newIndex > farthestIndex:
                            farthestIndex = newIndex
                            pageWithFarthestIndex = memory[frameIndex]
                    indexOfPageToRemove = memory.index(pageWithFarthestIndex)

                    memory.remove(pageWithFarthestIndex)
                    memory.insert(indexOfPageToRemove, pages[i])
                    numPageFaults += 1

    return numPageFaults

def main():

    if len(sys.argv) == 2:
        numFrames = int(sys.argv[1])
        numPages = random.randint(10, 100)
        pages = [random.randint(0, 9) for ranNum in range(numPages)]
        print('Random Page References: ', pages)
        print('Number of Page References: ', len(pages))
        print('Number of Frames: ', numFrames)
        print('FIFO', FIFO(numFrames, pages), 'page faults.')
        print('LRU', LRU(numFrames, pages), 'page faults.')
        print('OPT', OPT(numFrames, pages), 'page faults.')
    else:
        numFrames = int(sys.argv[1])
        try:
            numPages = int(sys.argv[2])
        except:
            print("Invalid second argument provided.")

        pages = [random.randint(0, 9) for ranNum in range(numPages)]
        print('Random Page References: ', pages)
        print('Number of Page References: ', len(pages))
        print('Number of Frames: ', numFrames)
        print('FIFO', FIFO(numFrames, pages), 'page faults.')
        print('LRU', LRU(numFrames, pages), 'page faults.')
        print('OPT', OPT(numFrames, pages), 'page faults.')

if __name__ == "__main__":

    if len(sys.argv) < 2:
        print('Usage: python3 paging.py number_of_frames [number_of_pages]')
        print('Refer to README for additional information.')
    elif len(sys.argv) >= 2 and len(sys.argv) <= 3:

        numFrames = int(sys.argv[1])
        if numFrames in range(1, 8):
            main()
        else:
            print("Invalid number of frames entered. Provide integer from 1 to 7.")

    else:
        print(
            'Too many arguments provided. Refer to README for valid command line arguments.')


import datetime
from typing import Any, Dict, List, Optional, Type, TypeVar

from attrs import define as _attrs_define
from attrs import field as _attrs_field
from dateutil.parser import isoparse

T = TypeVar("T", bound="SequenceRevisionDetailsDto")

@_attrs_define
class SequenceRevisionDetailsDto:
    """
    Attributes:
        id (float):
        owner (str):
        published (bool):
        comment (str):
        created_date (Optional[datetime.datetime]):
    """

    id: float
    owner: str
    published: bool
    comment: str
    created_date: Optional[datetime.datetime]
    additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        id = self.id
        owner = self.owner
        published = self.published
        comment = self.comment
        created_date = self.created_date.isoformat() if self.created_date else None

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "id": id,
                "owner": owner,
                "published": published,
                "comment": comment,
                "createdDate": created_date,
            }
        )

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        id = d.pop("id")

        owner = d.pop("owner")

        published = d.pop("published")

        comment = d.pop("comment")

        _created_date = d.pop("createdDate")
        created_date: Optional[datetime.datetime]
        if _created_date is None:
            created_date = None
        else:
            created_date = isoparse(_created_date)

        sequence_revision_details_dto = cls(
            id=id,
            owner=owner,
            published=published,
            comment=comment,
            created_date=created_date,
        )

        sequence_revision_details_dto.additional_properties = d
        return sequence_revision_details_dto

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


from ftree import treedata
import pgbase
import pygame
import numpy as np
import moderngl
import sys
import shapely.geometry
import networkx as nx
import itertools
import scipy.optimize
import scipy.spatial
import math
import time
import random
import os

##from ftree import shapelayout
##def entity_format(entity):
##    if type(entity) == treedata.Person:
##        sex = entity.get_sex()
##        if sex == "male":
##            colour = (0, 0.5, 1, 1)
##        elif sex == "female":
##            colour = (1, 0.5, 0.5, 1)
##        else:
##            colour = (1, 0.5, 0, 1)
##        name = " ".join(entity.get_first_names())
##        return shapelayout.height_frame(shapelayout.string((name if len(name) != 0 else "?"), (0, 0, 0, 1)), 0.9, colour)
##    elif type(entity) == treedata.Partnership:
##        return shapelayout.height_frame(shapelayout.letter("M", (0, 0, 0, 1)), 0.9, (0.7, 0.7, 0, 1))
##    else:
##        return shapelayout.height_frame(shapelayout.letter("?", (0, 0, 0, 1)), 0.9)

class TreeView(pgbase.canvas2d.Window2D):
    def __init__(self, tree, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.tree = tree

        self.bg_prog = self.ctx.program(
            vertex_shader = """
                #version 430
                in vec2 unit_pos;
                out vec4 gl_Position;
                out vec2 v_pos;

                void main() {
                    gl_Position = vec4(unit_pos, 0.0, 1.0);
                    v_pos = vec2(unit_pos);
                }
            """,
            fragment_shader = """
                #version 430
                in vec2 v_pos;
                out vec4 f_colour;

                uniform vec2 cam_center;
                uniform mat2 cam_mat;

                void main() {
                    vec2 pos = cam_mat * v_pos + cam_center;
                    bool bx = mod(pos.x, 2) < 1;
                    bool by = mod(pos.y, 2) < 1;

                    if ((bx && by) || (!bx && !by)) {
                        f_colour = 0.98 * vec4(1, 1, 1, 0);
                    } else {
                        f_colour = 1 * vec4(1, 1, 1, 0);
                    }
                }

            """,
        )
        vertices = self.ctx.buffer(np.array([[-1, -1], [1, -1], [-1, 1], [1, 1]]).astype('f4'))
        indices = self.ctx.buffer(np.array([[0, 1, 3], [0, 2, 3]]))
        self.bg_vao = self.ctx.vertex_array(self.bg_prog,
                                         [(vertices, "2f4", "unit_pos")],
                                         indices)

        self.entity_prog = self.ctx.program(
            vertex_shader = """
                #version 430
                in vec2 pos;
                out vec2 g_pos;

                in float border;
                out float g_border;

                in uint v_tex_idx;
                flat out uint g_tex_idx;

                void main() {
                    g_pos = pos;
                    g_tex_idx = v_tex_idx;
                    g_border = border;
                }
            """,
            geometry_shader = """
                #version 430
                layout (points) in;
                layout (triangle_strip, max_vertices = 100) out;
                in vec2 g_pos[];

                flat in uint g_tex_idx[];
                flat out uint f_tex_idx;

                in float g_border[];
                out float f_border;

                uniform vec2 cam_center;
                uniform mat2 cam_mat_inv;

                uniform float w;
                uniform float h;

                out vec2 f_tex_pos;

                void main() {
                    f_tex_idx = g_tex_idx[0];
                    f_border = g_border[0];

                    vec2 pos;

                    float bw = w + 2 * g_border[0];
                    float bh = h + 2 * g_border[0];

                    f_tex_pos = vec2(-g_border[0] / w, -g_border[0] / h);
                    pos = g_pos[0] + vec2(-bw, -bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    f_tex_pos = vec2(1 + g_border[0] / w, -g_border[0] / h);
                    pos = g_pos[0] + vec2(bw, -bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    f_tex_pos = vec2(1 + g_border[0] / w, 1 + g_border[0] / h);
                    pos = g_pos[0] + vec2(bw, bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    EndPrimitive();

                    f_tex_pos = vec2(-g_border[0] / w, -g_border[0] / h);
                    pos = g_pos[0] + vec2(-bw, -bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    f_tex_pos = vec2(-g_border[0] / w, 1 + g_border[0] / h);
                    pos = g_pos[0] + vec2(-bw, bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    f_tex_pos = vec2(1 + g_border[0] / w, 1 + g_border[0] / h);
                    pos = g_pos[0] + vec2(bw, bh);
                    gl_Position = vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                    EmitVertex();
                    EndPrimitive();
                }
            """,
            fragment_shader = """
                #version 430
                in vec2 f_tex_pos;
                in float f_border;
                out vec4 f_colour;
                flat in uint f_tex_idx;

                uniform sampler2DArray tex;

                void main() {
                    if (f_tex_pos.x < 0 || f_tex_pos.x > 1 || f_tex_pos.y < 0 || f_tex_pos.y > 1) {
                        f_colour = vec4(0, 0, 0, 1);
                    } else {
                        f_colour = texture(tex, vec3(f_tex_pos, f_tex_idx));
                    }
                }

            """,
        )
        self.entity_prog["tex"] = 0
        self.entity_prog["w"] = 0.4
        self.entity_prog["h"] = 0.6
        self.entity_vao = None

        self.edge_prog = self.ctx.program(
            vertex_shader = """
                #version 430
                in vec2 top_pos;
                in vec2 bot_pos;
                in vec4 colour;
                out vec2 g_top_pos;
                out vec2 g_bot_pos;
                out vec4 g_colour;

                void main() {
                    g_top_pos = top_pos;
                    g_bot_pos = bot_pos;
                    g_colour = colour;
                }
            """,
            geometry_shader = """
                #version 430
                layout (points) in;
                layout (triangle_strip, max_vertices = 128) out;
                in vec2 g_top_pos[];
                in vec2 g_bot_pos[];
                in vec4 g_colour[];
                out vec4 g_colour_out;

                uniform vec2 cam_center;
                uniform mat2 cam_mat_inv;

                uniform float width;

                vec4 pos_to_gl(vec2 pos) {
                    return vec4(cam_mat_inv * (pos - cam_center), 0, 1);
                }

                void emit_line(vec2 p1, vec2 p2) {
                    vec2 vec = p2 - p1;
                    vec2 norm = width * normalize(vec2(-vec.y, vec.x));

                    gl_Position = pos_to_gl(p1 + norm);
                    EmitVertex();
                    gl_Position = pos_to_gl(p1 - norm);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 - norm);
                    EmitVertex();
                    EndPrimitive();

                    gl_Position = pos_to_gl(p1 + norm);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 + norm);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 - norm);
                    EmitVertex();
                    EndPrimitive();
                }

                void emit_joint(vec2 p1, vec2 p2, vec2 p3) {
                    vec2 vec12 = p2 - p1;
                    vec2 norm12 = width * normalize(vec2(-vec12.y, vec12.x));
                    vec2 vec23 = p3 - p2;
                    vec2 norm23 = width * normalize(vec2(-vec23.y, vec23.x));

                    gl_Position = pos_to_gl(p2);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 + norm12);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 + norm23);
                    EmitVertex();
                    EndPrimitive();

                    gl_Position = pos_to_gl(p2);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 - norm12);
                    EmitVertex();
                    gl_Position = pos_to_gl(p2 - norm23);
                    EmitVertex();
                    EndPrimitive();
                }

                float bez2(float f, float a, float b) {
                    return a + f * (b - a);
                }

                float bez3(float f, float a, float b, float c) {
                    return bez2(f, a + f * (b - a), b + f * (c - b));
                }

                float bez4(float f, float a, float b, float c, float d) {
                    return bez3(f, a + f * (b - a), b + f * (c - b), c + f * (d - c));
                }

                void main() {
                    g_colour_out = g_colour[0];

                    const int n = 10;
                    const float pi = 3.1415926535897932384626433;

                    vec2[n+2] pts;

                    vec2 p0 = g_top_pos[0];
                    vec2 p1 = g_top_pos[0] - vec2(0, 0.5);
                    vec2 p2 = g_top_pos[0] - vec2(0, 1);
                    vec2 p3 = g_bot_pos[0] + vec2(0, 1);
                    vec2 p4 = g_bot_pos[0] + vec2(0, 0.5);
                    vec2 p5 = g_bot_pos[0];

                    pts[0] = p0;
                    pts[n+1] = p5;

                    for (int i = 0; i < n; i++) {
                        float f = float(i) / float(n - 1);
                        f = 0.5 * (sin(pi * (f - 0.5)) + 1);
                        pts[i+1] = vec2(bez4(f, p1.x, p2.x, p3.x, p4.x), bez4(f, p1.y, p2.y, p3.y, p4.y));
                    }

                    for (int i = 0; i < n+1; i++) {
                        emit_line(pts[i], pts[i+1]);
                    }

                    for (int i = 0; i < n; i++) {
                        emit_joint(pts[i], pts[i+1], pts[i+2]);
                    }

                }
            """,
            fragment_shader = """
                #version 430
                in vec4 g_colour_out;
                out vec4 f_colour;

                void main() {
                    f_colour = g_colour_out;
                }

            """,
        )
        self.edge_prog["width"] = 0.07
        self.edge_vao = None

        self.tex = self.ctx.texture_array([400, 600, 2048], 4, dtype = "f1")
        self.node_surf_data = {}
        self.nodes_waiting_for_surf_update = set()
        self.visible_nodes = {} #node -> texture index

##        self.shapes = pgbase.canvas2d.ShapelyModel(self.ctx)

        self.node_heights = {}
        self.node_widths = {}

        self.root = None
        self.node_widths = {}
        self.node_heights = {}
        self.more_to_see_nodes = set([])
        self.cycle_edges = set([])

        #variables for changing root node animation
        self.node_draw_positions = {}

        self.moving_nodes = []
        self.moving_nodes_old_positions = np.zeros([0, 2])
        self.moving_nodes_new_positions = np.zeros([0, 2])
        self.last_rootchange_time = time.time()

        self.G = self.tree.digraph() #the full graph
        self.T = nx.DiGraph() #a tree subgraph of G which we will draw as much of as possible

        root = max(self.G.nodes(), key = lambda x : len(nx.ancestors(self.G, x)))
        for _ in range(0):
            root = next(iter(G.predecessors(root)))

        self.click_pos = None

        self.set_root(root)

    def set_root(self, root):
        self.root = root
        self.update_node_positions()

    def update_node_positions(self):
        old_node_draw_positions = {n : p for n, p in self.node_draw_positions.items()}

        #set self.node_heights and self.T
        self.T = nx.DiGraph()
        self.T.add_node(self.root)
        self.node_heights = {self.root : 0}
        if self.root in old_node_draw_positions:
            self.node_heights[self.root] = old_node_draw_positions[self.root][1]

        g_edges = set(self.G.edges())
        for t_edge in nx.bfs_edges(self.G.to_undirected(), self.root):
            if t_edge in g_edges:
                g_edge = t_edge
                self.T.add_edge(g_edge[0], g_edge[1])
                self.node_heights[t_edge[1]] = self.node_heights[t_edge[0]] - 1
            else:
                g_edge = (t_edge[1], t_edge[0])
                assert g_edge in g_edges
                self.T.add_edge(g_edge[0], g_edge[1])
                self.node_heights[t_edge[1]] = self.node_heights[t_edge[0]] + 1

        #set self.node_widths and self.more_to_see_nodes

        #successors
        #predecessors
        #ancestors
        #descendants

        ordering_node_widths = {node : self.node_widths.get(node, random.uniform(0, 1)) for node in self.T.nodes}

        def match(node, rw1, rw2):
            assert node in rw1
            assert node in rw2
            for a in rw1:
                for b in rw2:
                    if a != node and b != node:
                        assert a != b
            off = rw1[node] - rw2[node]
            rw2m = {b : w + off for b, w in rw2.items()}
            return rw1 | rw2m

        def center(rw, c):
            if len(rw) == 0:
                return rw
            else:
                min_w = min(list(rw.values()))
                max_w = max(list(rw.values()))
                off = 0.5 * (max_w + min_w)
                return {a : w - off for a, w in rw.items()}

        def stack_pair(rw1, rw2, side):
            #side = 0: place rw1 to the left of rw2
            #side = 1: place rw2 to the right of rw1
            assert 0 <= side <= 1

            rw1_max = {}
            for a in rw1:
                h = self.node_heights[a]
                if not h in rw1_max:
                    rw1_max[h] = rw1[a]
                else:
                    rw1_max[h] = max(rw1_max[h], rw1[a])
            rw2_min = {}
            for b in rw2:
                h = self.node_heights[b]
                if not h in rw2_min:
                    rw2_min[h] = rw2[b]
                else:
                    rw2_min[h] = min(rw2_min[h], rw2[b])

            for h in rw1_max.keys() | rw2_min.keys():
                assert not math.isnan(rw2_min.get(h, math.inf) - rw1_max.get(h, -math.inf))

            m = min(rw2_min.get(h, math.inf) - rw1_max.get(h, -math.inf) - 1 for h in rw1_max.keys() | rw2_min.keys())

            if m == math.inf:
                return center(rw1, 0) | center(rw2, 0)
            else:
                rw = {}
                for a, w in rw1.items():
                    rw[a] = w + m * (1 - side)
                for b, w in rw2.items():
                    rw[b] = w - m * side
                return rw

        def stack(rws):
            if len(rws) == 0:
                return {}
            elif len(rws) == 1:
                return rws[0]
            elif len(rws) == 2:
                return stack_pair(rws[0], rws[1], 0.5)
            else:
                k = len(rws) // 2
                return stack_pair(stack(rws[:k]), stack(rws[k:]), 0.5)

        def compute_widths_related(G, node):
            assert nx.is_tree(G.to_undirected())

            #decide on an order for things
            pred_lookup = {x : sorted(G.predecessors(x), key = lambda y : ordering_node_widths[y]) for x in G.nodes()}
            succ_lookup = {x : sorted(G.successors(x), key = lambda y : ordering_node_widths[y]) for x in G.nodes()}

            #heres how the algorithm works:
            #compute_upwards
            #    take a node and compute the positions of everything directly above it.
            #    directly above means all ancestors and all of their decendents lying inside the cone
            #compute_whole
            #    compute positions of all ancestors and _all_ of their decendents
            #    do this by using compute_upwards to find positions for all ancestors decendents inside the cone
            #    then go up the sides of the cone and place the remaining decendents fitted to each side of the cone

            def compute_widths_down_part(G, node, minh = -math.inf, excluded_succ = None):
                if excluded_succ is None:
                    excluded_succ = set([])
                assert self.node_heights[node] >= minh
                #add partnerships
                tops = [node]
                found_tops = set([node])
                for n in succ_lookup[node]:
                    if not n in excluded_succ:
                        if self.node_heights[n] >= minh:
                            for p in pred_lookup[n]:
                                if not p in found_tops:
                                    found_tops.add(p)
                                    if len(tops) % 2 == 0:
                                        tops = tops + [p]
                                    else:
                                        tops = [p] + tops
                tops = [{p : 0} for p in sorted(tops, key = lambda p : ordering_node_widths[p])]
                #add everything below
                bots = []
                for n in succ_lookup[node]:
                    if not n in excluded_succ:
                        if self.node_heights[n] >= minh:
                            bots.append(compute_widths_down_part(G, n, minh = minh))
                return center(stack(tops), 0) | center(stack(bots), 0)

            def compute_upwards(G, node, block_left, block_right, minh_left, minh_right):
                assert block_left or block_right
                preds = pred_lookup[node]
                minh_mid = self.node_heights[node] + 1
                tops = []
                for i, n in enumerate(preds):
                    if i == 0 == len(preds) - 1:
                        tops.append(compute_upwards(G, n, block_left, block_right, minh_left, minh_right))
                    elif i == 0 and block_left:
                        tops.append(compute_upwards(G, n, True, False, minh_left, minh_mid))
                    elif i == len(preds) - 1 and block_right:
                        tops.append(compute_upwards(G, n, False, True, minh_mid, minh_right))
                    else:
                        if i == 0:
                            tops.append(compute_whole(G, n, compute_widths_down_part(G, n, minh = minh_left, excluded_succ = {node}), minh_left = minh_left, minh_right = minh_mid))
                        elif i == len(preds) - 1:
                            tops.append(compute_whole(G, n, compute_widths_down_part(G, n, minh = minh_right, excluded_succ = {node}), minh_left = minh_mid, minh_right = minh_right))
                        else:
                            tops.append(compute_whole(G, n, compute_widths_down_part(G, n, minh = minh_mid, excluded_succ = {node}), minh_left = minh_mid, minh_right = minh_mid))

                top = center(stack(tops) , 0)
                if block_left and block_right:
                    return top | {node : 0}
                elif block_left:
                    ans = stack_pair({node : 0.0}, top, 0)
                    ans[node] = min(ans[node], 0.0)
                    return ans
                elif block_right:
                    ans = stack_pair(top, {node : 0.0}, 1)
                    ans[node] = max(ans[node], 0.0)
                    return ans
                else:
                    assert False #at least one of block_left/block_right should be True

            def compute_whole(G, node, base, minh_left, minh_right):
                assert node in base
                core = match(node, compute_upwards(G, node, True, True, minh_left, minh_right), base)

##                def yield_middle(x):
##                    ps = pred_lookup[x]
##                    if len(ps) == 1:
##                        yield x, ps[0]
##                        yield from yield_left_side(ps[0])

                def yield_left_side(x):
                    ps = pred_lookup[x]
                    if len(ps) >= 1:
                        yield x, ps[0]
                        yield from yield_left_side(ps[0])

                def yield_right_side(x):
                    ps = pred_lookup[x]
                    if len(ps) >= 1:
                        yield x, ps[-1]
                        yield from yield_right_side(ps[-1])

##                if (minh_both := minh_left) == minh_right:
##                    bottoms = []
##                    for a, b in yield_middle(node):
##                        for x in succ_lookup[b]:
##                            bottoms.append((x, compute_widths_down_part(G, x, minh = minh_both)))
##                    bottoms = sorted(bottoms, key = lambda pair : ordering_node_widths[pair[0]])
##                    bottoms = [piece for x, piece in bottoms]
##                    node = b #named becasue its at the top of the 1 wide stalk of the tree

                preds = pred_lookup[node]
                done = set([]) #so that we dont repeat descendants of things lying on the initial 1 wide stalk
                def gen_hang_left():
                    for a, b in yield_left_side(node):
                        if not b in done:
                            for x in reversed(succ_lookup[b]):
                                if not x is a:
                                    yield compute_widths_down_part(G, x, minh = minh_left)
                            done.add(b)

                def gen_hang_right():
                    for a, b in yield_right_side(node):
                        if not b in done:
                            for x in succ_lookup[b]:
                                if not x is a:
                                    yield compute_widths_down_part(G, x, minh = minh_right)
                            done.add(b)

                if minh_left <= minh_right:
                    hang_left = list(gen_hang_left())
                    hang_right = list(gen_hang_right())
                else:
                    hang_right = list(gen_hang_right())
                    hang_left = list(gen_hang_left())

                for hang in hang_left:
                    core = stack([hang, core])
                for hang in hang_right:
                    core = stack([core, hang])
                return core

            return center(compute_whole(G, node, compute_widths_down_part(G, node), -math.inf, -math.inf), 0)

        self.node_widths = compute_widths_related(self.T, self.root)

        self.more_to_see_nodes = set([])
        for node in self.node_widths:
            for x in itertools.chain(self.G.successors(node), self.G.predecessors(node)):
                if not x in self.node_widths:
                    self.more_to_see_nodes.add(node)

##        self.update_shapes_vao()

        self.moving_nodes = list(old_node_draw_positions.keys() & self.node_widths.keys())
        self.moving_nodes_old_positions = np.array([old_node_draw_positions[n] for n in self.moving_nodes])
        self.moving_nodes_new_positions = np.array([[self.node_widths[n], self.node_heights[n]] for n in self.moving_nodes])
        self.last_rootchange_time = time.time()

        self.visible_nodes = {ident : idx for idx, ident in enumerate(self.node_widths.keys() | self.moving_nodes)}
        self.nodes_waiting_for_surf_update = set(self.visible_nodes.keys())

    def get_surf_data(self, entity):
        if not entity in self.node_surf_data:
            if type(entity) == treedata.Person:
                surf = pygame.Surface([self.tex.width, self.tex.height], flags = pygame.SRCALPHA)

                sex = entity.get_sex()
                if sex is None:
                    bg_colour = [128, 128, 128, 255]
                elif sex == "male":
                    bg_colour = [64, 160, 255, 255]
                elif sex == "female":
                    bg_colour = [255, 64, 255, 255]
                else:
                    bg_colour = [112, 64, 255, 255]
                surf.fill(bg_colour)
                pgbase.surftools.write(surf, entity.name(), [0, 0, 1, 0.2], [0, 0, 0, 255])

                event_strs = entity.get_event_strings(["birth", "death"])

                y = 0.2
                for event_str in event_strs:
                    pgbase.surftools.write(surf, event_str, [0, y, 1, 0.1], [0, 0, 0, 255])
                    y += 0.1

            else:
                surf = pygame.Surface([self.tex.width, self.tex.height], flags = pygame.SRCALPHA)
                surf.fill([0, 0, 0, 0])
                pygame.draw.circle(surf, [0, 0, 0, 255], [surf.get_width() / 2, surf.get_height() / 2], surf.get_width() / 3)

            data = pygame.image.tostring(surf, "RGBA", 1)
            self.node_surf_data[entity] = data

        return self.node_surf_data[entity]

    def node_pos(self, node):
        #currently requires the x position to be preserved so that the interactive stuff works
        #can add a inverse node pos function if this needs to be changed in the future
        pos = self.node_draw_positions[node]
        return [pos[0], 2 * pos[1]]

    def update_vaos(self):
        import random

        H = self.G.subgraph(self.node_draw_positions.keys())

        nodes = list(H.nodes)
        buffer_vertices = self.ctx.buffer(np.array([self.node_pos(node) for node in nodes]).astype('f4'))
        tex_idx = self.ctx.buffer(np.array([self.visible_nodes[node] for node in nodes]))
        buffer_indices = self.ctx.buffer(np.array(range(len(nodes))))
        def get_border(node):
            if isinstance(self.tree.entity_lookup[node], treedata.Person):
                if node in self.more_to_see_nodes:
                    return 0.03
                else:
                    return 0.02
            else:
                return 0.0
        borders = self.ctx.buffer(np.array([get_border(node) for node in nodes]).astype('f4'))

        self.entity_vao = self.ctx.vertex_array(self.entity_prog,
                                                [(buffer_vertices, "2f4", "pos"),
                                                 (tex_idx, "u4", "v_tex_idx"),
                                                 (borders, "f4", "border")],
                                                buffer_indices)

        colour_tree_edge = [0, 0, 0, 1]
        colour_loop_edge = [1, 0, 1, 1]
        tree_edges = set(self.T.edges())

        edges = list(H.edges())
        buffer_top_vertices = self.ctx.buffer(np.array([self.node_pos(edge[0]) for edge in edges]).astype('f4'))
        buffer_bot_vertices = self.ctx.buffer(np.array([self.node_pos(edge[1]) for edge in edges]).astype('f4'))
        buffer_colours = self.ctx.buffer(np.array([(colour_tree_edge if edge in tree_edges else colour_loop_edge) for edge in edges]).astype('f4'))
        indices = self.ctx.buffer(np.array(range(len(edges))))
        self.edge_vao = self.ctx.vertex_array(self.edge_prog,
                                              [(buffer_top_vertices, "2f4", "top_pos"),
                                               (buffer_bot_vertices, "2f4", "bot_pos"),
                                               (buffer_colours, "4f4", "colour")],
                                              indices)

    def set_rect(self, rect):
        super().set_rect(rect)

    def tick(self, dt):
        d = 0.75
        if time.time() < self.last_rootchange_time + d and len(self.moving_nodes) != 0:
            f = (time.time() - self.last_rootchange_time) / d
            f = 0.5 * (math.cos(math.pi * (f - 1)) + 1)
            poses = f * self.moving_nodes_new_positions + (1 - f) * self.moving_nodes_old_positions
            self.node_draw_positions = {n : poses[i] for i, n in enumerate(self.moving_nodes)}
        else:
            self.node_draw_positions = {n : [self.node_widths[n], self.node_heights[n]] for n in self.node_widths}
        self.update_vaos()

        start_time = time.time()
        while time.time() - start_time < dt and len(self.nodes_waiting_for_surf_update) != 0:
            #generate some surfaces starting with the ones closest the center of the screen
            def distsq(a, b):
                return (a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2

            ident = min(self.nodes_waiting_for_surf_update, key = lambda ident : distsq(self.node_pos(ident), self.center)if ident in self.node_draw_positions else math.inf)
            self.nodes_waiting_for_surf_update.remove(ident)

            layer = self.visible_nodes[ident]
            self.tex.write(self.get_surf_data(self.tree.entity_lookup[ident]), viewport = (0, 0, layer, self.tex.width, self.tex.height, 1))

    def event(self, event):
        def dist(p1, p2):
            return math.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2)

        CLICK_DIST = 30

        super().event(event)
        if event.type == pygame.MOUSEBUTTONDOWN:
            if event.button == 1:
                self.click_pos = event.pos

        if event.type == pygame.MOUSEBUTTONUP:
            if event.button == 1:
                if not self.click_pos is None:
                    if dist(self.click_pos, event.pos) < CLICK_DIST:
                        pos = self.pygame_to_world(self.click_pos)
                        node = min(self.node_draw_positions.keys(), key = lambda node : dist(pos, self.node_pos(node)))
                        node_pos = self.node_pos(node)
                        scr_node_pos = self.world_to_pygame(node_pos)
                        if dist(self.click_pos, scr_node_pos) < CLICK_DIST:
                            self.set_root(node)
                            self.center = self.center - np.array(node_pos) + np.array(self.node_pos(node))
                            if not pgbase.tools.in_rect(self.world_to_pygame([0, 0]), self.rect):
                                self.center[0] = 0

                    else:
                        pos = self.pygame_to_world(self.click_pos)
                        node = min(self.node_draw_positions.keys(), key = lambda node : dist(pos, self.node_pos(node)))
                        node_pos = self.node_pos(node)
                        scr_node_pos = self.world_to_pygame(node_pos)
                        if dist(self.click_pos, scr_node_pos) < CLICK_DIST:
                            #move node and re-calculate positions using the new position to determine the ordering
                            self.node_widths[node] = self.pygame_to_world(event.pos)[0]
                            self.set_root(self.root)

                    self.click_pos = None

    def draw(self):
        super().set_uniforms([self.bg_prog, self.entity_prog, self.edge_prog])

        self.ctx.screen.use()
        self.ctx.clear(1, 1, 1, 1)

        self.ctx.enable_only(moderngl.BLEND)

        self.bg_vao.render(moderngl.TRIANGLES, instances = 1)
        if not self.edge_vao is None:
            self.edge_vao.render(moderngl.POINTS, instances = 1)
        if not self.entity_vao is None:
            self.tex.use(0)
            self.entity_vao.render(moderngl.POINTS, instances = 1)

def run(tree):
    assert type(tree) == treedata.Tree
    pgbase.core.Window.setup(size = [1600, 1000])
    pgbase.core.run_root(TreeView(tree))