import os, sys, os.path as op
import numpy as np
import cv2
import collections
import logging
from glob import glob
import shutil
import sqlite3
from progressbar import progressbar
from pprint import pformat

from lib.backend import backendDb
from lib.backend import backendMedia
from lib.utils import util

def add_parsers(subparsers):
    importKittiParser(subparsers)

# Data Format Description
# =======================

# The data for training and testing can be found in the corresponding folders.
# The sub-folders are structured as follows:

#   - image_02/ contains the left color camera images (png)
#   - label_02/ contains the left color camera label files (plain text files)
#   - calib/ contains the calibration for all four cameras (plain text file)

# The label files contain the following information, which can be read and
# written using the matlab tools (readLabels.m, writeLabels.m) provided within
# this devkit. All values (numerical or strings) are separated via spaces,
# each row corresponds to one object. The 15 columns represent:

# #Values    Name      Description
# ----------------------------------------------------------------------------
#    1    type         Describes the type of object: 'Car', 'Van', 'Truck',
#                      'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram',
#                      'Misc' or 'DontCare'
#    1    truncated    Float from 0 (non-truncated) to 1 (truncated), where
#                      truncated refers to the object leaving image boundaries
#    1    occluded     Integer (0,1,2,3) indicating occlusion state:
#                      0 = fully visible, 1 = partly occluded
#                      2 = largely occluded, 3 = unknown
#    1    alpha        Observation angle of object, ranging [-pi..pi]
#    4    bbox         2D bounding box of object in the image (0-based index):
#                      contains left, top, right, bottom pixel coordinates
#    3    dimensions   3D object dimensions: height, width, length (in meters)
#    3    location     3D object location x,y,z in camera coordinates (in meters)
#    1    rotation_y   Rotation ry around Y-axis in camera coordinates [-pi..pi]
#    1    score        Only for results: Float, indicating confidence in
#                      detection, needed for p/r curves, higher is better.

# Here, 'DontCare' labels denote regions in which objects have not been labeled,
# for example because they have been too far away from the laser scanner. To
# prevent such objects from being counted as false positives our evaluation
# script will ignore objects detected in don't care regions of the test set.
# You can use the don't care labels in the training set to avoid that your object
# detector is harvesting hard negatives from those areas, in case you consider
# non-object regions from the training images as negative examples.

# The coordinates in the camera coordinate system can be projected in the image
# by using the 3x4 projection matrix in the calib folder, where for the left
# color camera for which the images are provided, P2 must be used. The
# difference between rotation_y and alpha is, that rotation_y is directly
# given in camera coordinates, while alpha also considers the vector from the
# camera center to the object center, to compute the relative orientation of
# the object with respect to the camera. For example, a car which is facing
# along the X-axis of the camera coordinate system corresponds to rotation_y=0,
# no matter where it is located in the X/Z plane (bird's eye view), while
# alpha is zero only, when this object is located along the Z-axis of the
# camera. When moving the car away from the Z-axis, the observation angle
# will change.

# To project a point from Velodyne coordinates into the left color image,
# you can use this formula: x = P2 * R0_rect * Tr_velo_to_cam * y
# For the right color image: x = P3 * R0_rect * Tr_velo_to_cam * y

# Note: All matrices are stored row-major, i.e., the first values correspond
# to the first row. R0_rect contains a 3x3 matrix which you need to extend to
# a 4x4 matrix by adding a 1 as the bottom-right element and 0's elsewhere.
# Tr_xxx is a 3x4 matrix (R|t), which you need to extend to a 4x4 matrix
# in the same way!

# Note, that while all this information is available for the training data,
# only the data which is actually needed for the particular benchmark must
# be provided to the evaluation server. However, all 15 values must be provided
# at all times, with the unused ones set to their default values (=invalid) as
# specified in writeLabels.m. Additionally a 16'th value must be provided
# with a floating value of the score for a particular detection, where higher
# indicates higher confidence in the detection. The range of your scores will
# be automatically determined by our evaluation server, you don't have to
# normalize it, but it should be roughly linear. If you use writeLabels.m for
# writing your results, this function will take care of storing all required
# data correctly.

def _parseObject(c, line, imagefile):
    words = line.split(' ')
    # 1    type         Describes the type of object: 'Car', 'Van', 'Truck',
    #                   'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram',
    #                   'Misc' or 'DontCare'
    name = words[0]
    # 1    truncated    Float from 0 (non-truncated) to 1 (truncated), where
    #                   truncated refers to the object leaving image boundaries
    truncated = words[1]
    # 1    occluded     Integer (0,1,2,3) indicating occlusion state:
    #                   0 = fully visible, 1 = partly occluded
    #                   2 = largely occluded, 3 = unknown
    occluded = words[2]
    # 1    alpha        Observation angle of object, ranging [-pi..pi]
    alpha = words[3]
    # 4    bbox         2D bounding box of object in the image (0-based index):
    #                   contains left, top, right, bottom pixel coordinates
    x1 = int(float(words[4]))
    y1 = int(float(words[5]))
    width = int(float(words[6]) - float(words[4]) + 1)
    height = int(float(words[7]) - float(words[5]) + 1)
    # 3    dimensions   3D object dimensions: height, width, length (in meters)
    dim_height = words[8]
    dim_width = words[9]
    dim_length = words[10]
    # 3    location     3D object location x,y,z in camera coordinates (in meters)
    loc_x = words[11]
    loc_y = words[12]
    loc_z = words[13]
    # 1    rotation_y   Rotation ry around Y-axis in camera coordinates [-pi..pi]
    rotation_y = words[14]
    # 1    score        Only for results: Float, indicating confidence in
    #                   detection, needed for p/r curves, higher is better.
    score = float(words[15]) if len(words) == 16 else None
    c.execute(
        'INSERT INTO objects(imagefile,x1,y1,width,height,name,score) '
        'VALUES (?,?,?,?,?,?,?)',
        (imagefile, x1, y1, width, height, name, score))
    objectid = c.lastrowid
    s = 'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)'
    c.execute(s, (objectid, 'truncated', truncated))
    c.execute(s, (objectid, 'occluded', occluded))
    c.execute(s, (objectid, 'alpha', alpha))
    c.execute(s, (objectid, 'dim_height', dim_height))
    c.execute(s, (objectid, 'dim_width', dim_width))
    c.execute(s, (objectid, 'dim_length', dim_length))
    c.execute(s, (objectid, 'loc_x', loc_x))
    c.execute(s, (objectid, 'loc_y', loc_y))
    c.execute(s, (objectid, 'loc_z', loc_z))
    c.execute(s, (objectid, 'rotation_y', rotation_y))
    return objectid

def importKittiParser(subparsers):
    parser = subparsers.add_parser(
        'importKitti',
        description='Import KITTI annotations into a db. Note that the '
        'bounding box is stored as integer, and the decimal part is lost.')
    parser.set_defaults(func=importKitti)
    parser.add_argument('--images_dir',
                        required=True,
                        help='Directory with .png images. '
                        'E.g. "kitti/data_object_image_2/training/image_2".')
    parser.add_argument('--detection_dir',
                        help='Directory with .txt annotations of objects. '
                        'E.g. "kitti/data_object_image_2/training/label_2".')
    parser.add_argument(
        '--segmentation_dir',
        help='Directory with 8bit masks for semantic and 16bit masks '
        'for instance segmentation annotations. '
        'E.g. "kitti/data_object_image_2/training/label_2".')
    parser.add_argument('--with_display', action='store_true')

def importKitti(c, args):
    if args.with_display:
        imreader = backendMedia.MediaReader(args.rootdir)

    image_paths = sorted(glob(op.join(args.images_dir, '*.png')))
    logging.info('Found %d PNG images in %s' %
                 (len(image_paths), args.images_dir))

    for image_path in progressbar(image_paths):
        filename = op.splitext(op.basename(image_path))[0]
        logging.debug('Processing image: "%s"' % filename)

        # Add image to the database.
        imheight, imwidth = backendMedia.getPictureSize(image_path)
        imagefile = op.relpath(image_path, args.rootdir)
        c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)',
                  (imagefile, imwidth, imheight))

        if args.with_display:
            img = imreader.imread(imagefile)

        # Detection annotations.
        if args.detection_dir:
            detection_path = op.join(args.detection_dir, '%s.txt' % filename)
            if not op.exists(detection_path):
                raise FileNotFoundError('Annotation file not found at "%s".' %
                                        detection_path)

            # Read annotation file.
            with open(detection_path) as f:
                lines = f.read().splitlines()
            logging.debug('Read %d lines from detection file "%s"' %
                          (len(lines), detection_path))
            for line in lines:
                objectid = _parseObject(c, line, imagefile)

                if args.with_display:
                    c.execute('SELECT * FROM objects WHERE objectid=?',
                              (objectid, ))
                    object_entry = c.fetchone()
                    name = backendDb.objectField(object_entry, 'name')
                    roi = backendDb.objectField(object_entry, 'roi')
                    score = backendDb.objectField(object_entry, 'score')
                    util.drawScoredRoi(img, roi, name, score=score)

        # Segmentation annotations.
        if args.segmentation_dir:
            segmentation_path = op.join(args.segmentation_dir,
                                        '%s.png' % filename)
            if not op.exists(segmentation_path):
                raise FileNotFoundError('Annotation file not found at "%s".' %
                                        segmentation_path)

            # Add image to the database.
            maskfile = op.relpath(segmentation_path, args.rootdir)
            c.execute('UPDATE images SET maskfile=? WHERE imagefile=?',
                      (maskfile, imagefile))

            if args.with_display:
                mask = imreader.maskread(maskfile)
                img = util.drawMaskAside(img, mask, labelmap=None)

        # Maybe display.
        if args.with_display:
            cv2.imshow('importKitti', img[:, :, ::-1])
            if cv2.waitKey(-1) == 27:
                args.with_display = False
                cv2.destroyWindow('importKitti')


from typing import Annotated, Any, AsyncIterator

from fastapi import Depends, FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy.ext.asyncio import AsyncSession
from src.constants import UpdateAction
from src.database import async_session_factory
from src.schemas.todos import TodoResponseSchema, TodoSchema, UpdateTodoSchema
from src.schemas.users import UserResponseSchema, UserSchema
from src.utils.crud import (
    create_todo,
    create_user,
    delete_todo,
    get_todos,
    get_user,
    login_user,
    update_todo,
    validate_authtoken,
)

app = FastAPI(docs_url="/")

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

async def get_db() -> AsyncIterator[AsyncSession]:
    async with async_session_factory() as session:
        yield session

@app.post("/v1/register", response_model=UserResponseSchema)
async def register(
    user: UserSchema, db: Annotated[AsyncSession, Depends(get_db)]
) -> Any:
    user_exists = await get_user(db, user.email)
    if user_exists:
        raise HTTPException(status_code=400, detail="Email already registered")
    user = await create_user(db, user)
    return user

@app.post("/v1/login", response_model=UserResponseSchema)
async def login(user: UserSchema, db: Annotated[AsyncSession, Depends(get_db)]) -> Any:
    user_exists = await get_user(db, user.email)
    if not user_exists:
        raise HTTPException(status_code=404, detail="User does not exist")
    user = await login_user(db, user)
    if not user:
        raise HTTPException(status_code=400, detail="Invalid email or password")
    return user

@app.post("/v1/todos/create", response_model=TodoResponseSchema)
async def create_todos(
    request: Request,
    todo: TodoSchema,
    db: Annotated[AsyncSession, Depends(get_db)],
) -> Any:
    user = await validate_authtoken(db, request.headers.get("authorization"))
    if not user:
        raise HTTPException(status_code=422, detail="Invalid authorization token")

    todo = await create_todo(db, todo, user.id)
    if not todo:
        raise HTTPException(status_code=400, detail="Todo already exists for this user")

    return todo

@app.get("/v1/todos", response_model=list[TodoResponseSchema])
async def fetch_todos(
    request: Request,
    db: Annotated[AsyncSession, Depends(get_db)],
    search: str | None = None,
) -> Any:
    user = await validate_authtoken(db, request.headers.get("authorization"))
    if not user:
        raise HTTPException(status_code=422, detail="Invalid authorization token")
    print(search, "search")
    if search:
        todos = await get_todos(db, user.id, search_value=search)
    else:
        todos = await get_todos(db, user.id)
    return sorted(todos, key=lambda x: x.id)

@app.delete("/v1/todos/{todo_id}")
async def delete_todos(
    request: Request, todo_id: int, db: Annotated[AsyncSession, Depends(get_db)]
) -> Any:
    user = await validate_authtoken(db, request.headers.get("authorization"))
    if not user:
        raise HTTPException(status_code=422, detail="Invalid authorization token")
    todo = await delete_todo(db, todo_id, user.id)
    if not todo:
        raise HTTPException(status_code=404, detail="Todo instance not found")
    return "Todo deleted successfully"

@app.patch("/v1/todos/{todo_id}", response_model=TodoResponseSchema)
async def update_todos(
    request: Request,
    todo: UpdateTodoSchema,
    todo_id: int,
    db: Annotated[AsyncSession, Depends(get_db)],
) -> Any:
    user = await validate_authtoken(db, request.headers.get("authorization"))
    if not user:
        raise HTTPException(status_code=422, detail="Invalid authorization token")

    todo_action = UpdateAction.ARCHIVE if todo.archived else UpdateAction.COMPLETE
    if todo.archived:
        todo_value = todo.archived
    else:
        todo_value = todo.completed

    todo = await update_todo(db, todo_id, todo_value, user.id, todo_action)
    if not todo:
        raise HTTPException(status_code=404, detail="Todo instance not found")

    return todo


import setuptools

# Configure the required packages and scripts to install.
# Note that the Python Dataflow containers come with numpy already installed
# so this dependency will not trigger anything to be installed unless a version
# restriction is specified.
REQUIRED_PACKAGES = [
    'numpy',
    'beautifulsoup4',
    'pandas',
    'sendgrid',
    'lxml',
    'pandas_datareader'
    ]

setuptools.setup(
    name='modules',
    version='0.0.1',
    description='Edgar workflow package.',
    install_requires=REQUIRED_PACKAGES,
    packages=setuptools.find_packages()
    )


from django.urls import path

from . import views

app_name = "trip_builder"

urlpatterns = [
    path("", views.index, name="index"),
    path("new_vacation", views.new_vacation, name="new_vacation"),
    path("country_itineraries", views.country_itineraries, name="country_itineraries"),
]


from absl import app
from absl import flags
import data
import metric
import model as model_definition
import os
import storage
import torch
import torch.optim as optim
from torch.utils import data as torch_data
from torch.utils import tensorboard
from typing import Tuple

FLAGS = flags.FLAGS
flags.DEFINE_float("lr", default=0.01, help="Learning rate value.")
flags.DEFINE_integer("seed", default=1234, help="Seed value.")
flags.DEFINE_integer("batch_size", default=128, help="Maximum batch size.")
flags.DEFINE_integer("validation_batch_size", default=64, help="Maximum batch size during model validation.")
flags.DEFINE_integer("vector_length", default=50, help="Length of entity/relation vector.")
flags.DEFINE_float("margin", default=1.0, help="Margin value in margin-based ranking loss.")
flags.DEFINE_integer("norm", default=1, help="Norm used for calculating dissimilarity metric (usually 1 or 2).")
flags.DEFINE_integer("epochs", default=2000, help="Number of training epochs.")
flags.DEFINE_string("dataset_path", default="./synth_data", help="Path to dataset.")
flags.DEFINE_bool("use_gpu", default=True, help="Flag enabling gpu usage.")
flags.DEFINE_integer("validation_freq", default=10, help="Validate model every X epochs.")
flags.DEFINE_string("checkpoint_path", default="", help="Path to model checkpoint (by default train from scratch).")
flags.DEFINE_string("tensorboard_log_dir", default="./runs", help="Path for tensorboard log directory.")

flags.DEFINE_string("en_reln_mapping", default="file", help="Function to use for creating embedding mappings.") # 'file', 'wn9', 'fb15k'
flags.DEFINE_bool("retrain_text_layer", default=False, help="Retrain the PV-DM model.")
flags.DEFINE_bool("test_only", default=False, help="Just running the saved model on test dataset.")
flags.DEFINE_float("beta", default=0.4, help="weight of margin ranking loss.")

HITS_AT_1_SCORE = float
HITS_AT_3_SCORE = float
HITS_AT_10_SCORE = float
MRR_SCORE = float
METRICS = Tuple[HITS_AT_1_SCORE, HITS_AT_3_SCORE, HITS_AT_10_SCORE, MRR_SCORE]

def test(model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int,
         summary_writer: tensorboard.SummaryWriter, device: torch.device, epoch_id: int, metric_suffix: str,
         ) -> METRICS:
    examples_count = 0.0
    hits_at_1 = 0.0
    hits_at_3 = 0.0
    hits_at_10 = 0.0
    mrr = 0.0

    entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(0) # Returns a 1-D tensor of size entities_count with values from 0 to entities_count, and then Returns a new tensor with a dimension of size one inserted at the specified position/ basically adding another dimension.
    for head, relation, tail in data_generator:
        # print(head, relation, tail)
        current_batch_size = head.size()[0]

        head, relation, tail = head.to(device), relation.to(device), tail.to(device)
        all_entities = entity_ids.repeat(current_batch_size, 1) # with torch.repeat(), you can specify the number of repeats for each dimension
        heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1])
        relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1])
        tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1])

        # Check all possible tails
        triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3)
        # print(triplets)
        tails_predictions = model.predict(triplets).reshape(current_batch_size, -1)
        # Check all possible heads
        triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3)
        heads_predictions = model.predict(triplets).reshape(current_batch_size, -1)

        # Concat predictions
        predictions = torch.cat((tails_predictions, heads_predictions), dim=0)
        ground_truth_entity_id = torch.cat((tail.reshape(-1, 1), head.reshape(-1, 1)))

        # Each prediction is an array of N size, where N is no_of_Entity_in_KB, and there are no_of_samples_in_batch * 2 (head & tail) ground_truth and column level prediction

        # https://medium.com/@m_n_malaeb/recall-and-precision-at-k-for-recommender-systems-618483226c54
        hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1)
        hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3)
        hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10)
        mrr += metric.mrr(predictions, ground_truth_entity_id)

        examples_count += predictions.size()[0]

    hits_at_1_score = hits_at_1 / examples_count * 100
    hits_at_3_score = hits_at_3 / examples_count * 100
    hits_at_10_score = hits_at_10 / examples_count * 100
    mrr_score = mrr / examples_count * 100
    summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id)
    summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id)
    summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id)
    summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id)

    return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score

def collate_fn(data):
    # print(data) # prints one batch of data, i.e., [('4684', 1, '1363'), ('2814', 1, '5384'), ('3045', 1, '3039')]
    heads, relations, tails = list(), list(), list()
    for i in range(len(data)):
        heads.append(int(data[i][0]))
        relations.append(int(data[i][1]))
        tails.append(int(data[i][2])) # coz they are string
    return torch.tensor(heads), torch.tensor(relations), torch.tensor(tails)

def main(_):
    torch.random.manual_seed(FLAGS.seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

    path = FLAGS.dataset_path
    train_path = os.path.join(path, "train.txt")
    validation_path = os.path.join(path, "valid.txt")
    test_path = os.path.join(path, "test.txt")

    if FLAGS.en_reln_mapping is 'file':
        entity2id, relation2id = data.create_mappings_for_WN9(path)
    elif FLAGS.en_reln_mapping is 'wn9':
        entity2id, relation2id = data.create_mappings(train_path, 'WN9')
    elif FLAGS.en_reln_mapping is 'fb15k':
        entity2id, relation2id = data.create_mappings(train_path, 'FB15K')
    else:
        entity2id, relation2id = data.create_mappings(train_path, 'WN9')

    #for key, value in sorted(entity2id.items(), key=lambda x: x[1]):
    #    print("{} : {}".format(key, value))   # No OOKB entityid for now
        # input("wait")

    batch_size = FLAGS.batch_size
    vector_length = FLAGS.vector_length
    margin = FLAGS.margin
    norm = FLAGS.norm
    learning_rate = FLAGS.lr
    epochs = FLAGS.epochs
    device = torch.device('cuda') if FLAGS.use_gpu else torch.device('cpu')

    # train_set = data.FB15KDataset(train_path, entity2id, relation2id)
    train_set = data.WN9Dataset(train_path, entity2id, relation2id)
    # collate_fn (callable, optional) – merges a list of samples to form a mini-batch of Tensor(s). Used when using batched loading from a map-style dataset.
    train_generator = torch_data.DataLoader(train_set, batch_size=batch_size, collate_fn=collate_fn)
    # validation_set = data.FB15KDataset(validation_path, entity2id, relation2id)
    validation_set = data.WN9Dataset(validation_path, entity2id, relation2id)
    validation_generator = torch_data.DataLoader(validation_set, batch_size=FLAGS.validation_batch_size, collate_fn=collate_fn)
    # test_set = data.FB15KDataset(test_path, entity2id, relation2id)
    test_set = data.WN9Dataset(test_path, entity2id, relation2id)
    test_generator = torch_data.DataLoader(test_set, batch_size=FLAGS.validation_batch_size, collate_fn=collate_fn)

    autoencoder =  model_definition.AutoEncoder(entity2id, retrain_text_layer=FLAGS.retrain_text_layer, hidden_dimension=vector_length) # for autoencoder embedding layer weight initialization
    model = model_definition.TransE(entity_count=len(entity2id), relation_count=len(relation2id), dim=vector_length,
                                    margin=margin, beta=FLAGS.beta,
                                    device=device, norm=norm,
                                    autoencoder=autoencoder)  # type: torch.nn.Module
    model = model.to(device)
    # optimizer = optim.SGD(model.parameters(), lr=learning_rate)
    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate)

    summary_writer = tensorboard.SummaryWriter(log_dir=FLAGS.tensorboard_log_dir)
    start_epoch_id = 1
    step = 0
    best_score = 0.0

    if FLAGS.checkpoint_path:
        start_epoch_id, step, best_score = storage.load_checkpoint(FLAGS.checkpoint_path, model, optimizer)

    print(model)

    if FLAGS.test_only:
        epochs = -1
        start_epoch_id = 0
    # Training loop
    for epoch_id in range(start_epoch_id, epochs + 1):
        print("Starting epoch: ", epoch_id)
        loss_impacting_samples_count = 0
        samples_count = 0
        model.train()

        for local_heads, local_relations, local_tails in train_generator:
            # data was not well prepared, had to use collate_fn in Dataloaders to fix it, will find it above
            local_heads, local_relations, local_tails = (local_heads.to(device), local_relations.to(device),
                                                         local_tails.to(device))

            positive_triples = torch.stack((local_heads, local_relations, local_tails), dim=1)

            # Preparing negatives.
            # Generate binary tensor to replace either head or tail. 1 means replace head, 0 means replace tail.
            head_or_tail = torch.randint(high=2, size=local_heads.size(), device=device)
            random_entities = torch.randint(high=len(entity2id), size=local_heads.size(), device=device)
            broken_heads = torch.where(head_or_tail == 1, random_entities, local_heads)
            broken_tails = torch.where(head_or_tail == 0, random_entities, local_tails)
            negative_triples = torch.stack((broken_heads, local_relations, broken_tails), dim=1)

            optimizer.zero_grad()

            loss, pd, nd = model(positive_triples, negative_triples)
            # loss.mean().backward()
            loss.backward()

            summary_writer.add_scalar('Loss/train', loss.data.cpu().numpy(), global_step=step)
#           summary_writer.add_scalar('Loss/train', loss.mean().data.cpu().numpy(), global_step=step)
            summary_writer.add_scalar('Distance/positive', pd.sum().data.cpu().numpy(), global_step=step)
            summary_writer.add_scalar('Distance/negative', nd.sum().data.cpu().numpy(), global_step=step)

            loss = loss.data.cpu()
            #loss_impacting_samples_count += loss.nonzero().size()[0]
            #samples_count += loss.size()[0]

            optimizer.step()
            step += 1

        #summary_writer.add_scalar('Metrics/loss_impacting_samples', loss_impacting_samples_count / samples_count * 100,
        #                          global_step=epoch_id)

        if epoch_id % FLAGS.validation_freq == 0:
            model.eval()
            _, _, hits_at_10, _ = test(model=model, data_generator=validation_generator,
                                       entities_count=len(entity2id),
                                       device=device, summary_writer=summary_writer,
                                       epoch_id=epoch_id, metric_suffix="val")
            score = hits_at_10
            print(score)
            if score > best_score:
                best_score = score
                storage.save_checkpoint(model, optimizer, epoch_id, step, best_score)

    # Testing the best checkpoint on test dataset
    storage.load_checkpoint("checkpoint.tar", model, optimizer)
    best_model = model.to(device)
    best_model.eval()
    scores = test(model=best_model, data_generator=test_generator, entities_count=len(entity2id), device=device,
                  summary_writer=summary_writer, epoch_id=1, metric_suffix="test")
    print("Test scores: ", scores)

if __name__ == '__main__':
    app.run(main)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Run this flask app on localhost and post image as follows:

     curl -X POST -F "file=@<IMAGE FILE PATH>" localhost:5000/catdog

The response will be "cat", "dog", "unknown_class" or "unsupported_file".
"""

from flask import Flask, request

from cats_dogs.classifier import CatDogClassifier

app = Flask(__name__)

@app.route('/catdog', methods=['POST'])
def predict_cat_dog() -> dict:
    """
    View function receiving POST request with image file.

    Returns
    -------
    dict : {"class": <IMAGE CLASS>}
    """
    buff = request.files['file'].read()
    pred = clf.predict_file(buff)
    return {"class": pred}

if __name__ == '__main__':
    clf = CatDogClassifier('model/model.hd5', 0.9999)
    app.run()


from django.conf.urls import url
from django.contrib import admin

from .views import (

	phishing_new,
	phishing_edit,
	phishing_remove,
	)

urlpatterns = [

    url(r'^create/$', phishing_new, name='phishing_create'),
    url(r'^(?P<pk>\d+)/edit/$', phishing_edit, name='phishing_update'),
    url(r'^(?P<pk>\d+)/delete/$', phishing_remove, name='phishing_delete'),
]


import subprocess

import logging

import os

from deployer.exceptions import PythonSetupException, BinNotFoundException

def get_python_bin():
    logging.info('Looking for python binary...')

    for binfile in [
        '/usr/bin/python3.7m',
        '/usr/bin/python3.7',
        '/usr/bin/python3.6m',
        '/usr/bin/python3.7',
        '/usr/bin/python3.6m',
        '/usr/bin/python3.6',
        '/usr/bin/python2.7'
    ]:
        if os.path.isfile(binfile):
            return binfile

    raise BinNotFoundException('Could not find a python binary.')

def create_python_venv(basepath):
    logging.info('Creating python virtualenv...')

    binfile = get_python_bin()

    logging.info(f'Using python binary: {binfile}')

    subprocess.Popen(
        f'virtualenv -p {binfile} ' + os.path.join(basepath, './venv'),
        shell=True,
        stdout=subprocess.PIPE
    ).stdout.read()

def run_python_setup(basepath):
    logging.info('Running python setup.py')

    if not os.path.isfile('setup.py'):
        raise PythonSetupException('setup.py not found')

    logging.info(subprocess.Popen(
        f'''
        cd {basepath}
        ./venv/bin/python setup.py install
        ./venv/bin/python setup.py develop
        ''',
        shell=True,
        stdout=subprocess.PIPE
    ).stdout.read())


from mydb import db
from sqlalchemy import ForeignKey, UniqueConstraint

class User(db.Model):
    """User Model for storing user related details"""

    __tablename__ = "users"

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    username = db.Column(db.String(50), unique=True)
    password_hash = db.Column(db.String(255))
    token = db.Column(db.String(299))
    avatar = db.Column(db.String(100))

class Item(db.Model):
    """Item Model for storing item details"""

    __tablename__ = "items"

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    article = db.Column(db.String(20))
    id_user = db.Column(db.Integer, ForeignKey("users.id"))
    name = db.Column(db.String(100))
    deleted = db.Column(db.Integer, default=0)
    item_image = db.Column(db.String(100))
    __table_args__ = (UniqueConstraint("article", "id_user", "deleted"),)

class Price(db.Model):
    """Item Model for storing item details"""

    __tablename__ = "prices"

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    item_id = db.Column(db.Integer, ForeignKey("items.id"))
    price = db.Column(db.Integer, nullable=False)
    currency = db.Column(db.Integer, ForeignKey("currencies.id"))
    __table_args__ = (UniqueConstraint("item_id", "price", "currency"),)

class Currency(db.Model):
    """Item Model for storing item details"""

    __tablename__ = "currencies"

    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    currency = db.Column(db.String(3), nullable=False)


from typing import List, Any, Final, Dict
import json
import copy

from linebot.models \
    import FlexSendMessage, TextSendMessage, QuickReply, QuickReplyButton, MessageAction

from covid_data_getter import PatientsType, prefectures_dict

def get_patients_message(
        patients: PatientsType,
        target_prefectures: List[str],
        month: int,
        day: int,
        now: Any
) -> FlexSendMessage:
    with open('messages/templates/flex_message.json') as f:
        flex_message = json.load(f)

    with open('messages/templates/prefecture_row.json') as f:
        prefecture_row = json.load(f)

    with open('messages/templates/reference_content.json') as f:
        reference_content = json.load(f)

    flex_message['header']['contents'][0]['contents'][0]['text'] \
        = f'{month}月{day}日 COVID-19 感染状況'

    month_now = str(now.month).zfill(2)
    day_now = str(now.day).zfill(2)
    hour_now = str(now.hour).zfill(2)
    minute_now = str(now.minute).zfill(2)
    flex_message['header']['contents'][0]['contents'][1]['text'] \
        = f'{month_now}月{day_now}日 {hour_now}:{minute_now} 時点'

    for prefecture in target_prefectures:

        if prefecture == 'ALL':
            # 全国データの更新
            flex_message['body']['contents'][2]['contents'][1]['text'] \
                = f'{str(patients[prefecture]["patients"])}人'
            flex_message['body']['contents'][2]['contents'][2]['text'] \
                = f'{str(patients[prefecture]["severe"])}人'
            continue

        # 各県データの追加
        prefecture_row['contents'][1]['contents'][0]['text'] = prefectures_dict[prefecture]
        prefecture_row['contents'][1]['contents'][1]['text'] \
            = f'{str(patients[prefecture]["patients"])}人'
        prefecture_row['contents'][1]['contents'][2]['text'] \
            = f'{str(patients[prefecture]["severe"])}人'

        flex_message['body']['contents'].append(copy.deepcopy(prefecture_row))

    flex_message['body']['contents'].append(reference_content)

    return FlexSendMessage(alt_text=f'{month}月{day}日 COVID-19 感染者数', contents=flex_message)

def get_quick_reply_buttons(prefix: str, values: List[str]) -> QuickReply:
    items = [
        QuickReplyButton(
            action=MessageAction(label=value, text=f'{prefix} {value}')
        ) for value in values
    ]
    return QuickReply(items=items)


import argparse
import json
import os
import urllib.parse as p
from csv import writer

import processors as ps
from youtube import YouTubeClient

def get_video_id_by_url(url):
    """
        Get the YouTube video ID from its URL
        https://www.thepythoncode.com/code/using-youtube-api-in-python

        Args:
            url: A video URL
        Returns:
            The video ID from the video `url`
    """
    # split URL parts
    parsed_url = p.urlparse(url)
    # get the video ID by parsing the query of the URL
    video_id = p.parse_qs(parsed_url.query).get("v")
    if video_id:
        return video_id[0]
    else:
        raise Exception(f"Wasn't able to parse video URL: {url}")

def load_credentials(file_path):
    """
        Get YouTube Data API crendetials from the credentials file

        Args:
            file_path: A path to the credentials file
        Returns:
            the YouTube Data API credentials
    """
    with open(file_path) as f:
        credentials = json.load(f)
        if credentials['apiKey'] != '':
            return credentials
        else:
            raise Exception("No API key found in credentials file")

def initialize_artefacts(videoId, directory):
    """
        Initialize artefacts for a video

        Args:
            videoId: A video ID
            directory: A directory to store files
        Returns:
            results_file_path: A file path to the results file
    """
    video_dir = os.path.join(directory, videoId.lower())

    # Check if the file already exists
    if os.path.isdir(video_dir):
        print("Directory already exists, do you want to overwrite? (y/n)")
        if input().lower() == 'y':
            os.system(f"rm -rf {video_dir}")
        else:
            print("You can load the dataset in a notebook or an interactive session, and analyse via pandas.")
            print("============== Aborting ==============")
            exit(1)

    # Create the results file
    os.mkdir(video_dir)
    print("Directory '% s' created" % video_dir)

    results_file_path = os.path.join(video_dir, 'results.csv')
    with open(results_file_path, 'a+') as f:
        csv_writer = writer(f)
        # Write only the headers
        csv_writer.writerow(["comment", "xlmr_label", "xlmr_score", "mbert_label", "mbert_score", "language", "raw_comment"])

    return results_file_path

def classify_comment(comment, lang, raw_comment, model_xlmr, model_mbert, results_csv_path):
    """
        Classify comment using the pretrained models

        Args:
            comment: A clean comment to be classified
            lang: Language of the comment
            raw_comment: The original comment without cleaning
            model_xlmr: fine-tuned XLM-R model
            model_mbert: fine-tuned mBERT model
            results_csv_path: File path to store classification results
        Returns:
            xlmr_label: Classification label by XLM-R model
            mbert_label: Classification label by mBERT model
    """
    # Score with XLM-R
    xlmr_pred = model_xlmr([comment])[0]
    xlmr_label = xlmr_pred['label']
    xlmr_score = xlmr_pred['score']

    # Score with mBERT
    mbert_pred = model_mbert([comment])[0]
    mbert_label = mbert_pred['label']
    mbert_score = mbert_pred['score']

    # Write the classificatio results into the results file
    with open(results_csv_path, 'a+') as f:
        csv_writer = writer(f)
        csv_writer.writerow([comment, xlmr_label, xlmr_score, mbert_label, mbert_score, lang, raw_comment])

    return (xlmr_label, mbert_label)

def stats_counter(label, lang, model_type, datastore):
    """
        Count classification statistics

        Args:
            label: Classification label of the comment
            lang: Language of the comment
            model_type: The model that generates this classification
            datastore: A dictionary to store the statistics
    """
    # Number of positive comments increases if the label is 'VERY POSITIVE' or 'SOMEWHAT POSITIVE'
    if label in ['VERY POSITIVE', 'SOMEWHAT POSITIVE']:
        datastore[model_type][lang]['positive'] += 1
    # Number of negative comments increases if the label is 'VERY NEGATIVE' or 'SOMEWHAT NEGATIVE'
    elif label in ['VERY NEGATIVE', 'SOMEWHAT NEGATIVE']:
        datastore[model_type][lang]['negative'] += 1
    # Otherwise number of neutral comments increases
    else:
        datastore[model_type][lang]['neutral'] += 1

def cal_percentage(num, total_num):
    """
        Calculate percentage helper

        Args:
            num: The dividend number
            total_num: The divisor number
        Returns:
            The percentage rounded to 1 decimal place
    """
    percentage = num / total_num * 100
    return round(percentage, 1)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--video_url', default='https://www.youtube.com/watch?v=gdZLi9oWNZg')
    parser.add_argument('--output_dir', default='videos')
    parser.add_argument('--verbose', default=False)
    parser.add_argument('--max_pages', help='Max number of pages to fetch', default=100)
    parser.add_argument('--results_per_page', help='Results per page', default=100)
    parser.add_argument('--order', default='time')
    parser.add_argument('--credentials', default='./credentials.json')
    args = parser.parse_args()

    # Connect YouTube Data API
    credentials = load_credentials(args.credentials)
    youtube_video_id = get_video_id_by_url(args.video_url)
    parameters = {
        'videoId': youtube_video_id,
        'part': 'snippet',
        'textFormat': 'plainText',
        'maxResults': args.results_per_page,
        'order': args.order,
    }
    youtube_client = YouTubeClient(credentials, parameters, args.verbose)

    # Initialize classifiers and stats
    xlmr = ps.load_xlmr_model()
    mbert = ps.load_mbert_model()
    language_classifier = ps.load_language_classifier()
    results_file_path = initialize_artefacts(youtube_video_id, args.output_dir)
    stats_disqualified = 0
    stats_total_comments_processed = 0

    # A dictionary to store classification statistics
    run_stats = {
        'xlmr': {
            'en': {
                'positive': 0,
                'negative': 0,
                'neutral': 0
            },
            'es': {
                'positive': 0,
                'negative': 0,
                'neutral': 0
            },
        },
        'mbert': {
            'en': {
                'positive': 0,
                'negative': 0,
                'neutral': 0
            },
            'es': {
                'positive': 0,
                'negative': 0,
                'neutral': 0
            },
        },
    }

    # Data processing
    for comment in youtube_client.get_comments(int(args.max_pages)):
        stats_total_comments_processed += 1

        # Data cleaning
        processed_comment = ps.preprocess_pipeline(comment, verbose=args.verbose)
        qualified, lang = ps.qualify(processed_comment, language_classifier)
        if not qualified:
            stats_disqualified += 1
            continue

        autocorrected = ps.autocorrect_comment(processed_comment, lang, args.verbose)

        # Classify comments and store classifcation results
        xlmr_label, mbert_label = classify_comment(autocorrected, lang, comment, xlmr, mbert, results_file_path)

        stats_counter(xlmr_label, lang, 'xlmr', run_stats)
        stats_counter(mbert_label, lang, 'mbert', run_stats)

    # Print classification summary on the terminal
    print("=============================================== Results ===============================================\n")
    xlmr_total_positive = run_stats['xlmr']['en']['positive'] + run_stats['xlmr']['es']['positive']
    xlmr_total_negative = run_stats['xlmr']['en']['negative'] + run_stats['xlmr']['es']['negative']
    xlmr_total_neutral = run_stats['xlmr']['en']['neutral'] + run_stats['xlmr']['es']['neutral']
    mbert_total_positive = run_stats['mbert']['en']['positive'] + run_stats['mbert']['es']['positive']
    mbert_total_negative = run_stats['mbert']['en']['negative'] + run_stats['mbert']['es']['negative']
    mbert_total_neutral = run_stats['mbert']['en']['neutral'] + run_stats['mbert']['es']['neutral']
    stats_qualified = stats_total_comments_processed - stats_disqualified

    print(f"Disqualified: {stats_disqualified} ({cal_percentage(stats_disqualified, stats_total_comments_processed)}%)\nQualified: {stats_qualified} ({cal_percentage(stats_qualified, stats_total_comments_processed)}%)\n")
    print(f"XLM-R: {xlmr_total_positive} positive ({cal_percentage(xlmr_total_positive, stats_total_comments_processed)}%), {xlmr_total_negative} negative ({cal_percentage(xlmr_total_negative, stats_total_comments_processed)}%), {xlmr_total_neutral} neutral ({cal_percentage(xlmr_total_neutral, stats_total_comments_processed)}%)\n", end="")
    print(f"mBERT: {mbert_total_positive} positive ({cal_percentage(mbert_total_positive, stats_total_comments_processed)}%), {mbert_total_negative} negative ({cal_percentage(mbert_total_negative, stats_total_comments_processed)}%), {mbert_total_neutral} neutral ({cal_percentage(mbert_total_neutral, stats_total_comments_processed)}%)\n")
    print(f"LANGUAGE STATS:")
    print(f"XLM-R (EN): {run_stats['xlmr']['en']['positive']} positive, {run_stats['xlmr']['en']['negative']} negative, {run_stats['xlmr']['en']['neutral']} neutral\n", end="")
    print(f"XLM-R (ES): {run_stats['xlmr']['es']['positive']} positive, {run_stats['xlmr']['es']['negative']} negative, {run_stats['xlmr']['es']['neutral']} neutral\n")
    print(f"mBERT (EN): {run_stats['mbert']['en']['positive']} positive, {run_stats['mbert']['en']['negative']} negative, {run_stats['mbert']['en']['neutral']} neutral\n", end="")
    print(f"mBERT (ES): {run_stats['mbert']['es']['positive']} positive, {run_stats['mbert']['es']['negative']} negative, {run_stats['mbert']['es']['neutral']} neutral\n")

if __name__ == '__main__':
  main()


from cement import App, TestApp, init_defaults, Interface, Handler
from cement.core.exc import CaughtSignal

from core.exc import BotError
from interfaces.manage import ManageHandler, ManageInterface

from importlib import import_module
import datetime
from os import scandir
import os
import socket
import pika
import json
import time

# configuration defaults
CONFIG = init_defaults('bot')
RUTA = './interfaces/'
host_queue = 'localhost'
hand_list = []
if_list = []

def get_ip(domain):
    """ get the ip from the domain """
    try:
        ip = socket.gethostbyname(domain)
        return ip
    except socket.gaierror as e:
        print(e)
        return

def ls(ruta=RUTA):
    """ Get all bots in the interfaces directory """
    return [arch.name for arch in scandir(ruta) if arch.is_file()]

def do_imports():
    files = ls()
    for file in files:
        if '__init__' not in file and 'manage' not in file:
            s = file.replace('.py', '')
            upper = s.title()
            first = 'interfaces.' + s
            handler = upper + 'Handler'
            interface = upper + 'Interface'
            # Dynamic import
            i = dynamic_import(first, interface)
            h = dynamic_import(first, handler)
            # Add to list
            hand_list.append(h)
            if_list.append(i)

def dynamic_import(abs_module_path, class_name):
    module_object = import_module(abs_module_path)

    target_class = getattr(module_object, class_name)

    return target_class

def launch_scan(app, type_bots, scan):
    scan = json.loads(scan)
    # Start geolocation handler
    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    geo = app.handler.get('geoIf', 'geo', setup=True)

    scan_list = []
    if not scan['launch']:
        app.log.info("The scan already done")
        return None
    elif scan['executionTime'] < now and scan['launch']:
        for host in scan['hosts']:
            results = []
            # Get the ip
            ip = get_ip(host)
            # Save data geolocation
            geo_data = geo.get_geo(ip, host)
            for tp in type_bots:
                # Launch bot scan
                app.log.info("BOT " + tp)
                b = app.handler.get(tp + "If", tp, setup=True)
                bot_data = b.bot_scan(host, ip)
                d = dict({tp: bot_data})
                app.log.info("Scanning completed")
                results.append(d)

            # All data in object for send
            scan_finished = geo_data
            scan_finished['results'] = results
            scan_list.append(scan_finished)
        return scan_list

def start_consumer(app, manage, type_bots):
    connection = pika.BlockingConnection(
        pika.ConnectionParameters(host=host_queue))
    channel = connection.channel()

    channel.queue_declare(queue='task_queue', durable=True)

    print(' [*] Waiting for messages. To exit press CTRL+C')

    def callback(ch, method, properties, body):
        data = launch_scan(app, type_bots, body)
        if not data:
            return
        # Send data to api
        scan = json.loads(body)
        manage.send_data(url='data', data=data, id=scan['id'])
        # Update done field in bots collection
        manage.update_done(scan['id'])
        ch.basic_ack(delivery_tag=method.delivery_tag)

    channel.basic_qos(prefetch_count=1)
    channel.basic_consume(queue='task_queue', on_message_callback=callback)

    channel.start_consuming()

class Bot(App):
    """ Mi BOT App primary application. """

    class Meta:
        label = 'bot'
        plugin_dirs = ['./plugins']

        # configuration dir
        dir = os.path.realpath('properties.conf')
        config_files = [dir]

        # configuration defaults
        config_defaults = CONFIG

        # call sys.exit() on close
        exit_on_close = True

        # load additional framework extensions
        extensions = [
            'yaml',
            'colorlog',
        ]

        # configuration handler
        config_handler = 'yaml'

        # configuration file suffix
        config_file_suffix = '.yml'

        # set the log handler
        log_handler = 'colorlog'

        # Funcion que llame al back para recuperar los handlers y los interfaces

        # Register handlers and interfaces
        handlers = [ManageHandler] + hand_list
        interfaces = [ManageInterface] + if_list

class BotTest(TestApp, Bot):
    """A sub-class of Bot that is better suited for testing."""

    class Meta:
        label = 'bot'

def main():
    with Bot() as app:
        try:
            app.run()
            # Create the handler that it will make the calls to api
            manage = app.handler.get('manageIf', 'manage', setup=True)
            # Load all bots and scans and then do imports in the interface dir
            type_bots, scans = manage.get_scan()
            x = 1
            n_bots = len(type_bots)
            for bot in type_bots:
                app.log.info("Downloading %d of %d files" % (x, n_bots))
                manage.download_files(type_bot=bot)
                app.log.info("File %d downloaded" % x)
                x += 1
            do_imports()
            for e in if_list:
                app.interface.define(e)
            for e in hand_list:
                app.handler.register(e)

            # ---- Start consumer ----
            start_consumer(app, manage, type_bots)

        except AssertionError as e:
            print('AssertionError > %s' % e.args[0])
            app.exit_code = 1

            if app.debug is True:
                import traceback
                traceback.print_exc()

        except BotError as e:
            print('BotError > %s' % e.args[0])
            app.exit_code = 1

            if app.debug is True:
                import traceback
                traceback.print_exc()

        except CaughtSignal as e:
            # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error)
            print('\n%s' % e)
            app.exit_code = 0

if __name__ == '__main__':
    main()


import json
import time

from BaseClass import ProtocolObject

class NetworkPlayer(ProtocolObject):
	def __init__(self, conn, addr):
		# Assigns self.buffer and self.conn
		super().__init__()
		self.conn = conn
		self.ip_address = addr[0]
		self.port = addr[1]

		self.nickname = None
		self.init = None
		self.waiting = 0
		self.game_player = None
		self.index = None

	def validate(self):
		# Called before player is assigned a lobby
		init = self.get_command()
		if init == "1":
			self.send_command("9")
			return True
		else:
			return False

	def get_nickname(self):
		# Called before player is assigned a lobby
		command = self.get_command()
		if command == "disconnected":
			# TODO
			raise Exception

		if not command:
			# If we didn't get anything
			return False, None
		else:
			command = json.loads(command)
			self.nickname = command["name"]
			return True, self.nickname

	def accept_name(self):
		to_send = json.dumps({"cf": True})
		self.send_command(to_send)

	def reject_name(self):
		to_send = json.dumps({"cf": False})
		self.send_command(to_send)

	def send_names(self, names, lobby_id):
		# Called once after a player has been added to a lobby
		to_send = json.dumps({"names": names, "id": lobby_id}, separators=(',', ':'))
		self.send_command(to_send)

	def update(self, time_left, no_of_players, new_players, lost_players, started):
		# Called once every few seconds while we're in the lobby
		# Send info to the client
		to_send = {}
		to_send["time"] = time_left
		to_send["players"] = no_of_players
		to_send["new"] = new_players
		to_send["lost"] = lost_players
		to_send["st"] = started

		to_send = json.dumps(to_send, separators=(',', ':'))

		self.send_command(to_send)
		self.waiting += 1

	def receive(self):
		# Called while we are in the lobby to check if the client is responding
		if self.waiting > 0:
			s = self.get_command()
			if s == "disconnected" or self.waiting > 5:
				return "disconnected"
			elif not s:
				return False
			s = json.loads(s)
			if s and (s["cf"] != 5):
				raise Exception("5 not returned from client")
			if s["cf"] == 5:
				self.waiting -= 1
				return True
		return False

	def start_game(self, index, names):
		self.index = index
		to_send = {"index": index, "names": names}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def start_round(self, dice_list, dice_quantities):
		# Called at the beginning of each round to tell the client how many dice
		# there are, and what their dice are.
		# Convert a list of ints into a string
		to_send = {"i": "@", "dice": dice_list, "dice_q": dice_quantities}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def send_current_player(self, player_no):
		# Tell every player who's turn it is
		to_send = {"i": "?", "player": player_no}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def get_turn(self):
		# Called when it is this player's turn, so we wait until they respond
		while True:
			command = self.get_command()
			if command:
				if command == "disconnected":
					# TODO something
					print("Client " + self.nickname + " disconnected")
					exit()

				else:
					turn = json.loads(command)
					bid = turn["bid"]
					if bid == "dudo":
						return "dudo"
					bid = tuple(bid)
					print("Bid received from " + self.nickname + ": " + str(bid))
					return bid

	def send_turn(self, bid):
		# Called after a turn has happened, we are telling our player what
		# the turn was.
		to_send = {"i": "sendturn", "bid": bid}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def send_dudo(self, player_no, is_out, all_dice):
		to_send = {"i": "senddudo", "player": player_no, "out": is_out, "alldice": all_dice}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def send_player_out(self, player_no):
		to_send = {"i": "sendplayerout", "player": player_no}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)

	def end_game(self):
		# TODO
		to_send = {"i": "endgame"}
		to_send = json.dumps(to_send, separators=(',', ':'))
		self.send_command(to_send)


import pathlib
from itertools import product
from PIL import Image
from numpy import asarray

input_path = pathlib.Path(__file__).parent / "input.txt"

class GridIterator:
    def __init__(self):
        self.iterator = product(range(100), range(100))

    def __iter__(self):
        return self.iterator

print("first star:")
grid = [[0] * 100 for _ in range(100)]
with open(input_path, "r") as input_file:
    for row, line in enumerate(input_file):
        for column, number in enumerate(line.strip()):
            grid[column][row] = int(number)

print(grid[0])
total_danger = 0
EDGES = {0, 99}
for row, column in GridIterator():
    left = max(0, column - 1)
    right = min(99, column + 1)
    up = max(0, row - 1)
    down = min(99, row + 1)
    middle = grid[column][row]
    greaters = 0
    if row in EDGES:
        greaters += 1
    if column in EDGES:
        greaters += 1
    if grid[column][up] > middle:
        greaters += 1
    if grid[column][down] > middle:
        greaters += 1
    if grid[left][row] > middle:
        greaters += 1
    if grid[right][row] > middle:
        greaters += 1
    if greaters == 4:
        total_danger += middle + 1
print(f"total danger: {total_danger}")

print("second star:")
# turn the table into a binary grid; replace all 9s with high values,
# everything else with 0.
HIGH = 100000
for row, column in GridIterator():
    grid[column][row] = HIGH if grid[column][row] == 9 else 0

# "paint" in the areas with 2s, keeping count of how many cells
# were filled in per connected area.
def get_neighbors(column, row):
    retval = []
    if column > 0:
        retval.append((column - 1, row))
    if column < 99:
        retval.append((column + 1, row))
    if row > 0:
        retval.append((column, row - 1))
    if row < 99:
        retval.append((column, row + 1))
    return retval

paint = 1
stack = []

for row, column in GridIterator():
    if grid[column][row] != 0:
        continue
    stack.append((column, row))
    while len(stack) > 0:
        current_c, current_r = stack.pop()
        if grid[current_c][current_r] == 0:
            grid[current_c][current_r] = paint
            stack.extend(get_neighbors(current_c, current_r))
    paint += 1
    print(f" done painting {paint-1}")

# Grid is now painted in; time to see which color "won."
counts = {}

for row, column in GridIterator():
    val = grid[column][row]
    if val != HIGH:
        counts[val] = counts.get(val, 0) + 1

paints = counts.keys()
cell_count = counts.values()
sorted_counts = [
    (paint_, count)
    for (paint_, count) in sorted(
        zip(paints, cell_count), key=lambda x: x[1], reverse=True
    )
]
print(f"Top three: {sorted_counts[0]}, {sorted_counts[1]}, {sorted_counts[2]}.")
print(
    f"Product of top three: {sorted_counts[0][1] * sorted_counts[1][1] * sorted_counts[2][1]}"
)

# Silly extra thing that is really not needed but scratch
# it, I'm doing it anyway.
output_image = Image.new("HSV", (100, 100))
editable_image = asarray(output_image)
for row, column in GridIterator():
    val = grid[column][row]
    if val != HIGH:
        hue = (val / paint) * 255
        editable_image[column, row] = (hue, 250, 250)
output_path = pathlib.Path(__file__).parent / "out.png"
Image.fromarray(editable_image, mode="HSV").convert(mode="RGB").save(output_path)