import itertools
from packaging.version import Version

import pytest
import numpy as np
import tensorflow as tf

from tensorflow_addons import text
from tensorflow_addons.utils import test_utils
from numpy.testing import assert_array_equal

def calculate_sequence_score(inputs, transition_params, tag_indices, sequence_lengths):
    expected_unary_score = sum(
        inputs[i][tag_indices[i]] for i in range(sequence_lengths)
    )
    expected_binary_score = sum(
        transition_params[tag_indices[i], tag_indices[i + 1]]
        for i in range(sequence_lengths - 1)
    )
    return expected_unary_score + expected_binary_score

def brute_force_decode(sequence_lengths, inputs, transition_params):
    num_words = inputs.shape[0]
    num_tags = inputs.shape[1]

    all_sequence_scores = []
    all_sequences = []

    tag_indices_iterator = itertools.product(range(num_tags), repeat=sequence_lengths)
    inputs = tf.expand_dims(inputs, 0)
    sequence_lengths = tf.expand_dims(sequence_lengths, 0)
    transition_params = tf.constant(transition_params)

    # Compare the dynamic program with brute force computation.
    for tag_indices in tag_indices_iterator:
        tag_indices = list(tag_indices)
        tag_indices.extend([0] * (num_words - sequence_lengths))
        all_sequences.append(tag_indices)
        sequence_score = text.crf_sequence_score(
            inputs=inputs,
            tag_indices=tf.expand_dims(tag_indices, 0),
            sequence_lengths=sequence_lengths,
            transition_params=transition_params,
        )
        sequence_score = tf.squeeze(sequence_score, [0])
        all_sequence_scores.append(sequence_score)

    expected_max_sequence_index = np.argmax(all_sequence_scores)
    expected_max_sequence = all_sequences[expected_max_sequence_index]
    expected_max_score = all_sequence_scores[expected_max_sequence_index]
    return expected_max_sequence, expected_max_score

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_filtered_inputs(dtype):
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [np.array(3, dtype=np.int32), np.array(1, dtype=np.int32)]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[4, 5, -3]], dtype=dtype),
    ]
    tag_bitmap_list = [
        np.array(
            [
                [True, False, False],
                [False, True, True],
                [False, True, True],
                [False, True, True],
            ],
            dtype=bool,
        ),
        np.array([[False, True, True]], dtype=bool),
    ]
    neg_inf = float("-inf")
    expected_filtered_inputs_list = [
        np.array(
            [[4, neg_inf, neg_inf], [neg_inf, -1, 3], [neg_inf, 2, 1], [neg_inf, 0, 0]],
            dtype=dtype,
        ),
        np.array([[neg_inf, 5, -3]], dtype=dtype),
    ]
    for sequence_lengths, inputs, tag_bitmap, expected_filtered_inputs in zip(
        sequence_lengths_list,
        inputs_list,
        tag_bitmap_list,
        expected_filtered_inputs_list,
    ):
        filtered_inputs = text.crf_filtered_inputs(
            inputs=tf.expand_dims(inputs, 0), tag_bitmap=tf.expand_dims(tag_bitmap, 0)
        )
        filtered_inputs = tf.squeeze(filtered_inputs, [0])

        test_utils.assert_allclose_according_to_type(
            filtered_inputs, expected_filtered_inputs
        )

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_sequence_score(dtype):
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [
        np.array(3, dtype=np.int32),
        np.array(1, dtype=np.int32),
    ]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[4, 5, -3]], dtype=dtype),
    ]
    tag_indices_list = [
        np.array([1, 2, 1, 0], dtype=np.int32),
        np.array([1], dtype=np.int32),
    ]
    for sequence_lengths, inputs, tag_indices in zip(
        sequence_lengths_list, inputs_list, tag_indices_list
    ):
        sequence_score = text.crf_sequence_score(
            inputs=tf.expand_dims(inputs, 0),
            tag_indices=tf.expand_dims(tag_indices, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            transition_params=tf.constant(transition_params),
        )
        sequence_score = tf.squeeze(sequence_score, [0])

        expected_sequence_score = calculate_sequence_score(
            inputs, transition_params, tag_indices, sequence_lengths
        )
        test_utils.assert_allclose_according_to_type(
            sequence_score, expected_sequence_score
        )

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_multi_tag_sequence_score(dtype):
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [
        np.array(3, dtype=np.int32),
        np.array(1, dtype=np.int32),
    ]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[4, 5, -3]], dtype=dtype),
    ]
    tag_bitmap_list = [
        np.array(
            [
                [True, True, False],
                [True, False, True],
                [False, True, True],
                [True, False, True],
            ],
            dtype=bool,
        ),
        np.array([[True, True, False]], dtype=bool),
    ]
    for sequence_lengths, inputs, tag_bitmap in zip(
        sequence_lengths_list, inputs_list, tag_bitmap_list
    ):
        sequence_score = text.crf_multitag_sequence_score(
            inputs=tf.expand_dims(inputs, 0),
            tag_bitmap=tf.expand_dims(tag_bitmap, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            transition_params=tf.constant(transition_params),
        )
        sequence_score = tf.squeeze(sequence_score, [0])
        all_indices_list = [
            single_index_bitmap.nonzero()[0]
            for single_index_bitmap in tag_bitmap[:sequence_lengths]
        ]
        expected_sequence_scores = [
            calculate_sequence_score(
                inputs, transition_params, indices, sequence_lengths
            )
            for indices in itertools.product(*all_indices_list)
        ]
        expected_log_sum_exp_sequence_scores = np.logaddexp.reduce(
            expected_sequence_scores
        )
        test_utils.assert_allclose_according_to_type(
            sequence_score, expected_log_sum_exp_sequence_scores
        )

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_unary_score(dtype):
    inputs = np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype)
    for dtype in (np.int32, np.int64):
        tag_indices = np.array([1, 2, 1, 0], dtype=dtype)
        sequence_lengths = np.array(3, dtype=np.int32)
        unary_score = text.crf_unary_score(
            tag_indices=tf.expand_dims(tag_indices, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            inputs=tf.expand_dims(inputs, 0),
        )
        unary_score = tf.squeeze(unary_score, [0])
        expected_unary_score = sum(
            inputs[i][tag_indices[i]] for i in range(sequence_lengths)
        )
        test_utils.assert_allclose_according_to_type(unary_score, expected_unary_score)

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_binary_score(dtype):
    tag_indices = np.array([1, 2, 1, 0], dtype=np.int32)
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    sequence_lengths = np.array(3, dtype=np.int32)
    binary_score = text.crf_binary_score(
        tag_indices=tf.expand_dims(tag_indices, 0),
        sequence_lengths=tf.expand_dims(sequence_lengths, 0),
        transition_params=tf.constant(transition_params),
    )
    binary_score = tf.squeeze(binary_score, [0])
    expected_binary_score = sum(
        transition_params[tag_indices[i], tag_indices[i + 1]]
        for i in range(sequence_lengths - 1)
    )
    test_utils.assert_allclose_according_to_type(binary_score, expected_binary_score)

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_log_norm(dtype):
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [
        np.array(3, dtype=np.int32),
        np.array(1, dtype=np.int64),
    ]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[3, -1, 3]], dtype=dtype),
    ]
    tag_indices_list = [
        np.array([1, 2, 1, 0], dtype=np.int32),
        np.array([2], dtype=np.int32),
    ]

    for sequence_lengths, inputs, tag_indices in zip(
        sequence_lengths_list, inputs_list, tag_indices_list
    ):
        num_words = inputs.shape[0]
        num_tags = inputs.shape[1]
        all_sequence_scores = []

        # Compare the dynamic program with brute force computation.
        for tag_indices in itertools.product(range(num_tags), repeat=sequence_lengths):
            tag_indices = list(tag_indices)
            tag_indices.extend([0] * (num_words - sequence_lengths))
            all_sequence_scores.append(
                text.crf_sequence_score(
                    inputs=tf.expand_dims(inputs, 0),
                    tag_indices=tf.expand_dims(tag_indices, 0),
                    sequence_lengths=tf.expand_dims(sequence_lengths, 0),
                    transition_params=tf.constant(transition_params),
                )
            )

        brute_force_log_norm = tf.reduce_logsumexp(all_sequence_scores)
        log_norm = text.crf_log_norm(
            inputs=tf.expand_dims(inputs, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            transition_params=tf.constant(transition_params),
        )
        log_norm = tf.squeeze(log_norm, [0])

        test_utils.assert_allclose_according_to_type(log_norm, brute_force_log_norm)

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_log_norm_zero_seq_length(dtype):
    """Test `crf_log_norm` when `sequence_lengths` contains one or more
    zeros."""
    inputs = tf.constant(np.ones([2, 10, 5], dtype=dtype))
    transition_params = tf.constant(np.ones([5, 5], dtype=dtype))
    sequence_lengths = tf.constant(np.zeros([2], dtype=np.int32))
    expected_log_norm = np.zeros([2], dtype=dtype)
    log_norm = text.crf_log_norm(inputs, sequence_lengths, transition_params)
    test_utils.assert_allclose_according_to_type(log_norm, expected_log_norm)

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_log_likelihood(dtype):
    inputs = np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype)
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    sequence_lengths = np.array(3, dtype=np.int32)

    num_words = inputs.shape[0]
    num_tags = inputs.shape[1]
    all_sequence_log_likelihoods = []

    # Make sure all probabilities sum to 1.
    for tag_indices in itertools.product(range(num_tags), repeat=sequence_lengths):
        tag_indices = list(tag_indices)
        tag_indices.extend([0] * (num_words - sequence_lengths))
        sequence_log_likelihood, _ = text.crf_log_likelihood(
            inputs=tf.expand_dims(inputs, 0),
            tag_indices=tf.expand_dims(tag_indices, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            transition_params=tf.constant(transition_params),
        )
        all_sequence_log_likelihoods.append(sequence_log_likelihood)
    total_log_likelihood = tf.reduce_logsumexp(all_sequence_log_likelihoods)
    test_utils.assert_allclose_according_to_type(
        total_log_likelihood, 0.0, rtol=1e-6, atol=1e-6, half_rtol=2e-3, half_atol=2e-3
    )

    # check if `transition_params = None` raises an error
    text.crf_log_likelihood(
        inputs=tf.expand_dims(inputs, 0),
        tag_indices=tf.expand_dims(tag_indices, 0),
        sequence_lengths=tf.expand_dims(sequence_lengths, 0),
    )

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_viterbi_decode(dtype):
    inputs = np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype)
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    sequence_lengths = np.array(3, dtype=np.int32)
    num_words = inputs.shape[0]
    num_tags = inputs.shape[1]

    all_sequence_scores = []
    all_sequences = []

    # Compare the dynamic program with brute force computation.
    for tag_indices in itertools.product(range(num_tags), repeat=sequence_lengths):
        tag_indices = list(tag_indices)
        tag_indices.extend([0] * (num_words - sequence_lengths))
        all_sequences.append(tag_indices)
        sequence_score = text.crf_sequence_score(
            inputs=tf.expand_dims(inputs, 0),
            tag_indices=tf.expand_dims(tag_indices, 0),
            sequence_lengths=tf.expand_dims(sequence_lengths, 0),
            transition_params=tf.constant(transition_params),
        )
        sequence_score = tf.squeeze(sequence_score, [0])
        all_sequence_scores.append(sequence_score)

    expected_max_sequence_index = np.argmax(all_sequence_scores)
    expected_max_sequence = all_sequences[expected_max_sequence_index]
    expected_max_score = all_sequence_scores[expected_max_sequence_index]

    actual_max_sequence, actual_max_score = text.viterbi_decode(
        inputs[:sequence_lengths], transition_params
    )

    test_utils.assert_allclose_according_to_type(actual_max_score, expected_max_score)
    assert actual_max_sequence == expected_max_sequence[:sequence_lengths]

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_decode(dtype):
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [
        np.array(3, dtype=np.int32),
        np.array(1, dtype=np.int64),
    ]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[-1, 2, 1]], dtype=dtype),
    ]
    tag_indices_list = [
        np.array([1, 2, 1, 0], dtype=np.int32),
        np.array([2], dtype=np.int32),
    ]

    for sequence_lengths, inputs, tag_indices in zip(
        sequence_lengths_list, inputs_list, tag_indices_list
    ):
        expected_max_sequence, expected_max_score = brute_force_decode(
            sequence_lengths, inputs, transition_params
        )

        actual_max_sequence, actual_max_score = text.crf_decode(
            tf.expand_dims(inputs, 0),
            tf.constant(transition_params),
            tf.expand_dims(sequence_lengths, 0),
        )
        actual_max_sequence = tf.squeeze(actual_max_sequence, [0])
        actual_max_score = tf.squeeze(actual_max_score, [0])

        test_utils.assert_allclose_according_to_type(
            actual_max_score, expected_max_score, 1e-6, 1e-6
        )
        assert (
            list(actual_max_sequence[:sequence_lengths])
            == expected_max_sequence[:sequence_lengths]
        )

@pytest.mark.parametrize("dtype", [np.float16, np.float32])
def test_crf_constrained_decode(dtype):
    transition_params = np.array([[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=dtype)
    # Test both the length-1 and regular cases.
    sequence_lengths_list = [np.array(3, dtype=np.int32), np.array(1, dtype=np.int32)]
    inputs_list = [
        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=dtype),
        np.array([[4, 5, -3]], dtype=dtype),
    ]
    tag_bitmap_list = [
        np.array(
            [
                [True, False, False],
                [False, True, True],
                [False, True, True],
                [False, True, True],
            ],
            dtype=bool,
        ),
        np.array([[False, True, True]], dtype=bool),
    ]
    for sequence_lengths, inputs, tag_bitmap in zip(
        sequence_lengths_list, inputs_list, tag_bitmap_list
    ):
        filtered_inputs = text.crf_filtered_inputs(
            inputs=tf.expand_dims(inputs, 0), tag_bitmap=tf.expand_dims(tag_bitmap, 0)
        )

        expected_max_sequence, expected_max_score = text.crf_decode(
            filtered_inputs,
            tf.constant(transition_params),
            tf.expand_dims(sequence_lengths, 0),
        )

        expected_max_sequence = tf.squeeze(expected_max_sequence, [0])
        expected_max_score = tf.squeeze(expected_max_score, [0])

        actual_max_sequence, actual_max_score = text.crf_constrained_decode(
            tf.expand_dims(inputs, 0),
            tf.expand_dims(tag_bitmap, 0),
            tf.constant(transition_params),
            tf.expand_dims(sequence_lengths, 0),
        )

        actual_max_sequence = tf.squeeze(actual_max_sequence, [0])
        actual_max_score = tf.squeeze(actual_max_score, [0])

        test_utils.assert_allclose_according_to_type(
            actual_max_score, expected_max_score, 1e-6, 1e-6
        )
        assert list(actual_max_sequence[:sequence_lengths]) == list(
            expected_max_sequence[:sequence_lengths]
        )

def test_crf_decode_zero_seq_length():
    """Test that crf_decode works when sequence_length contains one or more
    zeros."""
    inputs = tf.constant(np.ones([2, 10, 5], dtype=np.float32))
    transition_params = tf.constant(np.ones([5, 5], dtype=np.float32))
    sequence_lengths = tf.constant(np.zeros([2], dtype=np.int32))
    tags, scores = text.crf_decode(inputs, transition_params, sequence_lengths)
    assert len(tags.shape) == 2
    assert len(scores.shape) == 1

def test_different_dtype():
    inputs = np.ones([16, 20, 5], dtype=np.float32)
    tags = tf.convert_to_tensor(np.ones([16, 20], dtype=np.int64))
    seq_lens = np.ones([16], dtype=np.int64) * 20

    loss, _ = text.crf_log_likelihood(
        inputs=inputs, tag_indices=tags, sequence_lengths=seq_lens
    )

def test_tf_function():
    batch_size = 4
    num_tags = 10
    input_signature = (
        tf.TensorSpec([None, None, num_tags]),
        tf.TensorSpec([num_tags, num_tags]),
        tf.TensorSpec([None], dtype=tf.int32),
    )
    crf_decode = tf.function(input_signature=input_signature)(text.crf_decode)
    potentials = tf.random.uniform([batch_size, 1, num_tags])
    transition_params = tf.random.uniform([num_tags, num_tags])
    sequence_length = tf.ones([batch_size], dtype=tf.int32)
    crf_decode(potentials, transition_params, sequence_length)

class CRFDecode(tf.keras.layers.Layer):
    def __init__(self):
        super().__init__()

    def call(self, potentials, transition_params, sequence_length):
        return text.crf_decode(potentials, transition_params, sequence_length)

@pytest.mark.skipif(
    tf.__version__[:3] == "2.4",
    reason="CRF Decode doesn't work in TF2.4, the issue was fixed in TF core, but didn't make the release",
)
def test_crf_decode_save_load(tmpdir):
    class DummyLoss(tf.keras.losses.Loss):
        def call(self, y_true, y_pred):
            return tf.zeros(shape=())

    tf.keras.backend.clear_session()
    input_tensor = tf.keras.Input(shape=(10, 3), dtype=tf.float32, name="input_tensor")
    seq_len = tf.keras.Input(shape=(), dtype=tf.int32, name="seq_len")
    transition = tf.constant([[1, 1, 0], [0, 1, 1], [1, 0, 1]], dtype=tf.float32)

    output = tf.multiply(input_tensor, tf.constant(1.0))
    decoded, _ = CRFDecode()(input_tensor, transition, seq_len)

    model = tf.keras.Model(
        inputs=[input_tensor, seq_len], outputs=[output, decoded], name="example_model"
    )
    model.compile(optimizer="Adam", loss=DummyLoss())

    x_data = {
        "input_tensor": np.random.random_sample((5, 10, 3)).astype(dtype=np.float32),
        "seq_len": np.array([10] * 5, dtype=np.int32),
    }

    tensor_name = (
        "tf.math.multiply"
        if Version(tf.__version__) >= Version("2.5.0")
        else "tf_op_layer_Mul"
    )
    y_data = {tensor_name: np.random.randint(0, 3, (5, 10))}

    model.fit(x_data, y_data)
    model.predict(
        {
            "input_tensor": tf.expand_dims(x_data["input_tensor"][0], 0),
            "seq_len": np.array([10]),
        }
    )

    temp_dir = str(tmpdir.mkdir("model"))
    model.save(temp_dir)

    tf.keras.backend.clear_session()
    model = tf.keras.models.load_model(
        temp_dir,
        custom_objects={
            "CrfDecodeForwardRnnCell": text.crf.CrfDecodeForwardRnnCell,
            "DummyLoss": DummyLoss,
        },
    )
    model.fit(x_data, y_data)
    model.predict(
        {
            "input_tensor": tf.expand_dims(x_data["input_tensor"][0], 0),
            "seq_len": np.array([10]),
        }
    )

@pytest.mark.parametrize(
    "potentials,sequence_length",
    [
        # performs masking
        pytest.param(
            tf.random.normal([2, 12, 3]),
            tf.constant([8, 10]),
        ),
        # does not perform masking
        pytest.param(
            tf.random.normal([4, 8, 10]),
            tf.constant([8, 8, 8, 8]),
        ),
    ],
)
def test_crf_decode_forward_mask(potentials, sequence_length):
    # mimics setup of the `_multi_seq_fn` closure in `crf_decode`
    initial_state = tf.slice(potentials, [0, 0, 0], [-1, 1, -1])
    initial_state = tf.squeeze(initial_state, axis=[1])
    inputs = tf.slice(potentials, [0, 1, 0], [-1, -1, -1])

    sequence_length_less_one = tf.maximum(
        tf.constant(0, dtype=tf.int32), sequence_length - 1
    )

    n_tags = potentials.shape[-1]
    transition_params = tf.random.normal([n_tags, n_tags])

    backpointers, _ = text.crf_decode_forward(
        inputs, initial_state, transition_params, sequence_length_less_one
    )

    # everything masked by `sequence_length_less_one` should be equal to 0.
    mask = tf.sequence_mask(sequence_length_less_one, tf.shape(inputs)[1])

    # the indices that _should_ have been masked in the RNN operation
    masked_indices = tf.cast(tf.logical_not(mask), tf.int32)

    # sum of each row in the mask should equal timedim - seq lens
    exp_mask_sums = (
        tf.repeat(inputs.shape[1], inputs.shape[0]) - sequence_length_less_one
    )
    mask_sums = tf.reduce_sum(masked_indices, axis=1)
    assert_array_equal(
        exp_mask_sums.numpy(),
        mask_sums.numpy(),
    )

    # now apply the inverse mask to the backpointers and show that ALL are zeros. this is proof that
    # we appropriately masked timesteps
    masked_indices = tf.expand_dims(masked_indices, [2])
    zeros = masked_indices * backpointers
    assert tf.reduce_all(zeros == 0).numpy(), "Mask not applied correctly: {0}".format(
        zeros
    )


from argparse import ArgumentParser
from recsys.utils import parse_config
from recsys.core.handler import RequestHandler

CONFIG_FILE = 'config.yaml'

def parse_arguments():
    parser = ArgumentParser()
    parser.add_argument(
        '-m', '--movies', nargs='+',
        help=('Favorite movies you want to'
              ' get recommendations based on.')
    )
    args = parser.parse_args()
    return args

def main():
    arguments = parse_arguments()
    config = parse_config(CONFIG_FILE)
    handler = RequestHandler(config['inference'], config['logger'])
    print(handler.handle_request(arguments.movies))

if __name__ == '__main__':
    main()


# !/usr/bin/python
import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
from tensorflow.keras import Sequential, layers, regularizers, models, optimizers
from sklearn.model_selection import train_test_split
import cv2
from sklearn.metrics import confusion_matrix, classification_report

# ********************************* Read in the data ********************************

# define folder paths for the data
train_humans_path = "../data/horse-or-human/train/humans/"
train_horses_path = "../data/horse-or-human/train/horses/"

paths = [train_humans_path, train_horses_path]

X_data = []
y_data = []

for path in paths:
    for file in os.listdir(path):
        image = cv2.imread(os.path.join(path, file), cv2.IMREAD_COLOR)
        X_data.append(image)
        if 'human' in file:
            label = 1
        else:
            label = 0
        y_data.append(label)

X_data = np.asarray(X_data)
y_data = np.asarray(y_data)

# *************************************** Inspect features and labels *****************

print("\nShape of X_data:", X_data.shape)
print("Shape of y_data:", y_data.shape)

print("\nData type of X_data:", type(X_data))
print("Data type of y_data:", type(y_data))

print("\nData type of 1st sample in X_data:", type(X_data[0]))
print("Data type of 1st label in y_data:", type(y_data[0]))

print("\nThe first X_data sample: \n", X_data[0])

print("\nThe y_data labels: \n", y_data)
print(f"\nNumber of y_data labels that are ones (expected 527): {y_data.sum()}")

# ****************************** Scale features ****************************************

X_data = X_data / 255.0
print("\nThe first X_data sample after normalization: \n", X_data[0])

# ****************************** Split into train and test sets *************************

X_train, X_valid, y_train, y_valid = train_test_split(X_data, y_data, test_size=0.2, random_state=42)

print(f"\nShape of X_train: {X_train.shape}")
print(f"Shape of X_valid: {X_valid.shape}")
print(f"Shape of y_train: {y_train.shape}")
print(f"Shape of y_valid: {y_valid.shape}\n")

# ****************************** Build model and fit ***********************************

# Build model
clf = Sequential()
clf.add(layers.Conv2D(16, (3, 3), strides=(1, 1), activation='relu', input_shape=(300, 300, 3)))
clf.add(layers.MaxPooling2D(2, 2))
clf.add(layers.Conv2D(32, (3, 3), strides=(1, 1), activation='relu'))
clf.add(layers.MaxPooling2D(2, 2))
clf.add(layers.Conv2D(64, (3, 3), strides=(1, 1), activation='relu'))
clf.add(layers.MaxPooling2D(2, 2))
clf.add(layers.Conv2D(64, (3, 3), strides=(1, 1), activation='relu'))
clf.add(layers.MaxPooling2D(2, 2))
clf.add(layers.Flatten())
clf.add(layers.Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.03)))
# clf.add(layers.Dense(16, activation='relu'))
clf.add(layers.Dropout(0.5))
clf.add(layers.Dense(1, activation='sigmoid'))

# Print summary of model
clf.summary()

# Compile model and fit
opt = optimizers.Adam(lr=0.001)
clf.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])
history = clf.fit(X_train, y_train, validation_split=0.2, epochs=25, verbose=2)

# ********************************** Save Model ***************************************

clf.save('model')

# ************************** Evaluate metrics on training data ************************

_, accuracy = clf.evaluate(X_train, y_train)
print('\nAccuracy on training data is: {:.1%}'.format(accuracy))

# ******************************* Visualize results ***********************************

params = {'legend.fontsize': 'x-large',
          'axes.labelsize': 'x-large',
          'axes.titlesize': 'x-large',
          'xtick.labelsize': 'x-large',
          'ytick.labelsize': 'x-large'}
pylab.rcParams.update(params)

fig, axes = plt.subplots(2, figsize=(15, 15), sharex=True)
axes[0].plot(history.history['accuracy'])
axes[0].plot(history.history['val_accuracy'])
axes[0].set_title('Model Accuracy')
axes[0].set_ylabel('Accuracy')
axes[0].set_ylim(ymax=1.0)
axes[0].grid()
axes[0].legend(['Train', 'Validation'], loc='lower right')
axes[1].plot(history.history['loss'])
axes[1].plot(history.history['val_loss'])
axes[1].set_title('Model Loss')
axes[1].set_ylabel('Loss')
# axes[1].set_yscale("log")
axes[1].set_xlabel('Epoch')
axes[1].grid()
axes[1].legend(['Train', 'Validation'], loc='upper right')
plt.savefig('../results/fig_acc_loss.svg', dpi=400, bbox_inches='tight')

# ******************************* Load Model *************************************************

reconstructed_model = models.load_model('model')  # comment out if not loading saved model

# ******************************* Evaluate metrics on validation data *******************************************

_, accuracy = reconstructed_model.evaluate(X_valid, y_valid)  # only use when loading saved model
# _, accuracy = clf.evaluate(X_test, y_test)  # only use when using model in memory
print('\nAccuracy on validation hold-out set is: {:.1%}\n'.format(accuracy))

y_pred_valid = clf.predict_classes(X_valid)

# Confusion matrix and classification report
cm = confusion_matrix(y_valid, y_pred_valid)
print("Confusion matrix:\n", cm)

cr = classification_report(y_valid, y_pred_valid)
print("\nClassification report:\n", cr)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 10 01:00:07 2019

@author: thugwithyoyo
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import os

import tkinter as tk
from tkinter.filedialog import askopenfilename

def GetSessionDecodingMaximum():

    # Acquire path of workspace to load.
    root = tk.Tk()
    RestoreFilePath = askopenfilename()
    root.withdraw()

    # Open workspace.
    #exec(open('./RestoreShelvedWorkspaceScript.py').read())
    try:

        exec(open('./RestoreShelvedWorkspaceScript.py').read())

    except:

        print('Unshelving error.  Will attempt to continue...')

    # Determine parent directory and filename from complete path.
    drive, path_and_file = os.path.splitdrive(RestoreFilePath)
    path, file = os.path.split(path_and_file)

    PerfIndices = np.arange(1, Performance.shape[0])
    MaxPerfVal = Performance[0]['performance']
    MaxPerfIndex = 0

    SessionName = file[0:19]

    for i in PerfIndices:

        if Performance[i]['performance'] > MaxPerfVal:

            MaxPerfIndex = i
            MaxPerfVal = Performance[i]['performance']

    return {
            'MaxPerfIndex': np.array([MaxPerfIndex]),
            'MaxPerfVal': np.array([MaxPerfVal]),
            'MaxPerfErrorBar': np.array([ConfInts[MaxPerfIndex]['performance_SE']]),
            'ShuffledPerfMedian': np.array([EventsShuffled[MaxPerfIndex]['performance_median']]),
            'ShuffledPerfErrorBar': np.array([EventsShuffled[MaxPerfIndex]['performance_SE']]),
            'SessionName': np.array([SessionName])
            }

# Integer width to position clusters.
ClusterIncrement = 1.

# The proportion of space to use to separate bar groups.
GroupSepWidthProportion = 0.25

# Calculate barwidth
BarWidth = ClusterIncrement*(1. - GroupSepWidthProportion)/2.

# Set colorscheme.  Is this necessary?
mpl.rc('image', cmap='nipy_spectral')

SessionPerfMeans = np.empty((0,))
SessionPerfSEs = np.empty((0,))
SessionShuffledPerfMeans = np.empty((0,))
SessionShuffledPerfSEs = np.empty((0,))

SessionNames = np.array([])

#EntriesFilt = np.array([1, 3, 5, 7, 9, -1])
#EntriesFilt = EntriesFilt[::-1]

GetAnotherSession = True

while (GetAnotherSession == True):

    SessionDict = GetSessionDecodingMaximum()

#    SessionPerfMeans = np.hstack([SessionPerfMeans, np.array([MaxPerfVal])])
#    SessionPerfSEs = np.hstack([SessionPerfSEs, np.array([ConfInts[i]['performance_SE']])])
#    SessionShuffledPerfMeans = np.hstack([SessionShuffledPerfMeans, np.array([EventsShuffled[i]['performance_median']])])
#    SessionShuffledPerfSEs = np.hstack([SessionShuffledPerfSEs, np.array([EventsShuffled[i]['performance_SE']])])
#
#    SessionNames = np.hstack([SessionNames, SessionName])

    SessionPerfMeans = np.hstack([SessionPerfMeans, SessionDict['MaxPerfVal']])
    SessionPerfSEs = np.hstack([SessionPerfSEs, SessionDict['MaxPerfErrorBar']])
    SessionShuffledPerfMeans = np.hstack([SessionShuffledPerfMeans, SessionDict['ShuffledPerfMedian']])
    SessionShuffledPerfSEs = np.hstack([SessionShuffledPerfSEs, SessionDict['ShuffledPerfErrorBar']])

    SessionNames = np.hstack([SessionNames, SessionDict['SessionName']])

    root = tk.Tk()

    GetAnotherSession = tk.messagebox.askyesno(message='Include another session?')

    root.withdraw()

#
# Extract recording IDs from filenames
NiceSessionNames = SessionNames

for i in np.arange(0, SessionNames.shape[0]):

    NiceSessionNames[i] = SessionNames[i][0:10]

SortMap = np.argsort(NiceSessionNames)

# Begin plot generation
(NumSessions, ) = SessionPerfMeans.shape
fig1, ax1 = plt.subplots(nrows=1, ncols=1)

fig1.suptitle('Peak trace decoding accuracy across sessions')

xLocs = np.arange(ClusterIncrement, (NumSessions + 1)*ClusterIncrement,
                  ClusterIncrement)

# Plot bars and errorbars of OBSERVED peak decoder performance across sessions.
ax1.bar(xLocs - BarWidth/2., SessionPerfMeans[SortMap],
        color='orange', ecolor='orange', label='Observed outcomes',
        width=BarWidth)

ax1.errorbar(xLocs - BarWidth/2., SessionPerfMeans[SortMap],
             yerr=SessionPerfSEs[SortMap],
             color='black', ecolor='black', fmt=',')

# Plot bars and errorbars of SHUFFLED peak decoder performance across sessions.
ax1.bar(xLocs + BarWidth/2., SessionShuffledPerfMeans[SortMap],
        color='gray', ecolor='gray', label='Shuffled outcomes',
        width=BarWidth)

ax1.errorbar(xLocs + BarWidth/2., SessionShuffledPerfMeans[SortMap],
             yerr=SessionShuffledPerfSEs[SortMap],
             color='black', ecolor='black', fmt=',')

ax1.set_xlabel('Session')
ax1.set_ylabel('Peak Decoding Accuracy (%)')
ax1.set_ylim([0., 1.])
ax1.set_yticks([0., 0.5, 1.0])
ax1.set_yticklabels(['0', '50', '100'])

#ax1.set_xticks(range(1, NumSessions+1, 1))
ax1.set_xticks(xLocs)
#ax1.set_xticklabels(NiceSessionNames[SortMap], rotation=45, ha='center')
ax1.set_xticklabels(np.arange(1,12,1), rotation=0)
ax1.legend(loc='lower right')
ax1.spines['top'].set_visible(False)
ax1.spines['right'].set_visible(False)
ax1.set_title('PLS-DA: 400ms window, slid over 100ms increments')


#Filename:getZhihu.py
import re,os
import urllib2
from bs4 import BeautifulSoup
import sys
import time

reload(sys)
sys.setdefaultencoding("utf-8")

def getHtml(url):
    header={'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1','Referer' : '******'}
    request=urllib2.Request(url,None,header)
    response=urllib2.urlopen(request)
    text=response.read()
    return text

def mkDir():
    date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
    os.mkdir(str(date))

def saveText(text):
    date=time.strftime('%Y-%m-%d',time.localtime(time.time()))
    dir_name="/home/wang/Documents/py/Zhihu/"+date
    soup=BeautifulSoup(text)
#    i=1
#    for i in soup.h2:
#        i=i+1
    if soup.h2.get_text()=='':
        filename=dir_name+"/ad.txt"
        fp=file(filename,'w')
        content=soup.find('div',"content")
        content=content.get_text()
        fp.write(content)
        fp.close()

#    elif i > 1:
#        filename=dir_name+"/kiding.txt"
#        contents=soup.findAll('div',"content")+soup.findAll("div","question")
#        contents=contents.get_text()
#        fp=file(filename,'w')
#        fp.write(contents)
#        fp.close()

    else:
        filename=dir_name+"/"+soup.h2.get_text()+".txt"
        fp=file(filename,'w')
        content=soup.find('div',"content")
        content=content.get_text()
        fp.write(content)
        fp.close()

#   print content #test

def getUrl(url):
    html=getHtml(url)

#   print html
    soup=BeautifulSoup(html)
    urls_page=soup.find('div',"post-body")
#   print urls_page

    urls=re.findall('"((http)://.*?)"',str(urls_page))
    return urls

def main():
    mkDir()
    page="http://zhihudaily.ahorn.me"
    urls=getUrl(page)
    for url in urls:
        text=getHtml(url[0])
        saveText(text)

if __name__=="__main__":
    main()

#url="http://daily.zhihu.com/story/4671450"
#url="http://zhihudaily.ahorn.me"
#text=getHtml(url)
#saveText(text)
#getUrl(url)


"""
# TODO:
    - Make work with more than 2 players
    - Make work with uneven teams
    - Make GUI a thing (command line ofc)
"""

import json

STARTING_ELO = 1000

def expected_score(playera: str, playerb: str):
    """
    Calculates the percentage chance that Player A has to win against Player B
    :param playera: The first player
    :param playerb: The second player
    :return: The chance that the first player has to win against Player B
    :raise IndexError: Either Player A or Player B is not in ranking.json
    """
    rb = STARTING_ELO
    ra = STARTING_ELO
    init_rankings = json.load(open("ranking.json"))
    try:
        ra = init_rankings[playera]
    except KeyError:
        data = json.load(open("ranking.json"))
        data.update({playera: STARTING_ELO})

        with open("ranking.json", "w") as fout:
            json.dump(data, fout, sort_keys=True, indent=4)
        expected_score(playera, playerb)
    try:
        rb = init_rankings[playerb]
    except KeyError:
        data = json.load(open("ranking.json"))
        data.update({playerb: STARTING_ELO})

        with open("ranking.json", "w") as fout:
            json.dump(data, fout, sort_keys=True, indent=4)
        expected_score(playera, playerb)

    return 1 / (1 + 10 ** ((rb - ra) / 400))

def expected_score_as_percent(playera: str, playerb: str):
    # return expected_score(playera, playerb) * 100
    return "{}% chance {} will win".format(round((expected_score(playera, playerb) * 100), 2), playera)

def add_game(playera: str, playerb: str, winner: int):
    """
    Adds a game to games.json
    :param playera: The first player
    :param playerb: The second player
    :param winner: The player that won
    :return: The new elo of both players
    """
    data = json.load(open("games.json"))
    if type(data) is dict:
        data = [data]

    outcome = [0] * 2
    if winner == 1:
        outcome[0] = 1
    else:
        outcome[1] = 1

    data.append({
        "Player A": {
            "Name": playera,
            "Outcome": outcome[0]
        },
        "Player B": {
            "Name": playerb,
            "Outcome": outcome[1]
        }
    })

    with open("games.json", "w") as fout:
        json.dump(data, fout, sort_keys=True, indent=4)

    update_rankings(playera, playerb, winner)

def update_rankings(playera: str, playerb: str, winner: int):
    """
    Updates elo values in ranking.json
    :param playera: The first player
    :param playerb: The second player
    :param winner: The player that won
    :return: N/A
    """
    outcome = [0] * 2
    if winner == 1:
        outcome[0] = 1
    else:
        outcome[1] = 1

    init_rankings = json.load(open("ranking.json"))
    expecteda = expected_score(playera, playerb)
    expectedb = expected_score(playerb, playera)
    try:
        olda = init_rankings[playera]
    except KeyError:
        olda = STARTING_ELO
    try:
        oldb = init_rankings[playerb]
    except KeyError:
        oldb = STARTING_ELO
    newa = round((olda + 32 * (outcome.pop(0) - expecteda)), 2)
    newb = round((oldb + 32 * (outcome.pop(0) - expectedb)), 2)

    data = json.load(open("ranking.json"))
    data[playera] = newa
    data[playerb] = newb

    with open("ranking.json", "w") as fout:
        json.dump(data, fout, sort_keys=True, indent=4)

def gui():
    pass

add_game("test1", "test2", 1)


from rest_framework import serializers
from account.models import User, Plan, ConfirmationCode, SerialNumber, Opt

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = (
            'id', 'full_name',
            'phone',
        )

class ConfirmationCodeSerializer(serializers.ModelSerializer):
    serial_number = serializers.SlugRelatedField(slug_field='number', read_only=True)

    class Meta:
        model = ConfirmationCode
        fields = (
            'id', 'serial_number', 'code',
            'expiry', 'activation_date',
        )

class SerialNumberSerializer(serializers.ModelSerializer):
    user = serializers.SlugRelatedField(slug_field='full_name', read_only=True)
    # code = serializers.SerializerMethodField()
    code = ConfirmationCodeSerializer(many=False)

    class Meta:
        model = SerialNumber
        fields = (
            'id', 'number', 'user',
            'created_at', 'days_charge', 'code'
        )

    # def create(self, validated_data):
    #     """
    #     Overriding the default create method of the Model serializer.
    #     :param validated_data: data containing all the details of student
    #     :return: returns a successfully created student record
    #     """
    #     user_data = validated_data.pop('user')
    #     user = UserSerializer.create(UserSerializer(), validated_data=user_data)
    #     student, created = UnivStudent.objects.update_or_create(user=user,
    #                         subject_major=validated_data.pop('subject_major'))
    #     return student

class PlanSerializer(serializers.ModelSerializer):
    class Meta:
        model = Plan
        fields = (
            'id', 'title', 'price', 'days',
        )


import matplotlib.pyplot as plt
import pandas as pd
import geopandas as gpd
from shapely.geometry import Point

fname = "data/calif/calif.ast"

# ASCII_data "CALIF ASCII data format"
fmt = ["station_id 1 8 char 0",
       "latitude_deg 10 11 short 0",
       "latitude_min 12 15 short 2",
       "longitude_deg 16 19 short 0",
       "longitude_min 20 23 short 2",
       "sea_level_elev_ft 24 29 long 1",
       "obs_grav 30 36 long 2",
       "accuracy_code 37 40 char 0",
       "Free_air_anom 41 46 long 2",
       "Bouguer_anom_simp 47 52 long 2",
       "terr_corr_inner 53 57 long 2",
       "terr_corr_total 58 62 long 2",
       "Hammer_zone 63 63 char 0",
       "Bouguer_anom_267 64 69 long 2",
       "isostatic_anom 70 75 long 2",
       "IGSN71_conv_factor 76 80 char 0"]

dtypes = []

f = open(fname, "r+")
data_content = f.readlines()
f.close()

allrows = []
for lineno, line in enumerate(data_content):
    row = []
    for fmtcol in fmt:
        colname, start_col, end_col, dtype, num = fmtcol.split()

        # from one-indexing to zero-indexing,=
        start_col, end_col = int(start_col)-1, int(end_col)
        num_decimal = int(num)

        if dtype == "char":
            item = line[start_col:end_col]
            if lineno == 0:
                dtypes.append((colname))

        elif num_decimal > 0:
            end_col -= num_decimal
            new_end_col = end_col + num_decimal
            stritem = line[start_col:end_col].strip()
            neg = False
            if stritem:
                if stritem[0] == "-":
                    neg = True
                    if len(stritem) == 1:
                        whole = 0
                    else:
                        whole = float(stritem[1:])
                else:
                    whole = float(stritem)
            else:
                whole = 0
            frac = float(line[end_col:new_end_col]) / (10**num_decimal)
            item = whole + frac
            if lineno == 0:
                dtypes.append((colname))
        else:
            whole = float(line[start_col:end_col])
            if line[start_col] == "-":
                whole = float(line[start_col+1:end_col])
            item = float(whole)
            if lineno == 0:
                dtypes.append((colname))

        row.append(item)
    allrows.append(row)

allrows_df = pd.DataFrame(allrows, columns=dtypes)

# 60 minutes per degree of latitude/longitude
# Converting Latitude/Longitude to decimal degrees
allrows_df["latitude"] = (allrows_df["latitude_deg"] +
                          allrows_df["latitude_min"] / 60)
allrows_df["longitude"] = -1*( allrows_df["longitude_deg"] +
                               allrows_df["longitude_min"] / 60)

df = allrows_df.drop(columns=["latitude_deg", "latitude_min",
                              "longitude_deg", "longitude_min"])

# Converting to a georeferenced data object and saving as geojson
geom = [Point(x, y) for (x, y) in df[["longitude", "latitude"]].values]
gdf = gpd.GeoDataFrame(df, geometry=geom, crs="epsg:4326")
gdf.to_file("data/geojson/calif_nev_ncei_grav.geojson", driver="GeoJSON")

# Very basic scatter plot of the data
fig, ax = plt.subplots(figsize=(12,12))
ax.scatter(df["longitude"], df["latitude"], c=df["isostatic_anom"], s=0.8)
ax.set_xlabel("Longitude")
ax.set_ylabel("Latitude")
ax.set_aspect("equal")

# Save to file
fig.savefig("figures/canv_data.png")


import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
'''
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['mape'])
history = model.fit(X_train, y_train, epochs=10, batch_size=16)           # default: epochs=1, batch_size=32
print(hisotry.history.keys())                   # dict_keys(['loss', 'mape', 'val_loss', 'val_mape'])
'''
# X_train: numpy array: (num_samples, num_features)
# y_train: numpy array: (num_samples, num_classes)  (categorical_crossentropy) (label is a vector)
# y_train: numpy array: (num_samples, )  (sparse_categorical_crossentropy)  (label is a single integer)
# verbose: 0 (silent), 1 (all), 2 (one line for one epoch)

''' Example '''
from tensorflow.keras.preprocessing import image
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# 1. load data
fashion_mnist_data = tf.keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist_data.load_data()
print(train_images.shape)
print(train_labels[0])

train_images = train_images / 255
test_images = test_images / 255

labels = [
    'T-shirt/top',
    'Trouser',
    'Pullover',
    'Dress',
    'Coat',
    'Sandal',
    'Shirt',
    'Sneaker',
    'Bag',
    'Ankle boot'
]
i=0
img=train_images[i,:,:]
plt.imshow(img)
print("img: \n", img)

# 2. Build model
model = Sequential([
    Conv2D(16, (3,3), activation='relu' ,input_shape=(28, 28, 1)),
    MaxPooling2D((3, 3)),
    Flatten(),
    Dense(10, activation='softmax')
])
model.summary()

# 3. Compile and train
opt = tf.keras.optimizers.Adam(learning_rate=0.005)
acc = tf.keras.metrics.SparseCategoricalAccuracy()
mae = tf.keras.metrics.MeanAbsoluteError()
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=[acc, mae])

history = model.fit(train_images[...,np.newaxis], train_labels, epochs=8, batch_size=256)       # need a channel_dim
# X_train: numpy array: (num_samples, num_features)
# y_train: numpy array: (num_samples, num_classes)  (categorical_crossentropy) (label is a vector)
# y_train: numpy array: (num_samples, )  (sparse_categorical_crossentropy)  (label is a single integer)
# verbose: 0 (silent), 1 (all), 2 (one line for one epoch)

# 4. Plot
df = pd.DataFrame(history.history)
print(df.head())

loss_plot = df.plot(y='loss', title="Loss vs. Epoch", legend=False)
loss_plot = df.plot(y='mean_absolute_error', title="Mae vs. Epoch", legend=False)
loss_plot.set(xlabel='Epochs', ylabel='Loss')
plt.show()


from PIL import Image
import numpy as np
from pathlib import Path
import os, webbrowser

img_path = Path("./test_set/data/")
save_path = Path("./test_set/eval/")

for i in range(20):
    print("Processing pic " + str(i))
    x_img = Image.open(img_path / 'x' / (str(i) + '_x.png'))
    y_img = Image.open(img_path / 'y' / (str(i) + '_y.png'))

    y_np = np.array(y_img.convert('RGBA'))

    y_np[:,:, 3] = (255 * (y_np[:, :, :3] != 255).any(axis=2)).astype(np.uint8)/4

    #y_np = np.where(y_np == [255, 255, 255], [0,0,0,255], y_np)

    y_img = Image.fromarray(y_np, mode='RGBA')
    x_img.convert('RGBA')
    #y_img = Image.blend(x_img, y_img, 0.25)

    comb_img = Image.alpha_composite(x_img, y_img)
    path = (save_path / (str(i) + "_comb.png"))
    comb_img.save(path, format='PNG')


import PyPDF2

def read_native_pdf(pdf_file_path):
    pdf_file = open(pdf_file_path, 'rb')
    pdf_reader = PyPDF2.PdfReader(pdf_file)

    text = ''

    for page in pdf_reader.pages:
        page_text = page.extract_text()
        # print('\n\n\nTEXT:', page_text)
        text = text + page_text

    #print(text)
    return text


import os, sys
import gettext

from twisted.web.woven import page, widgets
from twisted.web import microdom

t = gettext.translation(
    'remusserver', '/usr/local/share/locale', fallback=True)
_ = t.gettext

document = microdom.parseString("<xml />", caseInsensitive=0, preserveCase=0)

class Stylesheet(widgets.Widget):
    tagName = 'link'

    def initialize(self, link=""):
        self.link = link

    def setLink(self, link):
        self.link = link

    def setUp(self, request, node, data):
        href = self.link or data
        self['rel'] = "stylesheet"
        self['type'] = "text/css"
        self['href'] = href

class Menu(widgets.Widget):
    tagName = 'div'

    def setUp(self, request, node, data):
        for menuitem in self.model:
            link = document.createElement("a")
            link.attributes.update({
                'href':  menuitem.orig["link"],
                'title': menuitem.orig["tooltip"]})
            link.appendChild(document.createTextNode(menuitem.orig["text"]))

            span = document.createElement("span")
            span.attributes.update({
                'class':  "menuitem"})
            span.appendChild(link)
            self.appendChild(span)
            self["class"] = "topmenu"

def remus_page_hierarchy(cls, l=None):
    "Returns a list of inherited classes up to RemusPage"
    if l == None:
        l = []
    if issubclass(cls, RemusPage):
        l.append(cls)
        for base in cls.__bases__:
            remus_page_hierarchy(base, l)
    return l

def get_template(cls):
    if cls.template:
        template = cls.template
    elif cls.templateFile:
        if not cls.templateDirectory:
            mod = sys.modules[cls.__module__]
            if hasattr(mod, '__file__'):
                cls.templateDirectory = os.path.split(mod.__file__)[0]

        template = open(os.path.join(
            cls.templateDirectory, cls.templateFile)).read()
    return template

def merge_templates(cls, template):
    base_template = get_template(cls)
    return base_template.replace("<child/>", template)

class RemusPage(page.Page):

    title = "Remus page"

    templateFile = "RemusPage.html"

    templateDirectory = "/usr/local/libdata/remus"

    menu = (
        {
            'text': _('Home'),
            'link': '/',
            'tooltip': _('To Remus homepage')
        },
        {
            'text': _('Browse'),
            'link': '/music/',
            'tooltip': _('Browse the music database')
        },
        {
            'text': _('Playlists'),
            'link': '/playlists/',
            'tooltip': _('Manage playlists')
        },
        {
            'text': _('Server statistics'),
            'link': '/stats/',
            'tooltip': _('Various server statistics')
        }
        )

    def __init__(self, *args, **kwargs):
        page.Page.__init__(self, *args, **kwargs)

        hier = remus_page_hierarchy(self.__class__)

        template = get_template(self)

        for cls in hier[1:]:
            template = merge_templates(cls, template)
        self.template = template

    def wvfactory_Stylesheet(self, request, node, model):
        return Stylesheet(model)

    def wmfactory_base_url(self, request):
        return request.prePathURL()

    def wmfactory_title(self, request):
        return self.title

    def wmfactory_stylesheet(self, request):
        return "/styles/remus.css"

    def wvfactory_Menu(self, request, node, model):
        return Menu(model)

    def wmfactory_menu(self, request):
        return self.menu


import numpy as np
import json
from nuscenes.nuscenes import NuScenes
from nuscenes.utils.geometry_utils import BoxVisibility
import yaml

with open("./config.yaml") as f:
    config = yaml.load(f, Loader=yaml.FullLoader)

# Settings
sensor = config["sensor"]
use_category = config["use_category"]
use_visibility = config["use_visibility"]
scene_num_train = config["scene_num_train"]
min_dist = config["min_dist"]
max_dist = config["max_dist"]

use_scene_path = config["use_scene_path"]
nuscenes_path = config["nuscenes_path"]
rawlst_paths = config["rawlst_path"]

# Compute distance between box and ego.
def box_distance(sensor_data, ann_data):
    ego_location = nusc.get("ego_pose", sensor_data["ego_pose_token"])[
        "translation"]
    box_world_location = ann_data["translation"]
    distance = np.linalg.norm(
        np.array(box_world_location) - np.array(ego_location))
    return distance

with open(use_scene_path, "r") as f:
    use_scene = json.load(f)
img_seq = []
scene_lsts = []
scene_lsts.append(use_scene[:scene_num_train])  # [0, scene_num_train) for training set
scene_lsts.append(use_scene[scene_num_train:])  # [scene_num_train, last] for validation set
seq_head = True
found_box = False
nusc = NuScenes(version='v1.0-trainval', dataroot=nuscenes_path, verbose=False)

# Iterate over selected scenes to record frame pairs
# that contain obj(s) which meet requirements.
for i, scene_lst in enumerate(scene_lsts):

    for scene in nusc.scene:

        if scene["token"] in scene_lst:

            sample = nusc.get("sample", scene["first_sample_token"])
            sensor_data = nusc.get("sample_data", sample['data'][sensor])
            curr_sensor_token = sensor_data["token"]

            # Iterate over all CAM_FRONT images in current scene (all frames)
            while curr_sensor_token:

                sensor_data = nusc.get("sample_data", curr_sensor_token)
                # Last frame of current scene.
                if not sensor_data["next"]:
                    found_box = False
                    seq_head = True
                    break
                # Ignore non-keyframe.
                if not sensor_data["is_key_frame"]:
                    curr_sensor_token = sensor_data["next"]
                    continue

                # Get boxes in CAM_FRONT of corresponding frame
                data_path, curr_boxlst, _ = nusc.get_sample_data(
                    curr_sensor_token, box_vis_level=BoxVisibility.ALL)

                for cbox in curr_boxlst:

                    if cbox.name not in use_category:
                        continue
                    annotation = nusc.get("sample_annotation", cbox.token)
                    visibility_token = annotation["visibility_token"]
                    if nusc.get("visibility", visibility_token)["level"] != use_visibility:
                        continue
                    distance = box_distance(sensor_data, annotation)
                    if distance >= min_dist and distance <= max_dist:
                        curr_instance_token = annotation["instance_token"]
                        next_sensor_token = sensor_data["next"]
                        next_sensor_data = nusc.get(
                            "sample_data", next_sensor_token)
                        while not next_sensor_data["is_key_frame"]:
                            next_sensor_token = next_sensor_data["next"]
                            if not next_sensor_token:
                                break
                            next_sensor_data = nusc.get(
                                "sample_data", next_sensor_token)

                        if not next_sensor_data:
                            break

                        next_data_path, next_boxlst, _ = nusc.get_sample_data(
                            next_sensor_data["token"], box_vis_level=BoxVisibility.ALL)

                        for nbox in next_boxlst:

                            next_annotation = nusc.get(
                                "sample_annotation", nbox.token)
                            next_instance_token = next_annotation["instance_token"]

                            # Same obj in two frames.
                            if next_instance_token == curr_instance_token:
                                next_visibility_token = next_annotation["visibility_token"]
                                if nusc.get("visibility", next_visibility_token)["level"] != use_visibility:
                                    break
                                next_sensor_data = nusc.get(
                                    "sample_data", next_sensor_token)
                                next_distance = box_distance(
                                    next_sensor_data, next_annotation)
                                if next_distance >= min_dist and next_distance <= max_dist:
                                    # If beginning of a new image sequence, record current frame also.
                                    if seq_head:
                                        img_seq.append(list())
                                        curr_lst = img_seq[-1]
                                        curr_lst.append(dict())
                                        curr_lst[-1]["imgpath"] = nusc.get_sample_data_path(
                                            curr_sensor_token)
                                        curr_lst[-1]["name"] = curr_sensor_token
                                        seq_head = False
                                    curr_lst = img_seq[-1]
                                    curr_lst.append(dict())
                                    curr_lst[-1]["imgpath"] = nusc.get_sample_data_path(
                                        next_sensor_token)
                                    curr_lst[-1]["name"] = next_sensor_token
                                    found_box = True
                                    break

                        if found_box:
                            break

                if not found_box:
                    seq_head = True

                found_box = False
                curr_sensor_token = sensor_data["next"]

    with open(rawlst_paths[i], 'w') as f:
        json.dump(img_seq, f, sort_keys=True, indent=4)


import matplotlib.pyplot as plt
import numpy as np

def graficar(fileName,title,xlabel,ylabel,intervaloX,maximoY,color):
    x = []
    y = []
    file = open(fileName,"r")
    for line in file:
        if line[0] == '/':
            break
        firstPart = True
        first =""
        second =""
        for char in line:
            if char == ',':
                firstPart = False
                continue
            if char == ':':
                break
            if firstPart:
                first = first + char
            else:
                second = second + char
        x1 = int(first)
        x.append(x1)
        y1 = float(second) /float(x1*x1*x1)
        y.append(y1)
    plt.plot(x,y,color)
    plt.xticks(np.arange(min(x)-2, max(x)+4, intervaloX))
    plt.axis([min(x)-2,max(x)+4,min(y),maximoY])
    plt.ylabel(ylabel)
    plt.xlabel(xlabel)
    plt.title(title);
    #plt.legend(("óptimo instancias", "óptimo aleatorio"), loc='upper right')
    imageName = fileName + ".png"
    plt.savefig(imageName)
    #plt.clf()
    file.close()
    return;

#graficar("ResGolosa","Porcentaje de ahorro promedio para instancias aleatorias","n","Porcentaje de ahorro",50,60,'ro')
graficar("Tgolosa","Tiempo de ejecución del algoritmo","n","milisegundos",100,0.0003,'-b')
#graficar("ResGolosaSerieX","Distancia porcentual al óptimo Set X","n","Distancia al óptimo",50,100,'bo')

Codigo/golosa/Resultados/Instancias aleatorias/Medicion de tiempo/graficador.py


# -*- coding: utf-8 -*-

from __future__ import absolute_import, with_statement

import sys

import storm.tracer
from storm.locals import create_database

from . import schema
from .store import StormStore
from .schema.utils import CheckingSchema
from .utils import StoreWithCache
from kittystore import SchemaUpgradeNeeded
from kittystore.caching import setup_cache

def _get_native_store(settings):
    database = create_database(settings.KITTYSTORE_URL)
    return StoreWithCache(database)

def _get_schema(settings):
    dbtype = settings.KITTYSTORE_URL.partition(":")[0]
    return CheckingSchema(schema.CREATES[dbtype], [], [], schema)

def create_storm_db(settings, debug=False):
    if debug:
        storm.tracer.debug(True, stream=sys.stdout)
    store = _get_native_store(settings)
    dbschema = _get_schema(settings)
    dbschema.upgrade(store)
    version = list(store.execute(
                "SELECT patch.version FROM patch "
                "ORDER BY version DESC LIMIT 1"
                ))[0][0]
    return version

def get_storm_store(settings, search_index=None, debug=False, auto_create=False):
    if debug:
        storm.tracer.debug(True, stream=sys.stdout)
    store = _get_native_store(settings)
    dbschema = _get_schema(settings)
    if dbschema.has_pending_patches(store):
        if auto_create:
            dbschema.upgrade(store)
        else:
            store.close()
            raise SchemaUpgradeNeeded()
    setup_cache(store.cache, settings)
    return StormStore(store, search_index, settings, debug=debug)


import datetime
import json
from multiprocessing import Pool

import pandas as pd
import requests
from tqdm import tqdm

def write_log(message):
    with open('/tmp/chrome_extensions.log', 'a') as f:
        f.write(f'{datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")} -- {message}\n')

class ChromeExtension(object):
    def __init__(self, ext_id):
        self.ext_id = ext_id
        self.report = None

    @staticmethod
    def get_headers():
        with open('apikey.txt', 'r') as f:
            return {'API-Key': f.read().strip()}

    def submit_for_scan(self):
        api_url = 'https://api.crxcavator.io/v1/submit'
        try:
            response = requests.post(
                api_url,
                headers=self.get_headers(),
                json={'extension_id': f'{self.ext_id}'}
            )
            if response.status_code == 200:
                # write_log(f'Successfully submitted extension {self.ext_id} for scanning.')
                return True
            else:
                write_log(f'ERROR: Status code [{response.status_code}] while submitting extension: {self.ext_id}'
                          f'\r\n\t\tResponse:{response.text}')
                return False
        except requests.exceptions.ConnectionError as e:
            write_log(f'Connection error received when submitting {self.ext_id} to be scanned, trying again.')
            return self.submit_for_scan()

    def crxcavator_lookup(self):
        # Lookup the risk score in crxcavator as it has everything that I planned to do and more.
        api_url = f'https://api.crxcavator.io/v1/report/{self.ext_id}'
        try:
            response = requests.get(
                api_url,
                headers=self.get_headers()
            )
            if response.status_code == 200 and response.json():
                self.report = response.json()[-1]
            else:
                self.report = {}
                self.submit_for_scan()
        except requests.exceptions.ConnectionError as e:
            write_log(f'Connection error received when querying {self.ext_id}, trying again.')
            self.crxcavator_lookup()

    def get_risk_info(self, submit=False):
        if self.report is None:
            self.crxcavator_lookup()
        if self.report == {}:
            write_log(f'No data for extension {self.ext_id}')
            if submit is not False:
                self.submit_for_scan()
            return {'extension_id': self.ext_id, 'name': 'No data available.'}
        risk = self.report['data']['risk'] if 'risk' in self.report['data'].keys() else None
        report_data = {
            'extension_id': self.ext_id,
            'version': self.report['version'],
            'total_risk': risk['total'] if (risk and ('total' in risk.keys())) else None,
            'URLs': self.report['data']['extcalls'] if 'extcalls' in self.report['data'].keys() else None,
            'dangerous_fns': list(self.report['data']['dangerousfunctions'].keys()) if 'dangerousfunctions' in
                                                                                self.report['data'].keys() else None,
            'entrypoints': self.report['data']['entrypoints'] if 'entrypoints' in self.report['data'].keys() else None,
            'csp_risk': risk['csp']['total'] if 'csp' in risk.keys() else None,
            'permissions_risk': risk['permissions']['total'] if 'permissions' in risk.keys() else None,
            'webstore_risk': risk['webstore']['total'] if 'webstore' in risk.keys() else None,
            'manifest': self.report['data']['manifest'] if 'manifest' in self.report['data'].keys() else None
        }
        webstore = self.report['data']['webstore']
        if webstore:
            report_data.update({
                'name': webstore['name'],
                'short_description': webstore['short_description'],
                'version': webstore['version'],  # available in manifest and base object
                'last_updated': webstore['last_updated'],
                'offered_by': webstore['offered_by'],  # Looks worse than mine
                'rating': webstore['rating'],
                'rating_users': webstore['rating_users'],
                'users': webstore['users'],
                'size': webstore['size'],
                'type': webstore['type'],
                'permission_warnings': webstore['permission_warnings'],
                'email': webstore['email'],
                'address': webstore['address'],
                'privacy_policy': webstore['privacy_policy']
            })
        else:
            report_data.update({
                'name': None,
                'short_description': None,
                'version': None,  # available in manifest and base object
                'last_updated': None,
                'offered_by': None,  # Looks worse than mine
                'rating': None,
                'rating_users': None,
                'users': None,
                'size': None,
                'type': None,
                'permission_warnings': None,
                'email': None,
                'address': None,
                'privacy_policy': None
            })
        return report_data

class ExtensionList(object):
    def __init__(self, ext_ids, pool_size=20):
        self.extensions = [ChromeExtension(ext_id) for ext_id in ext_ids]
        self.pool_size = pool_size

    def run_async(self, func, data, progress=True, task=None):
        with Pool(self.pool_size) as p:
            if progress:
                return_data = list(tqdm(p.imap(func, data), total=len(data), desc=task))
            else:
                return_data = list(p.imap(func, data))
        return return_data

    def get_extension_data(self):
        print(f'#### Getting CRXcavator data for {len(self.extensions)} extensions ####')
        self.extensions = self.run_async(self._lookup, self.extensions, progress=True, task='API Queries')

    @staticmethod
    def _lookup(extension) -> ChromeExtension:
        extension.crxcavator_lookup()
        return extension

    def get_data_as_df(self):
        return pd.DataFrame(self.run_async(self._export_data, self.extensions, progress=True, task='Getting CRX data'))

    @staticmethod
    def _export_data(extension) -> ChromeExtension:
        return extension.get_risk_info()

    def submit_for_scans(self):
        results = self.run_async(self._scan_crx, self.extensions, progress=True, task='Submitting CRX for scanning.')
        print(f'Submission Successes: {len([x for x in results if x is True])}')
        print(f'Submission Failures: {len([x for x in results if x is False])}')

    @staticmethod
    def _scan_crx(extension) -> ChromeExtension:
        return extension.submit_for_scan()

    def to_csv(self, csv_path):
        ext_df = self.get_data_as_df()
        columns = [
            'extension_id', 'name', 'short_description', 'version', 'last_updated', 'total_risk',
            'offered_by', 'rating', 'rating_users', 'users', 'size', 'type', 'permission_warnings', 'email',
            'address', 'privacy_policy', 'entrypoints', 'csp_risk', 'permissions_risk',
            'webstore_risk'
        ]
        ext_df.to_csv(csv_path, columns=columns, index=False, encoding='utf-8')


import numpy as np
import matplotlib.pyplot as plt
import math

from sklearn.cluster import KMeans
from sklearn.datasets import load_sample_image

images = ['china.jpg', 'flower.jpg']
#images = ['flower.jpg']

K = 50

# tests sur les 2 images
for image in images:
	img = load_sample_image(image)

	(width,height,channels) = img.shape

	# Création de la matrice des pixels
	X = img.reshape(width*height, channels)

	# création du clustering
	clustering = KMeans(n_clusters=K, init='random', n_init=100)
	clustering.fit(X)

	# récupération des étiquettes sous forme matricielle :
	Y = clustering.labels_.reshape(width, height)

	# Création de l'image couleur à 3 canaux
	segmentation = np.zeros((width,height, channels))  # taille identique à img

	# à compléter pour remplir le tableau à 3 dimensions 'segmentation'
	# avec les couleurs des centres
	centers = clustering.cluster_centers_
	segmentation = centers[Y]

	# afficher l'originale, les étiquettes Y et la segmentation avec la couleur moyenne :
	# plt.figure()
	# plt.imshow(img)
	# plt.draw()
	# plt.figure()
	# plt.imshow(Y)
	# plt.draw()
	plt.figure()
	plt.imshow(segmentation / 255)
	# les centres sont à valeurs réelles,
	# donc imshow s'attend à avoir des valeurs entre 0 et 1 pour segmentation
	# au lieu de 0 et 255 pour les entiers
	plt.draw()

plt.show()


import pandas as pd
from src.config.config import CLIENT_ID, CLIENT_SECRET
from src.config.auth import authenticate
from flask import Flask, jsonify, request
from flask_cors import CORS
import joblib

app = Flask(__name__)
cors = CORS(app)

sp = authenticate(client_id=CLIENT_ID, client_secret=CLIENT_SECRET)

model = joblib.load("src/predictor.joblib")

def predict_moods(track_id):
    # get the tracks information
    track_info = sp.track(track_id)

    # create a dict with the track info
    track_data = {
        "title": track_info["name"],
        "artist": track_info["artists"][0]["name"],
        "album": track_info["album"]["name"],
        "album_cover": track_info["album"]["images"][0]["url"]
    }

    # get the audio features
    features = sp.audio_features(tracks=[track_id])[0]

    # create a dateframe with the audio features
    df = pd.DataFrame.from_dict(features, orient='index').T

    # drop non feature cols
    df.drop(
        ['type', 'id', 'uri', 'track_href', 'liveness', 'mode', 'analysis_url', 'duration_ms', 'time_signature', 'key'],
        axis=1,
        inplace=True)

    # predict the mood probabilities for the track
    probs = model.predict_proba(df)[0]

    # get top 2
    top_moods = sorted(zip(model.classes_, probs), key=lambda x: x[1], reverse=True)

    # add track data to result dictionary
    result = {
        "track": track_data,
        "moods": []
    }

    # adds the mood predictions to result dict
    for mood, prob in top_moods:
        result["moods"].append({
            "name": mood,
            "probability": round(prob, 2)
        })

        # get similar tracks
        seed_artists = [track_info["artists"][0]["id"]]
        seed_tracks = [track_id]
        recommendations = sp.recommendations(seed_artists=seed_artists, seed_tracks=seed_tracks, limit=5)['tracks']
        recommended_tracks = []
        for rec in recommendations:
            recommended_tracks.append({
                "title": rec["name"],
                "artist": rec["artists"][0]["name"],
                "album": rec["album"]["name"],
                "album_cover": rec["album"]["images"][0]["url"],
                "track_id": rec["id"]
            })

        result["recommendations"] = recommended_tracks

    return result

@app.route("/", methods=["GET"])
def index():
    return "
Welcome to our Spotify app
"

@app.route("/predict", methods=["POST"])
def predict():
    track_id = request.args.get("track_id")
    result = predict_moods(track_id)
    return jsonify(result)

if __name__ == "__main__":
    app.debug = True
    app.run()


from glob import glob
import numpy as np
import math
import cv2

from utils.data_loader import DataLoader
from utils.gate_corner import chollet_gate_fix_missing
from utils.data_formater import DataFormater

def remove_empty_frames(raw_data):
    # Clean les frames de debut si pas de diff entre (n) et (n+1)
    for key, val in raw_data.items():
        data = raw_data[key] # data est une liste avec toutes les frames
        # clean empty rateThrust
        i = 0
        while i < len(data):
            if not data[i]["rateThrust"]:
                del data[i]
            else:
                i += 1
        # Clean static drone frames
        while len(data) > 1:
            frame = data[1]
            if frame and frame["droneSpeed"][0] == 0 and frame["droneSpeed"][1] == 0 and frame["droneSpeed"][2] == 0:
                del data[0]
            else:
                break
        raw_data[key] = data
    return raw_data

class GroundTruth(object):
    """docstring for GroundTruthUtils."""
    def __init__(self, path=None, json_file=None, yaml_file=None, split=0.1, nb_timesteps=6):
        self.split = split

        # Search json and yaml in path if path not None
        # else use directly json_file and yaml_file
        if path:
            path = path.rstrip('/')
            json_file = [file for file in glob("{}/*.json".format(path))]
            yaml_file = [file for file in glob("{}/*.yaml".format(path))]
            if len(json_file) != 1:
                print("No json or more than one file in the specified path.")
                exit(1)
            if len(yaml_file) != 1:
                print("No yaml or more than one file in the specified path.")
                exit(1)
            json_file = json_file[0]
            yaml_file = yaml_file[0]

        self.path = path
        self.json_file = json_file
        self.yaml_file = yaml_file

        self.data_formater = DataFormater(nb_timesteps=nb_timesteps)
        self.data_loader = DataLoader(json_file=json_file, yaml_file=yaml_file, nb_gates_nominal=2, nb_gates_ir=1)
        data = np.array(self.data_loader.data)
        self.init(data)

    def init(self, data):
        bad_runs = [
            3, 11, 25, 28, 46, 50, 51,
            54, 55, 58, 59, 76, 80,
        ]

        # Clean data and store it in a nice dictionnary
        self.data = {}
        for frame in data:
            if int(frame['run_id']) in bad_runs:
                continue
            if frame['run_id'] not in self.data:
                self.data[frame['run_id']] = []
            id = frame['run_id']
            del frame['run_id']
            self.data[id].append(frame)

        self.data = self.__cleanRuns(self.data)

        # Store informations
        self.nb_runs = len(self.data)
        self.nb_frames = 0 # TODO: Add this

        # Set the idxs for the input generator
        self.idxs = []
        for idx in self.data.keys():
            self.idxs += [(idx, n)for n in range(len(self.data[idx]))]
        self.idxs = np.array(self.idxs)

        self.__split()

    def __formatFrames(self, raw_data):
        for key, val in raw_data.items():
            data = raw_data[key] # data est une liste avec toutes les frames
            data_array = []
            for frame in data:
                # INPUT
                if not frame['eulerDroneNominal'][0]:
                    continue
                input_array = self.data_formater.format_input(frame)
                # OUTPUT
                output_array = self.data_formater.format_output(frame)
                data_array.append([input_array, output_array])
            raw_data[key] = np.array(data_array)
        return raw_data

    def __cleanRuns(self, raw_data):
        raw_data = remove_empty_frames(raw_data)
        raw_data = self.__formatFrames(raw_data)
        return raw_data

    def __split(self):

        self.nb_sample = self.idxs.shape[0]

        size_valid = int(self.split * self.nb_sample)
        size_train = self.nb_sample - size_valid

        self.idxs_train = self.idxs[size_valid:]
        self.idxs_valid = self.idxs[0:size_valid]

if __name__ == "__main__":
    import time
    import pickle
    import cv2

    PATH = "../resources/"
    json_file = PATH + "test_dumps_map1.json"
    yaml_file = PATH + "nominal_gate_locations.yaml"
    gt = GroundTruth(json_file=json_file, yaml_file=yaml_file)

    #with open('data_killian.pkl', 'r') as f:
    #    gt = pickle.load(f)

    data = []
    for k, v in gt.data.items():
        for x in v[:, 0]:
            #print('Euler 1', ["{0:0.2f}".format(i) for i in x[:3]])
            #print('Euler 2', ["{0:0.2f}".format(i) for i in x[3:6]])
            #print('drl', ["{0:0.2f}".format(i) for i in x[6:7]])
            #print('dist', ["{0:0.2f}".format(i) for i in x[7:9]])
            #print('speed', ["{0:0.2f}".format(i) for i in x[9:12]])
            #print('corners', ["{0:0.2f}".format(i) for i in x[12:20]])
            print("Next")
            w, h = 400, 400
            img = np.zeros((w, h, 3), np.uint8)
            bbox = x[12:20]
            xy = bbox.reshape(4, -1) * [w, h]
            xy = xy[:,[1, 0]]
            xy = np.round(xy)
            cv2.polylines(img, np.int32([xy]), True, (0,0,255), thickness=2)
            cv2.imshow("Gate detection", (img))
            cv2.waitKey(0)

            data += [x]
    np.array(data, dtype=np.float32)


import argparse
import datetime
import json
import logging
import os
import sys
import traceback

from BootOption import BootOption

obj_boot_option = BootOption()
log_file = ""

try:

    parser = argparse.ArgumentParser(description='Boot Device Options ')
    parser.add_argument("-H", "--host", action="store", type=str, required=True,
                        default=None, help="BMC IP address", metavar='')
    parser.add_argument("-u", "--username", action="store", type=str,
                        default="Administrator", help="Remote manager user name", metavar='')
    parser.add_argument("-p", "--password", action="store", type=str,
                        default="Administrator", help="Remote manager user password", metavar='')
    parser.add_argument("-M", "--module", action="store", type=int,
                        default=9999, help="Module number 0..17", metavar='', choices=range(0, 17))

    parser.add_argument("-a", "--action", action="store", type=str,
                        default="get", help=" action [ get | set ]", metavar='', choices=['get', 'set'])
    parser.add_argument("-d", "--device", action="store", type=str, default="None",
                        help="Boot device. Poassible values are : {}".format(obj_boot_option.device_values),
                        metavar='', choices=obj_boot_option.device_values)
    parser.add_argument("-i", "--instance", action="store", type=str, default="0",
                        help="instance number of boot devices disk and pxe (possible values: from 0 to 15)",
                        metavar='', choices=obj_boot_option.inst_values)
    parser.add_argument("-P", "--persistant", action="store", type=str, default="no",
                        help="Make this choice persistant. Possible values are [ no | yes ]",
                        metavar='', choices=obj_boot_option.boolean_values)
    parser.add_argument("-c", "--clear_cmos", action="store", type=str, default="no",
                        help="Clear CMOS at next boot. Possible values are [ no | yes ]",
                        metavar='', choices=obj_boot_option.boolean_values)
    # parser.add_argument("-v", "--verbose", action="store_false",  help="verbosity")

    args = parser.parse_args()

    obj_boot_option.create(args.host, args.username, args.password)

    os.makedirs("/var/log/redfish/", exist_ok=True)
    log_file = "/var/log/redfish/" + __file__ + "_" + obj_boot_option.bmc_ip + "_" + args.action + "_" \
               + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + ".log"
    obj_boot_option.set_log(log_file)
    logging.info(__file__)

    obj_boot_option.set_device(args.device)
    obj_boot_option.set_instance(args.instance)
    obj_boot_option.set_clr_cmos(args.clear_cmos)
    obj_boot_option.set_persistant(args.persistant)

    if args.action == "set":
        obj_boot_option.apply_option()

    obj_boot_option.get_options()
    obj_boot_option.print_option()

except KeyError:
    logging.exception(repr(traceback.extract_stack()))
except json.JSONDecodeError:
    logging.exception(repr(traceback.extract_stack()))
except ValueError:
    logging.exception(repr(traceback.extract_stack()))
except KeyboardInterrupt:
    print("Program interrupt by user ")
except Exception:
    logging.exception(repr(traceback.extract_stack()))
finally:
    print("/!\\ Something bad happened. See log logfile {}".format(log_file))

sys.exit()


import os
import sys
import logging
import logging.handlers
from datetime import datetime
from blacksheep.utils.folders import ensure_folder
from blacksheep.exceptions import HttpException, MessageAborted

def _get_loggers():
    process_id = os.getpid()

    access_logger = logging.getLogger('blacksheep.server-access')
    app_logger = logging.getLogger('blacksheep.server')

    app_logger.setLevel(logging.INFO)
    access_logger.setLevel(logging.DEBUG)

    max_bytes = 24 * 1024 * 1024

    file_handler = logging.handlers.RotatingFileHandler

    now = datetime.now()
    ts = now.strftime('%Y%m%d')
    hour_ts = now.strftime('%H%M%S')

    ensure_folder(f'logs/{ts}')

    access_handler = file_handler(f'logs/{ts}/{hour_ts}-blacksheep-access-{process_id}.log',
                                  maxBytes=max_bytes,
                                  backupCount=5)

    app_handler = file_handler(f'logs/{ts}/{hour_ts}-blacksheep-{process_id}.log',
                               maxBytes=max_bytes,
                               backupCount=5)

    access_handler.setLevel(logging.DEBUG)
    app_handler.setLevel(logging.DEBUG)
    app_handler.setFormatter(logging.Formatter(
        '%(levelname)s @ %(asctime)s @ %(filename)s '
        '%(funcName)s %(lineno)d: %(message)s'))

    access_logger.addHandler(access_handler)
    app_logger.addHandler(app_handler)

    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.DEBUG)
    app_logger.addHandler(handler)
    access_logger.addHandler(handler)

    access_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))

    return access_logger, app_logger

def get_logged_url(request):
    if request.url.query:
        return request.url.path.decode() + '?<query is hidden>'
    return request.url.path.decode()

def setup_sync_logging():
    access_logger, app_logger = _get_loggers()

    async def logging_middleware(request, handler):
        access_logger.debug(f'{request.method.ljust(8)} {get_logged_url(request)}')
        try:
            response = await handler(request)
        except HttpException:
            raise
        except MessageAborted:
            app_logger.warning(f'The connection was lost or aborted while the request was being sent. '
                               f'{request.method.ljust(8)} {get_logged_url(request)}')
            raise
        except Exception:
            app_logger.exception(f'{"*" * 30}\nunhandled exception while handling: {request.method} {get_logged_url(request)}')
            raise
        return response

    return logging_middleware, access_logger, app_logger


from django.urls import path

from . import views

app_name = 'trigger3'

urlpatterns = [
    path('createpesanansumberdaya', views.createPesananSD, name='CreatePesananSumberDaya'),
    path('readpesanansumberdaya', views.readPesanan, name='ReadPesananSumberDaya'),
    path('readpesanansumberdaya/detail/<int:idPesanan>', views.detailPesanan, name='DetailPesanan'),
    path('readpesanansumberdaya/riwayatstatus/<int:idPesanan>',views.riwayatStatus,name='RiwayatStatus'),
    path('readpesanansumberdaya/updatepesanan/<int:idPesanan>', views.updatePesanan, name='UpdatePesanan'),
    path('SupplierReadPesanan',views.SupplierReadPesanan,name='SupplierReadPesanan'),
    path('SupplierReadPesanan/detail/<int:idPesanan>',views.detailPesananSupplier,name='DetailPesananSupplier'),
    path('SupplierReadPesanan/riwayatstatus/<int:idPesanan>',views.riwayatStatusSupplier,name='RiwayatStatusSupplier'),
    path('createstockfaskes',views.createStockFaskes, name='CreateStockFaskes'),
    path('liststockfaskes',views.listStockFaskes, name='ListStockFaskes'),
    path('liststockfaskes/updatestock/<str:kodeFaskes>/<str:namaFaskes>/<str:namaItem>', views.updateStockFaskes, name='UpdateStockFaskes'),
    path('liststockfaskespetugas',views.ListStockFaskesPetugas,name='ListStockFaskesPetugas')
]


import setuptools

pkg_vars = dict()
with open("sleeper/_version.py") as f:
    exec(f.read(), pkg_vars)

package_version = pkg_vars["__version__"]
minimum_python_version_required = pkg_vars["__version_minimum_python__"]

with open("requirements.txt", "r", encoding="utf8") as reqs:
    required_packages = reqs.read().splitlines()

with open("README.md") as f:
    read_me = f.read()

setuptools.setup(
    name="sleeper",
    version=package_version,
    author="Joey Greco",
    author_email="joeyagreco@gmail.com",
    description="A Python wrapper for the Sleeper API.",
    long_description_content_type="text/markdown",
    long_description=read_me,
    license="MIT",
    url="https://github.com/joeyagreco/sleeper",
    include_package_data=True,
    packages=setuptools.find_packages(exclude=("test", "docs")),
    install_requires=required_packages,
    python_requires=f">={minimum_python_version_required}",
    keywords="nfl football sleeper sleeper-api sleeper-fantasy-football fantasy-football wrapper wrapper-api",
)


import collections
N, M = map(int, input().split())
xy = [list(map(int, input().split())) for i in range(M)]
table = collections.defaultdict(int)
table2 = collections.defaultdict(int)
table2[1] += 1

for i in range(N):
    table[i+1] = 1
reached = [1]
for i in range(M):
    table[xy[i][1]] += 1
    table[xy[i][0]] -= 1
    if xy[i][0] in table2 and table2[xy[i][0]] > 0:
        table2[xy[i][0]] = table[xy[i][0]]
        table2[xy[i][1]] += table[xy[i][1]] + 1

table2 = list(table2.items())
table2 = [x for x in table2 if x[1] > 0]
print(len(table2))


#%%
from genalog.pipeline import AnalogDocumentGeneration
from genalog.degradation.degrader import ImageState

sample_text = "sample/generation/example.txt"

# Common CSS properties
STYLE_COMBINATIONS = {
    "font_family"   : ["Times"], # sans-serif, Times, monospace, etc
    "font_size"     : ["12px"],
    "text_align"    : ["justify"], # left, right, center, justify
    "language"      : ["en_US"],  # controls how words are hyphenated
    "hyphenate"     : [True],
}
# <columns|letter|text_block>.html.jinja
HTML_TEMPLATE = "columns.html.jinja"
# Degration effects applied in sequence
DEGRADATIONS = [
    ("blur", {"radius": 5}),    # needs to be an odd number
    ("bleed_through", {
        "src": ImageState.CURRENT_STATE, "background": ImageState.ORIGINAL_STATE,
        "alpha": 0.8,
        "offset_y": 9, "offset_x": 12
    }),
    ("morphology", {"operation": "open", "kernel_shape":(5,5)}),
    ("pepper", {"amount": 0.05}),
    ("salt", {"amount": 0.2}),
]

doc_generation = AnalogDocumentGeneration(styles=STYLE_COMBINATIONS, degradations=DEGRADATIONS)
img_array = doc_generation.generate_img(sample_text, HTML_TEMPLATE, target_folder=None)

import cv2
from IPython.core.display import Image, display

_, encoded_image = cv2.imencode('.png', img_array)
display(Image(data=encoded_image, width=600))