from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import uuid

from six.moves import xrange  # pylint: disable=redefined-builtin

import tensorflow as tf
from tensorflow.contrib.linear_optimizer.python.ops.sdca_ops import SdcaModel
from tensorflow.python.framework.test_util import TensorFlowTestCase
from tensorflow.python.platform import googletest

_MAX_ITERATIONS = 100

def make_example_proto(feature_dict, target, value=1.0):
  e = tf.train.Example()
  features = e.features

  features.feature['target'].float_list.value.append(target)

  for key, values in feature_dict.items():
    features.feature[key + '_indices'].int64_list.value.extend(values)
    features.feature[key + '_values'].float_list.value.extend([value] *
                                                              len(values))

  return e

def make_example_dict(example_protos, example_weights):

  def parse_examples(example_protos):
    features = {
        'target': tf.FixedLenFeature(shape=[1],
                                     dtype=tf.float32,
                                     default_value=0),
        'age_indices': tf.VarLenFeature(dtype=tf.int64),
        'age_values': tf.VarLenFeature(dtype=tf.float32),
        'gender_indices': tf.VarLenFeature(dtype=tf.int64),
        'gender_values': tf.VarLenFeature(dtype=tf.float32)
    }
    return tf.parse_example(
        [e.SerializeToString() for e in example_protos], features)

  sparse_merge = lambda ids, values: tf.sparse_merge(ids, values, ids.shape[1])

  parsed = parse_examples(example_protos)
  sparse_features = [
      sparse_merge(parsed['age_indices'], parsed['age_values']),
      sparse_merge(parsed['gender_indices'], parsed['gender_values'])
  ]
  return dict(sparse_features=sparse_features,
              dense_features=[],
              example_weights=example_weights,
              example_labels=tf.reshape(parsed['target'], [-1]),
              example_ids=['%d' % i for i in xrange(0, len(example_protos))])

def make_dense_examples_dict(dense_feature_values, weights, labels):
  dense_feature_tensors = ([
      tf.convert_to_tensor(values,
                           dtype=tf.float32) for values in dense_feature_values
  ])
  return dict(sparse_features=[],
              dense_features=dense_feature_tensors,
              example_weights=weights,
              example_labels=labels,
              example_ids=['%d' % i for i in xrange(0, len(labels))])

def make_variable_dict(max_age, max_gender):
  # TODO(sibyl-toe9oF2e):  Figure out how to derive max_age & max_gender from
  # examples_dict.
  age_weights = tf.Variable(tf.zeros([max_age + 1], dtype=tf.float32))
  gender_weights = tf.Variable(tf.zeros([max_gender + 1], dtype=tf.float32))
  return dict(sparse_features_weights=[age_weights, gender_weights],
              dense_features_weights=[])

def make_dense_variable_dict(num_dense_features, num_examples):
  feature_weights = ([
      tf.Variable(tf.zeros([1],
                           dtype=tf.float32))
      for _ in xrange(0, num_dense_features)
  ])
  return dict(sparse_features_weights=[],
              dense_features_weights=feature_weights,
              dual=tf.Variable(tf.zeros(
                  [num_examples],
                  dtype=tf.float32)),
              primal_loss=tf.Variable(tf.zeros(
                  [],
                  dtype=tf.float64)))

def get_binary_predictions_for_logistic(predictions, cutoff=0.5):
  return tf.cast(
      tf.greater_equal(predictions, tf.ones_like(predictions) * cutoff),
      dtype=tf.int32)

def get_binary_predictions_for_hinge(predictions):
  return tf.cast(
      tf.greater_equal(predictions, tf.zeros_like(predictions)),
      dtype=tf.int32)

# Setup the single container shared across all tests. This is testing proper
# isolation across optimizers instantiated in each of the tests below.
CONTAINER = uuid.uuid4().hex

# Clear the shared container.
def tearDown():
  # TODO(sibyl-Mooth6ku): Proper cleanup of Containers when possible.
  pass

# TODO(sibyl-Mooth6ku): Add tests that exercise L1 and Shrinking.
# TODO(sibyl-vie3Poto): Refactor tests to avoid repetition of boilerplate code.
class SdcaOptimizerTest(TensorFlowTestCase):
  """Base SDCA optimizer test class for any loss type."""

  def _single_threaded_test_session(self):
    config = tf.ConfigProto(inter_op_parallelism_threads=1,
                            intra_op_parallelism_threads=1)
    return self.test_session(use_gpu=False, config=config)

  # The following tests, check that operations raise errors when certain
  # preconditions on the input data are not satisfied. These errors are raised
  # regardless of the loss type.
  def testNoWeightedExamples(self):
    # Setup test data with 1 positive, and 1 negative example.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    # Zeroed out example weights.
    example_weights = [0.0, 0.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      self.assertAllClose([0.5, 0.5], lr.predictions(examples).eval())
      lr.minimize().run()
      self.assertAllClose([0.5, 0.5], lr.predictions(examples).eval())
      with self.assertRaisesOpError(
          'No examples found or all examples have zero weight.'):
        lr.approximate_duality_gap().eval()

class SdcaWithLogisticLossTest(SdcaOptimizerTest):
  """SDCA optimizer test class for logistic loss."""

  def testSimple(self):
    # Setup test data
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      self.assertAllClose(0.693147, unregularized_loss.eval())
      self.assertAllClose(0.693147, loss.eval())
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()
      # The high tolerance in unregularized_loss comparisons is due to the
      # fact that it's possible to trade off unregularized_loss vs.
      # regularization and still have a sum that is quite close to the
      # optimal regularized_loss value.  SDCA's duality gap only ensures that
      # the regularized_loss is within 0.01 of optimal.
      # 0.525457 is the optimal regularized_loss.
      # 0.411608 is the unregularized_loss at that optimum.
      self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05)
      self.assertAllClose(0.525457, loss.eval(), atol=0.01)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllEqual([0, 1], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testSimpleNoL2(self):
    # Same as test above (so comments from above apply) but without an L2.
    # The algorithm should behave as if we have an L2 of 1 in optimization but
    # 0 in regularized_loss.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=0,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      self.assertAllClose(0.693147, unregularized_loss.eval())
      self.assertAllClose(0.693147, loss.eval())
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # There is neither L1 nor L2 loss, so regularized and unregularized losses
      # should be exactly the same.
      self.assertAllClose(0.40244, unregularized_loss.eval(), atol=0.01)
      self.assertAllClose(0.40244, loss.eval(), atol=0.01)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllEqual([0, 1], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testSomeUnweightedExamples(self):
    # Setup test data with 4 examples, but should produce the same
    # results as testSimple.
    example_protos = [
        # Will be used.
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        # Will be ignored.
        make_example_proto(
            {'age': [1],
             'gender': [0]}, 0),
        # Will be used.
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
        # Will be ignored.
        make_example_proto(
            {'age': [1],
             'gender': [0]}, 1),
    ]
    example_weights = [1.0, 0.0, 1.0, 0.0]
    with self._single_threaded_test_session():
      # Only use examples 0 and 2
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05)
      self.assertAllClose(0.525457, loss.eval(), atol=0.01)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllClose([0, 1, 1, 1], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testFractionalExampleLabel(self):
    # Setup test data with 1 positive, and 1 mostly-negative example.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0.1),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      with self.assertRaisesOpError(
          'Only labels of 0.0 or 1.0 are supported right now.'):
        lr.minimize().run()

  def testImbalanced(self):
    # Setup test data with 1 positive, and 3 negative examples.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [2],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [3],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [1.0, 1.0, 1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(3, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      self.assertAllClose(0.226487 + 0.102902,
                          unregularized_loss.eval(),
                          atol=0.08)
      self.assertAllClose(0.328394 + 0.131364, loss.eval(), atol=0.01)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllEqual([0, 0, 0, 1], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testImbalancedWithExampleWeights(self):
    # Setup test data with 1 positive, and 1 negative example.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [3.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      self.assertAllClose(0.284860, unregularized_loss.eval(), atol=0.08)
      self.assertAllClose(0.408044, loss.eval(), atol=0.012)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllEqual([0, 1], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testInstancesOfOneClassOnly(self):
    # Setup test data with 1 positive (ignored), and 1 negative example.
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [0]}, 1),  # Shares gender with the instance above.
    ]
    example_weights = [1.0, 0.0]  # Second example "omitted" from training.
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='logistic_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      unregularized_loss = lr.unregularized_loss(examples)
      loss = lr.regularized_loss(examples)
      predictions = lr.predictions(examples)
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()
      self.assertAllClose(0.411608, unregularized_loss.eval(), atol=0.05)
      self.assertAllClose(0.525457, loss.eval(), atol=0.01)
      predicted_labels = get_binary_predictions_for_logistic(predictions)
      self.assertAllEqual([0, 0], predicted_labels.eval())
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  # TODO(katsiaspis): add a test for the case when examples at the end of an
  # epoch are repeated, since example id may be duplicated.

class SdcaWithLinearLossTest(SdcaOptimizerTest):
  """SDCA optimizer test class for linear (squared) loss."""

  def testSimple(self):
    # Setup test data
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, -10.0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 14.0),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='squared_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = lr.predictions(examples)
      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Predictions should be 2/3 of label due to minimizing regularized loss:
      #   (label - 2 * weight)^2 / 2 + L2 * 2 * weight^2
      self.assertAllClose([-20.0 / 3.0, 28.0 / 3.0],
                          predictions.eval(),
                          rtol=0.005)
      self.assertAllClose(0.01,
                          lr.approximate_duality_gap().eval(),
                          rtol=1e-2,
                          atol=1e-2)

  def testL2Regularization(self):
    # Setup test data
    example_protos = [
        # 2 identical examples
        make_example_proto(
            {'age': [0],
             'gender': [0]}, -10.0),
        make_example_proto(
            {'age': [0],
             'gender': [0]}, -10.0),
        # 2 more identical examples
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 14.0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 14.0),
    ]
    example_weights = [1.0, 1.0, 1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=16,
                     symmetric_l1_regularization=0,
                     loss_type='squared_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = lr.predictions(examples)

      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Predictions should be 1/5 of label due to minimizing regularized loss:
      #   (label - 2 * weight)^2 + L2 * 16 * weight^2
      optimal1 = -10.0 / 5.0
      optimal2 = 14.0 / 5.0
      self.assertAllClose(
          [optimal1, optimal1, optimal2, optimal2],
          predictions.eval(),
          rtol=0.01)

  def testL1Regularization(self):
    # Setup test data
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, -10.0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 14.0),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1.0,
                     symmetric_l1_regularization=4.0,
                     loss_type='squared_loss')
      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      prediction = lr.predictions(examples)
      loss = lr.regularized_loss(examples)

      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Predictions should be -4.0, 48/5 due to minimizing regularized loss:
      #   (label - 2 * weight)^2 / 2 + L2 * 2 * weight^2 + L1 * 4 * weight
      self.assertAllClose([-4.0, 20.0 / 3.0], prediction.eval(), rtol=0.08)

      # Loss should be the sum of the regularized loss value from above per
      # example after plugging in the optimal weights.
      self.assertAllClose(308.0 / 6.0, loss.eval(), atol=0.01)

  def testFeatureValues(self):
    # Setup test data
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, -10.0, -2.0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 14.0, 2.0),
    ]
    example_weights = [1.0, 1.0]
    with self._single_threaded_test_session():
      examples = make_example_dict(example_protos, example_weights)

      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='squared_loss')

      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = lr.predictions(examples)

      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Predictions should be 8/9 of label due to minimizing regularized loss:
      #   (label - 2 * 2 * weight)^2 / 2 + L2 * 2 * weight^2
      self.assertAllClose([-10.0 * 8 / 9, 14.0 * 8 / 9],
                          predictions.eval(),
                          rtol=0.07)

  def testDenseFeatures(self):
    with self._single_threaded_test_session():
      examples = make_dense_examples_dict(
          dense_feature_values=[[-2.0, 0.0], [0.0, 2.0]],
          weights=[1.0, 1.0],
          labels=[-10.0, 14.0])
      variables = make_dense_variable_dict(2, 2)
      options = dict(symmetric_l2_regularization=1,
                     symmetric_l1_regularization=0,
                     loss_type='squared_loss')
      lr = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = lr.predictions(examples)

      train_op = lr.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Predictions should be 4/5 of label due to minimizing regularized loss:
      #   (label - 2 * weight)^2 / 2 + L2 * weight^2
      self.assertAllClose([-10.0 * 4 / 5, 14.0 * 4 / 5],
                          predictions.eval(),
                          rtol=0.01)

      loss = lr.regularized_loss(examples)
      self.assertAllClose(148.0 / 10.0, loss.eval(), atol=0.01)

class SdcaWithHingeLossTest(SdcaOptimizerTest):
  """SDCA optimizer test class for hinge loss."""

  def testSimple(self):
    # Setup test data
    example_protos = [
        make_example_proto(
            {'age': [0],
             'gender': [0]}, 0),
        make_example_proto(
            {'age': [1],
             'gender': [1]}, 1),
    ]
    example_weights = [1.0, 1.0]
    with self.test_session(use_gpu=False):
      examples = make_example_dict(example_protos, example_weights)
      variables = make_variable_dict(1, 1)
      options = dict(symmetric_l2_regularization=1.0,
                     symmetric_l1_regularization=0,
                     loss_type='hinge_loss')
      model = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()

      # Before minimization, the weights default to zero. There is no loss due
      # to regularization, only unregularized loss which is 0.5 * (1+1) = 1.0.
      predictions = model.predictions(examples)
      self.assertAllClose([0.0, 0.0], predictions.eval())
      unregularized_loss = model.unregularized_loss(examples)
      regularized_loss = model.regularized_loss(examples)
      self.assertAllClose(1.0, unregularized_loss.eval())
      self.assertAllClose(1.0, regularized_loss.eval())

      # After minimization, the model separates perfectly the data points. There
      # are 4 sparse weights: 2 for age (say w1, w2) and 2 for gender (say w3
      # and w4). Solving the system w1 + w3 = 1.0, w2 + w4 = -1.0 and minimizing
      # wrt to \|\vec{w}\|_2, gives w1=w3=1/2 and w2=w4=-1/2. This gives 0.0
      # unregularized loss and 0.25 L2 loss.
      train_op = model.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      binary_predictions = get_binary_predictions_for_hinge(predictions)
      self.assertAllEqual([-1.0, 1.0], predictions.eval())
      self.assertAllEqual([0, 1], binary_predictions.eval())
      self.assertAllClose(0.0, unregularized_loss.eval())
      self.assertAllClose(0.25, regularized_loss.eval(), atol=0.05)

  def testDenseFeaturesPerfectlySeparable(self):
    with self._single_threaded_test_session():
      examples = make_dense_examples_dict(
          dense_feature_values=[[1.0, 1.0], [1.0, -1.0]],
          weights=[1.0, 1.0],
          labels=[1.0, 0.0])
      variables = make_dense_variable_dict(2, 2)
      options = dict(symmetric_l2_regularization=1.0,
                     symmetric_l1_regularization=0,
                     loss_type='hinge_loss')
      model = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = model.predictions(examples)
      binary_predictions = get_binary_predictions_for_hinge(predictions)

      train_op = model.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      self.assertAllClose([1.0, -1.0], predictions.eval(), atol=0.05)
      self.assertAllEqual([1, 0], binary_predictions.eval())

      # (1.0, 1.0) and (1.0, -1.0) are perfectly separable by x-axis (that is,
      # the SVM's functional margin >=1), so the unregularized loss is ~0.0.
      # There is only loss due to l2-regularization. For these datapoints, it
      # turns out that w_1~=0.0 and w_2~=1.0 which means that l2 loss is ~0.25.
      unregularized_loss = model.unregularized_loss(examples)
      regularized_loss = model.regularized_loss(examples)
      self.assertAllClose(0.0, unregularized_loss.eval(), atol=0.02)
      self.assertAllClose(0.25, regularized_loss.eval(), atol=0.02)

  def testDenseFeaturesSeparableWithinMargins(self):
    with self._single_threaded_test_session():
      examples = make_dense_examples_dict(
          dense_feature_values=[[1.0, 1.0], [0.5, -0.5]],
          weights=[1.0, 1.0],
          labels=[1.0, 0.0])
      variables = make_dense_variable_dict(2, 2)
      options = dict(symmetric_l2_regularization=1.0,
                     symmetric_l1_regularization=0,
                     loss_type='hinge_loss')
      model = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = model.predictions(examples)
      binary_predictions = get_binary_predictions_for_hinge(predictions)

      train_op = model.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # (1.0, 0.5) and (1.0, -0.5) are separable by x-axis but the datapoints
      # are within the margins so there is unregularized loss (1/2 per example).
      # For these datapoints, optimal weights are w_1~=0.0 and w_2~=1.0 which
      # gives an L2 loss of ~0.25.
      self.assertAllClose([0.5, -0.5], predictions.eval(), rtol=0.05)
      self.assertAllEqual([1, 0], binary_predictions.eval())
      unregularized_loss = model.unregularized_loss(examples)
      regularized_loss = model.regularized_loss(examples)
      self.assertAllClose(0.5, unregularized_loss.eval(), atol=0.02)
      self.assertAllClose(0.75, regularized_loss.eval(), atol=0.02)

  def testDenseFeaturesWeightedExamples(self):
    with self._single_threaded_test_session():
      examples = make_dense_examples_dict(
          dense_feature_values=[[1.0, 1.0], [0.5, -0.5]],
          weights=[3.0, 1.0],
          labels=[1.0, 0.0])
      variables = make_dense_variable_dict(2, 2)
      options = dict(symmetric_l2_regularization=1.0,
                     symmetric_l1_regularization=0,
                     loss_type='hinge_loss')
      model = SdcaModel(CONTAINER, examples, variables, options)
      tf.initialize_all_variables().run()
      predictions = model.predictions(examples)
      binary_predictions = get_binary_predictions_for_hinge(predictions)
      train_op = model.minimize()
      for _ in xrange(_MAX_ITERATIONS):
        train_op.run()

      # Point (1.0, 0.5) has higher weight than (1.0, -0.5) so the model will
      # try to increase the margin from (1.0, 0.5). Due to regularization,
      # (1.0, -0.5) will be within the margin. For these points and example
      # weights, the optimal weights are w_1~=0.4 and w_2~=1.2 which give an L2
      # loss of 0.5 * 0.25 * 0.25 * 1.6 = 0.2. The binary predictions will be
      # correct, but the boundary will be much closer to the 2nd point than the
      # first one.
      self.assertAllClose([1.0, -0.2], predictions.eval(), atol=0.05)
      self.assertAllEqual([1, 0], binary_predictions.eval())
      unregularized_loss = model.unregularized_loss(examples)
      regularized_loss = model.regularized_loss(examples)
      self.assertAllClose(0.2, unregularized_loss.eval(), atol=0.02)
      self.assertAllClose(0.4, regularized_loss.eval(), atol=0.02)

if __name__ == '__main__':
  googletest.main()


import argparse

import numpy as np
import cv2

from src.models import ORTModelLoader, DNNModelLoader
from src.general import run_yolov5_seg
from src.utils import check_file

def parse_opt():
    parser = argparse.ArgumentParser(description="Detect using YOLOv5 Segmentation model")
    required = parser.add_argument_group("required arguments")
    required.add_argument(
        "-m", "--model", type=str, required=True, help="YOLOv5 Segmentation onnx model path"
    )
    source = parser.add_argument_group("source arguments")
    source.add_argument("-i", "--image", type=str, help="Image source")
    source.add_argument("-v", "--video", type=str, help="Video source")

    parser.add_argument(
        "--topk",
        type=int,
        default=100,
        help="Integer representing the maximum number of boxes to be selected per class",
    )
    parser.add_argument(
        "--conf-tresh",
        type=float,
        default=0.2,
        help="Float representing the threshold for deciding when to remove boxes based on confidence score",
    )
    parser.add_argument(
        "--iou-tresh",
        type=float,
        default=0.45,
        help="Float representing the threshold for deciding whether boxes overlap too much with respect to IOU",
    )
    parser.add_argument(
        "--score-tresh",
        type=float,
        default=0.25,
        help="Float representing the threshold for deciding whether render boxes or not",
    )
    parser.add_argument(
        "--mask-tresh",
        type=float,
        default=0.5,
        help="Float representing the threshold for deciding mask area",
    )
    parser.add_argument(
        "--mask-alpha",
        type=float,
        default=0.4,
        help="Float representing the opacity of mask layer",
    )
    parser.add_argument(
        "--dnn",
        action="store_true",
        help="Use OpenCV DNN module [if false using onnxruntime] for backend",
    )

    opt = parser.parse_args()
    if opt.image is None and opt.video is None:
        raise argparse.ArgumentError("Please specify image or video source!")
    elif opt.image and opt.video:
        raise argparse.ArgumentError("Please specify either image or video source!")
    return opt

def main(opt) -> None:
    if opt.dnn:
        model = DNNModelLoader(opt.model)  # use Opencv DNN module
    else:
        model = ORTModelLoader(opt.model)  # use onnxruntime

    # warmup model
    _ = run_yolov5_seg(
        model,
        (np.random.rand(model.width, model.height, 3) * 255).astype(np.uint8),  # random image
        opt.conf_tresh,
        opt.iou_tresh,
        opt.score_tresh,
        opt.topk,
        opt.mask_tresh,
        opt.mask_alpha,
    )

    if opt.image:
        check_file(opt.image, "Image file not found!")

        # Image preprocessing
        img = cv2.imread(opt.image)
        img = run_yolov5_seg(
            model,
            img,
            opt.conf_tresh,
            opt.iou_tresh,
            opt.score_tresh,
            opt.topk,
            opt.mask_tresh,
            opt.mask_alpha,
        )

        cv2.imshow("output", img)
        cv2.waitKey(0)
    elif opt.video:
        # Video processing
        vid_source = 0 if opt.video == "0" else opt.video
        cap = cv2.VideoCapture(vid_source)

        while cap.isOpened():
            ret, frame = cap.read()

            if not ret:
                break

            frame = run_yolov5_seg(
                model,
                frame,
                opt.conf_tresh,
                opt.iou_tresh,
                opt.score_tresh,
                opt.topk,
                opt.mask_tresh,
                opt.mask_alpha,
            )

            cv2.imshow("output", frame)

            if cv2.waitKey(1) == ord("q"):
                break

        cap.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    opt = parse_opt()
    main(opt)


# construct pipeline
pipe_knn = make_pipeline(
    MinMaxScaler(), # used to normalize data onto a similar scale
    SelectPercentile(), # used to filter out features that add noise
    KNeighborsRegressor()
)

# create the parameter grid for hyperparameter tuning
param_grid_knn = {
    'selectpercentile__percentile':range(10, 30, 5), # what upper percentile of features to take
    'kneighborsregressor__n_neighbors':range(1, 20), # the number of neighbors to take
    'kneighborsregressor__weights':["uniform", "distance"] # how to weight the connections between neighbors
}

# perform grid search of pipeline
knn_grid = GridSearchCV(pipe_knn, param_grid_knn)

# use results to create model on training data
knn_grid.fit(train_features, train_outcome)

# find the best parameters from the grid search
knn_best_params = knn_grid.best_params_

# find the score of our model on the test data
knn_grid_score = knn_grid.score(test_features, test_outcome)

# find the mean absolute error of our model on the test data
knn_mae = mean_absolute_error(knn_grid.predict(test_features), test_outcome)

# find the explained variance score of our model on the test data
knn_evs = explained_variance_score(knn_grid.predict(test_features), test_outcome)


from typing import List

class Solution:
    def removeDuplicates(self, nums: List[int]) -> int:
        if len(nums) < 2: return len(nums)

        j = 0

        for n in nums[1:]:
            if n != nums[j]:
                j += 1
                nums[j] = n

        print(nums)
        return j + 1

if __name__ == "__main__":
    l = [[1, 1, 2], [0, 0, 1, 1, 1, 2, 2, 3, 3, 4]]

    for nums in l:
        print(Solution().removeDuplicates(nums))

LeetcodeEasy/26. Remove Duplicates from Sorted Array.py


from turtle import Screen, Turtle
import time
from snake import Snake
from food import Food
from score import Scoreboard

def draw_wall():
    wall = Turtle()
    wall.speed(0)
    wall.color("white")
    wall.penup()
    wall.goto(-292, 292)
    wall.pendown()
    wall.pensize(3)

    for _ in range(4):
        wall.forward(580)
        wall.right(90)

    wall.hideturtle()

def setup_scoreboard():
    scoreboard = Scoreboard()
    scoreboard.penup()
    scoreboard.color("white")
    scoreboard.goto(0, 296)
    scoreboard.hideturtle()
    return scoreboard

screen = Screen()
screen.setup(width=655, height=655)
screen.bgcolor("#000000")
screen.title("My Snake Game")
screen.tracer(0)

draw_wall()

snake = Snake()
food = Food()
score = setup_scoreboard()

screen.listen()
screen.onkey(snake.up, "Up")
screen.onkey(snake.down, "Down")
screen.onkey(snake.left, "Left")
screen.onkey(snake.right, "Right")

game_is_on = True

while game_is_on:
    screen.update()
    time.sleep(0.1)
    snake.move()

    if snake.head.distance(food) < 15:
        food.refresh()
        snake.extend()
        score.increase_score()

    if (
        snake.head.xcor() > 280
        or snake.head.xcor() < -290
        or snake.head.ycor() > 290
        or snake.head.ycor() < -290
    ):
        score.reset()
        snake.reset()

# Detect collision with tail.
for segment in snake.segments[1:]:
    if snake.head.distance(segment) < 10:
        score.reset()
        snake.reset()

screen.exitonclick()


import platform
from typing import Dict, List
from docplex.cp.model import *
from pandas import DataFrame
from const.weights import WEIGHT_OF_ADJUSTMENT_TIME, WEIGHT_OF_TARDY_JOB
from libs.settings import settings

from services.production_planning.job_duration_calculator import JobDurationCalculator
from libs.utils import resource_path
from libs.loggers import logging

logger = logging.getLogger('planner')

class Planner:
    def __init__(
        self,
        jobs_dict: Dict[int, int],
        machines_dict: Dict[int, int],
        pending_task: DataFrame,
        duration_calculator: JobDurationCalculator,
        due_date_dict: Dict[int, int],
        setup_time_dict: Dict[int, int] = None,
    ):
        logger.info('Start planning ...')

        self.mdl = CpoModel(name='productionPlanning')

        self.jobs = list(jobs_dict.keys())
        self.machines = list(machines_dict.keys())
        self.jobs_dict = jobs_dict
        self.due_date_dict = due_date_dict
        self.machines_dict = machines_dict
        self.pending_task = pending_task
        self.duration_calculator = duration_calculator
        self.setup_time_dict = setup_time_dict
        self.processing_itv_vars = []
        self.__solution_status = False

    def __prepare_processing_interval(self):
        processing_itv_vars = []

        for j in self.jobs:
            processing_itv_job_vars = []
            for m in self.machines:
                job_id = self.jobs_dict.get(j)
                mat_id = self.pending_task.loc[job_id]['mat_id']
                pending_volume = float(
                    self.pending_task.loc[job_id]['res_draft_volume'])
                machine = self.machines_dict.get(m)

                self.duration_calculator.register(
                    machine_id=machine,
                    mat_id=mat_id
                )

                duration = self.duration_calculator.calculate_duration(
                    pending_volume=pending_volume
                )

                self.duration_calculator.clear()

                if duration:
                    int_var = self.mdl.interval_var(
                        optional=True, size=duration, name="interval_job{}_machine{}".format(j, m))

                    processing_itv_job_vars.append(
                        int_var
                    )

                else:
                    processing_itv_job_vars.append(
                        None
                    )

            processing_itv_vars.append(processing_itv_job_vars)

        return processing_itv_vars

    def __create_setup_matrix(self, processing_itv_vars):
        setup_matrix = [*range(0, len(self.machines))]
        for m in self.machines:
            setup_time = self.setup_time_dict.get(m)
            temp1 = []
            for j1 in self.jobs:
                temp2 = []
                for j2 in self.jobs:
                    if (isinstance(processing_itv_vars[j1][m], expression.CpoIntervalVar) & isinstance(processing_itv_vars[j2][m], expression.CpoIntervalVar)):
                        if self.pending_task.loc[self.jobs_dict.get(j1)]['mat_id'] == self.pending_task.loc[self.jobs_dict.get(j2)]['mat_id']:
                            temp2.append(0)
                        else:
                            temp2.append(setup_time)
                if len(temp2) > 0:
                    temp1.append(temp2)
            setup_matrix[m] = temp1

        return setup_matrix

    def __add_job_must_be_done_constraint(self, processing_itv_vars):
        for j in self.jobs:
            processing_itv_vars_at_j = processing_itv_vars[j]
            non_none_processing_itv_vars_at_j = []

            for m in self.machines:
                if isinstance(processing_itv_vars_at_j[m], expression.CpoIntervalVar):
                    non_none_processing_itv_vars_at_j.append(
                        processing_itv_vars_at_j[m])

            self.mdl.add(
                self.mdl.sum(
                    [self.mdl.presence_of(var)
                     for var in non_none_processing_itv_vars_at_j]
                ) == 1
            )

    def __add_no_overlap_and_set_up_overhead_constraint(self, processing_itv_vars):
        sequence_vars = [
            self.mdl.sequence_var(
                [processing_itv_vars[j][m] for j in self.jobs if isinstance(
                    processing_itv_vars[j][m], expression.CpoIntervalVar)],
                name="sequences_machine{}".format(m))
            for m in self.machines
        ]

        if self.setup_time_dict:
            setup_matrix = self.__create_setup_matrix(processing_itv_vars)

            for m in self.machines:
                if len(setup_matrix[m]) > 0:
                    self.mdl.add(self.mdl.no_overlap(
                        sequence_vars[m], setup_matrix[m]))
                else:
                    self.mdl.add(self.mdl.no_overlap(
                        sequence_vars[m]))

        else:
            for m in self.machines:
                self.mdl.add(self.mdl.no_overlap(sequence_vars[m]))

        return sequence_vars

    def __add_objective_function(self, sequence_vars: List[expression.CpoSequenceVar]):
        adjustment_time_list = []

        for m in self.machines:
            for var in sequence_vars[m].get_interval_variables():
                start_next = self.mdl.start_of_next(
                    sequence_vars[m], var, lastValue=-1, absentValue=-1)

                binary = self.mdl.binary_var(
                    name=f"binary_adjustment_time{var.name}")
                self.mdl.add(binary == (start_next >= 0))

                adjustment_time = (start_next - self.mdl.end_of(var)) * binary
                adjustment_time.set_name(f"adjustment_time_{var.name}")

                adjustment_time_list.append(adjustment_time)

        adjustment_time_obj = self.mdl.sum(adjustment_time_list)

        n_tardy_day_list = []
        for m in self.machines:
            for j in self.jobs:
                if isinstance(self.processing_itv_vars[j][m], expression.CpoIntervalVar):
                    due_date = self.due_date_dict.get(j)
                    if due_date:
                        if due_date > 0:
                            n_tardy_day_list.append(self.mdl.max(
                                [0, self.mdl.end_of(self.processing_itv_vars[j][m]) - due_date]))

        n_tardy_day_obj = self.mdl.sum(n_tardy_day_list)
        self.mdl.add(self.mdl.minimize(adjustment_time_obj *
                     WEIGHT_OF_ADJUSTMENT_TIME + n_tardy_day_obj * WEIGHT_OF_TARDY_JOB))

    def get_processing_itv_vars(self):
        return self.processing_itv_vars

    def get_solution_status(self):
        return self.__solution_status

    def __update_solution_status(self, status=True):
        self.__solution_status = status

    def __calculate_objective_value(self, overall_objective_value: int, end_time_unit_dict: Dict[int, int]):
        tardy_job_objective_value = (
            self.pending_task.index.map(end_time_unit_dict) - self.pending_task['due_time_unit']
        ).apply(lambda x: x if x > 0 else 0).sum()
        tardy_job_objective_value = tardy_job_objective_value * WEIGHT_OF_TARDY_JOB

        return {
            "tardy_job_objective_value": tardy_job_objective_value,
            "adjustment_time_objective_value": overall_objective_value - tardy_job_objective_value
        }

    def __create_end_time_unit_dict(self, msol):
        end_time_unit_dict = {}
        for m in self.machines:
            for j in self.jobs:
                itv: CpoIntervalVarSolution = msol.get_var_solution(
                    self.processing_itv_vars[j][m])
                if itv:
                    if itv.is_present():
                        end_time_unit_dict.update(
                            {
                                j: itv.get_end()
                            }
                        )

        return end_time_unit_dict

    def generate(self):
        processing_itv_vars = self.__prepare_processing_interval()
        self.processing_itv_vars = processing_itv_vars

        self.__add_job_must_be_done_constraint(processing_itv_vars)
        sequence_var = self.__add_no_overlap_and_set_up_overhead_constraint(
            processing_itv_vars)
        self.__add_objective_function(sequence_var)

        if platform.system() in (['Linux', 'Darwin']):
            # Linux or MAC OS X
            execfile = './cpoptimizer'

        elif platform.system() == 'Windows':
            # Windows
            execfile = './cpoptimizer.exe'

        else:
            raise Exception('Invalid platform')

        msol = self.mdl.solve(
            log_output=True if settings.get_setting(
                "STAGE") == 'dev' else None,
            execfile=resource_path(execfile),
            TimeLimit=settings.get_setting('run_time_limit')
        )

        self.__update_solution_status()
        end_time_unit_dict = self.__create_end_time_unit_dict(msol)

        obj_value_details = self.__calculate_objective_value(
            msol.get_objective_value(), end_time_unit_dict)

        logger.info('Success.')
        logger.info('Objective value is {}'.format(msol.get_objective_value()))
        logger.info('Tardy job objective value: {}'.format(
            obj_value_details['tardy_job_objective_value']))
        logger.info('Adjustment time objective value: {}'.format(
            obj_value_details['adjustment_time_objective_value']))

        return msol


from pyexpat import model
from django import forms
from django.db.models import fields
from .models import quizquestion, quiztopics, useranswers, users
from django.core import validators

class AddQuiz(forms.ModelForm):
    class Meta:
        model = quiztopics
        fields = ['quiz_cat','quiz_title','quiz_desc','quiz_url','quiz_thumb']

class AddQuestion(forms.ModelForm):
    class Meta:
        model = quizquestion
        fields = ['qn_cat','qn_title','qn_name','qn_opt1','qn_opt2','qn_opt3','qn_opt4','qn_ans']

class UserAnswer(forms.ModelForm):
    class Meta:
        model = useranswers
        fields = [
            'full_name',
            'username',
            'email_id',
            'qn_cat',
            'qn_name',
            'qn_ans',
        ]


'''
from the subjects videos of raw h36m dataset to frames
'''

import os
import sys

import cv2

def makedir(folder_path):
    if not os.path.isdir(folder_path):
        os.mkdir(folder_path)

def extract_frames(img_folder, vidcap, prefix):
    if (prefix[-1] == '') or (prefix[-1] == '\n'):
        prefix = prefix[:-1]

    subject = int(prefix[5:7])
    print (prefix, subject)

    imgdir = img_folder + '/' + str(prefix)
    makedir(imgdir)

    f = 0
    while True:
        success, image = vidcap.read()
        f += 1
        if not success:
            print ('End of file!')
            break

        if subject in [2, 3, 4, 10]: # Test
            continue
        elif subject in [1, 5, 6, 7, 8]: # Train
            if (f % 5) != 1:
                continue
        elif subject in [9, 11]: # Validation
            if (f % 64) != 1:
                continue
        else:
            raise Exception('Unexpected subject number ({})'.format(subject))

        print ('%s %d' % (prefix, f))
        fname = imgdir + '/%05d.jpg' % f
        cv2.imwrite(fname, image, [cv2.IMWRITE_JPEG_QUALITY, 90])

# python2 <PATH>/datasets/h36m/h36m_frames_map.txt <PATH>/datasets/h36m/images
if __name__ == '__main__':
    try:
        map_path = sys.argv[1]
        output_dir = sys.argv[2]
    except Exception as e:
        print (str(e) + '\n\nExpected the first argument to be the input file, second to be output images folder path')
        sys.exit()

    with open(map_path, 'r') as f_map:
        try:
            while True:
                line = f_map.readline()
                if line == '':
                    break
                mp4_name, prefix = line.split(':')
                vidcap = cv2.VideoCapture(mp4_name)
                extract_frames(output_dir, vidcap, prefix)
                vidcap.release()
        except Exception as e:
            print (e)


import cv2
from onnx_ort import Detector
image = cv2.imread("2.png")
# print(image)
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import queue
import subprocess
import copy
# 创建一个5个线程的消费
class ThreadPoolExecutor(ThreadPoolExecutor):
    def __init__(self, max_workers=None, thread_name_prefix=''):
        super().__init__(max_workers, thread_name_prefix)
        # 队列大小为最大线程数的两倍
        self._work_queue = queue.Queue(self._max_workers * 2)

threadPool = ThreadPoolExecutor(max_workers=4)
obj=Detector(640,0.2,0.6,"zju-smokingFace.onnx")
obj1=Detector(640,0.2,0.6,"zju-smokingFace.onnx")
obj2=Detector(640,0.2,0.6,"zju-smokingFace.onnx")
obj3=Detector(640,0.2,0.6,"zju-smokingFace.onnx")
obj4=Detector(640,0.2,0.6,"zju-smokingFace.onnx")
start=time.time()
def job():
    obj.detect_out(image)
def job1():
    obj1.detect_out(image)
def job2():
    obj2.detect_out(image)
def job3():
    obj3.detect_out(image)
    #print(obj.detect_out(image))

pool=ThreadPoolExecutor(max_workers=4)
for i in range(1000):
    # job(obj)
    job1()
end=time.time()
print("耗费时间是%s"%(end-start))
# print(det.detect_out(image))
# print(det.detect_out(image))
# print(det.detect_out(image))
# print(det.detect_out(image))


import mnist_loader
import network
import numpy as np

def get_data(filename, num_labels, delimiter=','):
    train_data = []
    with open(filename) as fin:
        for line in fin:
            line_arr = np.array(list(map(float, line.strip().split(delimiter))))
            x, y = line_arr[:-num_labels], line_arr[-num_labels:]
            x = x.reshape(len(x), 1)
            y = y.reshape(len(y), 1)
            train_data.append((x, y))
    return train_data

def train_images():
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    net = network.Network([784, 30, 10])
    net.SGD(training_data, epochs=3, mini_batch_size=5, eta=3.0)

def train_2dim():
    training_data = get_data("data/data2.txt", num_labels=1)
    net = network.Network([2, 5, 1])
    num_epochs, batch_size, learn_rate = 90, 10, 3.0
    net.SGD(training_data, num_epochs, batch_size, learn_rate)  # works
    print("output at 0, 0:" + str(net.feedforward(np.array([0., 0.]).reshape(2, 1))))
    print("output at 10, 10:" + str(net.feedforward(np.array([10., 10.]).reshape(2, 1))))
    print("num_epochs, batch_size, learn_rate = {0}, {1}, {2}".format(num_epochs, batch_size, learn_rate))
    for b, w in zip(net.biases, net.weights):
        print(np.concatenate((b, w), 1))

if __name__ == "__main__":
    train_2dim()


def plot_phased_magseries(times,
                          mags,
                          period,
                          epoch='min',
                          fitknotfrac=0.01,
                          errs=None,
                          magsarefluxes=False,
                          normto='globalmedian',
                          normmingap=4.0,
                          sigclip=30.0,
                          phasewrap=True,
                          phasesort=True,
                          phasebin=None,
                          plotphaselim=(-0.8,0.8),
                          yrange=None,
                          xtimenotphase=False,
                          xaxlabel='phase',
                          yaxlabel=None,
                          modelmags=None,
                          modeltimes=None,
                          modelerrs=None,
                          outfile=None,
                          plotdpi=100):
    '''Plots a phased magnitude/flux time-series using the period provided.

    Parameters
    ----------

    times,mags : np.array
        The mag/flux time-series to plot as a function of phase given `period`.

    period : float
        The period to use to phase-fold the time-series. Should be the same unit
        as `times` (usually in days)

    epoch : 'min' or float or None
        This indicates how to get the epoch to use for phasing the light curve:

        - If None, uses the `min(times)` as the epoch for phasing.

        - If epoch is the string 'min', then fits a cubic spline to the phased
          light curve using `min(times)` as the initial epoch, finds the
          magnitude/flux minimum of this phased light curve fit, and finally
          uses the that time value as the epoch. This is useful for plotting
          planetary transits and eclipsing binary phased light curves so that
          phase 0.0 corresponds to the mid-center time of primary eclipse (or
          transit).

        - If epoch is a float, then uses that directly to phase the light
          curve and as the epoch of the phased mag series plot.

    fitknotfrac : float
        If `epoch='min'`, this function will attempt to fit a cubic spline to
        the phased light curve to find a time of light minimum as phase
        0.0. This kwarg sets the number of knots to generate the spline as a
        fraction of the total number of measurements in the input
        time-series. By default, this is set so that 100 knots are used to
        generate a spline for fitting the phased light curve consisting of 10000
        measurements.

    errs : np.array or None
        If this is provided, contains the measurement errors associated with
        each measurement of flux/mag in time-series. Providing this kwarg will
        add errbars to the output plot.

    magsarefluxes : bool
        Indicates if the input `mags` array is actually an array of flux
        measurements instead of magnitude measurements. If this is set to True,
        then the plot y-axis will be set as appropriate for mag or fluxes.

    normto : {'globalmedian', 'zero'} or a float
        Sets the normalization target::

          'globalmedian' -> norms each mag to the global median of the LC column
          'zero'         -> norms each mag to zero
          a float        -> norms each mag to this specified float value.

    normmingap : float
        This defines how much the difference between consecutive measurements is
        allowed to be to consider them as parts of different timegroups. By
        default it is set to 4.0 days.

    sigclip : float or int or sequence of two floats/ints or None
        If a single float or int, a symmetric sigma-clip will be performed using
        the number provided as the sigma-multiplier to cut out from the input
        time-series.

        If a list of two ints/floats is provided, the function will perform an
        'asymmetric' sigma-clip. The first element in this list is the sigma
        value to use for fainter flux/mag values; the second element in this
        list is the sigma value to use for brighter flux/mag values. For
        example, `sigclip=[10., 3.]`, will sigclip out greater than 10-sigma
        dimmings and greater than 3-sigma brightenings. Here the meaning of
        "dimming" and "brightening" is set by *physics* (not the magnitude
        system), which is why the `magsarefluxes` kwarg must be correctly set.

        If `sigclip` is None, no sigma-clipping will be performed, and the
        time-series (with non-finite elems removed) will be passed through to
        the output.

    phasewrap : bool
        If this is True, the phased time-series will be wrapped around phase
        0.0.

    phasesort : bool
        If this is True, the phased time-series will be sorted in phase.

    phasebin : float or None
        If this is provided, indicates the bin size to use to group together
        measurements closer than this amount in phase. This is in units of
        phase. The binned phased light curve will be overplotted on top of the
        phased light curve. Useful for when one has many measurement points and
        needs to pick out a small trend in an otherwise noisy phased light
        curve.

    plotphaselim : sequence of two floats or None
        The x-axis limits to use when making the phased light curve plot. By
        default, this is (-0.8, 0.8), which places phase 0.0 at the center of
        the plot and covers approximately two cycles in phase to make any trends
        clear.

    yrange : list of two floats or None
        This is used to provide a custom y-axis range to the plot. If None, will
        automatically determine y-axis range.

    xtimenotphase : bool
        If True, the x-axis gets units of time (multiplies phase by period).

    xaxlabel : str
        Sets the label for the x-axis.

    yaxlabel : str or None
        Sets the label for the y-axis. If this is None, the appropriate label
        will be used based on the value of the `magsarefluxes` kwarg.

    modeltimes,modelmags,modelerrs : np.array or None
        If all of these are provided, then this function will overplot the
        values of modeltimes and modelmags on top of the actual phased light
        curve. This is useful for plotting variability models on top of the
        light curve (e.g. plotting a Mandel-Agol transit model over the actual
        phased light curve. These arrays will be phased using the already
        provided period and epoch.

    outfile : str or StringIO/BytesIO or matplotlib.axes.Axes or None
        - a string filename for the file where the plot will be written.
        - a StringIO/BytesIO object to where the plot will be written.
        - a matplotlib.axes.Axes object to where the plot will be written.
        - if None, plots to 'magseries-phased-plot.png' in current dir.

    plotdpi : int
        Sets the resolution in DPI for PNG plots (default = 100).

    Returns
    -------

    str or StringIO/BytesIO or matplotlib.axes.Axes
        This returns based on the input:

        - If `outfile` is a str or None, the path to the generated plot file is
          returned.
        - If `outfile` is a StringIO/BytesIO object, will return the
          StringIO/BytesIO object to which the plot was written.
        - If `outfile` is a matplotlib.axes.Axes object, will return the Axes
          object with the plot elements added to it. One can then directly
          include this Axes object in some other Figure.

    '''

    # sigclip the magnitude timeseries
    stimes, smags, serrs = sigclip_magseries(times,
                                             mags,
                                             errs,
                                             magsarefluxes=magsarefluxes,
                                             sigclip=sigclip)

    # check if we need to normalize
    if normto is not False:
        stimes, smags = normalize_magseries(stimes, smags,
                                            normto=normto,
                                            magsarefluxes=magsarefluxes,
                                            mingap=normmingap)

        if ( isinstance(modelmags, np.ndarray) and
             isinstance(modeltimes, np.ndarray) ):

            stimes, smags = normalize_magseries(modeltimes, modelmags,
                                                normto=normto,
                                                magsarefluxes=magsarefluxes,
                                                mingap=normmingap)

    # figure out the epoch, if it's None, use the min of the time
    if epoch is None:
        epoch = stimes.min()

    # if the epoch is 'min', then fit a spline to the light curve phased
    # using the min of the time, find the fit mag minimum and use the time for
    # that as the epoch
    elif isinstance(epoch, str) and epoch == 'min':

        try:
            spfit = spline_fit_magseries(stimes, smags, serrs, period,
                                         knotfraction=fitknotfrac)
            epoch = spfit['fitinfo']['fitepoch']
            if len(epoch) != 1:
                epoch = epoch[0]
        except Exception as e:
            LOGEXCEPTION('spline fit failed, using min(times) as epoch')
            epoch = npmin(stimes)

    # now phase the data light curve (and optionally, phase bin the light curve)
    if errs is not None:

        phasedlc = phase_magseries_with_errs(stimes, smags, serrs, period,
                                             epoch, wrap=phasewrap,
                                             sort=phasesort)
        plotphase = phasedlc['phase']
        plotmags = phasedlc['mags']
        ploterrs = phasedlc['errs']

        # if we're supposed to bin the phases, do so
        if phasebin:

            binphasedlc = phase_bin_magseries_with_errs(plotphase, plotmags,
                                                        ploterrs,
                                                        binsize=phasebin)
            binplotphase = binphasedlc['binnedphases']
            binplotmags = binphasedlc['binnedmags']
            binploterrs = binphasedlc['binnederrs']

    else:

        phasedlc = phase_magseries(stimes, smags, period, epoch,
                                   wrap=phasewrap, sort=phasesort)
        plotphase = phasedlc['phase']
        plotmags = phasedlc['mags']
        ploterrs = None

        # if we're supposed to bin the phases, do so
        if phasebin:

            binphasedlc = phase_bin_magseries(plotphase,
                                              plotmags,
                                              binsize=phasebin)
            binplotphase = binphasedlc['binnedphases']
            binplotmags = binphasedlc['binnedmags']
            binploterrs = None

    # phase the model light curve
    modelplotphase, modelplotmags = None, None

    if ( isinstance(modelerrs,np.ndarray) and
         isinstance(modeltimes,np.ndarray) and
         isinstance(modelmags,np.ndarray) ):

        modelphasedlc = phase_magseries_with_errs(modeltimes, modelmags,
                                                  modelerrs, period, epoch,
                                                  wrap=phasewrap,
                                                  sort=phasesort)
        modelplotphase = modelphasedlc['phase']
        modelplotmags = modelphasedlc['mags']

    # note that we never will phase-bin the model (no point).
    elif ( not isinstance(modelerrs,np.ndarray) and
           isinstance(modeltimes,np.ndarray) and
           isinstance(modelmags,np.ndarray) ):

        modelphasedlc = phase_magseries(modeltimes, modelmags, period, epoch,
                                        wrap=phasewrap, sort=phasesort)
        modelplotphase = modelphasedlc['phase']
        modelplotmags = modelphasedlc['mags']

    # finally, make the plots

    # check if the outfile is actually an Axes object
    if isinstance(outfile, matplotlib.axes.Axes):
        ax = outfile

    # otherwise, it's just a normal file or StringIO/BytesIO
    else:
        fig = plt.figure()
        fig.set_size_inches(7.5,4.8)
        ax = plt.gca()

    if xtimenotphase:
        plotphase *= period

    if phasebin:
        ax.errorbar(plotphase, plotmags, fmt='o',
                    color='#B2BEB5',
                    yerr=ploterrs,
                    markersize=3.0,
                    markeredgewidth=0.0,
                    ecolor='#B2BEB5',
                    capsize=0)
        if xtimenotphase:
            binplotphase *= period
        ax.errorbar(binplotphase, binplotmags, fmt='bo', yerr=binploterrs,
                    markersize=5.0, markeredgewidth=0.0, ecolor='#B2BEB5',
                    capsize=0)

    else:
        ax.errorbar(plotphase, plotmags, fmt='ko', yerr=ploterrs,
                    markersize=3.0, markeredgewidth=0.0, ecolor='#B2BEB5',
                    capsize=0)

    if (isinstance(modelplotphase, np.ndarray) and
        isinstance(modelplotmags, np.ndarray)):

        if xtimenotphase:
            modelplotphase *= period
        ax.plot(modelplotphase, modelplotmags, zorder=5, linewidth=2,
                alpha=0.9, color='#181c19')

    # make a grid
    ax.grid(color='#a9a9a9',
            alpha=0.9,
            zorder=0,
            linewidth=1.0,
            linestyle=':')

    # make lines for phase 0.0, 0.5, and -0.5
    ax.axvline(0.0,alpha=0.9,linestyle='dashed',color='g')
    if not xtimenotphase:
        ax.axvline(-0.5,alpha=0.9,linestyle='dashed',color='g')
        ax.axvline(0.5,alpha=0.9,linestyle='dashed',color='g')
    else:
        ax.axvline(-period*0.5,alpha=0.9,linestyle='dashed',color='g')
        ax.axvline(period*0.5,alpha=0.9,linestyle='dashed',color='g')

    # fix the ticks to use no offsets
    ax.get_yaxis().get_major_formatter().set_useOffset(False)
    ax.get_xaxis().get_major_formatter().set_useOffset(False)

    # get the yrange
    if yrange and isinstance(yrange,(list,tuple)) and len(yrange) == 2:
        ymin, ymax = yrange
    else:
        ymin, ymax = ax.get_ylim()

    # set the y axis labels and range
    if not yaxlabel:
        if not magsarefluxes:
            ax.set_ylim(ymax, ymin)
            yaxlabel = 'magnitude'
        else:
            ax.set_ylim(ymin, ymax)
            yaxlabel = 'flux'

    # set the x axis limit
    if not plotphaselim:
        ax.set_xlim((npmin(plotphase)-0.1,
                     npmax(plotphase)+0.1))
    else:
        if xtimenotphase:
            ax.set_xlim((period*plotphaselim[0],period*plotphaselim[1]))
        else:
            ax.set_xlim((plotphaselim[0],plotphaselim[1]))

    # set up the axis labels and plot title
    ax.set_xlabel(xaxlabel)
    ax.set_ylabel(yaxlabel)
    ax.set_title('period: %.6f d - epoch: %.6f' % (period, epoch))

    LOGINFO('using period: %.6f d and epoch: %.6f' % (period, epoch))

    # check if the output filename is actually an instance of StringIO
    if sys.version_info[:2] < (3,0):

        is_Strio = isinstance(outfile, cStringIO.InputType)

    else:

        is_Strio = isinstance(outfile, Strio)

    # make the figure
    if (outfile and
        not is_Strio and
        not isinstance(outfile, matplotlib.axes.Axes)):

        if outfile.endswith('.png'):
            fig.savefig(outfile, bbox_inches='tight', dpi=plotdpi)
        else:
            fig.savefig(outfile, bbox_inches='tight')
        plt.close()
        return period, epoch, os.path.abspath(outfile)

    elif outfile and is_Strio:

        fig.savefig(outfile, bbox_inches='tight', dpi=plotdpi, format='png')
        return outfile

    elif outfile and isinstance(outfile, matplotlib.axes.Axes):

        return outfile

    elif not outfile and dispok:

        plt.show()
        plt.close()
        return period, epoch

    else:

        LOGWARNING('no output file specified and no $DISPLAY set, '
                   'saving to magseries-phased-plot.png in current directory')
        outfile = 'magseries-phased-plot.png'
        plt.savefig(outfile, bbox_inches='tight', dpi=plotdpi)
        plt.close()
        return period, epoch, os.path.abspath(outfile)


from dash import Dash, dcc, html
from dash.dependencies import Input, Output, State

app = Dash(__name__)

app.layout = html.Div([
    dcc.Input(id='input-1-state', type='text', value='Montréal'),
    dcc.Input(id='input-2-state', type='text', value='Canada'),
    html.Button(id='submit-button-state', n_clicks=0, children='Submit'),
    html.Div(id='output-state')
])

@app.callback(Output('output-state', 'children'),
              Input('submit-button-state', 'n_clicks'),
              State('input-1-state', 'value'),
              State('input-2-state', 'value'))
def update_output(n_clicks, input1, input2):
    return u'''
        The Button has been pressed {} times,
        Input 1 is "{}",
        and Input 2 is "{}"
    '''.format(n_clicks, input1, input2)

if __name__ == '__main__':
    app.run_server(debug=True)

3. Dashboards Interativos com Python/Callbacks Avançados/states2.py


import sys

from ovito.io import export_file
from ovito.pipeline import Pipeline

from utils.argument_parser import parse_arguments
from utils.config import get_config
from utils.parser_utils import get_neighbors_data, get_particles_data
from utils.parser_utils import get_particles_static_source

def visualization_ovito(config_file: str):
    config = get_config(config_file)
    df = get_particles_data(config.dynamic_file, config.static_file)
    static_source = get_particles_static_source(df,
                                                get_neighbors_data(config.neighbors_file).get(config.particle),
                                                config.particle,
                                                config.R)

    pipeline = Pipeline(source=static_source)

    export_file(pipeline, '../results/visualization.dump', 'lammps/dump',
                columns=["Particle Identifier", "Position.X", "Position.Y", "Position.Z", "Radius", "Neighbor"])

if __name__ == "__main__":
    arguments = parse_arguments(sys.argv[1:])

    config_file = arguments['config_file']

    try:
        visualization_ovito(config_file)
    except FileNotFoundError as e:
        print("File not found")
        print(e)
    except OSError:
        print("Error occurred.")
    except KeyboardInterrupt:
        print('Program interrupted by user.')
    except Exception as e:
        print(e)


import os
import sys
import pandas as pd
from dataclasses import dataclass
from src.logger import logging
from src.exceptions import CustomException
from src.utils import *

class PredictionPipeline:
    def __init__(self):
        pass

    def predict(self, features):
        try:
            logging.info("PredictPipeline.predict initiated at predictionpipeline.py")
            preprocessor_path = os.path.join ('artifacts', 'preprocessor.pkl')
            model_path = os.path.join('artifacts', 'model.pkl')

            # Loading preprocessor and model
            preprocessor = load_object(preprocessor_path)
            model = load_object(model_path)
            logging.info("Preprocessor and model objects loaded from artifacts")
            logging.info(f"Model metrics: intercept = {model.intercept_}; coefficients= \n {model.coef_}")

            # Data Preprocessing
            data_scaled = preprocessor.transform(features)
            logging.info("Data scaled and transformed using preprocessor")
            logging.info(data_scaled)

            # Prediction
            predicted_value = model.predict(X=data_scaled)
            logging.info("Predicted value/s obtained")

            return predicted_value

        except Exception as e:
            logging.info("Exception occured in predict at predictionpipeline.py")
            raise CustomException(e,sys)

class CustomData:
    def __init__(self,
                 carat:float,
                 depth:float,
                 table:float,
                 x:float,
                 y:float,
                 z:float,
                 cut:str,
                 color:str,
                 clarity:str):

        self.carat = carat
        self.depth = depth
        self.table = table
        self.x = x
        self.y = y
        self.z = z
        self.cut = cut
        self.color = color
        self.clarity = clarity

    def get_data_as_dataframe(self):
        logging.info("get_data_as_dataframe initiated at predictionpipeline.py")
        try:
            custom_data_input_dict = {
                'carat':[self.carat],
                'depth':[self.depth],
                'table':[self.table],
                'x':[self.x],
                'y':[self.y],
                'z':[self.z],
                'cut':[self.cut],
                'color':[self.color],
                'clarity':[self.clarity]
            }

            df = pd.DataFrame(custom_data_input_dict)
            logging.info("Inputs gathered in dataframe")

            return df

        except Exception as e:
            logging.info("Exception occured in get_data_as_dataframe at predictionpipeline.py")
            raise CustomException(e,sys)


import torch.optim as optim
from tqdm import tqdm
import torch.nn.functional as F

class TrainCifar10(object):
    def __init__(self, device, model, train_loader):
        self.device = device
        self.train_loader = train_loader
        self.model = model
        self.train_acc = []
        self.train_losses = []

    def train(self, limit, test_cifar, learning_rate=0.01, momentum=0.9):
        optimizer = optim.SGD(self.model.parameters(), lr=learning_rate,
                              momentum=momentum)

        for epoch in range(limit):
            self.model.train()
            p_bar = tqdm(self.train_loader)
            correct = 0
            processed = 0

            print("EPOCH:", epoch+1)

            for batch_idx, (data, labels) in enumerate(p_bar):
                data, labels = data.to(self.device), labels.to(self.device)

                # Init
                optimizer.zero_grad()
                # In PyTorch, we need to set the gradients to zero before starting to do backpropagation because PyTorch accumulates the gradients on subsequent backward passes.
                # Because of this, when you start your training loop, ideally you should zero out the gradients so that you do the parameter update correctly.

                # Predict
                y_pred = self.model(data)

                # Calculate loss
                loss = F.nll_loss(y_pred, labels)
                self.train_losses.append(loss)

                # Backpropagation
                loss.backward()
                optimizer.step()

                # Update p_bar-tqdm
                pred = y_pred.argmax(dim=1,
                                     keepdim=True)  # get the index of the max log-probability
                correct += pred.eq(labels.view_as(pred)).sum().item()
                processed += len(data)

                p_bar.set_description(
                    desc=f'Loss={loss.item()} Batch_id={batch_idx} Accuracy={100 * correct / processed:0.2f}')
                self.train_acc.append(100 * correct / processed)
            test_cifar.test()


from typing import Union
from datetime import datetime
import requests
from xbrr.edinet.client.base_client import BaseClient
from xbrr.edinet.models import MetaData, Documents

class BaseDocumentListClient(BaseClient):
    """Base client to handle Document List API."""

    def __init__(self, response_type: str):
        """
        Arguments:
            response_type {str} -- Response type of document list api.
        """
        super().__init__(target="documents.json")
        self.response_type = response_type

    def _get(self, date: Union[str, datetime]) -> dict:
        """Get Document List API response.

        Arguments:
            date {(str, datetime)} -- Request date.

        Raises:
            Exception: Date format exception.

        Returns:
            dict -- EDINET Response (JSON).
        """
        url = self.endpoint
        _date = date
        if isinstance(date, str):
            try:
                _date = datetime.strptime(date, "%Y-%m-%d")
            except ValueError:
                raise Exception("Date format should be yyyy-mm-dd.")

        _date = _date.strftime("%Y-%m-%d")

        params = {
            "date": _date,
            "type": self.response_type
        }

        r = requests.get(url, params=params, verify=False)  # Caution

        if not r.ok:
            r.raise_for_status()
        else:
            body = r.json()
            return body

class MetaDataClient(BaseDocumentListClient):
    """Client to get metadata of document list."""

    def __init__(self):
        super().__init__(response_type="1")  # 1 = only metadata.

    def get(self, date: Union[str, datetime]) -> MetaData:
        """Get metadeta response.

        Arguments:
            date {(str, datetime)} -- Request date.

        Returns:
            MetaData -- Metadata of document list.
        """
        body = self._get(date)
        instance = MetaData.create(body)
        return instance

class DocumentListClient(BaseDocumentListClient):
    """Client to get document list."""

    def __init__(self):
        super().__init__(response_type="2")  # 2 = metadata and document list.

    def get(self, date: Union[str, datetime]) -> Documents:
        """Get metadeta response.

        Arguments:
            date {(str, datetime)} -- Request date.

        Returns:
            Documents -- Document list and its metadata.
        """
        body = self._get(date)
        instance = Documents.create(body)
        return instance


import torch
from matplotlib import pyplot as plt

def plot_activation(fn):
    x = torch.arange(-3.0, 3.0, 0.01, requires_grad=True)
    y = fn(x)
    fn_name = fn.__name__
    plt.plot(x.detach(), y.detach(), label=fn_name)
    y.backward(torch.ones_like(x), retain_graph=True)
    plt.plot(x.detach(), x.grad, label=f'{fn_name} grad')
    plt.legend()


## time - O(n), space - O(k)
class Solution:
    def lengthOfLongestSubstringKDistinct(self, s: str, k: int) -> int:
        if len(s) <= k:
            return len(s)

        window = {}
        cnt, res = 0, 0
        left = 0
        for right, c in enumerate(s):
            if c not in window or window[c] == 0:
                cnt += 1
                window[c] = 0
            window[c] += 1
            if cnt <= k:
                res = max(res, right - left + 1)
            while cnt > k:
                window[s[left]] -= 1
                if window[s[left]] == 0:
                    cnt -= 1
                left += 1
        return res

340. Longest Substring with At Most K Distinct Characters _ string.py


# Databricks notebook source
# MAGIC %md This notebook sets up the companion cluster(s) to run the solution accelerator with. It also creates the Workflow to create a Workflow DAG and illustrate the order of execution. Feel free to interactively run notebooks with the cluster or to run the Workflow to see how this solution accelerator executes. Happy exploring!
# MAGIC
# MAGIC The pipelines, workflows and clusters created in this script are user-specific, so you can alter the workflow and cluster via UI without affecting other users. Running this script again after modification resets them.
# MAGIC
# MAGIC **Note**: If the job execution fails, please confirm that you have set up other environment dependencies as specified in the accelerator notebooks. Accelerators sometimes require the user to set up additional cloud infra or data access, for instance.

# COMMAND ----------

# DBTITLE 0,Install util packages
# MAGIC %pip install git+https://github.com/databricks-academy/dbacademy-rest git+https://github.com/databricks-academy/dbacademy-gems

# COMMAND ----------

from dbacademy.dbrest import DBAcademyRestClient
from dbacademy.dbgems import get_username, get_cloud, get_notebook_dir
import hashlib
import json
import re

# COMMAND ----------

# DBTITLE 1,Key cluster configs
spark_version = "10.4.x-cpu-ml-scala2.12"
num_workers = 2
node_type_dict = {"AWS": "i3.xlarge", "MSA": "Standard_D3_v2", "GCP": "n1-highmem-4"}

# COMMAND ----------

# DBTITLE 1,Companion job and cluster(s) definition
def get_job_param_json(env, solacc_path, job_name, node_type_id, spark_version, spark):

    # This job is not environment specific, so `env` is not used
    num_workers = 2

    # 1st DLT
    pipeline_name1 = f"SOLACC_rtb_lite_{current_user_sql}"
    dlt_definition_dict1 = {
          "clusters": [
              {
                  "label": "default",
                  "autoscale": {
                      "min_workers": 1,
                      "max_workers": 3
                  }
              }
          ],
          "development": True,
          "continuous": False,
          "edition": "advanced",
          "libraries": [
              {
                  "notebook": {
                      "path": f"{solacc_path}/01_dlt_real_time_bidding"
                  }
              }
          ],
          "name": pipeline_name1,
          "storage": f"{config_database_path}rtb_lite/dlt_{current_user_sql}",
          "target": f"SOLACC_rtb_lite_{current_user_sql}",
          "allow_duplicate_names": "true"
      }
    pipeline_id1 = create_or_update_pipeline_by_name(client, dlt_config_table, pipeline_name1, dlt_definition_dict1, spark)

    return {
        "timeout_seconds": 7200,
        "name": job_name,
        "max_concurrent_runs": 1,
        "tags": {
            "usage": "solacc_automation",
            "group": "CME_solacc_automation"
        },
        "tasks": [
            {
                "pipeline_task": {
                    "pipeline_id": pipeline_id1
                },
                "task_key": "rtb_lite_01",
                "description": ""
            },
          {
                "job_cluster_key": "rtb_lite_cluster",
                "libraries": [],
                "notebook_task": {
                    "notebook_path": f"{solacc_path}/02_real_time_bidding_ml"
                },
                "task_key": "rtb_lite_02",
                "description": "",
                "depends_on": [
                    {
                        "task_key": "rtb_lite_01"
                    }
                ]
            }
        ],
        "job_clusters": [
            {
                "job_cluster_key": "rtb_lite_cluster",
                "new_cluster": {
                    "spark_version": spark_version,
                "spark_conf": {
                    "spark.databricks.delta.formatCheck.enabled": "false"
                    },
                    "num_workers": num_workers,
                    "node_type_id": node_type_id,
                    "custom_tags": {
            "usage": "solacc_automation",
            "group": "CME_solacc_automation"
        }
                }
            }
        ]
    }

# COMMAND ----------

# DBTITLE 1,Automated configs
current_cloud = get_cloud()
current_user = get_username()
client = DBAcademyRestClient()
current_user_sql = re.sub('\W', '_', current_user.split("@")[0])
solacc_path = get_notebook_dir()[:-7]
hash_code = hashlib.sha256(solacc_path.encode()).hexdigest()
solacc_name = solacc_path.split("/")[-1]
node_type_id = node_type_dict[current_cloud]
job_name = f"[SOLACC] {current_user_sql} {solacc_name} | {hash_code}"
config_database = f"databricks_solacc_{current_user_sql}"
dlt_config_table = f"{config_database}.dlt"
config_database_path = f"/databricks_solacc/{current_user}/"
pipeline_name = f"SOLACC_POS_DLT_{current_user_sql}"
env = "dev"

# COMMAND ----------

# DBTITLE 1,Initialize special config table for DLT-based accelerators
spark.sql(f"CREATE DATABASE IF NOT EXISTS {config_database} LOCATION '{config_database_path}'")
spark.sql(f"CREATE TABLE IF NOT EXISTS {dlt_config_table} (path STRING, pipeline_id STRING, solacc STRING)")

# COMMAND ----------

# DBTITLE 1,Util functions
# Note these functions assume that names for solacc jobs/cluster/pipelines are unique, which is guaranteed if solacc jobs/cluster/pipelines are created from this script only

def create_or_update_pipeline_by_name(client, dlt_config_table, pipeline_name, dlt_definition_dict, spark):
    """Look up a companion pipeline by name and edit with the given param and return pipeline id; create a new pipeline if a pipeline with that name does not exist"""
    dlt_id_pdf = spark.table(dlt_config_table).filter(f"solacc = '{pipeline_name}'").toPandas()
    assert len(dlt_id_pdf) <= 1, f"two pipelines with the same name {pipeline_name} exist in the {dlt_config_table} table; please manually inspect the table to make sure pipelines names are unique"
    pipeline_id = dlt_id_pdf['pipeline_id'][0] if len(dlt_id_pdf) > 0 else None
    if pipeline_id:
        dlt_definition_dict['id'] = pipeline_id
        print(f"Found dlt {pipeline_name} at '{pipeline_id}'; updating it with latest config if there is any change")
        client.execute_put_json(f"{client.endpoint}/api/2.0/pipelines/{pipeline_id}", dlt_definition_dict)
    else:
        response = DBAcademyRestClient().pipelines().create_from_dict(dlt_definition_dict)
        pipeline_id = response["pipeline_id"]
        # log pipeline id to the cicd dlt table: we use this delta table to store pipeline id information because looking up pipeline id via API can sometimes bring back a lot of data into memory and cause OOM error; this table is user-specific
        # Reusing the DLT pipeline allows for DLT run history to accumulate over time rather than to be wiped out after each deployment. DLT has some UI components that only show up after the pipeline is executed at least twice.
        spark.createDataFrame([{"solacc": pipeline_name, "pipeline_id": pipeline_id}]).write.mode("append").option("mergeSchema", "True").saveAsTable(dlt_config_table)

    return pipeline_id

def create_or_update_job_by_name(client, params):
  """Look up the companion job by name and resets it with the given param and return job id; create a new job if a job with that name does not exist"""
  jobs = client.jobs().list()
  jobs_matched = list(filter(lambda job: job["settings"]["name"] == params["name"], jobs))
  assert len(jobs_matched) <= 1, f"""Two jobs with the same name {params["name"]} exist; please manually inspect them to make sure solacc job names are unique"""
  job_id = jobs_matched[0]["job_id"] if len(jobs_matched)  == 1 else None
  if job_id:
    reset_params = {"job_id": job_id,
                   "new_settings": params}
    json_response = client.execute_post_json(f"{client.endpoint}/api/2.1/jobs/reset", reset_params) # returns {} if status is 200
    assert json_response == {}, "Job reset returned non-200 status"
    print(f"""Reset the {params["name"]} job with job_id {job_id} to original definition""")
  else:
    json_response = client.execute_post_json(f"{client.endpoint}/api/2.1/jobs/create", params)
    job_id = json_response["job_id"]
    print(f"""Created {params["name"]} job with job_id {job_id}""")

  return

def create_or_update_cluster_by_name(client, params):
    """Look up a companion cluster by name and edit with the given param and return cluster id; create a new cluster if a cluster with that name does not exist"""
    clusters = client.execute_get_json(f"{client.endpoint}/api/2.0/clusters/list")["clusters"]
    clusters_matched = list(filter(lambda cluster: params["cluster_name"] == cluster["cluster_name"], clusters)) # cluster names are guaranteed unique by the Databricks platform
    cluster_id = clusters_matched[0]["cluster_id"] if len(clusters_matched) == 1 else None
    if cluster_id:
      params["cluster_id"] = cluster_id
      json_response = client.execute_post_json(f"{client.endpoint}/api/2.0/clusters/edit", params) # returns {} if status is 200
      assert json_response == {}, "Job reset returned non-200 status"
      print(f"""Reset the {params["cluster_name"]} cluster with cluster_id {cluster_id} to original definition""")
    else:
      json_response = client.execute_post_json(f"{client.endpoint}/api/2.0/clusters/create", params)
      cluster_id = json_response["cluster_id"]
      print(f"""Created {params["cluster_name"]} cluster with cluster_id {cluster_id}""")
    return

def convert_job_cluster_to_cluster(job_cluster_params):
  params = job_cluster_params["new_cluster"]
  params["cluster_name"] = f"""{current_user_sql}_{job_cluster_params["job_cluster_key"]}"""
  params["autotermination_minutes"] = 45
  return params

# COMMAND ----------

# DBTITLE 1,Create companion job and cluster(s)
job_params = get_job_param_json(env, solacc_path, job_name, node_type_id, spark_version, spark)
create_or_update_job_by_name(client, job_params)
for job_cluster_params in job_params["job_clusters"]:
  create_or_update_cluster_by_name(client, convert_job_cluster_to_cluster(job_cluster_params))


from django.urls import path
from django.contrib.auth.views import LoginView
from . import views
app_name = 'dairy'

urlpatterns = [
	path('', views.homePage, name='home'),
	path('milklist/', views.milkList, name='milklist'),
	path('addmilk/', views.addMilk, name='addmilk'),
	path('search/', views.searchFilter, name='search'),
	path('logout/', views.logoutView, name='logout'),
	path('login/',LoginView.as_view(template_name='login.html'), name='login'),
]


import math
import sys

def get_fasta_records(lines):
    dna_strings = []
    is_first_dna = True
    id, dna = '', ''
    for line in lines:
        line = line.strip();
        if line[0] == '>':
            if is_first_dna:
                is_first_dna = False
            else:
                dna_strings.append([id, dna])
            id, dna = line[1:], ''
        else:
            dna += line
    dna_strings.append([id, dna])
    return dna_strings

def get_num_maximum_matchings(rna):
    au_pair = [rna.count(_) for _ in 'AU']
    cg_pair = [rna.count(_) for _ in 'CG']
    return get_num_pair_matchings(au_pair) * get_num_pair_matchings(cg_pair)

def get_num_pair_matchings(pair):
    pair.sort()
    return math.factorial(pair[1]) // math.factorial(pair[1] - pair[0])

def main():
    fasta_records = get_fasta_records(sys.stdin.readlines())
    rna = fasta_records[0][1]
    print(get_num_maximum_matchings(rna))

if __name__ == '__main__':
    sys.exit(main())


import spacy

def get_most_similar_movie(description):
    nlp = spacy.load("en_core_web_sm")
    movies = []
    with open("movies.txt", "r") as file:
        movies = file.readlines()

    similarity_scores = []
    for movie in movies:
        movie = movie.strip()
        doc1 = nlp(movie)
        doc2 = nlp(description)
        similarity_scores.append(doc1.similarity(doc2))

    most_similar_movie = movies[similarity_scores.index(max(similarity_scores))]
    return most_similar_movie.strip()

if __name__ == "__main__":
    description = "Will he save their world or destroy it? When the Hulk becomes too dangerous for the Earth, the Illuminati trick Hulk into a shuttle and launch him into space to a planet where the Hulk can live in peace. Unfortunately, Hulk land on the planet Sakaar where he is sold into slavery and trained as a gladiator."
    most_similar_movie = get_most_similar_movie(description)
    print("Next movie recommendation:", most_similar_movie)


import gym
import gfootball.env as grf_env
import numpy as np

class GRFWrapperEnv(gym.Env):

    def __init__(self,):
        self.env = None
        self.num_controlled_lagents = 0
        self.num_controlled_ragents = 0
        self.num_controlled_agents = 0
        self.num_lagents = 0
        self.num_ragents = 0
        self.action_space = None
        self.observation_space = None

    def init_args(self, parser):
        env = parser.add_argument_group('GRF')
        env.add_argument('--scenario', type=str, default='academy_3_vs_1_with_keeper',
                         help="Scenario of the game")
        env.add_argument('--num_controlled_lagents', type=int, default=3,
                         help="Number of controlled agents on the left side")
        env.add_argument('--num_controlled_ragents', type=int, default=0,
                         help="Number of controlled agents on the right side")
        env.add_argument('--reward_type', type=str, default='scoring',
                         help="Reward type for training")
        env.add_argument('--render', action="store_true", default=False,
                         help="Render training or testing process")

    def multi_agent_init(self, args):
        self.env = grf_env.create_environment(
            env_name=args.scenario,
            stacked=False,
            representation='multiagent',
            rewards=args.reward_type,
            write_goal_dumps=False,
            write_full_episode_dumps=False,
            render=args.render,
            dump_frequency=0,
            logdir='/tmp/test',
            extra_players=None,
            number_of_left_players_agent_controls=args.num_controlled_lagents,
            number_of_right_players_agent_controls=args.num_controlled_ragents,
            channel_dimensions=(3, 3))
        self.num_controlled_lagents = args.num_controlled_lagents
        self.num_controlled_ragents = args.num_controlled_ragents
        self.num_controlled_agents = args.num_controlled_lagents + args.num_controlled_ragents
        self.num_lagents = self.env.num_lteam_players
        self.num_ragents = self.env.num_rteam_players
        if self.num_controlled_agents > 1:
            action_space = gym.spaces.Discrete(self.env.action_space.nvec[1])
        else:
            action_space = self.env.action_space
        if self.num_controlled_agents > 1:
            observation_space = gym.spaces.Box(
                low=self.env.observation_space.low[0],
                high=self.env.observation_space.high[0],
                dtype=self.env.observation_space.dtype)
        else:
            observation_space = gym.spaces.Box(
                low=self.env.observation_space.low,
                high=self.env.observation_space.high,
                dtype=self.env.observation_space.dtype)

        # check spaces
        self.action_space = action_space
        self.observation_space = observation_space
        return

    # check epoch arg
    def reset(self):
        self.stat = dict()
        obs = self.env.reset()
        if self.num_controlled_agents == 1:
            obs = obs.reshape(1, -1)
        return obs

    def step(self, actions):
        o, r, d, i = self.env.step(actions)
        if self.num_controlled_agents == 1:
            o = o.reshape(1, -1)
            r = r.reshape(1, -1)
        next_obs = o
        rewards = r
        dones = d
        infos = i
        self.stat['success'] = infos['score_reward']

        return next_obs, rewards, dones, infos

    def seed(self):
        return

    def render(self):
        self.env.render()

    def exit_render(self):
        self.env.disable_render()