from typing import Callable, Tuple

import tensorflow.compat.v2 as tf
import tf_quant_finance as tff

from tf_quant_finance import types
from tf_quant_finance import utils
from tf_quant_finance.math import make_val_and_grad_fn
from tf_quant_finance.math import optimizer
from tf_quant_finance.models.heston import approximations

__all__ = [
    'CalibrationResult',
    'calibration',
]

@utils.dataclass
class CalibrationResult:
  """Collection of calibrated Heston parameters.

  For a review of the Heston model and the conventions used, please see the
  docstring for `HestonModel`, or for `calibration` below.

  Attributes:
    initial_variance: Rank-1 `Tensor` specifying the initial volatility levels.
    mean_reversion: Rank-1 `Tensor` specifying the mean reversion rate.
    theta: Rank-1 `Tensor` specifying the long run price variance.
    volvol: Rank-1 `Tensor` specifying the vol-vol parameters.
    rho: Rank-1 `Tensor` specifying the correlations between the forward and
      the stochastic volatility.
  """
  initial_variance: types.RealTensor
  mean_reversion: types.RealTensor
  theta: types.RealTensor
  volvol: types.RealTensor
  rho: types.RealTensor

def calibration(
    *,
    prices: types.RealTensor,
    strikes: types.RealTensor,
    expiries: types.RealTensor,
    spots: types.RealTensor = None,
    forwards: types.RealTensor = None,
    is_call_options: types.BoolTensor,
    discount_rates: types.RealTensor = None,
    dividend_rates: types.RealTensor = None,
    discount_factors: types.RealTensor = None,
    mean_reversion: types.RealTensor,
    initial_variance: types.RealTensor,
    theta: types.RealTensor,
    volvol: types.RealTensor,
    rho: types.RealTensor,
    mean_reversion_lower_bound: types.RealTensor = 0.001,
    mean_reversion_upper_bound: types.RealTensor = 0.5,
    initial_variance_lower_bound: types.RealTensor = 0.0,
    initial_variance_upper_bound: types.RealTensor = 1.0,
    theta_lower_bound: types.RealTensor = 0.0,
    theta_upper_bound: types.RealTensor = 1.0,
    volvol_lower_bound: types.RealTensor = 0.0,
    volvol_upper_bound: types.RealTensor = 1.0,
    rho_lower_bound: types.RealTensor = -1.0,
    rho_upper_bound: types.RealTensor = 1.0,
    optimizer_fn: Callable[..., types.RealTensor] = None,
    tolerance: types.RealTensor = 1e-6,
    maximum_iterations: types.RealTensor = 100,
    validate_args: bool = False,
    dtype: tf.DType = None,
    name: str = None
    ) -> Tuple[CalibrationResult, types.BoolTensor, types.IntTensor]:

  """Calibrates the Heston model using European option prices.

  Represents the Ito process:

  ```None
    dX(t) = -V(t) / 2 * dt + sqrt(V(t)) * dW_{X}(t),
    dV(t) = mean_reversion(t) * (theta(t) - V(t)) * dt
            + volvol(t) * sqrt(V(t)) * dW_{V}(t)
  ```

  where `W_{X}` and `W_{V}` are 1D Brownian motions with a correlation `rho(t)`.
  `mean_reversion`, `theta`, `volvol`, and `rho` are positive piecewise constant
  functions of time. Here `V(t)` represents the process variance at time `t` and
  `X` represents logarithm of the spot price at time `t`.

  `mean_reversion` corresponds to the mean reversion rate, `theta` is the long
  run price variance, and `volvol` is the volatility of the volatility.

  #### Example

  ```python
  import tf_quant_finance as tff
  import tensorflow.compat.v2 as tf

  dtype = np.float64

  # Set some market conditions.
  observed_prices = np.array(
      [[29.33668202, 23.98724723, 19.54631658, 15.9022847, 12.93591534],
       [15.64785924, 21.05865247, 27.11907971, 33.74249536, 40.8485591]],
      dtype=dtype)
  strikes = np.array(
      [[80.0, 90.0, 100.0, 110.0, 120.0], [80.0, 90.0, 100.0, 110.0, 120.0]],
      dtype=dtype)
  expiries = np.array([[0.5], [1.0]], dtype=dtype)
  forwards = 100.0
  is_call_options = np.array([[True], [False]])

  # Calibrate the model.
  # In this example, we are calibrating a Heston model.
  models, is_converged, _ = tff.models.heston.calibration(
      prices=observed_prices,
      strikes=strikes,
      expiries=expiries,
      forwards=forwards,
      is_call_options=is_call_options,
      mean_reversion=np.array([0.3], dtype=dtype),
      initial_variance=np.array([0.8], dtype=dtype),
      theta=np.array([0.75], dtype=dtype),
      volvol=np.array([0.1], dtype=dtype),
      rho=np.array(0.0, dtype=dtype),
      optimizer_fn=tff.math.optimizer.bfgs_minimize,
      maximum_iterations=1000)

  # This will return two `HestonModel`s, where:
  # Model 1 has mean_reversion = 0.3, initial_variance = 0.473, volvol = 0.1,
  # theta = 0.724 and rho = 0.028
  # Model 2 has mean_reversion = 0.3, initial_variance = 0.45, volvol = 0.1,
  # theta = 0.691 and rho = -0.073

  ```

  Args:
    prices: Real `Tensor` of shape [batch_size, num_strikes] specifying the
      observed options prices. Here, `batch_size` refers to the number of Heston
      models calibrated in this invocation.
    strikes: Real `Tensor` of shape [batch_size, num_strikes] specifying the
      strike prices of the options.
    expiries: Real `Tensor` of shape compatible with [batch_size, num_strikes]
      specifying the options expiries.
    spots: A real `Tensor` of any shape that broadcasts to the shape of the
      `volatilities`. The current spot price of the underlying. Either this
      argument or the `forwards` (but not both) must be supplied.
    forwards: A real `Tensor` of any shape that broadcasts to the shape of
      `strikes`. The forwards to maturity. Either this argument or the
      `spots` must be supplied but both must not be supplied.
    is_call_options: A boolean `Tensor` of shape compatible with
      [batch_size, num_strikes] specifying whether or not the prices correspond
      to a call option (=True) or a put option (=False).
    discount_rates: An optional real `Tensor` of same dtype as the
      `strikes` and of the shape that broadcasts with `strikes`.
      If not `None`, discount factors are calculated as e^(-rT),
      where r are the discount rates, or risk free rates. At most one of
      discount_rates and discount_factors can be supplied.
      Default value: `None`, equivalent to r = 0 and discount factors = 1 when
      discount_factors also not given.
    dividend_rates: An optional real `Tensor` of same dtype as the
      `strikes` and of the shape that broadcasts with `volatilities`.
      Default value: `None`, equivalent to q = 0.
    discount_factors: An optional real `Tensor` of same dtype as the
      `strikes`. If not `None`, these are the discount factors to expiry
      (i.e. e^(-rT)). Mutually exclusive with `discount_rates`. If neither is
      given, no discounting is applied (i.e. the undiscounted option price is
      returned). If `spots` is supplied and `discount_factors` is not `None`
      then this is also used to compute the forwards to expiry. At most one of
      `discount_rates` and `discount_factors` can be supplied.
      Default value: `None`, which maps to e^(-rT) calculated from
      discount_rates.
    mean_reversion: Real `Tensor` of shape [batch_size], specifying the initial
      estimate of the mean reversion parameter.
    initial_variance: Real `Tensor` of shape [batch_size], specifying the
      initial estimate of the variance parameter.
      Values must satisfy `0 <= initial_variance`.
    theta: Real `Tensor` of shape [batch_size], specifying the initial estimate
      of the long run variance parameter. Values must satisfy `0 <= theta`.
    volvol: Real `Tensor` of shape [batch_size], specifying the initial estimate
      of the vol-vol parameter. Values must satisfy `0 <= volvol`.
    rho: Real `Tensor` of shape [batch_size], specifying the initial estimate of
      the correlation between the forward price and the volatility. Values must
      satisfy -1 < `rho` < 1.
    mean_reversion_lower_bound: Real `Tensor` compatible with that of
      `mean_reversion`, specifying the lower bound for the calibrated value.
      Default value: 0.001.
    mean_reversion_upper_bound: Real `Tensor` compatible with that of
      `mean_reversion`, specifying the lower bound for the calibrated value.
      Default value: 0.5.
    initial_variance_lower_bound: Real `Tensor` compatible with that of
      `initial_variance`, specifying the lower bound for the calibrated value.
      Default value: 0.0.
    initial_variance_upper_bound: Real `Tensor` compatible with that of
      `initial_variance`, specifying the lower bound for the calibrated value.
      Default value: 1.0.
    theta_lower_bound: Real `Tensor` compatible with that of `theta`,
      specifying the lower bound for the calibrated value.
      Default value: 0.0.
    theta_upper_bound: Real `Tensor` compatible with that of `theta`,
      specifying the lower bound for the calibrated value.
      Default value: 1.0.
    volvol_lower_bound: Real `Tensor` compatible with that of `volvol`,
      specifying the lower bound for the calibrated value.
      Default value: 0.0.
    volvol_upper_bound: Real `Tensor` compatible with that of `volvol`,
      specifying the lower bound for the calibrated value.
      Default value: 1.0.
    rho_lower_bound: Real `Tensor` compatible with that of `rho`, specifying the
      lower bound for the calibrated value.
      Default value: -1.0.
    rho_upper_bound: Real `Tensor` compatible with that of `rho`, specifying the
      upper bound for the calibrated value.
      Default value: 1.0.
    optimizer_fn: Optional Python callable which implements the algorithm used
      to minimize the objective function during calibration. It should have
      the following interface: result =
        optimizer_fn(value_and_gradients_function, initial_position, tolerance,
        max_iterations) `value_and_gradients_function` is a Python callable that
        accepts a point as a real `Tensor` and returns a tuple of `Tensor`s of
        real dtype containing the value of the function and its gradient at that
        point. 'initial_position' is a real `Tensor` containing the starting
        point of the optimization, 'tolerance' is a real scalar `Tensor` for
        stopping tolerance for the procedure and `max_iterations` specifies the
        maximum number of iterations.
      `optimizer_fn` should return a namedtuple containing the items: `position`
        (a tensor containing the optimal value), `converged` (a boolean
        indicating whether the optimize converged according the specified
        criteria), `failed` (a boolean indicating if the optimization resulted
        in a failure), `num_iterations` (the number of iterations used), and
        `objective_value` ( the value of the objective function at the optimal
        value). The default value for `optimizer_fn` is None and conjugate
        gradient algorithm is used.
      Default value: `None` - indicating LBFGS minimizer.
    tolerance: Scalar `Tensor` of real dtype. The absolute tolerance for
      terminating the iterations.
      Default value: 1e-6.
    maximum_iterations: Scalar positive integer `Tensor`. The maximum number of
      iterations during the optimization.
      Default value: 100.
    validate_args: Boolean value indicating whether or not to validate the shape
      and values of the input arguments, at the potential expense of performance
      degredation.
      Default value: False.
    dtype: The default dtype to use when converting values to `Tensor`s.
      Default value: `None`, which means that default dtypes inferred by
        TensorFlow are used.
    name: String. The name to give to the ops created by this function.
      Default value: `None`, which maps to the default name 'heston_calibration'

  Returns:
    A Tuple of three elements:
    * The first is a `CalibrationResult` holding the calibrated alpha, beta,
      volvol, and rho, where alpha[i] corresponds to the calibrated `alpha` of
      the i-th batch, etc.
    * A `Tensor` of optimization status for each batch element (whether the
      optimization algorithm has found the optimal point based on the specified
      convergance criteria).
    * A `Tensor` containing the number of iterations performed by the
      optimization algorithm.
  """

  if (spots is None) == (forwards is None):
    raise ValueError('Either spots or forwards must be supplied but not both.')
  if (discount_rates is not None) and (discount_factors is not None):
    raise ValueError('At most one of discount_rates and discount_factors may '
                     'be supplied')

  name = name or 'heston_calibration'
  with tf.name_scope(name):
    prices = tf.convert_to_tensor(prices, dtype=dtype, name='prices')
    dtype = dtype or prices.dtype

    # Extract batch shape
    batch_shape = prices.shape.as_list()[:-1]
    if None in batch_shape:
      batch_shape = tff.utils.get_shape(prices)[:-1]

    strikes = tf.convert_to_tensor(strikes, dtype=dtype, name='strikes')
    expiries = tf.convert_to_tensor(expiries, dtype=dtype, name='expiries')

    if discount_factors is not None:
      discount_factors = tf.convert_to_tensor(
          discount_factors, dtype=dtype, name='discount_factors')

    if discount_rates is not None:
      discount_rates = tf.convert_to_tensor(
          discount_rates, dtype=dtype, name='discount_rates')
    elif discount_factors is not None:
      discount_rates = -tf.math.log(discount_factors) / expiries
    else:
      discount_rates = tf.convert_to_tensor(
          0.0, dtype=dtype, name='discount_rates')

    if dividend_rates is None:
      dividend_rates = 0.0
    dividend_rates = tf.convert_to_tensor(
        dividend_rates, dtype=dtype, name='dividend_rates')

    if forwards is not None:
      forwards = tf.convert_to_tensor(forwards, dtype=dtype, name='forwards')
    else:
      spots = tf.convert_to_tensor(spots, dtype=dtype, name='spots')
      cost_of_carries = discount_rates - dividend_rates
      forwards = spots * tf.exp(cost_of_carries * expiries)

    is_call_options = tf.convert_to_tensor(
        is_call_options, name='is_call', dtype=tf.bool)

    if optimizer_fn is None:
      optimizer_fn = optimizer.lbfgs_minimize

    # Pre-processing mean_reversion
    mean_reversion = tf.convert_to_tensor(mean_reversion, dtype=dtype)
    mean_reversion_lower_bound = tf.convert_to_tensor(
        mean_reversion_lower_bound, dtype=dtype)
    mean_reversion_upper_bound = tf.convert_to_tensor(
        mean_reversion_upper_bound, dtype=dtype)
    mean_reversion = _assert_parameter_valid(
        validate_args,
        mean_reversion,
        lower_bound=mean_reversion_lower_bound,
        upper_bound=mean_reversion_upper_bound,
        message='`mean_reversion` is invalid!')
    # Broadcast mean_reversion to the correct batch shape
    mean_reversion = tf.broadcast_to(mean_reversion, batch_shape,
                                     name='broadcast_mean_reversion')
    initial_mean_reversion = _to_unconstrained(
        mean_reversion, mean_reversion_lower_bound, mean_reversion_upper_bound)

    # Pre-processing theta
    theta = tf.convert_to_tensor(theta, dtype=dtype)
    theta_lower_bound = tf.convert_to_tensor(theta_lower_bound, dtype=dtype)
    theta_upper_bound = tf.convert_to_tensor(theta_upper_bound, dtype=dtype)
    theta = _assert_parameter_valid(
        validate_args,
        theta,
        lower_bound=theta_lower_bound,
        upper_bound=theta_upper_bound,
        message='`theta` is invalid!')
    # Broadcast theta to the correct batch shape
    theta = tf.broadcast_to(theta, batch_shape, name='broadcast_theta')
    initial_theta = _to_unconstrained(theta, theta_lower_bound,
                                      theta_upper_bound)

    # Pre-processing volvol
    volvol_lower_bound = tf.convert_to_tensor(volvol_lower_bound, dtype=dtype)
    volvol_upper_bound = tf.convert_to_tensor(volvol_upper_bound, dtype=dtype)
    volvol = _assert_parameter_valid(
        validate_args,
        volvol,
        lower_bound=volvol_lower_bound,
        upper_bound=volvol_upper_bound,
        message='`volvol` is invalid!')
    # Broadcast volvol to the correct batch shape
    volvol = tf.broadcast_to(volvol, batch_shape, name='broadcast_volvol')
    initial_volvol = _to_unconstrained(volvol, volvol_lower_bound,
                                       volvol_upper_bound)

    initial_variance_lower_bound = tf.convert_to_tensor(
        initial_variance_lower_bound, dtype=dtype)
    initial_variance_upper_bound = tf.convert_to_tensor(
        initial_variance_upper_bound, dtype=dtype)

    initial_variance = _assert_parameter_valid(
        validate_args, initial_variance,
        lower_bound=initial_variance_lower_bound,
        upper_bound=initial_variance_upper_bound,
        message='`initial_variance` is invalid!')
    # Broadcast initial_variance to the correct batch shape
    initial_variance = tf.broadcast_to(initial_variance, batch_shape,
                                       name='broadcast_initial_variance')
    initial_variance_unconstrained = _to_unconstrained(
        initial_variance, initial_variance_lower_bound,
        initial_variance_upper_bound)

    # Pre-processing rho
    rho_lower_bound = tf.convert_to_tensor(rho_lower_bound, dtype=dtype)
    rho_upper_bound = tf.convert_to_tensor(rho_upper_bound, dtype=dtype)
    rho = _assert_parameter_valid(
        validate_args,
        rho,
        lower_bound=rho_lower_bound,
        upper_bound=rho_upper_bound,
        message='`rho` is invalid!')
    # Broadcast rho to the correct batch shape
    rho = tf.broadcast_to(rho, batch_shape, name='broadcast_rho')
    initial_rho = _to_unconstrained(rho, rho_lower_bound, rho_upper_bound)

    # Construct initial state for the optimizer
    # shape[batch_size, 5]
    initial_x = tf.stack(
        [initial_variance_unconstrained, initial_theta, initial_volvol,
         initial_mean_reversion, initial_rho], axis=-1)

    optimizer_arg_handler = _OptimizerArgHandler(
        volvol_lower_bound=volvol_lower_bound,
        volvol_upper_bound=volvol_upper_bound,
        rho_lower_bound=rho_lower_bound,
        rho_upper_bound=rho_upper_bound,
        mean_reversion_lower_bound=mean_reversion_lower_bound,
        mean_reversion_upper_bound=mean_reversion_upper_bound,
        theta_lower_bound=theta_lower_bound,
        theta_upper_bound=theta_upper_bound,
        initial_variance_lower_bound=initial_variance_lower_bound,
        initial_variance_upper_bound=initial_variance_upper_bound)

    loss_function = _get_loss_for_price_based_calibration(
        prices=prices,
        strikes=strikes,
        expiries=expiries,
        forwards=forwards,
        discount_rates=discount_rates,
        dividend_rates=dividend_rates,
        is_call_options=is_call_options,
        dtype=dtype,
        optimizer_arg_handler=optimizer_arg_handler)

    optimization_result = optimizer_fn(
        loss_function,
        initial_position=initial_x,
        tolerance=tolerance,
        max_iterations=maximum_iterations)

    calibration_parameters = optimization_result.position
    calibrated_theta = optimizer_arg_handler.get_theta(calibration_parameters)
    calibrated_volvol = optimizer_arg_handler.get_volvol(calibration_parameters)
    calibrated_rho = optimizer_arg_handler.get_rho(calibration_parameters)
    calibrated_initial_variance = optimizer_arg_handler.get_initial_variance(
        calibration_parameters)
    calibrated_mean_reversion = optimizer_arg_handler.get_mean_reversion(
        calibration_parameters)

    return (CalibrationResult(mean_reversion=calibrated_mean_reversion,
                              volvol=calibrated_volvol,
                              rho=calibrated_rho,
                              theta=calibrated_theta,
                              initial_variance=calibrated_initial_variance),
            optimization_result.converged,
            optimization_result.num_iterations)

def _get_loss_for_price_based_calibration(
    *, prices, strikes, expiries, forwards, is_call_options,
    discount_rates, dividend_rates, optimizer_arg_handler, dtype):
  """Creates a loss function to be used in volatility-based calibration."""

  def _price_transform(x):
    return tf.math.log1p(x)

  scaled_target_values = _price_transform(prices)

  @make_val_and_grad_fn
  def loss_function(x):
    """Loss function for the price-based optimization."""
    candidate_initial_variance = optimizer_arg_handler.get_initial_variance(x)
    candidate_theta = optimizer_arg_handler.get_theta(x)
    candidate_volvol = optimizer_arg_handler.get_volvol(x)
    candidate_rho = optimizer_arg_handler.get_rho(x)
    candidate_mean_reversion = optimizer_arg_handler.get_mean_reversion(x)

    values = approximations.european_option_price(
        strikes=strikes,
        expiries=expiries,
        forwards=forwards,
        is_call_options=is_call_options,
        variances=tf.expand_dims(candidate_initial_variance, axis=-1),
        mean_reversion=tf.expand_dims(candidate_mean_reversion, axis=-1),
        volvol=tf.expand_dims(candidate_volvol, axis=-1),
        rho=tf.expand_dims(candidate_rho, axis=-1),
        theta=tf.expand_dims(candidate_theta, axis=-1),
        discount_rates=discount_rates,
        dividend_rates=dividend_rates,
        dtype=dtype)

    scaled_values = _price_transform(values)
    return tf.math.reduce_mean(
        (scaled_values - scaled_target_values)**2, axis=-1)

  return loss_function

@utils.dataclass
class _OptimizerArgHandler:
  """Handles the packing/transformation of estimated parameters."""
  theta_lower_bound: types.RealTensor
  theta_upper_bound: types.RealTensor
  volvol_lower_bound: types.RealTensor
  volvol_upper_bound: types.RealTensor
  rho_lower_bound: types.RealTensor
  rho_upper_bound: types.RealTensor
  mean_reversion_lower_bound: types.RealTensor
  mean_reversion_upper_bound: types.RealTensor
  initial_variance_lower_bound: types.RealTensor
  initial_variance_upper_bound: types.RealTensor

  def get_initial_variance(
      self, packed_optimizer_args: types.RealTensor) -> types.RealTensor:
    """Unpack and return the rho parameter."""
    initial_variance = packed_optimizer_args[..., 0]
    return _to_constrained(
        initial_variance,
        self.initial_variance_lower_bound, self.initial_variance_upper_bound)

  def get_theta(self,
                packed_optimizer_args: types.RealTensor) -> types.RealTensor:
    """Unpack and return the volvol parameter."""
    theta = packed_optimizer_args[..., 1]
    return _to_constrained(
        theta,
        self.theta_lower_bound, self.theta_upper_bound)

  def get_volvol(self,
                 packed_optimizer_args: types.RealTensor) -> types.RealTensor:
    """Unpack and return the volvol parameter."""
    volvol = packed_optimizer_args[..., 2]
    return _to_constrained(
        volvol,
        self.volvol_lower_bound, self.volvol_upper_bound)

  def get_mean_reversion(
      self, packed_optimizer_args: types.RealTensor) -> types.RealTensor:
    """Unpack and return the mean_reversion parameter."""
    mean_reversion = packed_optimizer_args[..., 3]
    return _to_constrained(mean_reversion, self.mean_reversion_lower_bound,
                           self.mean_reversion_upper_bound)

  def get_rho(self,
              packed_optimizer_args: types.RealTensor) -> types.RealTensor:
    """Unpack and return the rho parameter."""
    rho = packed_optimizer_args[..., -1]
    return _to_constrained(
        rho,
        self.rho_lower_bound, self.rho_upper_bound)

def _scale(x, lb, ub):
  """Scales the values to be normalized to [lb, ub]."""
  return (x - lb) / (ub - lb)

def _to_unconstrained(x, lb, ub):
  """Scale and apply inverse-sigmoid."""
  x = _scale(x, lb, ub)
  return -tf.math.log((1.0 - x) / x)

def _to_constrained(x, lb, ub):
  """Sigmoid and unscale."""
  x = 1.0 / (1.0 + tf.math.exp(-x))
  return x * (ub - lb) + lb

def _assert_parameter_valid(validate_args, x, lower_bound, upper_bound,
                            message):
  """Helper to check that the input parameter is valid."""
  if validate_args:
    with tf.control_dependencies([
        tf.debugging.assert_greater_equal(x, lower_bound, message=message),
        tf.debugging.assert_less_equal(x, upper_bound, message=message),
    ]):
      return tf.identity(x)
  else:
    return x


import tweepy

consumer_key = 'xxxxxxxxxxx'
consumer_secret = 'xxxxxxxxxx'
access_token = 'xxxxxxxxxx'
access_token_secret = 'xxxxxxx'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth)

FILE_NAME = 'last.txt'
toReply = 'elonmusk'

mystring_us = f""" Testing these numbers
    Number 1
    Number 2
    Number 3"""

def read_last_seen(FILE_NAME):
    file_read = open(FILE_NAME, 'r')
    last_seen_id = int(file_read.read().strip())
    file_read.close()
    return last_seen_id

def store_last_seen(FILE_NAME, last_seen_id):
    file_write = open(FILE_NAME, 'w')
    file_write.write(str(last_seen_id))
    file_write.close()
    return

def reply():
    tweets = api.user_timeline(read_last_seen(FILE_NAME), screen_name = toReply, count=1)
    for tweet in tweets:
            api.update_status("@" + toReply + mystring_us, in_reply_to_status_id = tweet.id)

    store_last_seen(FILE_NAME, tweet.id)
    api.create_favorite(tweet.id) # Like the tweet with mentions

while True:
    reply()
    time.sleep(30)


import codecs
import csv
import os
import re
import sys
import urllib.request
import logging
from typing import Type, Tuple

from baseParser import BaseParser
from monstatparser import MonStatParser
from property import Property
from property import PropertyParse
from skills import SkillParser
from utils import isdebugging, getFileInsensitive
from urllib.error import URLError

logger = logging.getLogger(__name__)

getSkillId = (lambda par, skills: [skill for skill in skills if skill.id == par])
getSkillSkill = (lambda par, skills: [skill for skill in skills if skill.skill.lower() == par.lower()])
getMonster = (lambda x, monsters: [mon for mon in monsters if mon.id == x.par])
class Parser:
	skills: [SkillParser] = []
	definedProperties: [PropertyParse] = []
	monsters: [MonStatParser] = []
	version: str = ""

	def __init__(self, modName: str, version: str):
		self.modName = modName
		self.version = version
		self.skills = self.read(SkillParser)
		self.definedProperties = self.read(PropertyParse)
		self.monsters = self.read(MonStatParser)

	def read(self, cls: Type[BaseParser], dl=False) -> [BaseParser]:
		return BaseParserCreator.read(cls, self, self.modName, self.version, dl)

	def findInCommon(self, property: Property) -> str:
		classes = {}
		skillBeg = int(property.min)
		skillEnd = int(property.max)
		for skillI in range(skillBeg, skillEnd):
			sk = self.getSk(str(skillI))[0]
			if sk.char not in classes and len(classes) == 0:
				classes[sk.char] = True
			elif sk.char in classes and len(classes) == 1:
				pass
			else:
				# TODO
				return "Random Level {} skill ({} possibilities)".format(property.par, skillEnd-skillBeg)
		cls = list(classes.keys())[0]
		return "Random Level {} {} Skill".format(property.par, self.charAbbrvToFull(cls))

	def charAbbrvToFull(self, abbv: str) -> str:
		if abbv == "pal":
			return "Paladin"
		elif abbv == "ass":
			return "Assassin"
		elif abbv == "sor":
			return "Sorceress"
		elif abbv == "ama":
			return "Amazon"
		elif abbv == "dru":
			return "Druid"
		elif abbv == "bar":
			return "Barbarian"
		elif abbv == "nec":
			return "Necromancer"
		else:
			logger.error("Unknown char abbv %s" % abbv)
			sys.exit()

	def getSk(self, skillId) -> SkillParser:
		skill = getSkillId(skillId, self.skills)
		if not skill:
			skill = getSkillSkill(skillId, self.skills)
		if not skill:
			logger.error("Unable to get skill for property {}".format(skillId))
			sys.exit()
		return skill

	def parseSkillOnEvent(self, property) -> str:
		global skills
		if property.prop == "skill-rand":
			return self.findInCommon(property)

		skill = self.getSk(property.par)

		if property.prop == "skill":
			if(len(skill)):
				skill = skill[0]
				#TODO
				return "+{} to {} ({} Only)".format(generateRange(property), skill.skill, self.charAbbrvToFull(skill.char))
			else:
				return "+{} to {} (Class Only)".format(generateRange(property), property.par)
		elif property.prop == "att-skill":
			return "{}% Chance To Cast Level {} {} On Attack".format(property.min, property.max, property.par)
		elif property.prop == "oskill":
			if len(skill):
				skill = skill[0]
			return "+{} to {}".format(generateRange(property), skill.skill)
			#return "Level {} {} ({} Charges)".format(int(int(property.min)/3), skill.skill, property.max)
		elif property.prop == "kill-skill":
			when = "Kill An Enemy"
		elif property.prop == "levelup-skill":
			when = "Level-Up"
		elif property.prop == "death-skill":
			when = "Die"
		else:
			logger.error("Unknown skill property {}".format(property.prop))
			sys.exit()
		return "{}% Chance To Cast Level {} {} When you {}".format(property.min, property.max, property.par, when)

	def parseReanimate(self, property) -> str:
		if property.prop == "reanimate":
			monster = getMonster(property, self.monsters)[0]
			return "{} Reanimate As: {}".format(generateRange(property, "%"), monster)
		else:
			logger.error("Unknown reanimate {}".format(property.prop))
			sys.exit()

	def parseCharges(self, property) -> str:
		skills = self.getSk(property.par)
		if(len(skills) != 1):
			logger.error("Got %s skills" % len(skills))
			sys.exit()
		else:
			return "Level %s %s (%s Charges)" % (property.max, skills[0].skill, property.min)

	def validateProperty(self, property: Property) -> str:
		x = property
		matched = 0
		if x.prop.startswith("*"):
			if isdebugging():
				logger.info("Got star prop " + x.prop)
			x.prop = x.prop[1:]
		for prop in self.definedProperties:
			if prop.code == x.prop.lower():
				matched += 1
		if matched != 1:
			logger.error("Property {} is not valid".format(property.prop))
			sys.exit()

	def parse(self, properties: [Property]):
		poisonDmgProps: [Tuple[Property, int]] = []
		coldDmgProps: [Tuple[Property, int]] = []
		for k, property in enumerate(properties):
			parsed = None
			minMax = re.compile("[A-Za-z]+\-(min|max)")
			dmg = re.compile("dmg\-[a-z]+")
			pierce = re.compile('pierce\-[A-Za-z]+')
			res = re.compile('res\-[A-Za-z]+')
			resLen = re.compile('res\-[A-Za-z]+\-len')
			abs = re.compile('abs\-[A-Za-z]+')
			red = re.compile('red\-[A-Za-z]+')
			extra = re.compile('extra\-[A-Za-z]+')
			star = re.compile('\*.+')
			dot = re.compile("(pois|cold)-len")
			self.validateProperty(property)
			if property.prop is None:
				logger.error("cant")
			elif property.prop == "dmg":
				parsed = "{} Damage".format(generateRange(property))
			elif property.prop == "dmg%":
				parsed = "{} Enhanced Damage".format(generateRange(property, "%"))
			elif property.prop == "dmg%/lvl":
				parsed = "{} Enhanced Damage (Based on Character Level)".format(generateRange(property, "%"))
			elif property.prop == "stamdrain":
				parsed = "%s Slower stamina drain" % generateRange(property, "%")
			elif property.prop == "stam":
				parsed = "+%s to Maximum Stamina" % generateRange(property)
			elif property.prop == "stam/lvl":
				property.mult(1/8)
				parsed = "+%s to Maximum Stamina (Based on Character Level)" % generateRange(property)
			elif property.prop == "regen-stam":
				parsed ="Heal Stamina Plus %s" % generateRange(property, "%")
			elif property.prop == "regen-stam/lvl":
				property.mult(1/8)
				parsed = "Heal Stamina Plus %s (Based on Character Level)" % generateRange(property)
			elif re.match(dot, property.prop):
				property.mult(1/25)
				if property.prop.startswith("cold"):
					coldDmgProps.append((property, k))
				elif property.prop.startswith("pois"):
					poisonDmgProps.append((property, k))
				parsed = "This should not be seen"
			elif re.match(star, property.prop):

				logger.error("Got star property: {}".format(property.prop))
			elif re.match(red, property.prop):
				ret = "{}Damage Reduced By {}"
				if property.prop == "red-mag":
					type = "Magic "
				else:
					type = ""
				parsed = ret.format(type, generateRange(property))
			elif re.match(minMax, property.prop):
				type=""
				if(property.prop.startswith("cold")):
					coldDmgProps.append((property, k))
				elif(property.prop.startswith("pois")):
					poisonDmgProps.append((property, k))
				elif(property.prop.endswith("max")):
					type = "Maximum"
				elif(property.prop.endswith("min")):
					type = "Minimum"
				parsed = "+{} To {} Damage".format(generateRange(property), type)
			elif re.match(abs, property.prop):
				type = tagToString(property.prop)
				parsed = "{} Absorb +{}".format(type, generateRange(property, "%"))
			elif re.match(resLen, property.prop):
				if property.prop == "res-pois-len":
					parsed = "Poison Length Reduced by {}".format(generateRange(property, "%"))
				else:
					logger.error("Unknown reduce length {}".format(property.prop))
					sys.exit()
			elif re.match(res, property.prop):
				if property.prop == "res-all":
					parsed = "All Resistances +{}".format(generateRange(property))
				else:
					type = ""
					if property.prop == "res-all-max":
						type = "Maximum Poison/Fire/Cold/Lightning"
					elif "max" in property.prop:
						type = "Maximum "
					else:
						type += tagToString(property.prop)
					parsed = "{} Resistance +{}".format(type, generateRange(property, "%"))
					if "/lvl" in property.prop:
						parsed += " (Based on Character Level)"
			elif re.match(pierce, property.prop):
				type = tagToString(property.prop)
				parsed = "-{} To Enemy {} Resistance".format(generateRange(property, "%"), type)
			elif re.match(dmg, property.prop):
				if(property.prop == "dmg-undead"):
					parsed = "+{} Damage To Undead".format(generateRange(property, "%"))
				elif(property.prop == "dmg-und/lvl"):
					parsed = "+{} Damage To Undead (Based on Character Level)".format(generateRange(property, "%"))
				elif(property.prop == "dmg-demon"):
					parsed = "+{} Damage To Demons".format(generateRange(property, "%"))
				elif(property.prop == "dmg-dem/lvl"):
					parsed = "+{} Damage To Demons (Based On Character Level)".format(generateRange(property, "%"))
				elif property.prop == "dmg-to-mana":
					parsed = "{} Damage Taken Goes to Mana".format(generateRange(property, "%"))
				else:
					if (property.prop == "dmg-ac"):
						parsed = "{} To Monster Defense Per Hit".format(generateRange(property))
					elif property.prop == "dmg-elem":
						parsed = "+{} Fire/Cold/Lightning Damage".format(generateRange(property))
					else:
						ret = "Adds " + generateRange(property) + " "
						if (property.prop == "dmg-ltng"):
							ret += "Lightning"
						elif (property.prop == "dmg-cold"):
							ret += "Cold"
						elif (property.prop == "dmg-fire"):
							ret += "Fire"
						elif (property.prop == "dmg-pois"):
							ret += "Poison"
						elif (property.prop == "dmg-mag"):
							ret += "Magic"
						elif (property.prop == "dmg-norm"):
							pass
						else:
							logger.error("Unknown add damage: {}".format(property.prop))
							sys.exit()
						parsed = ret  + " Damage"
			elif re.match(extra, property.prop):
				if property.prop == "extra-pois":
					type = "Poison"
				elif property.prop == "extra-cold":
					type = "Cold"
				elif property.prop == "extra-fire":
					type = "Fire"
				elif property.prop == "extra-ltng":
					type = "Lightning"
				parsed = "+{} To {} Skill Damage".format(generateRange(property), type)

			elif property.prop in ["skill", "kill-skill", "levelup-skill", "death-skill", "att-skill", "oskill", "skill-rand"]:
				parsed = self.parseSkillOnEvent(property)
			elif property.prop == "randclassskill":
				# TODO
				parsed = "+3 To All Character Skills (Varies)"
			elif property.prop == "reanimate":
				self.parseReanimate(property)
			elif property.prop == "fireskill":
				parsed = "+{} To Fire Skills".format(generateRange(property))
			elif property.prop == "skilltab":
				# TODO
				parsed = "+{} To Skill Tab {}".format(generateRange(property), property.par)
			elif property.prop in ["sor", 'nec', 'bar', 'pal', 'ass', 'dru', 'ama']:
				parsed = "+%s To All %s Skills" % (generateRange(property), self.charAbbrvToFull(property.prop))
			elif property.prop == "charged":
				parsed = self.parseCharges(property)
			elif property.prop == "res-all":
				parsed = "All Resistances +{}".format(generateRange(property))
			elif property.prop == "aura":
				parsed = "Level {} {} Aura When Equipped".format(generateRange(property), property.par)
			elif property.prop in ["swing1","swing2","swing3"]:
				parsed = "{} Increased Attack Speed".format(generateRange(property, "%"))
			elif property.prop == "all-stats":
				parsed = "+{} To All Attributes".format(generateRange(property))
			elif property.prop.lower() == "light":
				parsed = "+{} To Light Radius".format(generateRange(property))
			elif property.prop == "stupidity":
				parsed = "Hit Blinds Target"
			elif property.prop == "allskills":
				parsed = "+{} To All Skills".format(generateRange(property))
			elif property.prop == "":
				parsed = ""
			elif property.prop == "balance1":
				parsed = "+{} Faster Hit Recovery".format(generateRange(property, "%"))
			elif property.prop == "balance2":
				parsed = "+{} Faster Hit Recovery".format(generateRange(property, "%"))
			elif property.prop == "balance3":
				parsed = "+{} Faster Hit Recovery".format(generateRange(property, "%"))
			elif property.prop == "hit-skill":
				parsed = "{}% Chance To Cast Level {} {} On Striking".format(min, max, property.par)
			elif property.prop == "gethit-skill":
				parsed = "{}% Chance To Cast Level {} {} When Struck".format(min, max, property.par)
			elif property.prop == "ethereal":
				parsed = "Ethereal"
			elif property.prop == "indestruct":
				parsed = "Indestructible"
			elif property.prop == "dur":
				parsed = "+%s Increased Durability" % generateRange(property)
			elif property.prop == "rep-dur":
				parsed = "Repairs Durability {} in 4 Seconds".format(property.par)
			elif property.prop == "knock":
				parsed = "Knockback"
			elif property.prop == "mana/lvl":
				val = int(property.par) / 8
				parsed = "+{}-{} To Mana (Based On Character Level)".format(val, val * 111)
			elif property.prop == "regen-mana":
				parsed = "Regenerate Mana {}".format(generateRange(property, "%"))
			elif property.prop == "mana%":
				parsed = "Increase Maximum Mana {}".format(generateRange(property, "%"))
			elif property.prop in ["cast1", "cast2", "cast3"]:
				parsed = "{} Faster Cast Rate".format(generateRange(property, "%"))
			elif property.prop == "hp" or property.prop == "mana":
				if property.prop == "hp":
					type = "Life"
				else:
					type = "Mana"
				parsed = "+{} To {}".format(generateRange(property), type)
			elif property.prop == "hp/lvl":
				parsed = "+{} To Life (Based on Character Level)".format(generateRange(property))
			elif property.prop == "explosivearrow":
				parsed = "FIres Explosive Arrows or Bolts ({})".format(generateRange(property))
			elif property.prop == "cheap":
				parsed = "Reduces All Vendor Prices {}".format(generateRange(property, "%"))
			elif property.prop == "mag%":
				parsed = "{} Better Chance of Getting Magic Items".format(generateRange(property, "%"))
			elif property.prop == "mag%/lvl":
				parsed = "+{} Better Chance of Getting Magical Items (Based On Character Level)".format(
					generateRange(property))
			elif property.prop == "red-dmg%":
				parsed = "Damage Reduced By {}".format(generateRange(property, "%"))
			elif property.prop == "str":
				parsed = "+{} to Strength".format(generateRange(property))
			elif property.prop == "str/lvl":
				parsed = "+{} To Strength (Based On Character Level)".format(property)
			elif property.prop == "dex":
				parsed = "+{} to Dexterity".format(generateRange(property))
			elif property.prop == "dex/lvl":
				parsed = "+{} To Dexterity (Based On Character Level)".format(property)
			elif property.prop == "enr":
				parsed = "+{} to Energy".format(generateRange(property))
			elif property.prop == "enr/lvl":
				parsed = "+{} To Energy (Based On Character Level)".format(property)
			elif property.prop == "vit":
				parsed = "+{} to Vitality".format(generateRange(property))
			elif property.prop == "vit/lvl":
				parsed = "vit/lvl"
			elif property.prop == "slow":
				parsed = "Slows Target By {}".format(generateRange(property, "%"))
			elif property.prop == "ac":
				parsed = "+{} Defense".format(generateRange(property))
			elif property.prop == "ac/lvl":
				if(property.min == "" or property.max == ""):
					property.min = property.par
					property.max = property.par
				property.min=str(int(int(property.min)/8))
				property.max=str(int(int(property.max)/8))
				parsed = "+{} To Defense (Based On Character Level)".format(generateRange(property))
			elif property.prop == "ac%":
				parsed = "+{} Enhanced Defense".format(generateRange(property, "%"))
			elif property.prop == "ac-miss":
				parsed = "+{} Defense Vs. Missle".format(generateRange(property))
			elif property.prop == "ignore-ac":
				parsed = "Ignores Target's Defense"
			elif property.prop == "reduce-ac":
				parsed = "-{} Target Defense".format(generateRange(property, "%"))
			elif property.prop == "hp%":
				parsed = "Increase Maximum Life {}".format(generateRange(property, "%"))
			elif property.prop == "att%":
				parsed = "{} Bonus To Attack Rating".format(generateRange(property, "%"))
			elif property.prop == "att":
				parsed = "+{} To Attack Rating".format(generateRange(property))
			elif property.prop == "att-undead":
				parsed = "+{} To Attack Rating Against Undead".format(generateRange(property))
			elif property.prop == "att-und/lvl":
				parsed = "+{} To Attack Rating Against Undead (Based on Character Level)".format(
					generateRange(property))
			elif property.prop == "att-demon":
				parsed = "+{} Damage to Demons".format(generateRange(property, "%"))
			elif property.prop == "att-dem/lvl":
				parsed = "+{} Damage to Demons (Based on Character Level)".format(generateRange(property, "%"))
			elif property.prop == "demon-heal":
				parsed = "+{} Life After Each Demon Kill".format(generateRange(property))
			elif property.prop == "noheal":
				parsed = "Prevent Monster Heal"
			elif property.prop == "heal-kill":
				parsed = "+{} Life After Each Kill".format(generateRange(property))
			elif property.prop == "ease":
				parsed = "Requirements {}".format(generateRange(property, "%"))
			elif property.prop == "nofreeze":
				parsed = "Cannot be Frozen"
			elif property.prop == "freeze":
				parsed = "Freezes Target +{}".format(generateRange(property))
			elif property.prop == "half-freeze":
				parsed = "Half Freeze Duration"
			elif property.prop == "block1":
				parsed = "+{} Faster Block Rate".format(generateRange(property, "%"))
			elif property.prop == "block2":
				parsed = "+{} Faster Block Rate".format(generateRange(property, "%"))
			elif property.prop == "block3":
				parsed = "+{} Faster Block Rate".format(generateRange(property, "%"))
			elif property.prop == "block":
				parsed = "{} Increased Chance of Blocking".format(generateRange(property, "%"))
			elif property.prop == "howl":
				parsed = "Hit Causes Monster to Flee {}".format(generateRange(property, "%"))
			elif property.prop == "gold%":
				parsed = "{} Extra Gold From Monsters".format(generateRange(property, "%"))
			elif property.prop == "gold%/lvl":
				parsed = "{} Extra Gold From Monsters (Based on Character Level)".format(generateRange(property, "%"))
			elif property.prop == "manasteal":
				parsed = "{} Mana Stolen Per Hit".format(generateRange(property, "%"))
			elif property.prop == "lifesteal":
				parsed = "{} Life Stolen Per Hit".format(generateRange(property, "%"))
			elif property.prop == "deadly":
				parsed = "{} Deadly Strike".format(generateRange(property, "%"))
			elif property.prop == "deadly/lvl":
				parsed = "+{} Deadly Strike (Based on Character Level)".format(generateRange(property))
			elif property.prop == "regen":
				parsed = "Replenish Life {}".format(generateRange(property))
			elif property.prop == "regen-mana":
				parsed = "Regenerate Mana {}".format(generateRange(property, "%"))
			elif property.prop == "mana-kill":
				parsed = "+{} to Mana After Each Kill".format(generateRange(property))
			elif property.prop == "openwounds":
				parsed = "{} Chance of Open Wounds".format(generateRange(property, "%"))
			elif property.prop == "crush":
				parsed = "{} Chance of Crushing Blow".format(generateRange(property, "%"))
			elif property.prop == "light-thorns":
				parsed = "Attacker Takes Lightning Damage of %s" % generateRange(property)
			elif property.prop == "thorns":
				parsed = "Attacker Takes Damage of {}".format(generateRange(property))
			elif property.prop == "thorns/lvl":
				property.mult(1/8)
				parsed = "Attacker Takes Damage of %s (Based on Character Level)" % generateRange(property)
			elif property.prop == "rip":
				parsed = "Slain Monsters Rest in Peace"
			elif property.prop == "move2":
				parsed = "{} Faster Run Walk".format(generateRange(property, "%"))
			elif property.prop == "move3":
				parsed = "{} Faster Run Walk".format(generateRange(property, "%"))
			elif property.prop == "att/lvl":
				property.mult(.5)
				parsed = "{} to Attack Rating (Based on Chracter Level)".format(generateRange(property))
			elif property.prop == "dmg/lvl":
				property.mult(1/8)
				parsed = "{} to Maximum Damage (Based on Character Level)".format(generateRange(property))
			elif property.prop == "addxp":
				parsed = "{} to Experience Gained".format(generateRange(property, "%"))
			elif property.prop == "rep-quant":
				y = 100/int(property.par)
				z = y/int(property.par)
				property.mult(z)
				parsed = "Replenishes Quantity (1 in {} sec)".format(generateRange(property))
			elif property.prop  == "state":
				# TODO
				parsed = "Monster Set {} TODO".format(property.par.split("-")[-1])
			elif property.prop == "pierce":
				# TODO
				parsed = "Attacks Pierce target TODO"
			elif property.prop == "stack":
				parsed = "Increased stack size {}".format(generateRange(property))
			elif property.prop == "splash":
				# TODO
				if property.par == "proc_SplashDamage":
					parsed = "Attacks Spalsh TODO"
			elif property.prop == "magicarrow":
				parsed = "Fires Level {} Magic Arrows/Bolts".format(generateRange(property))
			elif property.prop == "ac-hth":
				parsed = "+{} Defense vs. Melee".format(generateRange(property))
			elif property.prop == "sock":
				parsed = "Socketed (%s)" % generateRange(property)
			elif property.prop == "bloody":
				# TODO not visible
				parsed = "Extra Blood {}".format(generateRange(property))
			else:
				logger.error("Unable to parse " + str(property))
				sys.exit()
			if parsed == "":
				logger.error("Property {} did not get parsed!".format(property.prop))
				sys.exit()
			property.parsed = parsed
			properties[k] = property

		byType = (lambda typ, props: [x for x in props if typ in x[0].prop])
		validProp = re.compile("(pois|cold)-(len|min|max)")
		def mergeDmgProps(props: [Property], typ: str):
			if len(props) == 3 and len([x for x in props if re.match(validProp, x[0].prop)]) == 3:
				lenProp = byType("len", props)[0]
				minProp = byType("min", props)[0]
				maxProp = byType("max", props)[0]
				properties[lenProp[1]].parsed = "Adds {}-{} {} Damage, Duration: {} seconds".format(
					generateRange(minProp[0]), generateRange(maxProp[0]), typ, generateRange(lenProp[0]))
				properties.remove(minProp[0])
				properties.remove(maxProp[0])
			elif len(props) != 0:
				logger.error("Mismatched number of %s properties %s" % (typ, len(props)))

		mergeDmgProps(poisonDmgProps, "Poison")
		mergeDmgProps(coldDmgProps, "Cold")

		return properties

def tagToString(prop) -> str:
	if "ltng" in prop:
		return "Lightning"
	elif "cold" in prop:
		return "Cold"
	elif "fire" in prop:
		return "Fire"
	elif "pois" in prop:
		return "Poison"
	elif "mag" in prop:
		return "Magic"
	else:
		logger.error("Unknown prop " + prop)
		sys.exit()

def generateRange(prop, append=""):
	if prop.min == "" and prop.max == "" and prop.par != "":
		return "{}{}".format(prop.par, append)
	else:
		if prop.min == prop.max:
			return prop.min + append
		return "{}-{}{}".format(prop.min, prop.max, append)

class BaseParserCreator:
	@staticmethod
	def read(cls: Type[BaseParser], parser: Parser, modName: str, version: str, dl=False) -> [BaseParser]:
		items = []
		dir = os.path.join(os.getcwd(), "versions", modName, version)
		path = os.path.join(dir, getFileInsensitive(dir, cls.getName()+".txt"))
		try:
			with codecs.open(path, 'r', encoding='utf-8', errors='ignore') as file:
				reader = csv.DictReader(file, delimiter='\t')
				for line in reader:
					item = cls(dict(line))
					item.parse(parser)
					if item.verify():
						item.setMod(modName, version)
						items.append(item)
						if dl:
							BaseParserCreator.Download(item)
					else:
						try:
							logger.debug("bad [%s] (%s) %s" % (cls.getName(), item.name, item))
						except:
							logger.debug("bad [%s] %s" % (cls.getName(), str(item)))
		except FileNotFoundError:
			logger.error("Unable to find %s" % path)
			sys.exit()

		return items

	@staticmethod
	def Download(cls: Type[BaseParser]):
		import socket
		dfto = socket.getdefaulttimeout()
		socket.setdefaulttimeout(1)
		name = cls.fileName()
		url = "http://classic.battle.net/images/battle/diablo2exp/images/{}".format(name) + "/{}.gif"
		gemName = cls.Name().replace(" ", "")
		nurl = url.format(gemName)
		try:
			try:
				os.mkdir(os.path.join('Images', name))
			except:
				pass
			imgName = os.path.join("Images", name, cls.FileName())
			urllib.request.urlretrieve(nurl, imgName)
			cls.img = imgName
		except URLError as e:
			logger.error("timedout getting %s" % nurl)
		except Exception:
			logger.error("Unable to parse {} url {}".format(gemName, nurl))
		socket.setdefaulttimeout(dfto)


from scipy import signal
import numpy as np
import numpy.matlib
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

# define the functions we would like to predict:
num_of_functions = 3
size = 4
W = 4 * (np.random.random((size, size)) - 0.5)
y = {
    0: lambda x: np.sum(np.dot(x, W), axis=1),
    1: lambda x: (x[:, 0] * W[0, 0] + x[:, 1] * W[1, 1]) * (
        x[:, 2] * W[2, 2] + x[:, 1] * W[3, 3]),
    2: lambda x: np.log(np.sum(np.exp(np.dot(x, W)), axis=1))
}

def mean_square_loss(p,y):
    '''
    calculates square loss
    :param p: the prediction (wx, batch of x)
    :param y: the training/test labels
    :return: mean of least square loss
    '''
    return np.mean((p - y) ** 2)

def mean_square_loss_reg(p,y,lamb,w):
    '''
    calculates square loss
    :param p: the prediction (wx, batch of x)
    :param y: the training/test labels
    :param lamb: the regularization parameter
    :param w: the weights matrix
    :return: mean of least square loss
    '''
    return np.mean((p - y) ** 2 + (lamb / 2)) * (w.dot(w))

def learn_linear(X, Y, batch_size, lamb, iterations, learning_rate):
    """
    learn a linear model for the given functions.
    :param X: the training and test input
    :param Y: the training and test labels
    :param batch_size: the batch size
    :param lamb: the regularization parameter
    :param iterations: the number of iterations
    :param learning_rate: the learning rate
    :return: a tuple of (w, training_loss, test_loss):
            w: the weights of the linear model
            training_loss: the training loss at each iteration
            test loss: the test loss at each iteration
    """

    training_loss = {func_id: [] for func_id in range(num_of_functions)}
    test_loss = {func_id: [] for func_id in range(num_of_functions)}
    w = {func_id: np.zeros(size) for func_id in range(num_of_functions)}

    for func_id in range(num_of_functions):
        for _ in range(iterations):

            # draw a random batch:
            idx = np.random.choice(len(Y[func_id]['train']), batch_size)
            x, y = X['train'][idx,:], Y[func_id]['train'][idx]

            # calculate the loss and derivatives:
            p = np.dot(x, w[func_id])
            loss = mean_square_loss_reg(p,y,lamb,w[func_id])
            test_p =  X['test'].dot(w[func_id])

            iteration_test_loss = mean_square_loss_reg(test_p,Y[func_id]['test'],lamb,w[func_id])
            # dl_dw = np.zeros(size)
            # for i in range(size):
            #
            #     dl_dw[i] = np.mean((2*((p - y)*x[:,i])))+lamb*w[func_id][i]
            #
            # update the model and record the loss:
            dl_dw = np.mean(np.multiply(x.T, 2 * (p - y)).T, axis=0) + (lamb * w[func_id])

            w[func_id] -= learning_rate * dl_dw
            training_loss[func_id].append(loss)
            test_loss[func_id].append(iteration_test_loss)

    return w, training_loss, test_loss

def test_linear_leaner(X,Y,batch_size, lamb, iterations, learning_rate ):
    '''

    Test a linear model for the given functions.
    :param X: the training and test input
    :param Y: the training and test labels
    :param batch_size: the batch size
    :param lamb: the regularization parameter
    :param iterations: the number of iterations
    :param learning_rate: the learning rate
    '''
    w,training_loss,test_loss = learn_linear(X, Y,batch_size, lamb, iterations, learning_rate )
    plt.figure(figsize=(10, 10))

    for i in range(num_of_functions):

        plt.plot(list(range(iterations)),training_loss[i],label = 'func%d'%i)

    plt.title('Test Loss for batch size %d and lambda %f' %(batch_size,lamb))
    plt.xlabel('Iterations')
    plt.ylabel('Loss')
    plt.legend()
    plt.savefig('Test_loss_%d_%f.png' %(batch_size,lamb))
    plt.show()

def forward(cnn_model, x):
    """
    Given the CNN model, fill up a dictionary with the forward pass values.
    :param cnn_model: the model
    :param x: the input of the CNN
    :return: a dictionary with the forward pass values
    """

    fwd = {}
    fwd['x'] = x
    fwd['o1'] = np.maximum(np.zeros(np.shape(x)), signal.convolve2d(x, [np.array(cnn_model['w1'])], mode='same'))
    fwd['o2'] = np.maximum(np.zeros(np.shape(x)), signal.convolve2d(x, [cnn_model['w2']], mode='same'))
    fwd['m1'] = np.zeros((size(x)))
    fwd['m'] = [np.maximum(fwd['o1'][:,:2]),np.maximum(fwd['o1'][:,3:]), np.maximum(fwd['o2'][:,:2]), np.maximum(fwd['o2'][:,3:])]
    fwd['m_argmax'] = [np.argmax(fwd['o1'][0:2]), np.argmax(fwd['o1'][3:]),np.argmax(fwd['o2'][0:2]), np.argmax(fwd['o2'][3:])]
    fwd['p'] = cnn_model['u'].dot(fwd['m'].T)

    return fwd

def backprop(model, y, fwd, batch_size):
    """
    given the forward pass values and the labels, calculate the derivatives
    using the back propagation algorithm.
    :param model: the model
    :param y: the labels
    :param fwd: the forward pass values
    :param batch_size: the batch size
    :return: a tuple of (dl_dw1, dl_dw2, dl_du)
            dl_dw1: the derivative of the w1 vector
            dl_dw2: the derivative of the w2 vector
            dl_du: the derivative of the u vector
    """
    p_w1 = np.dot(fwd['x'],model['w1'])
    p_w2 = np.dot(fwd['x'],model['w2'])

    u_1_3 = [fwd['u'][1],fwd['u'][3]]
    m_by_o = np.array([0,1,0,0],[0,0,])

    dl_dw1 = np.mean(np.multiply(x, 2 * (p_w1 - y)).T, axis=0)

    dl_dw2 = np.mean(np.multiply(fwd['x'], 2 * (p_w2 - y)).T, axis=0)

    dl_du =  np.mean(np.multiply(model['m'].T, 2 * (fwd['p'] - y)).T, axis=0)

    return (dl_dw1, dl_dw2, dl_du)

def learn_cnn(X, Y, batch_size, lamb, iterations, learning_rate):
    """
    learn a cnn model for the given functions.
    :param X: the training and test input
    :param Y: the training and test labels
    :param batch_size: the batch size
    :param lamb: the regularization parameter
    :param iterations: the number of iterations
    :param learning_rate: the learning rate
    :return: a tuple of (models, training_loss, test_loss):
            models: a model for every function (a dictionary for the parameters)
            training_loss: the training loss at each iteration
            test loss: the test loss at each iteration
    """

    training_loss = {func_id: [] for func_id in range(num_of_functions)}
    test_loss = {func_id: [] for func_id in range(num_of_functions)}
    models = {func_id: {} for func_id in range(num_of_functions)}

    for func_id in range(num_of_functions):

        # initialize the model:
        models[func_id]['w1'] = np.array([1]*4)
        models[func_id]['w2'] = np.array([1]*4)
        models[func_id]['u'] = np.array([1]*4)

        # train the network:
        for _ in range(iterations):

            # draw a random batch:
            idx = np.random.choice(len(Y[func_id]['train']), batch_size)
            x, y = X['train'][idx,:], Y[func_id]['train'][idx]

            # calculate the loss and derivatives using back propagation:
            fwd = forward(models[func_id], x)
            loss = mean_square_loss(fwd['p'], y)
            dl_dw1, dl_dw2, dl_du = backprop(models[func_id], y, fwd, batch_size)

            # record the test loss before updating the model:
            test_fwd = forward(models[func_id], X['test'])
            iteration_test_loss = mean_square_loss(test_fwd['p'], y)

            # update the model using the derivatives and record the loss:
            models[func_id]['w1'] -= learning_rate * dl_dw1
            models[func_id]['w2'] -= learning_rate * dl_dw2
            models[func_id]['u'] -= learning_rate * dl_du
            training_loss[func_id].append(loss)
            test_loss[func_id].append(iteration_test_loss)

    return models, training_loss, test_loss

def test_cnn(X, Y, batch_size, lamb, iterations, learning_rate):
    learn_cnn(X,Y,batch_size, lamb, iterations, learning_rate)

def print_linear_tensorflow_by_batch(amount_of_samples, train_loss,test_loss,accuracy_iter, iter_num, batch,learning_rate,acc):

        plt.figure(figsize=(10, 10))

        for i in range(amount_of_samples):
            plt.plot(list(range(0,iter_num,100)), accuracy_iter[i], label='batch size %d'%batch[i])

        plt.legend()
        plt.title('Test accuracy -  num of iteration %d  learning rate %f result(%f)' %(iter_num, learning_rate, acc[i]))
        plt.xlabel('Iterations')
        plt.ylabel('Accuracy')
        plt.savefig( 'Test accuracy -  num of iteration %d  learning rate %f _first_deep_comp.png' %(iter_num, learning_rate))

        plt.show()

        for i in range(amount_of_samples):
            plt.plot(list(range(iter_num)), train_loss[i], label='Train - batch size %d'%batch[i])

        plt.title('Training  Loss -  num of iteration %d learning rate %f' % (iter_num, learning_rate))
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig('Training Loss -  num of iteration %d  learning rate %f  _first_deep_comp.png' % (iter_num, learning_rate))
        plt.show()

        for i in range(amount_of_samples):

            plt.plot(list(range(0, iter_num, 100)), test_loss[i], label='Test - batch size %d' % batch[i])

        plt.title('Test Loss -  num of iteration %d learning rate %f' % (iter_num, learning_rate))
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig('Test Loss -  num of iteration %d  learning rate %f  _first_deep_comp.png' % (
        iter_num, learning_rate))
        plt.show()

def linear_tensorflow(iter_num, batch_size, learning_rate):
    mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

    # open a session
    sess = tf.InteractiveSession()

    # define input and output
    x = tf.placeholder(tf.float32, shape=[None, 784])
    y_ = tf.placeholder(tf.float32, shape=[None, 10])

    # define the weights and bias
    W = tf.Variable(tf.zeros([784, 10]))
    b = tf.Variable(tf.zeros([10]))

    #inialize the parameters
    sess.run(tf.global_variables_initializer())

    #building our regression model
    y = tf.matmul(x, W) + b

    # define a loss function
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))

    #training starts
    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)

    tf.summary.scalar('loss', cross_entropy)
    size_of_arr = int(iter_num/100)
    accuracy_iter = np.zeros(size_of_arr)
    train_loss = np.zeros(int(iter_num))
    test_loss = np.zeros(size_of_arr)
    for _ in range(iter_num):
        batch = mnist.train.next_batch(batch_size)
        tmp, train_loss[_] = sess.run([train_step, cross_entropy], feed_dict={x: batch[0], y_: batch[1]})
        if (_ % 100 == 0):
            curr_place = int(_/100)
            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
            accuracy_iter[curr_place] = accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels})
            test_loss[curr_place] = sess.run(cross_entropy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})
            print(_)

    # Evluating the results
    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))

    plt.figure(figsize=(10, 10))
    plt.plot(list(range(int(iter_num/100))), accuracy_iter, label='Accuracy')
    plt.title('Test accuracy - iter num %d batch size %d leaning rate %f' % (iter_num, batch_size, learning_rate))
    plt.xlabel('Iterations')
    plt.ylabel('Accuracy')
    plt.savefig('Test accuracy - iter num %d batch size %d leaning rate %f_linear_tn.png' % (iter_num, batch_size, learning_rate))
    plt.show()

    plt.figure(figsize=(10, 10))
    plt.plot(list(range(iter_num)), train_loss, label='Training Loss')
    plt.title('Training_Test Loss  iter num %d batch size %d leaning rate %f' % (iter_num, batch_size, learning_rate))
    plt.plot(list(range(0, iter_num, 100)), test_loss, label='Test Loss')
    plt.xlabel('Iterations')
    plt.ylabel('Loss')
    plt.legend()
    plt.savefig('Training_Test Loss  iter num %d batch size %d leaning rate %f_linear_tn.png' % (iter_num, batch_size, learning_rate))
    plt.show()

def first_deep_network(iter_num, batch_size, learning_rate):

    #creating a model for multilayer preceptron

    hidden_1 = 256  # 1st layer number of neurons
    hidden_2 = 256  # 2nd layer number of neurons
    input = 784  # MNIST data input (img shape: 28*28)
    output = 10  # MNIST total classes (0-9 digits)
    h_1 = 0
    h_2 = 1
    out = 2

    mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
    # open a session
    sess = tf.InteractiveSession()

    # define input and output
    x = tf.placeholder(tf.float32, shape=[None, input])
    y_ = tf.placeholder(tf.float32, shape=[None, output])

    # Store layers weight & bias
    weights = [tf.Variable(tf.random_normal([input, hidden_1])),tf.Variable(tf.random_normal([hidden_1, hidden_2])),
        tf.Variable(tf.random_normal([hidden_2, output]))]

    bias = [tf.Variable(tf.random_normal([hidden_1])),tf.Variable(tf.random_normal([hidden_2])),
    tf.Variable(tf.random_normal([output]))]

    # inialize the parameters
    sess.run(tf.global_variables_initializer())

    layer_1 = tf.nn.relu(tf.matmul(x, weights[h_1])+ bias[h_1])
    layer_2 =  tf.add(tf.matmul(layer_1, weights[h_2]) , bias[h_2])
    y = tf.matmul(layer_2, weights[out]) + bias[out]

    # define a loss function
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))

    # training starts
    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)

    tf.summary.scalar('loss', cross_entropy)
    size_of_arr = int(iter_num / 100)
    accuracy_iter = np.zeros((size_of_arr))

    train_loss = np.zeros((iter_num))
    test_loss = np.zeros((size_of_arr))
    for _ in range(iter_num):
        print(_)

        batch = mnist.train.next_batch(batch_size)
        tmp, train_loss[_] = sess.run([train_step, cross_entropy], feed_dict={x: batch[0], y_: batch[1]})
        if(_%100 == 0):
            curr_idx = int(_ / 100)
            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
            accuracy_iter[curr_idx] = accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels})
            test_loss[curr_idx] = sess.run(cross_entropy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})
            print(_)

    acc = accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels})
    print('accuracy of %f'%acc)

    plt.figure(figsize=(10, 10))
    plt.plot(list(range(0,iter_num,100)), accuracy_iter, label='Accuracy')
    plt.title('Test accuracy -  iter num %d batch size %d leaning rate %f' % (iter_num, batch_size, learning_rate))
    plt.xlabel('Iterations')
    plt.ylabel('Accuracy')
    plt.savefig(
        'Test accuracy - iter num %d batch size %d leaning rate %f_First_D_n.png' % (iter_num, batch_size, learning_rate))
    plt.show()

    plt.figure(figsize=(10, 10))
    plt.plot(list(range(iter_num)), train_loss, label='Train Loss')
    plt.title('Training_Test Loss  iter num %d batch size %d leaning rate %f' % (iter_num, batch_size, learning_rate))
    plt.plot(list(range(0,iter_num,100)), test_loss, label='Test Loss')

    plt.xlabel('Iterations')
    plt.ylabel('Loss')
    plt.legend()
    plt.savefig('Training_Test Loss  iter num %d batch size %d leaning rate %f_First_D_n.png' % (iter_num, batch_size, learning_rate))
    plt.show()

    return train_loss, test_loss, accuracy_iter,acc

def conv_neural_net(iter_num, batch_size, learning_rate):

    hidden_1 = 256  # 1st layer number of neurons
    hidden_2 = 256  # 2nd layer number of neurons
    input = 784  # MNIST data input (img shape: 28*28)
    output = 10  # MNIST total classes (0-9 digits)
    h_1 = 0
    h_2 = 1
    out = 2

    mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
    # open a session
    sess = tf.InteractiveSession()

    # define input and output
    x = tf.placeholder(tf.float32, shape=[None, input])
    y_ = tf.placeholder(tf.float32, shape=[None, output])

    # create weights vector
    def weight_variable(shape):
        initial = tf.truncated_normal(shape, stddev=0.1)
        return tf.Variable(initial)

    # create bias vector
    def bias_variable(shape):
        initial = tf.constant(0.1, shape=shape)
        return tf.Variable(initial)
    # convultion layer
    def conv2d(x, W):
        return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
    # pooling layer
    def max_pool_2x2(x):
        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                              strides=[1, 2, 2, 1], padding='SAME')

    #first layer
    W_conv1 = weight_variable([5, 5, 1, 32])
    b_conv1 = bias_variable([32])

    x_image = tf.reshape(x, [-1, 28, 28, 1])

    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
    h_pool1 = max_pool_2x2(h_conv1)

    # second layer
    W_conv2 = weight_variable([5, 5, 32, 64])
    b_conv2 = bias_variable([64])

    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    h_pool2 = max_pool_2x2(h_conv2)

    #densly connected layer
    W_fc1 = weight_variable([7 * 7 * 64, 1024])
    b_fc1 = bias_variable([1024])
    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)

    #Dropout

    keep_prob = tf.placeholder(tf.float32)
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

    # Readout Layer
    W_fc2 = weight_variable([1024, 10])
    b_fc2 = bias_variable([10])

    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2

    # train
    cross_entropy = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
    train_step =  tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)
    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    tf.summary.scalar('loss', cross_entropy)
    size_of_arr = int(iter_num/100)
    accuracy_iter = np.zeros((size_of_arr))
    train_loss = np.zeros((iter_num))
    test_loss = np.zeros((size_of_arr))
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(iter_num):

            batch = mnist.train.next_batch(batch_size)
            tmp, train_loss[i] = sess.run([train_step, cross_entropy],
                                          feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
            if i % 100 == 0:
                print(i)
                curr_idx = int(i / 100)
                accuracy_iter[curr_idx] = accuracy.eval(feed_dict={
                    x: batch[0], y_: batch[1], keep_prob: 1.0})
                test_loss[curr_idx] = sess.run(cross_entropy, feed_dict={
                    x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})

        plt.figure(figsize=(10, 10))
        plt.plot(list(range(0,iter_num,100)), accuracy_iter, label='Accuracy')
        plt.title('Test accuracy - iter num %d batch size %d learning rate %f' % (iter_num, batch_size, learning_rate))
        plt.xlabel('Iterations')
        plt.ylabel('Accuracy')
        plt.savefig('Test accuracy - iter num %d batch size %d leaning rate %f_conv_NN.png' % (iter_num, batch_size, learning_rate))
        plt.show()

        plt.figure(figsize=(10, 10))
        plt.plot(list(range(iter_num)), train_loss, label='Training Loss')
        plt.title('Training_Test Loss  iter num %d batch size %d learning rate %f' % (iter_num, batch_size, learning_rate))
        plt.plot(list(range(0,iter_num,100)), test_loss, label='Test Loss')
        plt.xlabel('Iterations')
        plt.ylabel('Loss')
        plt.legend()
        plt.savefig('Training_Test - iter num %d batch size %d learning rate %f_conv_NN.png' % (iter_num, batch_size, learning_rate))
        plt.show()

if __name__ == '__main__':

    for i in range(10):
        print(i)
    # generate the training and test data, adding some noise:
    X = dict(train=5 * (np.random.random((1000, size)) - .5),
             test=5 * (np.random.random((200, size)) - .5))
    Y = {i: {
        'train': y[i](X['train']) * (
        1 + np.random.randn(X['train'].shape[0]) * .01),
        'test': y[i](X['test']) * (
        1 + np.random.randn(X['test'].shape[0]) * .01)}
         for i in range(len(y))}

    #test_linear_leaner(X, Y, 50 , 0.5, 3500, 0.001)
    #test_cnn(X, Y, 10 , 0.5, 3500, 0.01)
 # *************************** checking linear tensor flow ***************************

    # linear_tensorflow(iter_num= 7000, batch_size = 10, learning_rate = 0.5)
    #
    # linear_tensorflow(iter_num = 7000, batch_size = 25, learning_rate = 0.5)
    #
    # linear_tensorflow(iter_num = 7000, batch_size =25, learning_rate = 0.2)
    #
    #linear_tensorflow(iter_num = 7000, batch_size =25, learning_rate = 0.9)

 # ******************************first deep model ************************************************

    # first_deep_network(iter_num = 5000, batch_size = 10, learning_rate= 0.001)
    #
    #first_deep_network(iter_num = 10000, batch_size = 25, learning_rate= 0.001)
    #
    # first_deep_network(iter_num=5000, batch_size=50, learning_rate=0.001)
    #
    # first_deep_network(iter_num = 5000, batch_size = 25, learning_rate= 0.01)
    #
    # first_deep_network(iter_num = 5000, batch_size = 25, learning_rate= 0.001)
    #
    # first_deep_network(iter_num=5000, batch_size=25, learning_rate=0.0001)

# ******************************conv neural net ************************************************

    conv_neural_net(iter_num =10000,batch_size= 25, learning_rate= 0.001)

# ***********************************

    iter_num = 10000
    batch_size = [1 , 25, 50]
    learning_rate = 0.001
    train_loss = []
    test_loss = []
    accuracy_iter = []
    acc = []

    for batch in batch_size:

        train_loss_i, test_loss_i, accuracy_iter_i,acc_i = first_deep_network(iter_num, batch, learning_rate)
        train_loss.append(train_loss_i)
        test_loss.append(test_loss_i)
        accuracy_iter.append(accuracy_iter_i)
        acc.append(acc_i)

    print_linear_tensorflow_by_batch(3, train_loss, test_loss, accuracy_iter, iter_num, batch_size,
                                     learning_rate,acc)


from enum import auto
from flask import Flask, request, jsonify
import xml.etree.ElementTree as ET
from xml.dom import minidom

import re
from datetime import datetime

#OBJETOS
from dte import Obj_DTE
from autorizado import Obj_aut

from validaciones import validaciones
from db import db

listaDB=[]
ret=[]

app = Flask(__name__)

@app.route('/ObtenerDatos',methods=['POST'])
def obtener_datos():
    xml_data = request.data
    xml=str(xml_data, 'utf-8')
    print(xml)
    root = ET.fromstring(xml)

    #****** SALIDA ******
    fecha=datetime.today().strftime('%d/%m/%Y')
    no_facturas=0
    errores=[]
    no_correctos=0
    no_emisores=0
    no_receptores=0
    aprob=[]
    #********************
    emi=[]
    rec=[]
    #****** SALIDA ******
    eFecha=0
    eReferencia=0
    ePrecio=0
    eIVA=0
    eTotal=0
    enite=0
    enitr=0
    eAut=0
    #********************

    for DTE in root.findall('DTE'):
        no_facturas+=1
        tiempo = DTE.find('TIEMPO').text.strip()
        referencia = DTE.find('REFERENCIA').text.strip()
        nit_emisor = DTE.find('NIT_EMISOR').text.replace(' ','')
        nit_receptor = DTE.find('NIT_RECEPTOR').text.replace(' ','')
        valor = DTE.find('VALOR').text.strip()
        iva = DTE.find('IVA').text.strip()
        total = DTE.find('TOTAL').text.strip()

        #print(tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total)
        res=validaciones(tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total)

        continuar=True

        if res.valf=='HORA INVALIDA'or res.valf=='FECHA INVALIDA':
            print('Error en fecha')
            eFecha+=1
        elif res.valr == 'REFERENCIA_INVALIDA':
            print('Error en referencia')
            eReferencia+=1
        elif res.valp=='VALOR_INCORRECTO' or res.valp=='IVA_MAL_CALCULADO' or res.valp=='TOTAL_MAL_CALCULADO':
            print('Error en valores')
            if res.valp=='VALOR_INCORRECTO':
                ePrecio+=1
            elif res.valp=='IVA_MAL_CALCULADO':
                eIVA+=1
            elif res.valp=='TOTAL_MAL_CALCULADO':
                eTotal+=1
        elif not res.valne:
            print('Error en nite')
            enite+=1
        elif not res.valnr:
            print('Error en nitr')
            enitr+=1
        else:
            listaDB=[]
            db.obtener(listaDB)
            continuar=True
            fech=tiempo.split('/')
            dia=fech[0].split(' ')[1]
            mes=fech[1]
            ano=fech[2].split(' ')[0]
            fe=ano+mes+dia
            co=1
            for i in listaDB:
                if str(fe) != str(i[0][:-8]):
                    pass
                else:
                    co+=1
            ins=True
            for i in listaDB:
                if str(referencia) != str(i[2]):
                    pass
                else:
                    ins=False
            if ins:
                correlativo='{num:08d}'.format(num=co)
                autorizacion=ano+mes+dia+correlativo
                no_correctos+=1
                aprob.append([autorizacion,res.valf, referencia, nit_emisor, nit_receptor, valor, iva, total])
                listaDB.append([autorizacion,res.valf, referencia, nit_emisor, nit_receptor, valor, iva, total])
                db.insertar(listaDB)
                if nit_emisor in emi:
                    pass
                else:
                    no_emisores+=1

                if nit_receptor in rec:
                    pass
                else:
                    no_receptores+=1
            else:
                print('Error en numero de referencia')
                eAut+=1
    errores=[eFecha,eReferencia,ePrecio,eIVA,eTotal,enite,enitr,eAut]
    ret=[fecha,no_facturas,errores,no_correctos,no_emisores,no_receptores,aprob]

    root = ET.Element("LISTAAUTORIZACIONES")
    doc = ET.SubElement(root, "AUTORIZACION")
    ET.SubElement(doc, "FECHA").text=str(fecha)
    ET.SubElement(doc, "FACTURAS_RECIBIDAS").text=str(no_facturas)
    EE=ET.SubElement(doc, "ERRORES")

    ET.SubElement(EE, "FECHAA").text=str(eFecha)
    ET.SubElement(EE, "REFERENCIA").text=str(eReferencia)
    ET.SubElement(EE, "PRECIO").text=str(ePrecio)
    ET.SubElement(EE, "IVA").text=str(eIVA)
    ET.SubElement(EE, "TOTAL").text=str(eTotal)
    ET.SubElement(EE, "NIT_EMISOR").text=str(enite)
    ET.SubElement(EE, "NIT_RECEPTOR").text=str(enitr)
    ET.SubElement(EE, "REFERENCIA_DUPLICADA").text=str(eAut)

    ET.SubElement(doc, "FACTURAS_CORRECTAS").text=str(no_correctos)
    ET.SubElement(doc, "CANTIDAD_EMISORES").text=str(no_emisores)
    ET.SubElement(doc, "CANTIDAD_RECEPTORES").text=str(no_receptores)
    LA=ET.SubElement(doc, "LISTADO_AUTORIZACIONES")
    for i in aprob:
        AP=ET.SubElement(LA, "APROBACION")
        ET.SubElement(AP, "NIT_EMISOR",ref=str(i[2])).text=str(i[3])
        ET.SubElement(AP, "CODIGO_APROBACION").text=str(i[0])
    ET.SubElement(LA, "TOTAL_APROBACIONES").text=str(no_correctos)

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")

    return str(xmlstr)

@app.route('/ReiniciarDatos',methods=['GET'])
def reiniciar_datos():
    db.reiniciar()
    return 'Datos eliminados'

@app.route('/ConsultaDatos',methods=['GET'])
def consulta_datos():

    return 'Consulta de Datos'

@app.route('/ConsultaNIT',methods=['GET'])
def consulta_NIT():
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoNIT=[]
    for i in listaDB:
        if i[3] not in listadoNIT:
            listadoNIT.append(i[3])
    return str(listadoNIT)

@app.route('/ObtenerNIT',methods=['POST'])
def obtener_NIT():
    xml_data = request.data
    xml=str(xml_data, 'utf-8')
    print(xml)
    root = ET.fromstring(xml)
    selec = root.find('SELECCIONADO').text.strip()
    #print(selec)
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoNIT=[]
    for i in listaDB:
        if i[3] not in listadoNIT:
            listadoNIT.append(i[3])

    listaTransas=[]
    for i in listaDB:
        if listadoNIT[int(selec)]==i[3]:
            listaTransas.append(i)
#[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    root = ET.Element("LISTAAUTORIZACIONES")

    for i in listaTransas:
        doc = ET.SubElement(root, "TRANSACCION")
        ET.SubElement(doc, "AUTORIZACION").text=str(i[0])
        ET.SubElement(doc, "TIEMPO").text=str(i[1])
        ET.SubElement(doc, "REFERENCIA").text=str(i[2])
        ET.SubElement(doc, "NIT_EMISOR").text=str(i[3])
        ET.SubElement(doc, "NIT_RECEPTOR").text=str(i[4])
        ET.SubElement(doc, "VALOR").text=str(i[5])
        ET.SubElement(doc, "IVA").text=str(i[6])
        ET.SubElement(doc, "TOTAL").text=str(i[7])

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")

    return str(xmlstr)

@app.route('/ConsultaNITR',methods=['GET'])
def consulta_NITR():
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoNIT=[]
    for i in listaDB:
        if i[4] not in listadoNIT:
            listadoNIT.append(i[4])
    return str(listadoNIT)

@app.route('/ObtenerNITR',methods=['POST'])
def obtener_NITR():
    xml_data = request.data
    xml=str(xml_data, 'utf-8')
    print(xml)
    root = ET.fromstring(xml)
    selec = root.find('SELECCIONADO').text.strip()
    #print(selec)
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoNIT=[]
    for i in listaDB:
        if i[4] not in listadoNIT:
            listadoNIT.append(i[4])

    listaTransas=[]
    for i in listaDB:
        if listadoNIT[int(selec)]==i[4]:
            listaTransas.append(i)
#[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    root = ET.Element("LISTAAUTORIZACIONES")

    for i in listaTransas:
        doc = ET.SubElement(root, "TRANSACCION")
        ET.SubElement(doc, "AUTORIZACION").text=str(i[0])
        ET.SubElement(doc, "TIEMPO").text=str(i[1])
        ET.SubElement(doc, "REFERENCIA").text=str(i[2])
        ET.SubElement(doc, "NIT_EMISOR").text=str(i[3])
        ET.SubElement(doc, "NIT_RECEPTOR").text=str(i[4])
        ET.SubElement(doc, "VALOR").text=str(i[5])
        ET.SubElement(doc, "IVA").text=str(i[6])
        ET.SubElement(doc, "TOTAL").text=str(i[7])

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")

    return str(xmlstr)

@app.route('/ConsultaFecha',methods=['GET'])
def consulta_Fecha():
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoFecha=[]
    for i in listaDB:
        if str(i[1]).split(' ')[0] not in listadoFecha:
            listadoFecha.append(str(i[1]).split(' ')[0])
            #print(str(i[1].split(' ')[0]))
    return str(listadoFecha)

@app.route('/ObtenerFecha',methods=['POST'])
def obtener_Fecha():
    xml_data = request.data
    xml=str(xml_data, 'utf-8')
    print(xml)
    root = ET.fromstring(xml)
    selec = root.find('SELECCIONADO').text.strip()
    #print(selec)
    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listadoFecha=[]
    for i in listaDB:
        if str(i[1]).split(' ')[0] not in listadoFecha:
            listadoFecha.append(str(i[1]).split(' ')[0])

    listaTransas=[]
    for i in listaDB:
        if str(listadoFecha[int(selec)].split(' ')[0])==str(i[1]).split(' ')[0]:
            listaTransas.append(i)
#[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    root = ET.Element("LISTAAUTORIZACIONES")

    for i in listaTransas:
        doc = ET.SubElement(root, "TRANSACCION")
        ET.SubElement(doc, "AUTORIZACION").text=str(i[0])
        ET.SubElement(doc, "TIEMPO").text=str(i[1])
        ET.SubElement(doc, "REFERENCIA").text=str(i[2])
        ET.SubElement(doc, "NIT_EMISOR").text=str(i[3])
        ET.SubElement(doc, "NIT_RECEPTOR").text=str(i[4])
        ET.SubElement(doc, "VALOR").text=str(i[5])
        ET.SubElement(doc, "IVA").text=str(i[6])
        ET.SubElement(doc, "TOTAL").text=str(i[7])

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")

    return str(xmlstr)

@app.route('/ResumenRango',methods=['POST'])
def resumen_rango():
    xml_data = request.data
    xml=str(xml_data, 'utf-8')
    print(xml)
    root = ET.fromstring(xml)
    f1 = root.find('FINICIO').text.strip()
    fe1 = f1.split('-')[2]+'/'+f1.split('-')[1]+'/'+f1.split('-')[0]
    fecha1 = datetime.strptime(fe1, '%d/%m/%Y')

    f2 = root.find('FFINAL').text.strip()
    fe2 = f2.split('-')[2]+'/'+f2.split('-')[1]+'/'+f2.split('-')[0]
    fecha2 = datetime.strptime(fe2, '%d/%m/%Y')

    listaDB=[]
    db.obtener(listaDB)
    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    listaTransas=[]
    for i in listaDB:
        fechaA = datetime.strptime(str(i[1]).split(' ')[0], '%d/%m/%Y')
        if fecha1 <= fechaA <= fecha2:
            listaTransas.append(i)

    #[autorizacion,tiempo, referencia, nit_emisor, nit_receptor, valor, iva, total]
    root = ET.Element("LISTAAUTORIZACIONES")

    for i in listaTransas:
        doc = ET.SubElement(root, "TRANSACCION")
        ET.SubElement(doc, "AUTORIZACION").text=str(i[0])
        ET.SubElement(doc, "TIEMPO").text=str(i[1])
        ET.SubElement(doc, "REFERENCIA").text=str(i[2])
        ET.SubElement(doc, "NIT_EMISOR").text=str(i[3])
        ET.SubElement(doc, "NIT_RECEPTOR").text=str(i[4])
        ET.SubElement(doc, "VALOR").text=str(i[5])
        ET.SubElement(doc, "IVA").text=str(i[6])
        ET.SubElement(doc, "TOTAL").text=str(i[7])

    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")
    print(xmlstr)
    return str(xmlstr)

@app.route('/Grafica',methods=['GET'])
def grafica():
    return 'Grafica'

@app.route('/Procesar',methods=['POST'])
def procesar():
    return 'Procesar'

if __name__=="__main__":
    app.run(host="0.0.0.0",port=3000,debug=True)


# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details
# http://www.gnu.org/licenses/gpl-3.0.txt

"""More complex types for torrent data, e.g. lists of peers or a file tree"""

import os
import time
from collections import abc, defaultdict, deque

from . import utils
from .base import TorrentBase  # noqa: F401

from ..logging import make_logger  # isort:skip
log = make_logger(__name__)

class TorrentFilePriority(str):
    _MAP = {-2: (-2, 'off'), -1: (-1, 'low'), 0: (0, 'normal'), 1: (1, 'high'),
            'off': (-2, 'off'), 'low': (-1, 'low'), 'normal': (0, 'normal'), 'high': (1, 'high')}
    valid_values = ('off', 'low', 'normal', 'high')

    def __new__(cls, prio):
        try:
            number, string = cls._MAP[prio]
        except KeyError:
            raise ValueError('Invalid %s value: %r' % (cls.__name__, prio))
        else:
            obj = super().__new__(cls, string)
            obj.as_int = number
            return obj

    def __eq__(self, other):
        o_int, o_str = self._MAP.get(other, (None, None))
        if isinstance(other, int) and o_int == self.as_int:
            return True
        elif isinstance(other, str) and o_str == str(self):
            return True
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __lt__(self, other):
        return self.as_int < int(other)

    def __gt__(self, other):
        return self.as_int > int(other)

    def __le__(self, other):
        return self.as_int <= int(other)

    def __ge__(self, other):
        return self.as_int >= int(other)

    def __int__(self):
        return self.as_int

    def __repr__(self):
        return '%s(%r)' % (type(self).__name__, str(self))

    def __hash__(self):
        return super().__hash__()

def _calc_percent(a, b):
    try:
        return a / b * 100
    except ZeroDivisionError:
        return 0

class TorrentFile(abc.Mapping):
    """Mapping that holds the values of a single file in a torrent"""

    # Distinguish subtrees from files without comparing classes everywhere
    nodetype = 'leaf'

    TYPES = {
        'id'              : None,
        'tid'             : None,
        'name'            : utils.SmartCmpStr,
        'path-absolute'   : utils.SmartCmpPath,
        'path-relative'   : utils.SmartCmpPath,
        'location'        : utils.SmartCmpPath,
        'size-total'      : utils.SizeInBytes,
        'size-downloaded' : utils.SizeInBytes,
        'is-wanted'       : bool,
        'priority'        : TorrentFilePriority,
        '%downloaded'     : utils.Percent,
    }

    _MODIFIERS = {
        'id'              : lambda raw: raw['id'],
        'tid'             : lambda raw: raw['tid'],
        'name'            : lambda raw: raw['name'],
        'path-absolute'   : lambda raw: os.path.join(raw['location'], raw['path'], raw['name']),
        'path-relative'   : lambda raw: os.path.join(raw['path'], raw['name']),
        'location'        : lambda raw: raw['location'],
        'size-total'      : lambda raw: raw['size-total'],
        'size-downloaded' : lambda raw: raw['size-downloaded'],
        'is-wanted'       : lambda raw: raw['is-wanted'],
        'priority'        : lambda raw: 'off' if not raw['is-wanted'] else raw['priority'],
        '%downloaded'     : lambda raw: _calc_percent(raw['size-downloaded'], raw['size-total']),
    }

    def __init__(self, tid, id, name, path, location, size_total, size_downloaded, is_wanted, priority):
        self._raw = {'tid': tid, 'id': id, 'name': name, 'path': path, 'location': location,
                     'is-wanted': is_wanted, 'priority': priority,
                     'size-total': size_total, 'size-downloaded': size_downloaded}
        self._cache = {}

    def __getitem__(self, key):
        if key not in self._cache:
            val = self._MODIFIERS[key](self._raw)
            typ = self.TYPES.get(key)
            if typ is not None:
                self._cache[key] = typ(val)
            else:
                self._cache[key] = val
        return self._cache[key]

    def update(self, raw):
        self._raw.update(raw)
        cache = self._cache
        for key,new_value in raw.items():
            cached_value = cache.get(key)
            if cached_value is not None and cached_value != new_value:
                del cache[key]

        # %downloaded is never in raw because it is calculated from
        # size-downloaded and size-total
        if 'size-downloaded' in raw:
            try:
                del cache['%downloaded']
            except KeyError:
                pass

    def __repr__(self):
        return '<{} {!r}>'.format(type(self).__name__, self['name'])

    def __iter__(self):
        return iter(self.TYPES)

    def __len__(self):
        return len(self.TYPES)

class TorrentPeer(abc.Mapping):
    TYPES = {
        'id'          : None,
        'tid'         : None,
        'tname'       : utils.SmartCmpStr,
        'tsize'       : utils.SizeInBytes,
        'ip'          : str,
        'port'        : int,
        'client'      : utils.SmartCmpStr,
        'downloaded'  : utils.SizeInBytes,
        '%downloaded' : utils.Percent,
        'rate-up'     : utils.BandwidthInBytes,
        'rate-down'   : utils.BandwidthInBytes,
        'eta'         : utils.Timedelta,
        'rate-est'    : utils.BandwidthInBytes,
    }

    _MODIFIERS = {
        'id'      : lambda p: (p['tid'], p['ip'], p['port']),
    }

    _MAX_PEER_PROGRESS_SAMPLE_AGE = 1800  # 30 minutes
    _PEER_PROGRESS_DATA = defaultdict(lambda: deque(maxlen=10))

    @classmethod
    def gc_peer_progress_data(cls):
        log.debug('Pruning peer progress data:')
        for peer_id,samples in tuple(cls._PEER_PROGRESS_DATA.items()):
            # Remove samples that are too old
            while samples and (samples[0][0] + cls._MAX_PEER_PROGRESS_SAMPLE_AGE) < time.monotonic():
                log.debug('Sample from %s is too old: %r', peer_id, samples[0])
                samples.popleft()

            # Remove samples where progress is 0.0.  A lot of the cache entries just have
            # a single sample from peers connecting for some reason but not downloading.
            if samples and samples[0][1] in (0.0, 1.0):
                samples.popleft()

            # Remove deque if there are no samples left
            if not samples:
                del cls._PEER_PROGRESS_DATA[peer_id]

        log.debug('Pruned peer progress data:')
        for peer_id,samples in tuple(cls._PEER_PROGRESS_DATA.items()):
            log.debug('%s: %s', peer_id, samples)

    @classmethod
    def _guess_peer_rate_and_eta(cls, peer_id, peer_progress, torrent_size):
        rate = 0
        if peer_progress >= 1:
            # Peer has already downloaded everything
            eta = utils.Timedelta.NOT_APPLICABLE
        else:
            eta = utils.Timedelta.UNKNOWN
            samples = cls._PEER_PROGRESS_DATA[peer_id]

            # Don't add the same progress twice
            if not samples or peer_progress != samples[-1][1]:
                samples.append((time.monotonic(), peer_progress))

            # We need at least 3 samples
            if len(samples) >= 3:
                # Use second and last sample to calculate rate.  The first sample is
                # ignored because its timestamp may be inaccurate: When we add the
                # first sample, the peer's progress is not current but the latest we
                # received, which happened likely tens of seconds ago.
                t_first, p_first = samples[1]
                t_last, p_last = samples[-1]
                p_diff = p_last - p_first
                t_diff = t_last - t_first

                # It's possible progress goes down, e.g. if a peer deletes a file
                if p_diff > 0:
                    torrent_size = int(torrent_size)  # Don't copy unit + unit prefix from torrent_size
                    size_diff = torrent_size * p_diff
                    rate = size_diff / t_diff
                    size_remaining = torrent_size - (torrent_size * peer_progress)
                    eta = size_remaining / rate

        return rate, eta

    def __init__(self, tid, tname, tsize, ip, port, client, downloaded, pdownloaded, rate_up, rate_down):
        self._cache = {}
        self._dct = {'tid': tid, 'tname': tname, 'tsize': tsize,
                     'ip': ip, 'port': port, 'client': client,
                     'downloaded': downloaded, '%downloaded': pdownloaded,
                     'rate-up': rate_up, 'rate-down': rate_down}
        self._dct['rate-est'], self._dct['eta'] = \
            self._guess_peer_rate_and_eta(self['id'], pdownloaded / 100, tsize)

    def __getitem__(self, key):
        cache = self._cache
        value = cache.get(key)
        if value is None:
            modifier = self._MODIFIERS.get(key)
            if modifier is not None:
                val = modifier(self._dct)
            else:
                val = self._dct[key]
            typ = self.TYPES.get(key)
            if typ is not None:
                cache[key] = typ(val)
            else:
                cache[key] = val
        return cache[key]

    def clearcache(self):
        self._cache.clear()

    def __repr__(self):
        return '<{} #{}, {}>'.format(type(self).__name__, self['tid'], self['ip'])

    def __iter__(self):
        return iter(self.TYPES)

    def __len__(self):
        return len(self.TYPES)

class TrackerStatus(utils.SmartCmpStr):
    def __new__(cls, status):
        if status not in ('stopped', 'idle', 'queued', 'announcing', 'scraping'):
            raise ValueError('Invalid tracker status: %r' % status)
        else:
            return super().__new__(cls, status)

class TorrentTracker(abc.Mapping):
    TYPES = {
        'id'                 : None,
        'tid'                : int,
        'tname'              : utils.SmartCmpStr,
        'tier'               : int,

        'url-announce'       : utils.URL,
        'url-scrape'         : utils.URL,
        'domain'             : utils.SmartCmpStr,

        'status-announce'    : TrackerStatus,
        'status-scrape'      : TrackerStatus,
        'status'             : TrackerStatus,

        'error-announce'     : utils.SmartCmpStr,
        'error-scrape'       : utils.SmartCmpStr,
        'error'              : utils.SmartCmpStr,

        'count-downloads'    : utils.Count,
        'count-leeches'      : utils.Count,
        'count-seeds'        : utils.Count,

        'time-last-announce' : utils.Timestamp,
        'time-next-announce' : utils.Timestamp,
        'time-last-scrape'   : utils.Timestamp,
        'time-next-scrape'   : utils.Timestamp,
    }

    _MODIFIERS = {
        'id'     : lambda self: hash((self['tid'], self['url-announce'])),
        'domain' : lambda self: self['url-announce'].domain,
        'status' : lambda self: (self['status-scrape'] if self['status-announce'] == 'idle'
                                 else self['status-announce']),
        'error'  : lambda self: ('Announce error: %s' % self['error-announce']
                                 if self['error-announce'] else
                                 'Scrape error: %s' % self['error-scrape']
                                 if self['error-scrape'] else '')
    }

    def __init__(self, trkdict):
        self._dct = trkdict
        self._cache = {}

    def __getitem__(self, key):
        cache = self._cache
        value = cache.get(key)
        if value is None:
            modifier = self._MODIFIERS.get(key)
            if modifier is not None:
                val = modifier(self)
            else:
                val = self._dct[key]
            typ = self.TYPES.get(key)
            if typ is not None:
                cache[key] = typ(val)
            else:
                cache[key] = val
        return cache[key]

    def __repr__(self):
        return '<%s %s>' % (type(self).__name__, self['url-announce'])

    def __iter__(self):
        return iter(self.TYPES)

    def __len__(self):
        return len(self.TYPES)


# -*- coding: utf-8 -*-
"""
Markdown Prism Highlight Extension For Pelican
==============================================
Extends Pelican's Markdown module and use Prism
code highlighting
"""

import sys
from pelican import signals

try:
    from .markdown_prism_fenced_code import PrismConfig, PrismFencedCodeExtension
except ImportError as e:
    PrismFencedCodeExtension = None
    print("\nMarkdown is not installed - Prism Highlight Markdown Extension disabled\n")

def init(pelicanobj):
    """Loads settings and instantiates the Python Markdown extension"""

    if not PrismFencedCodeExtension:
        return

    # Process settings
    config = process_settings(pelicanobj)

    # Configure Markdown Extension
    apply_markdown_extension(pelicanobj, config)

def process_settings(pelicanobj):
    """Handle user specified settings (see README for more details)"""

    # Default settings
    settings = {
        "default": {
            "lineno": True,
            "max-height": "30em",
        }
    }

    # Get the user specified settings
    try:
        user_settings = pelicanobj.settings['PRISM_PRESETS']
        settings.update(user_settings)
    except:
        pass

    return PrismConfig(settings)

def apply_markdown_extension(pelicanobj, config):
    """Instantiates the customized Markdown extension and disable conflict ones"""

    # Instantiate Markdown extension and append it to the current extensions
    try:
        pelicanobj.settings['MD_EXTENSIONS'].append(PrismFencedCodeExtension(config))
    except:
        sys.excepthook(*sys.exc_info())
        sys.stderr.write("\nError - md-prism-highlight failed to load\n")
        sys.stderr.flush()

def register():
    """Plugin registration"""
    signals.readers_init.connect(init)