from sklearn.model_selection import train_test_split
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from datetime import datetime
import tensorflow_datasets as tfds
import bert
from bert import run_classifier
from bert import optimization
from bert import tokenization
from bert import run_classifier_with_tfhub
#https://github.com/google-research/bert.git
import sys
from tensorflow import keras
import os
import re
from transformers import *
import numpy as np
from tensorflow.python.lib.io import file_io
import pickle
import gc
import threading
import logging
import argparse
"""
Usage
>> python -u runBert.py @args.txt

python -u runBert.py @params_model1.txt
------Example args.txt file -----

--tpuAddress node-3
--tpuZone us-central1-f
--outputDir test
--seqLen 15
--modelHub https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1
--batchSize 64
--epochs 40
--dropout .9

"""

####################################################
############ Setting output directory ##############
####################################################
def getDir(bucket, output_dir):
    return 'gs://{}/{}'.format(bucket, output_dir)

def setUp_output_dir():
    DO_DELETE = True
    USE_BUCKET =True

    if USE_BUCKET:
        OUTPUT_DIR = getDir(BUCKET, OUTPUT_DIR)

    if DO_DELETE:
        try:
            tf.gfile.DeleteRecursively(OUTPUT_DIR)
        except:
            # doesn't matter if the directory didn't exist
            pass
    tf.gfile.MakeDirs(OUTPUT_DIR)
    print('***** Model output directory: {} *****'.format(OUTPUT_DIR))

#################################################
############# Load Data set #####################
#################################################
def loadPdData(gsPath):
    return pd.read_csv(gsPath, sep = "\t")

def saveToGcloud(path,data,isPandas = False ):
    '''Saves to gcloud so we dont have to do this long ass step every time'''
    if isPandas:
        data.to_csv(path, index=False, sep="\t")
    else:
        with file_io.FileIO(path, mode='w') as f:
            pickle.dump(data,f)

def readFromGcloud(path, isPandas = False):
    if isPandas:
        return pd.read_csv(path,sep="\t" )
    else:
        with file_io.FileIO(path, mode='rb') as f:
            return pickle.load(f)

def worker_downloadTestData(name):
    """
    Worker so we can download test data asynch
    """
    logging.info("Thread %s: starting for loading test data", name)
    global test_features
    train_features = readFromGcloud(TEST_TFRecord_PATH)
    logging.info("Thread %s: finishing for loading test data", name)

#######################################################
############# Creating a model  #######################
#######################################################
def create_model(is_training, input_ids, input_mask, segment_ids, labels,
                 num_labels, bert_hub_module_handle, dropout):
    """Creates a classification model."""
    tags = set()
    if is_training:
        tags.add("train")
    bert_module = hub.Module(bert_hub_module_handle, tags=tags, trainable=True)
    bert_inputs = dict(
      input_ids=input_ids,
      input_mask=input_mask,
      segment_ids=segment_ids)
    bert_outputs = bert_module(
      inputs=bert_inputs,
      signature="tokens",
      as_dict=True)

    output_layer = bert_outputs["pooled_output"]

    hidden_size = output_layer.shape[-1].value

    output_weights = tf.get_variable(
      "output_weights", [num_labels, hidden_size],
      initializer=tf.truncated_normal_initializer(stddev=0.02))

    output_bias = tf.get_variable(
      "output_bias", [num_labels], initializer=tf.zeros_initializer())

    with tf.variable_scope("loss"):
        if is_training:
            # I.e., 0.1 dropout
            output_layer = tf.nn.dropout(output_layer, keep_prob=dropout)

        logits = tf.matmul(output_layer, output_weights, transpose_b=True)
        logits = tf.nn.bias_add(logits, output_bias)
        probabilities = tf.nn.softmax(logits, axis=-1)
        log_probs = tf.nn.log_softmax(logits, axis=-1)

        one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)

        per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
        loss = tf.reduce_mean(per_example_loss)

        return (loss, per_example_loss, logits, probabilities)

def model_fn_builder(num_labels, learning_rate, num_train_steps,
                     num_warmup_steps, use_tpu, bert_hub_module_handle, dropout):
    """Returns `model_fn` closure for TPUEstimator."""

    def model_fn(features, labels, mode, params):
        """The `model_fn` for TPUEstimator."""

        tf.logging.info("*** Features ***")
        for name in sorted(features.keys()):
            tf.logging.info("  name = %s, shape = %s" % (name, features[name].shape))

        input_ids = features["input_ids"]
        input_mask = features["input_mask"]
        segment_ids = features["segment_ids"]
        label_ids = features["label_ids"]

        is_training = (mode == tf.estimator.ModeKeys.TRAIN)

        (total_loss, per_example_loss, logits, probabilities) = create_model(
                is_training, input_ids, input_mask, segment_ids, label_ids, num_labels,
                bert_hub_module_handle, dropout)

        output_spec = None
        if mode == tf.estimator.ModeKeys.TRAIN:
            train_op = optimization.create_optimizer(
                    total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)

            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
                  mode=mode,
                  loss=total_loss,
                  train_op=train_op)
        elif mode == tf.estimator.ModeKeys.EVAL:

            def metric_fn(per_example_loss, label_ids, logits):
                predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)
                accuracy = tf.metrics.accuracy(label_ids, predictions)
                loss = tf.metrics.mean(per_example_loss)
                true_pos = tf.metrics.true_positives(
                            label_ids,
                            predictions)
                true_neg = tf.metrics.true_negatives(
                            label_ids,
                            predictions)
                false_pos = tf.metrics.false_positives(
                            label_ids,
                            predictions)
                false_neg = tf.metrics.false_negatives(
                            label_ids,
                            predictions)
                return {
                    "true_positives": true_pos,
                    "true_negatives": true_neg,
                    "false_positives": false_pos,
                    "false_negatives": false_neg,
                    "eval_accuracy": accuracy,
                    "eval_loss": loss,
                }

            eval_metrics = (metric_fn, [per_example_loss, label_ids, logits])
            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
              mode=mode,
              loss=total_loss,
              eval_metrics=eval_metrics)

        elif mode == tf.estimator.ModeKeys.PREDICT:
            output_spec = tf.contrib.tpu.TPUEstimatorSpec(
              mode=mode, predictions={"probabilities": probabilities})
        else:
            raise ValueError(
              "Only TRAIN, EVAL and PREDICT modes are supported: %s" % (mode))

        return output_spec

    return model_fn

####################################################
###### FUnctions to train + evaluate model #########
####################################################
def get_run_config(output_dir):
    """
    Used for run configuration when TPU used
    """
    return tf.contrib.tpu.RunConfig(
        cluster=tpu_cluster_resolver,
        model_dir=output_dir,
        save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,
        tpu_config=tf.contrib.tpu.TPUConfig(
            iterations_per_loop=ITERATIONS_PER_LOOP,
            num_shards=NUM_TPU_CORES,
            per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))

def getEstimator(mode_fn):
    """
    Returns the estimator used to train/eval model
    """
    return tf.estimator.tpu.TPUEstimator(
          use_tpu=True,
          model_fn=mode_fn,
          config=get_run_config(OUTPUT_DIR),
          train_batch_size=BATCH_SIZE,
          eval_batch_size=EVAL_BATCH_SIZE,
          predict_batch_size=PREDICT_BATCH_SIZE,
          eval_on_tpu = True
        )

def model_train(estimator):
    """
    Trains the model, rt only good for TPU
    """
    #Set drop_remainder =True to fix a TPU error
    #https://stackoverflow.com/questions/58029896/bert-fine-tuning-with-estimators-on-tpus-on-colab-typeerror-unsupported-operand

    print('***** Started training at {} *****'.format(datetime.now()))
    print('  Num examples = {}'.format(len(train_features)))
    print('  Batch size = {}'.format(BATCH_SIZE))
    tf.logging.info("  Num steps = %d", num_train_steps)

    current_time = datetime.now()
    train_input_fn = run_classifier.input_fn_builder(
        features=train_features,
        seq_length=MAX_SEQ_LENGTH,
        is_training=True,
        drop_remainder=True)
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
    print("Finished: Training took time ", datetime.now() - current_time)

    #train_features
def model_evaluate(estimator, data):
    """
    Evaluates the model
    """
    print('***** Started evaluation at {} *****'.format(datetime.now()))
    print('  Num examples = {}'.format(len(data)))
    print('  Batch size = {}'.format(EVAL_BATCH_SIZE))

    # Eval will be slightly WRONG on the TPU because it will truncate
    # the last batch.
    eval_steps = int(len(data) / EVAL_BATCH_SIZE)

    eval_input_fn = run_classifier.input_fn_builder(
        features=data,
        seq_length=MAX_SEQ_LENGTH,
        is_training=False,
        drop_remainder=True)

    result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)
    print('***** Finished evaluation at {} *****'.format(datetime.now()))
    output_eval_file = os.path.join(OUTPUT_DIR, "eval","eval_results.txt")
    tf.gfile.MakeDirs(os.path.join(OUTPUT_DIR, "eval"))
    with tf.gfile.GFile(output_eval_file, "w") as writer:
        print("***** Eval results *****")
        for key in sorted(result.keys()):
            print('  {} = {}'.format(key, str(result[key])))
            writer.write("%s = %s\n" % (key, str(result[key])))

####################################################
#################   Utility Functions ##############
####################################################
def saveModelParams(params, _dir):
    """
    Save model params to gCloud
    """
    model_params_file = os.path.join(_dir,"modelParams","model_parameters.txt")
    tf.gfile.MakeDirs(os.path.join(_dir, "modelParams"))

    with tf.gfile.GFile(model_params_file, "w") as writer:
        print("***** Model Parameters *****")
        for key in sorted(params.keys()):
            print('  {} = {}'.format(key, str(params[key])))
            writer.write("%s = %s\n" % (key, str(params[key])))
    print("Model paramters at: {}".format(os.path.join(_dir, "modelParams")))

def convert_arg_line_to_args(arg_line):
    """
    From: https://stackoverflow.com/questions/25084993/why-isnt-fromfile-prefix-chars-in-python-argparse-working
    """
    for arg in arg_line.split():
        if not arg.strip():
            continue
        yield arg

if __name__ == "__main__":
    my_parser = argparse.ArgumentParser(
            fromfile_prefix_chars='@',
            prog="runBert",
            description='Run bert on patent data!!')

    my_parser.convert_arg_line_to_args = convert_arg_line_to_args

    my_parser.add_argument(
            '-tpuAddress',
            action='store',
            type=str,
            required=True,
            help="The address of TPU node"
            )

    my_parser.add_argument(
            '-tpuZone',
            action='store',
            type=str,
            required=False,
            nargs='?',
            default="us-central1-f",
            help="The zone that the TPU is in: default us-central1-f"
            )

    my_parser.add_argument(
            '-outputDir',
            action='store',
            type=str,
            required=True,
            help="The output dir of results: will be stored in gs bucket `patents-research` under folder bertResults{outputDir}"
            )

    my_parser.add_argument(
            '-seqLen',
            action='store',
            type=int,
            required=True,
            help="The sequence length for the language model"
    )

    my_parser.add_argument(
            '-modelHub',
            action='store',
            type=str,
            required=False,
            nargs='?',
            default="https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1",
            help="The Bert model Hub"
    )

    my_parser.add_argument(
            '-batchSize',
            action='store',
            type=int,
            required=False,
            default=64,
            nargs='?',
            help="The training batch size"
    )

    my_parser.add_argument(
            '-epochs',
            action='store',
            type=float,
            required=False,
            default=40.0,
            nargs='?',
            help="The number of epochs"
    )

    my_parser.add_argument(
            '-dropout',
            action='store',
            type=float,
            required=False,
            default=0.7,
            nargs='?',
            help="Percent of data to keep"
    )

    args = my_parser.parse_args()

    ##### SET TPU CONSTANTS AND CONNECT TO IT #######
    TPU_ADDRESS = args.tpuAddress
    TPU_ZONE = args.tpuZone
    USE_TPU =True
    ITERATIONS_PER_LOOP = 1000
    tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_ADDRESS, zone=TPU_ZONE)
    tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver)
    tf.distribute.experimental.TPUStrategy(tpu_cluster_resolver)
    #NUM_TPU_CORES = len(tf.config.experimental.list_logical_devices('TPU'))
    NUM_TPU_CORES = 8

    if NUM_TPU_CORES==0:
        sys.exit("Problem with tpu make sure region is correct or tpu is runnign")

    ###################################
    ####### CONSTANTS ##################
    ####################################

    DATA_PATH = "gs://patents-research/patent_research/data_frwdcorrect.tsv"
    OUTPUT_DIR = "bertResults_{}".format(args.outputDir)# where the model will be saved
    BUCKET = "patents-research"

    DATA_COLUMN = 'text'
    LABEL_COLUMN = 'label'
    label_list = [0, 1, 2]

    MAX_SEQ_LENGTH = args.seqLen
    TRAIN_TFRecord_PATH= "gs://patents-research/patent_research/{}_{}.pickle".format("train_features",MAX_SEQ_LENGTH)
    TEST_TFRecord_PATH= "gs://patents-research/patent_research/{}_{}.pickle".format("test_features",MAX_SEQ_LENGTH)
    BERT_MODEL_HUB = args.modelHub

    #Set output directory
    setUp_output_dir()
    # Force TF Hub writes to the GS bucket we provide.
    os.environ['TFHUB_CACHE_DIR'] =  os.path.join(OUTPUT_DIR,"tfhub_cache")
    tf.gfile.MakeDirs(os.path.join(OUTPUT_DIR,"tfhub_cache"))

    # Model Parameters
    # These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)
    BATCH_SIZE = args.batchSize
    EVAL_BATCH_SIZE = NUM_TPU_CORES
    PREDICT_BATCH_SIZE = NUM_TPU_CORES
    LEARNING_RATE = 2e-5
    NUM_TRAIN_EPOCHS = args.epochs
    DROPOUT_KEEP_PROB = args.dropout
    # Warmup is a period of time where hte learning rate
    # is small and gradually increases--usually helps training.
    WARMUP_PROPORTION = 0.1
    # Model configs
    SAVE_CHECKPOINTS_STEPS = 1000
    SAVE_SUMMARY_STEPS = 100

    # Compute # train and warmup steps from batch size
    num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)
    num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)

    params={
            "TPU_ADDRESS":TPU_ADDRESS,
            "TPU_ZONE":TPU_ZONE,
            "TPU_ITERATIONS_PER_LOOP":ITERATIONS_PER_LOOP,
            "NUM_TPU_CORES":NUM_TPU_CORES,
            "TFHUB_CACHE_DIR":os.path.join(OUTPUT_DIR,"tfhub_cache"),
            "DATA_PATH":DATA_PATH,
            "OUTPUT_DIR":OUTPUT_DIR,
            "MAX_SEQ_LENGTH":MAX_SEQ_LENGTH,
            "TRAIN_TFRecord_PATH":TRAIN_TFRecord_PATH,
            "TEST_TFRecord_PATH":TEST_TFRecord_PATH,
            "BERT_MODEL_HUB":BERT_MODEL_HUB,
            "BATCH_SIZE":BATCH_SIZE,
            "EVAL_BATCH_SIZE":EVAL_BATCH_SIZE,
            "PREDICT_BATCH_SIZE":PREDICT_BATCH_SIZE,
            "LEARNING_RATE":LEARNING_RATE,
            "NUM_TRAIN_EPOCHS":NUM_TRAIN_EPOCHS,
            "DROPOUT_KEEP_PROB":DROPOUT_KEEP_PROB,
            "WARMUP_PROPORTION":WARMUP_PROPORTION,
            "SAVE_CHECKPOINTS_STEPS":SAVE_CHECKPOINTS_STEPS,
            "SAVE_SUMMARY_STEPS":SAVE_SUMMARY_STEPS,
            "num_train_steps":num_train_steps,
            "num_warmup_steps":num_warmup_steps
            }
    saveModelParams(params,OUTPUT_DIR)

    #####################################################
    ########### RUNNING SET UP FUNCTIONS ################
    #####################################################

    # Download train data
    print("Reading {} from gCloud".format(TRAIN_TFRecord_PATH))
    #train_features = readFromGcloud(TRAIN_TFRecord_PATH)
    print("Finished {} from gCloud!".format(TRAIN_TFRecord_PATH))

    # Download test data async - test data will be saved at test_features
    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,datefmt="%H:%M:%S")
    getTestData_thread = threading.Thread(target=worker_downloadTestData, args=(1,))
    #getTestData_thread.start() #async download of test data

    #####################################################
    ########## Train + Eval Model #######################
    #####################################################
    mode_fn = model_fn_builder(
      num_labels=len(label_list),
      learning_rate=LEARNING_RATE,
      num_train_steps=num_train_steps,
      num_warmup_steps=num_warmup_steps,
      dropout = DROPOUT_KEEP_PROB,
      use_tpu = USE_TPU,
      bert_hub_module_handle = BERT_MODEL_HUB
    )

    #estimator = getEstimator(mode_fn)
    #model_train(estimator)
    #gc.collect()
    #del train_features # Remove train_features might cause mem limit
    #gc.collect() # Remove train_features might cause mem limit

    #getTestData_thread.join()
    #gc.collect()
    #model_evaluate(estimator, train_features)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import lxml
import lxml.html.clean
import requests
import wikipedia as wp
from transliterate import translit

def get_html_from_text(raw_html):
    # clean_args = {
    #     "javascript": True,  # strip javascript
    #     "page_structure": False,  # leave page structure alone
    #     "style": True  # remove CSS styling
    # }
    # clean_html = lxml.html.clean.Cleaner(**clean_args).clean_html(raw_html)
    html = lxml.html.fromstring(raw_html)
    return html

def get_element_by_selector(b, selector):
    select_result = list(b.cssselect(selector))
    if len(select_result) == 0:
        return None
    return select_result[0]

def get_info_from_block(b):
    d = dict()
    p = get_element_by_selector(b, 'p[itemprop="description"]')
    d['short'] = '\n'.join(t for t in p.itertext()) if p is not None else None
    # short = '\n'.join(t for t in p.itertext())

    label_select = get_element_by_selector(b, 'h3 > a')
    d['label'] = label_select.text if label_select is not None else None

    lat_select = get_element_by_selector(b, 'meta[itemprop="latitude"]')
    d['lat'] = lat_select.attrib['content'] if lat_select is not None else None

    long_select = get_element_by_selector(b, 'meta[itemprop="longitude"]')
    d['long'] = long_select.attrib['content'] if long_select is not None else None

    return d
    # print(d)

def get_infos():
    response = requests.get('https://autotravel.ru/excite.php/1055/1')
    raw_html = response.text
    # BLOCK_SELECTOR = 'div[itemtype="http://schema.org/Place"]'
    BLOCK_SELECTOR = 'div[class="col-md-12 col-xs-12"] > div'

    html = get_html_from_text(raw_html)
    blocks = html.cssselect(BLOCK_SELECTOR)
    infos = list(filter(lambda d: d['long'] is not None and d['lat'] is not None, map(get_info_from_block, blocks)))
    return infos

def check_label(d, key):
    return type(d[key]) == str and d[key].find('\n') == -1

def search(query, lang):
    wp.set_lang(lang)
    wp_search = wp.search(query)
    if len(wp_search) == 0:
        return None
    return wp_search[0]

def get_page(d):
    label = d['label']
    ru_label = d['label_ru']

    exception = wp.exceptions.WikipediaException

    def try_different(suffix=''):
        try:
            p = wp.page(ru_label + suffix)
        except exception:
            try:
                p = wp.page(label + suffix)
            except exception:
                p = None
        return p
    p = try_different()
    if p is None:
        p = try_different(' (Санкт-Петербург)')
    return p

class ExtractorError(RuntimeError):
    def __init__(self, message):
        super(ExtractorError, self).__init__(message)
        self.message = message

OUTPUT_DIRECTORY = "sights/"

if __name__ == '__main__':
    infos = get_infos()
    bad_records = []
    for i, d in enumerate(infos):
        label = d['label']
        try:
            # en_search = search(label, 'en')
            # if en_search is None:
            en_search = translit(label, reversed=True)
            d['label_en'] = en_search
            assert check_label(d, 'label_en')

            ru_search = search(label, 'ru')
            if ru_search is None:
                raise ExtractorError('ru_search')
            d['label_ru'] = ru_search
            assert check_label(d, 'label_ru')

            p = get_page(d)
            if p is None:
                raise ExtractorError('get_page')
            if d['short'] is None:
                print(i, label, "does not have short description from site\n".format(d['label']))
                d['short'] = p.summary
            d['long_description'] = p.summary
            try:
                d['lat'] = float(p.coordinates[0])
                d['long'] = float(p.coordinates[1])
            except KeyError:
                pass
            d['url'] = p.url
            d['name'] = ''.join(filter(lambda c: c.isalnum() or c == '_', d['label_en'].replace(' ', '_')))
            if d['name'].startswith('List'):
                print(d)
            f = open(OUTPUT_DIRECTORY + d['name'] + '.sight', 'w')
            f.write('\n'.join([
                d['label'],
                d['label_ru'],
                d['label_en'],
                str(d['lat']) + ' ' + str(d['long']),
                d['url'],
                '===',
                d['short'],
                '===',
                d['long_description'],
            ]))
            f.close()
        except ExtractorError as e:
            print(i, label, e.message)
            bad_records.append(d)
    f = open('bad_records.txt', 'w')
    f.write('\n'.join([str(record) for record in bad_records]))
    f.close()


import gc
import sys
import wx
from weakref import ref
from testutil import check_collected

foo = 0
success = 0

def test_callafter_leak():
    def func():
        global foo
        foo = 42

    wr = ref(func)
    wx.CallAfter(func)
    del func

    # make sure that func runs
    wx.GetApp().Yield()

    assert wr() is None, gc.get_referrers(gc.get_referrers(wr())[0])
    assert foo == 42

    global success
    success = success + 1

def main():
    a = wx.PySimpleApp()

    N = 100

    for x in xrange(N):
        wx.CallAfter(test_callafter_leak)

    wx.CallAfter(gc.collect)

    for x in xrange(N):
        wx.CallAfter(test_callafter_leak)

    wx.CallAfter(a.ExitMainLoop)
    a.MainLoop()

    global success
    assert success == N*2

if __name__ == '__main__':
    main()


import json

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal, connect_nodes_bi, connect_nodes, sync_blocks, disconnect_nodes_bi
from test_framework.key import CECKey
from test_framework.blocktools import create_block, create_coinbase
from test_framework.script import hash160, CScript, OP_DUP, OP_HASH160, OP_EQUALVERIFY, OP_CHECKSIG, SignatureHashForkId, SIGHASH_ALL , SIGHASH_FORKID
from test_framework.mininode import CTransaction, CTxOut, CTxIn, COutPoint, ToHex
from test_framework.authproxy import JSONRPCException

class User:

    def __init__(self, secret_bytes):
        self.key = CECKey()
        self.key.set_secretbytes(secret_bytes)
        self.pubkey = self.key.get_pubkey()

    def spend_to_pkh (self, node, spend_tx, n, amount, to_pubkey):
        value = int(amount)
        scriptPubKey = CScript([OP_DUP, OP_HASH160, hash160(to_pubkey), OP_EQUALVERIFY, OP_CHECKSIG])

        tx = CTransaction()
        assert (n < len(spend_tx.vout))
        tx.vin.append(CTxIn(COutPoint(spend_tx.sha256, n), b"", 0xffffffff))
        tx.vout.append(CTxOut(value, scriptPubKey))
        tx.calc_sha256()

        self.__sign_tx(tx, spend_tx, n)
        tx.rehash()
        node.sendrawtransaction(ToHex(tx), False, True)

        if False: # if we want to get the tx as json formatted output for debugging
            tx_json = node.decoderawtransaction(ToHex(tx))
            for output in tx_json['vout']:
                output['value'] = float(output['value'])
            text = json.dumps(tx_json, indent=4)
            print("ds transaction:", text)

        return tx

    def __sign_tx(self, sign_tx, spend_tx, n):
        sighash = SignatureHashForkId( spend_tx.vout[n].scriptPubKey, sign_tx, 0, SIGHASH_ALL | SIGHASH_FORKID, spend_tx.vout[n].nValue )
        sign_tx.vin[0].scriptSig = CScript([self.key.sign(sighash) + bytes(bytearray([SIGHASH_ALL | SIGHASH_FORKID])), self.pubkey ])

class CompetingChainsTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 2
        self.nodeargs = ["-txindex=1", "-disablesafemode=0", "-debug=1"]
        self.extra_args = [self.nodeargs, self.nodeargs]
        self.nbDoubleSpends = 3
        self.lenChain0 = 8  # more than SAFE_MODE_MAX_VALID_FORK_LENGTH 7
        self.lenChain1 = 18  # less than SAFE_MODE_MAX_VALID_FORK_DISTANCE (72)
        self.FORK_ROOT_HEIGHT = 200

    def setup_network(self):
        self.setup_nodes()

    def make_coinbase(self, conn_rpc):
        tip = conn_rpc.getblock(conn_rpc.getbestblockhash())
        coinbase_tx = create_coinbase(tip["height"] + 1)
        block = create_block(int(tip["hash"], 16), coinbase_tx, tip["time"] + 1)
        block.solve()
        conn_rpc.submitblock(ToHex(block))
        return coinbase_tx

    def send_funds_to_attacker (self, node, attacker, coinbase_tx):
        funding_amount = int(coinbase_tx.vout[0].nValue / self.nbDoubleSpends)
        funding_tx = CTransaction()

        funding_tx.vin.append(CTxIn(COutPoint(coinbase_tx.sha256, 0), b"", 0xffffffff))

        scriptPubKey = CScript([OP_DUP, OP_HASH160, hash160(attacker.pubkey), OP_EQUALVERIFY, OP_CHECKSIG])
        for i in range(self.nbDoubleSpends):
            funding_tx.vout.append(CTxOut(funding_amount, scriptPubKey))

        funding_tx.rehash()
        funding_txid = node.sendrawtransaction(ToHex(funding_tx), False, True)
        assert_equal(node.getrawmempool(), [funding_txid])
        return funding_tx

    def contains_double_spends (self):
        spent_inputs = set([])
        seen_transactions = []
        ds_counter = 0
        for node in self.nodes:
            for height in range(node.getblockcount() + 1):
                blockhash = node.getblockhash(height)
                block = node.getblock(blockhash, 2)
                for txraw in block['tx']:
                    if txraw['txid'] in seen_transactions:
                        continue
                    else:
                        seen_transactions.append(txraw['txid'])
                    for i in txraw['vin']:
                        if 'coinbase' in i:
                            continue
                        new_element = (i['txid'], i['vout'])
                        if new_element in spent_inputs:
                            ds_counter += 1
                        else:
                            spent_inputs.add(new_element)
        return ds_counter

    def run_test(self):

        # Test 1:
        # 1. fund an attacker for the test on node0
        # 2. progress to block height 200
        # 3. sync all nodes
        # 4. disconnect the two nodes forking at block height 200
        # 5. spend attackers fund in node0 and double spend them in node1
        # 6. Assert that the two chains actually contain the attackers double-spends

        attacker = User(b"horsebattery")
        friend0_of_attacker = User(b"fatstack")
        friend1_of_attacker = User(b"fatheap")
        node0 = self.nodes[0]  # victim node
        node1 = self.nodes[1]  # node under control of attacker

        self.log.info("fund attacker. We fund him at height 200 -2")
        self.log.info("just for debugging convenience. We plan to fork at height 200")
        coinbase_tx = self.make_coinbase(node0)
        node0.generate(self.FORK_ROOT_HEIGHT - 2)
        assert (node0.getblockcount() == self.FORK_ROOT_HEIGHT - 1)

        self.log.info("fund attacker")
        funding_tx = self.send_funds_to_attacker (node0, attacker, coinbase_tx)
        node0.generate(1)
        assert (node0.getblockcount() == self.FORK_ROOT_HEIGHT + 0)

        self.log.info("sync nodes. All nodes have the same chain and funding transactions after syncing")
        connect_nodes_bi(self.nodes, 0, 1)
        sync_blocks(self.nodes)
        disconnect_nodes_bi(self.nodes, 0, 1)

        # fork from here
        assert (node0.getblockcount() == node1.getblockcount())

        self.log.info("spends attackers funds in node0")
        for i in range(self.nbDoubleSpends):
            attacker.spend_to_pkh(node0, funding_tx, i, funding_tx.vout[i].nValue, friend0_of_attacker.pubkey)
        node0.generate(1)
        assert (node0.getblockcount() == self.FORK_ROOT_HEIGHT + 1)

        self.log.info("double spend attacker funds in node1")
        for i in range(self.nbDoubleSpends):
            attacker.spend_to_pkh(node1, funding_tx, i, funding_tx.vout[i].nValue, friend1_of_attacker.pubkey)

        node1.generate(1)
        first_bad_block = node1.getbestblockhash()

        assert (node1.getblockcount() == self.FORK_ROOT_HEIGHT + 1)

        self.log.info("check that funds have been double spent to different addresses")
        assert(self.contains_double_spends () == self.nbDoubleSpends)

        # Test 2.
        # 1. Progress the two competing chains in node0 and node1 to different lengths (configurable).
        #    node1 shall hold the longer chain and is the one controlled by the attacker.
        #    The two nodes are not connected to each other directly or indirectly and at this point
        #    contain the doulbe-spends we have prapared.
        # 2. connect the nodes and sync them to force a reorg
        # 3. Assert that all double-spends disappeared - which nontheless means the attack succeeded.
        assert(self.lenChain0 <= self.lenChain1)
        self.log.info("Mine lenChain0 blocks on node0")

        node0.generate(self.lenChain0 - 1)
        assert(node0.getblockcount() == self.FORK_ROOT_HEIGHT + self.lenChain0)

        self.log.info("Mine competing lenChain1 blocks on node1")
        node1.generate(self.lenChain1 - 1)
        assert(node1.getblockcount() == self.FORK_ROOT_HEIGHT + self.lenChain1)

        self.log.info("Connect nodes to force a reorg")
        connect_nodes(self.nodes, 1, 0)
        sync_blocks(self.nodes[0:2])
        if self.lenChain1 > self.lenChain0:
            assert(node0.getblockcount() == self.FORK_ROOT_HEIGHT + self.lenChain1)
        else:
            assert(node1.getblockcount() == self.FORK_ROOT_HEIGHT + self.lenChain0)

        self.log.info("check that both nodes have the same chains")
        lastblock0 = node0.getbestblockhash()
        lastblock1 = node1.getbestblockhash()
        assert(lastblock0 == lastblock1)

        self.log.info("check that double-spends have been removed")
        assert (self.contains_double_spends () == 0)

        # Test 3: Assert that safemode has been reached
        try:
            node0.rpc.getbalance()
            assert False, "Should not come to here, should raise exception in line above."
        except JSONRPCException as e:
            assert e.error["message"] == "Safe mode: Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues. A large valid fork has been detected."

        # Test 4: Assert that safemode is exited if the offending chain is invalidated
        node0.invalidateblock(first_bad_block)
        node0.ignoresafemodeforblock(first_bad_block)
        balance = node0.rpc.getbalance()
        assert (balance != None)

if __name__ == '__main__':
    CompetingChainsTest().main()


from django.urls import path

from payment import views

app_name = 'payment'
urlpatterns = [

    path('canceled/$', views.payment_canceled, name='canceled'),
    path('done/$', views.payment_done, name='done'),
    path('(?P<id>\d+)/process', views.payment_process, name='process')
]


import json, requests, pytest
from pydantic import BaseModel, field_validator
from unittest.mock import Mock, MagicMock

class Location:
    def __init__(self, longitudecls, latitudecls):
        self._longitude = longitudecls
        self._latitude = latitudecls

    def get_weather(self):
        weather_data = requests.get(
            f'https://fcc-weather-api.glitch.me/api/current?lat={self._latitude}&lon={self._longitude}')
        data = json.loads(weather_data.text)
        try:
            if weather_data.status_code != 200:
                raise Exception(f'Error request, status code : {weather_data.status_code}')
            try:
                feels_like_temp = data.get['main']['feels_like']
            except:
                feels_like_temp = 0
            dict_for_weather = {
                'temperature': {
                    'temp': data['main']['temp'],
                    'feels_like': feels_like_temp,
                    'temp_min': data['main']['temp_min'],
                    'temp_max': data['main']['temp_max']
                },
                'pressure': data['main']['pressure'],
                'description': data['weather'][0]['description'],
                'name': data['name']
            }
        except Exception as e:
            return None
        return WeatherPydantic(**dict_for_weather)

class TemperaturePydantic(BaseModel):
    temp: float
    feels_like: float
    temp_min: float
    temp_max: float

    @field_validator('temp')
    def validate_temp(cls, temp: int):
        return round(temp * 1.8 + 32, 3)

class WeatherPydantic(BaseModel):
    temperature: TemperaturePydantic
    pressure: float
    description: str
    name: str

# longitude = int(input('Please enter longitude: '))
# latitude = int(input('Please inter latitude: '))
loc = Location(50, 38)
weather = loc.get_weather()
print(weather)

def test_get_weather(mocker):
    mocker.patch.object(
        requests,
        'get',
        return_value=Mock(
            status_code=200,
            text=json.dumps(

                    {
                        "coord": {
                            "lon": 50,
                            "lat": 28
                        }, "weather": [
                            {
                                "id": 800,
                                "main": "Clear",
                                "description": "clear sky",
                            }
                        ],
                        "base": "stations",
                        "main": {
                             "temp": 33.27,
                             "feels_like": 0.0,
                             "temp_min": 33.27,
                             "temp_max": 33.27,
                             "pressure": 1001,
                             "humidity": 72,
                             "sea_level": 1001,
                             "grnd_level": 1001
                         },
                        "visibility": 10000,
                        "wind": {
                             "speed": 3.25,
                             "deg": 258,
                             "gust": 3.6
                         },
                        "clouds": {
                             "all": 1
                         },
                        "dt": 1691623796,
                        "sys": {
                             "country": "SA",
                             "sunrise": 1691633230,
                             "sunset": 1691681029},
                        "timezone": 12600,
                        "id": 109435,
                        "name": "Jubail",
                        "cod": 200
                    }

            )
        )
    )
    test_loc = Location(50, 28)
    actual = test_loc.get_weather()
    expected = WeatherPydantic(
        temperature=TemperaturePydantic.model_construct(
            temp=round(33.27*1.8 + 32, 3),
            feels_like=0.00,
            temp_min=33.27,
            temp_max=33.27
        ),
        pressure=1001,
        description='clear sky',
        name='Jubail',
    )
    assert actual == expected


# __init__.py

__version__ = "1.2.2"  # Be sure to update version in setup.py as well

from difflib import SequenceMatcher
from sh3ll.command import command
from art import tprint

class IS(object):
    def __init__(self, name="", font="", prefix="CLA>"):
        self.name = name
        self.font = font
        self.prefix = prefix
        self.commands = []
        self.categories = []
        """
        Param: -variable value
        """

    def run(self):
        if self.font != "":
            tprint(self.name, font=self.font)
        else:
            tprint(self.name)

        while True:
            try:
                line = input(self.prefix).lower()
            except KeyboardInterrupt:
                exit()
            inputted_command = line.split()[0]

            # Parse command into command and arguments
            args = []
            tmp = line.split()[1:]
            count = 0
            for arg in range(len(tmp)):
                if count == 0:
                    if tmp[arg][0] == "'":
                        for arg2 in range(len(tmp[arg + 1:])):
                            if tmp[arg + 1 + arg2][-1] == "'":
                                args.append(''.join([s.strip("'") for s in tmp[arg:arg + arg2 + 2]]))
                                count = len(tmp[arg:arg + arg2 + 1])
                    else:
                        if tmp[arg][-1] != "'":
                            args.append(tmp[arg])
                else:
                    count -= 1

            cmds = [cmd.name for cmd in self.commands]
            categories = [cmd.category for cmd in self.commands]

            aliases = {}
            for command in self.commands:
                aliases[command.name] = command.aliases

            if inputted_command != "help" and inputted_command != "exit" and inputted_command != "q":
                if inputted_command in cmds and self.commands[cmds.index(inputted_command)].category == "":
                    self.commands[cmds.index(inputted_command)].execute(args)
                elif inputted_command in categories:
                    if line.split()[1] in cmds:
                        if self.commands[cmds.index(line.split()[1])].category == line.split()[0]:
                            self.commands[cmds.index(line.split()[1])].execute(args[1:])
                    else:
                        for command in self.commands:
                            if line.split()[1] in command.aliases:
                                command.execute(args[1:])
                else:
                    highestSimilarity = 0
                    mostSimilarCommand = ""
                    mostSimilarCommandCategory = ""

                    for command in self.commands:
                        similarity = SequenceMatcher(None, command.name, inputted_command).ratio()
                        if similarity > highestSimilarity:
                            highestSimilarity = SequenceMatcher(None, command.name, inputted_command).ratio()
                            mostSimilarCommand = command.name
                            mostSimilarCommandCategory = command.category

                    print(f"Command not recognized.\nDid you mean: '{mostSimilarCommandCategory} {mostSimilarCommand}'?")
            else:
                self.help() if inputted_command == "help" else exit()

    def help(self):
        print("help\tDisplays this menu")
        print("exit OR q\tExits the program")
        for command in self.commands:
            if command.category == "":
                print(f"{command.name}\t{command.help}")
        print()

        for category in self.categories:
            if category != "":
                print(f"\"{category}\" Commands:\n" + ("-" * (len(category) + 12)))
                cmds = []
                for command in self.commands:
                    if command.category == category:
                        cmds.append(command)

                longest_name = max([len(cmd.name) for cmd in cmds])
                longest_aliases = max([len(str(cmd.aliases)) for cmd in cmds])
                longest_help = max([len(cmd.help) for cmd in cmds])

                print("\tCommand" + (" " * (abs((len(category) + 1 + longest_name) - 7) + 4)) + "Aliases" + (
                            " " * (abs(longest_aliases - 7) + 4)) + "Help" + " " * (abs(longest_help - 4) + 4))
                print("\t" + ("-" * 7) + (" " * (abs((len(category) + 1 + longest_name) - 7) + 4)) + ("-" * 8) + (
                            " " * (abs(longest_aliases - 8) + 4)) + ("-" * 4))

                for command in cmds:
                    if abs(longest_name - len(command.name)) == 0:
                        print(f"\t{category} {command.name}" + (" " * (abs((len(category) + 1 + longest_name) - (
                                    len(category) + len(command.name) + 1)) + 4)), end="")
                    else:
                        print(f"\t{category} {command.name}" + (" " * (
                                    abs((len(category) + 1 + longest_name) - len(f"{category} {command.name}")) + 4)),
                              end="")
                    if abs(longest_aliases - len(str(command.aliases))) == 0:
                        print(f"{command.aliases}    ", end="")
                    else:
                        print(f"{command.aliases}" + (" " * (abs(longest_aliases - len(str(command.aliases))) + 4)),
                              end="")
                    print(f"{command.help}" + (" " * abs(longest_help - len(command.help))))
                print()

    def command(self, name="Unknown command", aliases=[], help="No help given", category="", progress=()):
        def wrap(function):
            if category not in self.categories:
                self.categories.append(category)  # Auto register cats
            self.commands.append(command(function, name=name, aliases=aliases, help=help, category=category, progress=progress))

            def wrapped_function(*args):
                return function(*args)

            return wrapped_function

        return wrap


#!/usr/bin/env python

# Demonstrate how to use the vtkBoxWidget to control volume rendering
# within the interior of the widget.

import vtk
from vtk.util.misc import vtkGetDataRoot
VTK_DATA_ROOT = vtkGetDataRoot()

# Load a volume, use the widget to control what's volume
# rendered. Basically the idea is that the vtkBoxWidget provides a box
# which clips the volume rendering.
v16 = vtk.vtkVolume16Reader()
v16.SetDataDimensions(64, 64)
v16.GetOutput().SetOrigin(0.0, 0.0, 0.0)
v16.SetDataByteOrderToLittleEndian()
v16.SetFilePrefix(VTK_DATA_ROOT+ "/Data/headsq/quarter")
v16.SetImageRange(1, 93)
v16.SetDataSpacing(3.2, 3.2, 1.5)

tfun = vtk.vtkPiecewiseFunction()
tfun.AddPoint(70.0, 0.0)
tfun.AddPoint(599.0, 0)
tfun.AddPoint(600.0, 0)
tfun.AddPoint(1195.0, 0)
tfun.AddPoint(1200, .2)
tfun.AddPoint(1300, .3)
tfun.AddPoint(2000, .3)
tfun.AddPoint(4095.0, 1.0)

ctfun = vtk.vtkColorTransferFunction()
ctfun.AddRGBPoint(0.0, 0.5, 0.0, 0.0)
ctfun.AddRGBPoint(600.0, 1.0, 0.5, 0.5)
ctfun.AddRGBPoint(1280.0, 0.9, 0.2, 0.3)
ctfun.AddRGBPoint(1960.0, 0.81, 0.27, 0.1)
ctfun.AddRGBPoint(4095.0, 0.5, 0.5, 0.5)

compositeFunction = vtk.vtkVolumeRayCastCompositeFunction()

volumeMapper = vtk.vtkVolumeRayCastMapper()
volumeMapper.SetInputConnection(v16.GetOutputPort())
volumeMapper.SetVolumeRayCastFunction(compositeFunction)

volumeProperty = vtk.vtkVolumeProperty()
volumeProperty.SetColor(ctfun)
volumeProperty.SetScalarOpacity(tfun)
volumeProperty.SetInterpolationTypeToLinear()
volumeProperty.ShadeOn()

newvol = vtk.vtkVolume()
newvol.SetMapper(volumeMapper)
newvol.SetProperty(volumeProperty)

outline = vtk.vtkOutlineFilter()
outline.SetInputConnection(v16.GetOutputPort())
outlineMapper = vtk.vtkPolyDataMapper()
outlineMapper.SetInputConnection(outline.GetOutputPort())
outlineActor = vtk.vtkActor()
outlineActor.SetMapper(outlineMapper)

# Create the RenderWindow, Renderer and both Actors
ren = vtk.vtkRenderer()
renWin = vtk.vtkRenderWindow()
renWin.AddRenderer(ren)
iren = vtk.vtkRenderWindowInteractor()
iren.SetRenderWindow(renWin)

# The SetInteractor method is how 3D widgets are associated with the
# render window interactor. Internally, SetInteractor sets up a bunch
# of callbacks using the Command/Observer mechanism (AddObserver()).
boxWidget = vtk.vtkBoxWidget()
boxWidget.SetInteractor(iren)
boxWidget.SetPlaceFactor(1.0)

# Add the actors to the renderer, set the background and size
ren.AddActor(outlineActor)
ren.AddVolume(newvol)

ren.SetBackground(0, 0, 0)
renWin.SetSize(300, 300)

# When interaction starts, the requested frame rate is increased.
def StartInteraction(obj, event):
    global renWin
    renWin.SetDesiredUpdateRate(10)

# When interaction ends, the requested frame rate is decreased to
# normal levels. This causes a full resolution render to occur.
def EndInteraction(obj, event):
    global renWin
    renWin.SetDesiredUpdateRate(0.001)

# The implicit function vtkPlanes is used in conjunction with the
# volume ray cast mapper to limit which portion of the volume is
# volume rendered.
planes = vtk.vtkPlanes()
def ClipVolumeRender(obj, event):
    global planes, volumeMapper
    obj.GetPlanes(planes)
    volumeMapper.SetClippingPlanes(planes)

# Place the interactor initially. The output of the reader is used to
# place the box widget.
boxWidget.SetInput(v16.GetOutput())
boxWidget.PlaceWidget()
boxWidget.InsideOutOn()
boxWidget.AddObserver("StartInteractionEvent", StartInteraction)
boxWidget.AddObserver("InteractionEvent", ClipVolumeRender)
boxWidget.AddObserver("EndInteractionEvent", EndInteraction)

outlineProperty = boxWidget.GetOutlineProperty()
outlineProperty.SetRepresentationToWireframe()
outlineProperty.SetAmbient(1.0)
outlineProperty.SetAmbientColor(1, 1, 1)
outlineProperty.SetLineWidth(3)

selectedOutlineProperty = boxWidget.GetSelectedOutlineProperty()
selectedOutlineProperty.SetRepresentationToWireframe()
selectedOutlineProperty.SetAmbient(1.0)
selectedOutlineProperty.SetAmbientColor(1, 0, 0)
selectedOutlineProperty.SetLineWidth(3)

iren.Initialize()
renWin.Render()
iren.Start()


from django.contrib.auth import get_user_model
from django.utils import timezone
from django.core.mail import send_mail

User = get_user_model()

def wish_birthday():
    today = timezone.now().date()
    user_list = User.objects.filter(birthday__day=today.day, birthday__month=today.month)

    for item in user_list:
        subject = 'Birthday Wish!'
        body = 'Hi {},\n Happy Birthday!!!'.format(item.username)
        send_mail(subject, body, 'contact@yourdomain.com', [item.email])


from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
def get_feature_matrix(N = 55):

    #initialize the feature vector with zeros.
    x_vec = np.zeros((N,3))

    x = []
    for i in range (N):
        im = Image.open("images/image_{number}.jpg".format(number=i+1))

        width, height = im.size

        rgb_im = im.convert('RGB')

        red = []
        green = []
        blue = []

        for y in range (height):
            for x in range (width):
                pixel = rgb_im.getpixel((x,y))
                red.append(pixel[0])
                green.append(pixel[1])
                blue.append(pixel[2])
        x_vec[i] = np.mean(red), np.mean(green), np.mean(blue)

    return x_vec;

def get_labels(N=55):
    y = np.zeros((N,1));

    y = np.zeros((N,1))
    #raise NotImplementedError()
    for i in range (0,20):
        y[i] = 1
    for i in range (21, N):
        y[i] = 0
    return y

def sigmoid_func(z):
    sigmoid = 1/(1+np.exp(-z))
    return sigmoid

def gradient(X,y,w):
    grad = []
    grad = np.transpose(np.dot(np.transpose(X), -(y-sigmoid_func(np.dot(X, np.transpose(w))))))/len(X)
    #raise NotImplementedError()
    return grad

def logisticRegression_func(X,y,step_size, K):
    N = X.shape[0]
    d = X.shape[1]
    # Initialize w as 1xd array.
    w = np.zeros((1,d))
    loss = float('inf')
    loss_list = []
    for i in range(K):
       # loss_list.append(gradient(X,y,w))

        cost = (1/N*np.dot(np.transpose(-y),np.log(sigmoid_func(np.dot(X,np.transpose(w)))))-np.dot(np.transpose((1-y)),np.log(1-sigmoid_func(np.dot(X,np.transpose(w))))))
        w = np.subtract(w, step_size*loss_list[i])
        loss_list.append(cost[0])
        #raise NotImplementedError()
    return loss_list, w

""" Predict Output """
def predict_output(X,w):
    y = []

    value = sigmoid_func(np.dot(X, np.transpose(w)))
    for i in range (len(value)):
        if value[i] >= 0.5:
            y.append(1)
        else:
            y.append(0)
    return y

y = get_labels()
X = get_feature_matrix()

# Full Vector
# Let s label : Grass = 1 , Soil = 0, Tiles = 0
assert X.shape == (55,3)
#axes = Visualize_data(X,y)
step_size = 1e-5
num_iter = 3000
e_list, w_opt = logisticRegression_func(X,y,step_size,num_iter)
print ('The optimal weight vector is:', w_opt)
y_hat = predict_output(X,w_opt)

def visualize_error(X, y, step_sizes, best = None, num_iter = 2000):
    plt.figure(figsize=(12, 4))
    fig, axes = plt.subplots(1, 2,figsize=(12, 4))
    for step in step_sizes:
        loss_list, w_opt = logisticRegression_func(X, y, step, num_iter)
        #raise NotImplementedError()
        n = len(loss_list) # Size of list remains the same.
        x_axes = np.linspace(0,n,n,endpoint=False)
        axes[0].plot(x_axes, loss_list, label=step)
    axes[0].set_xlabel('Number of Iterations')
    axes[0].set_ylabel('Loss Function')
    axes[0].legend()
    axes[0].set_title(r'$\bf{Figure\ 4.}$Converge of GD')

    for step in step_sizes:
        ### STUDENT TASK ###
        # Plot Error against Step Size.
        # Now mark the best converge in red. Use value from best as a correct step size.
        loss_list, w_opt = logisticRegression_func(X, y, step, num_iter)
        # YOUR CODE HERE
        #raise NotImplementedError()
        n = len(loss_list) # Size of list remains the same.
        x_axes = np.linspace(0,n,n,endpoint=False)
        if step == best:
            axes[1].plot(x_axes, loss_list, label=step, color="red")
        else:
            axes[1].plot(x_axes, loss_list, label=step, color="blue")

    axes[1].set_xlabel('Number of Iterations')
    axes[1].set_ylabel('Loss Function')
    axes[1].legend()
    axes[1].set_title(r'$\bf{Figure\ 5.}$Converge of GD')
    plt.tight_layout()
    return best, axes

### STUDENT TASK ###
# Change best=None into step size from the list that provides the fastest converge. e.g best=1
res0_1, axes = visualize_error(X/255, y, best=None, step_sizes=[0.1,0.5,1,5,10,16])
# YOUR CODE HERE
#raise NotImplementedError()


'''
started on 2022/06/13
end on 2022/xx/xx
@author zelo2
'''

import torch
import torch.nn as nn

class LightGCN(nn.Module):
    def __init__(self, n_user, n_item, norm_adj, device, args):
        super(LightGCN, self).__init__()
        self.device = device
        self.n_user = n_user
        self.n_item = n_item
        self.norm_adj = norm_adj

        self.embed_size = args.embed_size
        self.batch_size = args.batch_size
        self.layer_num = args.layer_num
        self.reg_value = eval(args.reg)[0]

        self.embeding_dict = self.init_weight()
        self.sp_norm_adj = self.convert_coo_matirix_2_sp_tensor(self.norm_adj).to(self.device)

    def init_weight(self):
        '''Embedding with xavier initialization'''
        initializer = nn.init.xavier_uniform_
        embedding_dict = nn.ParameterDict({
            'user_embed': nn.Parameter(initializer(torch.empty(self.n_user,
                                                               self.embed_size))),
            'item_embed': nn.Parameter(initializer(torch.empty(self.n_item,
                                                               self.embed_size)))
        })
        # self.user_embedding = nn.Embedding(self.n_user, self.embed_size)
        # self.item_embedding = nn.Embedding(self.n_user, self.embed_size)
        # nn.init.xavier_uniform_(self.user_embedding.weight)
        # nn.init.xavier_uniform_(self.item_embedding.weight)

        return embedding_dict

    def convert_coo_matirix_2_sp_tensor(self, X):
        coo = X.tocoo()
        # coo matrix--((data, (row, column)), shape)
        # data:矩阵中的数据， row, column表示这个数据在哪一行哪一列
        i = torch.LongTensor([coo.row, coo.col])  # [row, column]
        v = torch.from_numpy(coo.data).float()  # data
        return torch.sparse.FloatTensor(i, v, coo.shape)

    def sparse_dropout(self, x, rate, noise_shape):
        save_probability = 1 - rate
        # torch.rand: 均匀分布采样[0,1]
        # 因此加上它之后，大于1的概率即为 1 - node_dropout_rate
        save_probability += torch.rand(noise_shape)
        dropout_mask = torch.float(save_probability).type(torch.bool)
        i = x._indices()
        v = x._values()

        i = i[:, dropout_mask]
        v = v[dropout_mask]

        out = torch.sparse.FloatTensor(i, v, x.shape)

        return out * (1. / (1 - rate))  # dropout部分节点，重新正则化。
        # return out

    def forward(self, user, pos_item, neg_item, drop_flag=False):

        A = self.sp_norm_adj
        embedding_matrix = torch.cat([self.embeding_dict['user_embed'], self.embeding_dict['item_embed']]
                                     , 0)  # [M+N, embedding_size]
        embedding_matrix = embedding_matrix.to(self.device)

        all_embeddings = [embedding_matrix]

        for k in range(self.layer_num):

            # Graph Convolution operation without self connection
            embedding_matrix = torch.sparse.mm(A, embedding_matrix)

            # Normalization
            # norm_embeddings = F.normalize(embedding_matrix, p=2, dim=1)  # normalize each row

            all_embeddings += [embedding_matrix]

        all_embeddings = torch.stack(all_embeddings, dim=1)
        all_embeddings = torch.mean(all_embeddings, dim=1)
        user_embeddings = all_embeddings[:self.n_user, :]
        item_embeddings = all_embeddings[self.n_user:, :]

        user_embeddings = user_embeddings[user, :]
        pos_item_embeddings = item_embeddings[pos_item, :]
        neg_item_embeddings = item_embeddings[neg_item, :]

        return user_embeddings, pos_item_embeddings, neg_item_embeddings  # [batch_size, embed_size * layer_num] * 3

    def bpr_loss(self, users, pos_items, neg_items):
        '''
        :param users: user embeddings [batch_size, embed_size * layer_num]
        :param pos_items: positive item embeddings
        :param neg_items: negative item embeddings
        :return: Bayesian Personalized Ranking loss (BPR loss)
        '''
        pos_inner_product = torch.mul(users, pos_items)
        neg_inner_product = torch.mul(users, neg_items)

        pos_inner_product = torch.sum(pos_inner_product, axis=1)  # sum each row [batch_size]
        neg_inner_product = torch.sum(neg_inner_product, axis=1)

        loss_value = nn.LogSigmoid()(pos_inner_product - neg_inner_product)
        loss_value = -1 * torch.mean(loss_value)

        # L2范式：所有元素的平方和 开根号
        l2_value = torch.norm(users, p=2) ** 2 + torch.norm(pos_items, p=2) ** 2 + torch.norm(neg_items, p=2) ** 2
        l2_value /= 2

        # for k in range(self.layer_num):
        #     l2_value += torch.norm(self.weight_dict['W1_layer%d' % k], p=2) ** 2
        #     l2_value += torch.norm(self.weight_dict['b1_layer%d' % k], p=2) ** 2

        # l2_value /= (2 + self.layer_num * 2)

        l2_value = self.reg_value * l2_value / self.batch_size

        return loss_value + l2_value


"""
Module that can parse chess notation for individual moves. Mostly to debug
things and/or introduce chess states without having to wire up the entire
camera setup on a physical board.

Note that we're using standard Algebraic Notation:

https://en.wikipedia.org/wiki/Algebraic_notation_(chess)

Maybe we move on to FEN https://en.wikipedia.org/wiki/Forsyth%E2%80%93Edwards_Notation
to start from boards?

BUGS:
  - Doesn't handle pawn promotions
  - Doesn't handle disambiguations (when two knights can reach the same place)
"""
import re
from enum import Enum
from collections import namedtuple

# Data definitions. Currently don't allow for draws.
Piece = Enum('Piece', 'Pawn Rook Knight Bishop Queen King')
Action = Enum('Action', 'Move Capture CastleKingside CastleQueenside PawnPromotion')
Modifier = Enum('Modifier', 'Check CheckMate')

Col = Enum('Col', 'A B C D E F G H')
Row = Enum('Row', 'One Two Three Four Five Six Seven Eight')

Position = namedtuple('Position', 'row col')
Move = namedtuple('Move', 'piece action position modifiers')
# Black could be None in the case of a white Checkmate
Turn = namedtuple('Turn', 'white black')

LINE_REGEX = re.compile('(?:\d+\.\s+)\s*(\S+)(?:\s+(\S+)\s*)?$')
POSITION_PATTERN = '([a-h])(1|2|3|4|5|6|7|8)'
POSITION_REGEX = re.compile(POSITION_PATTERN)
PIECE_MAP = {
    'B': Piece.Bishop,
    'R': Piece.Rook,
    'Q': Piece.Queen,
    'K': Piece.King,
    'N': Piece.Knight
}
COL_MAP = {'a': Col.A, 'b': Col.B, 'c': Col.C, 'd': Col.D, 'e': Col.E, 'f': Col.F, 'g': Col.G, 'h': Col.H}
ROW_MAP = {
    '1': Row.One,
    '2': Row.Two,
    '3': Row.Three,
    '4': Row.Four,
    '5': Row.Five,
    '6': Row.Six,
    '7': Row.Seven,
    '8': Row.Eight
}
ACTION_MAP = {
    'x': Action.Capture,
    'O-O': Action.CastleKingside,
    'O-O-O': Action.CastleQueenside,
    '=': Action.PawnPromotion
}

def parse_file(filename):
    with open(filename) as f:
        lines = f.readlines()
        return [parse_line(line.rstrip('\n')) for line in lines]

def parse_line(line):
    components = LINE_REGEX.match(line)
    white_move = _parse_move(components.group(1))

    black_move = None
    black_move_spec = components.group(2)
    if black_move_spec:
        black_move = _parse_move(black_move_spec)
    return Turn(white=white_move, black=black_move)

def _parse_move(move):
    if re.match('O-O-O', move):
        return Move(piece=None, action=Action.CastleQueenside, position=None, modifiers=[])
    elif re.match('O-O', move):
        return Move(piece=None, action=Action.CastleKingside, position=None, modifiers=[])

    piece = _get_piece(move)
    action = _get_action(move)
    position = _get_position(move)
    modifiers = _get_modifiers(move)
    return Move(piece=piece, action=action, position=position, modifiers=modifiers)

def _get_piece(move):
    """
    The piece is realatively easy to determine: it's either a pawn, or directly
    determined by its first letter. Gets _a little_ weird for when pawns capture,
    so we default to that if the first character isnt' a recognized one.
    """
    match = re.search('^' + POSITION_PATTERN, move)
    if match:
        return Piece.Pawn
    else:
        return PIECE_MAP.get(move[0], Piece.Pawn)

def _get_action(move):
    for action in ACTION_MAP.iterkeys():
        if re.search(action, move):
            return ACTION_MAP[action]
    return Action.Move

def _get_position(move):
    """
    The position is pretty easily determined by one of the "acceptable letters" followed by
    an acceptable number.
    """
    match = POSITION_REGEX.search(move)
    return Position(col=COL_MAP[match.group(1)], row=ROW_MAP[match.group(2)])

def _get_modifiers(move):
    modifiers = []
    if re.search('\+', move):
        modifiers.append(Modifier.Check)
    elif re.search('#', move):
        modifiers.append(Modifier.CheckMate)
    return modifiers

def test_data():
    return [
           Turn(white=Move(piece=Piece.Pawn,
                           action=Action.Move,
                           position=Position(col=Col.E, row=Row.Four),
                           modifiers=[]),
                black=Move(piece=Piece.Pawn,
                           action=Action.Move,
                           position=Position(col=Col.E, row=Row.Five),
                           modifiers=[])),
            ]


# -*- coding: utf-8 -*-
"""
Helper functions for classification and quantization

Created on Mon Dec  5 14:50:27 2016

@author: brady
"""
import os
import numpy as np
from sklearn.tree import tree, _tree

def quantize(data, precision):
    """
    Turns floating point into fixed point data
    :param data: vector to quantize, assumes np-array
    :param precision: number of fixed points bits to used
    :returns: vector of length[data], with precision bits
    """
    data = np.array(data)
    data = data*1e5

    xmax = np.amax(np.abs(data))
    #if xmax <= 0:
    #    xmax = 0.000001 # helps with stability
    xq = xmax * np.minimum(
                    np.round(data*(2**(precision-1))/xmax) / (2**(precision-1)),
                    1-1/(2**(precision-1))
                )
    return xq/1e5

def tree_to_code(tree, feature_names, precision):
    tree_ = tree.tree_
    feature_name = [
        feature_names[i] if i != _tree.TREE_UNDEFINED else "undefined!"
        for i in tree_.feature
    ]

    valid_thresh = [
        t if t > 0 else np.min(np.abs(tree_.threshold))
        for t in tree_.threshold
    ]

    quant_thresh = quantize(valid_thresh, precision)

    def recurse(node, depth, quant_tree_str):
        indent = "  " * depth
        if tree_.feature[node] != _tree.TREE_UNDEFINED:
            name = feature_name[node]
            threshold = quant_thresh[node]

            quant_tree_str += "{}if {} <= {}:\n".format(indent, name, threshold)
            quant_tree_str += recurse(tree_.children_left[node], depth + 1, '')

            quant_tree_str += "{}else:  # if {} > {}\n".format(indent, name, threshold)
            quant_tree_str += recurse(tree_.children_right[node], depth + 1, '')

            return quant_tree_str
        else:
            quant_tree_str += "{}return {}\n".format(indent, np.argmax(tree_.value[node]))
            return quant_tree_str

    quant_tree_str = "def tree_{}b(features):\n".format(precision)
    quant_tree_str = recurse(0, 1, quant_tree_str)
    return quant_tree_str

def gen_quant_trees_str(tree, precisions):
    func_list_str = ''
    for p in precisions:
        names = ['features['+str(x)+']' for x in range(20)]
        func_list_str += tree_to_code(tree, names, p)
        func_list_str += "##################################################\n"

    return func_list_str

def make_quant_trees_module(filename, tree, precisions):
    trees_str = gen_quant_trees_str(tree, precisions)

    with open(filename, 'w') as f:
        f.write(trees_str)

def get_tree_results(tree, Xtest):
    """
    Runs data through a quantized DecisionTreeClassifier
    :param tree: DTC function handle
    :param Xtest: data to test
    :returns: predicted results
    """
    results = [tree(X) for X in Xtest]
    return np.array([results], ndmin=1).T

if __name__ == '__main__':
    DIR = r'C:\Users\brady\GitHub\MinVAD\feature_extract'

    tr_data = np.load(os.path.join(DIR, 'train_130k.npy'))
    tr_class = np.load(os.path.join(DIR, 'train_130k_class.npy'))

    myData = np.hstack((tr_data, tr_class))
    np.random.shuffle(myData)

    cutoff = int(np.floor(0.8 * len(tr_class)))
    clf = tree.DecisionTreeClassifier(max_depth = 5)
    clf = clf.fit(myData[:cutoff, :19], myData[:cutoff, 20])
    test_str = gen_quant_trees_str(clf, np.arange(16, 15, -1))
    print(test_str)


# %%
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import ks_2samp
import seaborn as sns
import pandas as pd
import random
from collections import defaultdict
from scipy.stats import ks_2samp, wasserstein_distance
from doubt import Boot
from nobias import ExplanationShiftDetector

random.seed(0)
# Scikit Learn
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import roc_auc_score, mean_squared_error
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPRegressor
from sklearn.linear_model import LogisticRegression, Lasso, LinearRegression
from sklearn.ensemble import (
    GradientBoostingRegressor,
    GradientBoostingClassifier,
)

plt.style.use("seaborn-whitegrid")
from matplotlib import rcParams

rcParams["axes.labelsize"] = 14
rcParams["xtick.labelsize"] = 12
rcParams["ytick.labelsize"] = 12
rcParams["figure.figsize"] = 16, 8
rcParams.update({"font.size": 22})

from xgboost import XGBRegressor, XGBClassifier
import shap
from alibi_detect.cd import ChiSquareDrift, TabularDrift, ClassifierDrift
from tqdm import tqdm
import lime.lime_tabular
import os
import sys

def blockPrint():
    sys.stdout = open(os.devnull, "w")

blockPrint()
# %%

# %%
def create_explanation(X, model):
    exp = X.copy()[:0]

    for i, _ in tqdm(enumerate(X.iterrows())):
        ex = explainer.explain_instance(X.iloc[i], model.predict)
        exx = pd.DataFrame(ex.local_exp[0], columns=["feature", "weight"]).sort_values(
            "feature"
        )
        exx.feature = X.columns
        exx = exx.T
        # Make header first row
        new_header = exx.iloc[0]  # grab the first row for the header
        exx = exx[1:]  # take the data less the header row
        exx.columns = new_header
        exx.reset_index(inplace=True)
        exp = pd.concat([exp, exx])

    return exp

def train_esd(X, X_ood, model, detector):
    aux = create_explanation(X, model)
    aux["y"] = 0
    aux_ood = create_explanation(X_ood, model)
    aux_ood["y"] = 1
    df = aux.append(aux_ood).drop(columns=["index"])
    X_tr, X_te, y_tr, y_te = train_test_split(
        df.drop("y", axis=1), df["y"], test_size=0.5, random_state=42
    )
    detector.fit(X_tr, y_tr)

    # return auc
    return roc_auc_score(y_te, detector.predict_proba(X_te)[:, 1])

# %%
res = []
for i in np.linspace(0, 1, 11):
    rho = i
    ## Sensitivity experiment
    sigma = 1

    mean = [0, 0]
    cov = [[sigma, 0], [0, sigma]]
    samples = 5_000
    x1, x2 = np.random.multivariate_normal(mean, cov, samples).T
    x3 = np.random.normal(0, sigma, samples)
    # Different values
    mean = [0, 0]
    cov = [[sigma, rho], [rho, sigma]]
    x11, x22 = np.random.multivariate_normal(mean, cov, samples).T
    x33 = np.random.normal(0, sigma, samples)

    # Create Data
    df = pd.DataFrame(data=[x1, x2, x3]).T
    df.columns = ["Var%d" % (i + 1) for i in range(df.shape[1])]
    # df["target"] = np.where(df["Var1"] * df["Var2"] > 0, 1, 0)
    df["target"] = (
        df["Var1"] * df["Var2"] + df["Var3"] + np.random.normal(0, 0.1, samples)
    )
    df["target"] = np.where(df["target"] > df["target"].mean(), 1, 0)
    X_ood = pd.DataFrame(data=[x11, x22, x33]).T
    X_ood.columns = ["Var%d" % (i + 1) for i in range(X_ood.shape[1])]

    ## Split Data
    X_tr, X_te, y_tr, y_te = train_test_split(df.drop(columns="target"), df["target"])
    ## Fit our ML model
    model = GradientBoostingClassifier()
    model_r = GradientBoostingRegressor()
    # model = LinearRegression()
    model.fit(X_tr, y_tr)
    model_r.fit(X_tr, y_tr)

    # Input KS Test
    input_ks = 1
    # Classifier Drift
    classifierDrift = 1
    # Output test
    output_ks = 1
    wass = 1
    # Uncertainty
    unc = 1

    # Explanation Shift
    ESD = ExplanationShiftDetector(
        model=XGBClassifier(),
        gmodel=Pipeline(
            [
                ("scaler", StandardScaler()),
                ("lr", LogisticRegression(penalty="l1", solver="liblinear")),
            ]
        ),
    )
    ESD.fit(X_tr, y_tr, X_ood)
    esd = ESD.get_auc_val()

    # Lime
    explainer = lime.lime_tabular.LimeTabularExplainer(
        X_tr.values,
        feature_names=X_tr.columns,
        class_names=["y"],
        discretize_continuous=True,
        verbose=True,
        mode="regression",
    )
    auc_lime = train_esd(
        X_te, X_ood, XGBClassifier().fit(X_tr, y_tr), LogisticRegression()
    )

    res.append([rho, input_ks, classifierDrift, output_ks, wass, unc, esd, auc_lime])

# %%
results = pd.DataFrame(
    res,
    columns=[
        "rho",
        "input_ks",
        "classifierDrift",
        "output_ks",
        "wass",
        "unc",
        "esd",
        "lime",
    ],
)
# %%
plt.figure()
plt.plot(results["rho"], results["esd"], label="Explanation Shift - SHAP")
ci = 1.96 * np.std(results["esd"]) / np.sqrt(len(results["rho"]))
plt.fill_between(
    results["rho"], (results["esd"] - ci), (results["esd"] + ci), alpha=0.1
)

plt.plot(results["rho"], results["lime"], label="Explanation Shift - Lime")
ci = 1.96 * np.std(results["lime"]) / np.sqrt(len(results["rho"]))
plt.fill_between(
    results["rho"], (results["lime"] - ci), (results["lime"] + ci), alpha=0.1
)

plt.legend()
plt.xlabel("Correlation coefficient")
plt.ylabel("AUC Explanation Shift Detector")
plt.title("Sensitivity to Multicovariate Shift")
plt.tight_layout()
plt.savefig("images/SOTAsensitivityLime.pdf", bbox_inches="tight")
plt.show()

# %%


from tkinter import *
import tkinter.font
from gpiozero import LED
import RPi.GPIO
RPi.GPIO.setmode(RPi.GPIO.BCM)

blue = LED(2)
green = LED(3)
red = LED(4)

win = Tk()
win.title("LED GUI Toggler")
myFont = tkinter.font.Font(family = 'Helvetica', size = 12, weight = "bold")

def ledToggleBlue():
    if blue.is_lit:
        blue.off()
        blueButton["text"] = "Turn Blue LED on"
    else:
        blue.on()
        blueButton["text"] = "Turn Blue LED off"

def ledToggleRed():
    if red.is_lit:
        red.off()
        redButton["text"] = "Turn Red LED on"
    else:
        red.on()
        redButton["text"] = "Turn Red LED off"

def ledToggleGreen():
    if green.is_lit:
        green.off()
        greenButton["text"] = "Turn Green LED on"
    else:
        green.on()
        greenButton["text"] = "Turn Green LED off"

def close():
    RPi.GPIO.cleanup()
    win.destroy()

blueButton = Button(win, text = 'Turn Blue LED on', font = myFont, command = ledToggleBlue, bg = 'blue', height = 1, width = 24)
blueButton.grid(row=0,column=1)

redButton = Button(win, text = 'Turn Red LED on', font = myFont, command = ledToggleRed, bg = 'red', height = 1, width = 24)
redButton.grid(row=1,column=1)

greenButton = Button(win, text = 'Turn Green LED on', font = myFont, command = ledToggleGreen, bg = 'green', height = 1, width = 24)
greenButton.grid(row=2,column=1)

exitButton = Button(win, text = 'Exit', font = myFont, command = close, bg = 'red2', height = 1, width = 6)
exitButton.grid(row=3,column=1)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

demand = [990,1980,3961,2971,1980]
d=0  # d% shortage allowance
Y_b = [1.3086,1.3671,1.4183,1.4538,1.5122]           # Fabric yield (consumption rate) rate per garment of size 饾浗

U = 0.85
l_max= 20
e= .07   # Fabric end allowance
f= 2.90 # Fabric cost

if len(demand)!=len(Y_b):
    raise ValueError('number of sizes and number of fabric consumption does not match')

# In[2]:

#Input variables (Marker)

M_d = 10                                # Average marker design time (minute)
z = 0.65                                # Printing speed per minute
v = 0.30                                #Standard cost per minute in marker making floor (labor, machine & electricity)

# In[3]:

#Input variables (Cutting Time)

T_G = 30                                  # General Preparation Time
x= .20           # Average spreading speed in minutes after taking account for the idle strokes.
T_M= 2           # Time for Placement of the marker
t_c= 4.5         # SMV of cutting time per garment pattern
T_S= 5           # preparation time for sticker placement
饾憽_饾憦 = 2.837                               # Standard minute value (SMV) of time takes to bundle.
饾憦 =  15                                   # pieces of garments in one bundle
饾懁 = 0.20                                 # standard cost per minute in cutting floor (labor, machine & electricity)
P_min, P_max= 10,350

# In[4]:

import numpy as np
import math
import pandas as pd
from copy import deepcopy
rng = np.random.default_rng()
import random
import time
import matplotlib.pylab as plt
import plotly.express as px

# In[5]:

def Update_Res(R,GG,PP):
    for s in range(len(GG)): #Updating Residual within the while loop
        R=R-np.dot(GG[s],PP[s])
    return R

# In[6]:

def Length(g_i_j):
    l_i = e+ np.dot(g_i_j,Y_b)/U
    return l_i

# In[7]:

def Shortage_allowance(Q,d=0.01):
    temp=np.dot((1-d),Q)
    return [round(i) for i in temp]
Q_b= Shortage_allowance(demand,d)

# Q_b

# In[8]:

from Heuristics import H1,H3,H5

# In[9]:

# Sol_1 = H5(Q=Q_b,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)
# Sol_1

# ## Objective Function

# In[10]:

def ObjectiveFunction (chromosome):

    temp_Chromosome=deepcopy(chromosome)
    G_a_b = temp_Chromosome['G']
    P_a = temp_Chromosome['P']
    Alpha = len(P_a)             # number of Sections

    '''                         Fabric Cost                      '''

    # Total fabric length = L # Total Fabric Cost = C_F

    l_a=[Length(G_a_b[alpha]) for alpha in range(Alpha) ] #Length function
    L= np.dot(l_a,P_a)       #Multiply then Sum
    C_F = L*f
    #print('Total Fabric Cost = C_F: ',C_F)

    '''                        Marker Cost                        '''

    #Marker Making Cost = C_M

    M_p_a = [(la-e)/z for la in l_a]     # devide each element of a 'l_a' by 'z'
    #M_p_a = Marker Printing time (minute) of section alpha

    '''
    饾憻 = {1 ; 饾憱饾憮 饾憽h饾憭 饾憵饾憥饾憻饾憳饾憭饾憻 饾憱饾憼 饾憦饾憭饾憱饾憶饾憯 饾憿饾憼饾憭饾憫 饾憮饾憸饾憻 饾憽h饾憭 饾憮饾憱饾憻饾憼饾憽 饾憽饾憱饾憵饾憭
        {0 ; 饾憱饾憮 饾憽h饾憭 饾憵饾憥饾憻饾憳饾憭饾憻 h饾憥饾憼 饾憦饾憭饾憭饾憶 饾憿饾憼饾憭饾憫 饾憦饾憭饾憮饾憸饾憻饾憭
    '''
    r=[]
    for i in range(Alpha):
        temp=0
        j=i-1
        while j>=0:
            if G_a_b[i]== G_a_b[j]:
                temp+=1
                break
            j-=1
        if temp==0:
            r.append(1)
        else:
            r.append(0)

    C_M = 0
    for 伪 in range(Alpha):
        if l_a[伪]>e:   # this makes sure that section has at least one garments
            C_M += (M_d*r[伪] + M_p_a[伪])*v

    # 'if la>e' makes sure that the section contain at least one garments,
    #  not all G_a_b values are zero

    '''                          Cutting Cost                            '''

    # Cutting Time of one section = T_T  # Total Cutting Cost = C_C

    #T_T  =T_G + T_F +T_M+ T_c+T_S +T_B

    T_C=[]   #Cutting time for every section
    for alpha in range(Alpha):
        T_C.append(sum(G_a_b[alpha])*t_c)

    T_F=[]  # Fab spreading time for each section
    for 伪 in range(Alpha):
        T_F.append(l_a[伪]*P_a[伪]/x)

    T_B=[] #Bundleing time for each section
    for 伪 in range(Alpha):
        T_B.append(math.ceil(P_a[伪]/b)*sum(G_a_b[伪])*t_b)

    T_T_T = 0  #Total cutting time
    for 伪 in range(Alpha):
        if l_a[伪]>e:   # this makes sure that section has at least one garments
            T_T_T+=T_G+T_F[伪]+T_M+T_C[伪]+T_S+ T_B[伪]

    C_C = T_T_T*w  #Total cutting cost

    '''                              Total Cost                 '''
    # Total Cost = C_T = C_F + C_M + C_C

    return C_F+C_M+C_C

# In[11]:

# ObjectiveFunction(Sol_1)

# ## Fitness Score

# In[12]:

def Fitness(chromosome):

    t_chromosome=deepcopy(chromosome)
    G_a_b= t_chromosome['G']
    P_a = t_chromosome['P']
    Beta= len(demand)

    score= ObjectiveFunction(t_chromosome)
    #print('score:',score)
    fitness_score=score

    '''       Penalty for shortage production           '''
    R= Update_Res(R=demand,GG=G_a_b,PP=P_a)
    for beta in range(Beta):
        if R[beta]>0:
            s_penalty= R[beta]/sum(demand)
            fitness_score +=score*s_penalty

    '''         Penalty for excess production           '''
    r=np.dot(1.02,demand)         # additional 2% allowance
    R= Update_Res(R=r,GG=G_a_b,PP=P_a)
    #print(R)
    for beta in range(Beta):
        if R[beta]<0:
            e_penalty= (-R[beta]/sum(demand))*2   # 2times than s_penalty
            fitness_score +=score*e_penalty

    '''       double check if the solution is valid       '''
    res= Update_Res(R=Q_b,GG=G_a_b,PP=P_a)
    if max(res)>0:
        '''solution is unvalid'''
        fitness_score +=10000   #this will eventualy make the solution extinct.

    return fitness_score

# Fitness(Sol_1)

# ## Function Initial Population Generation

# In[13]:

def GeneratePopulation(pop_size):
    P_of_S=[]
    for p in range(pop_size):
        rng = np.random.default_rng()
        h=rng.integers(0,3)
        #print('h:',h)

        if h==0:
            sol=H1(Q=Q_b,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)
        elif h==1:
            sol=H3(Q=Q_b,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)
        else:
            sol=H5(Q=Q_b,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)

        P_of_S.append(sol)
    return P_of_S
# Pool_of_Sol= GeneratePopulation(100)
# print(Pool_of_Sol)

# In[14]:

def S_with_F(p_o_s):
    p_o_s_with_f= deepcopy(p_o_s)
    for i in range(len(p_o_s)):
        if 'F' not in p_o_s[i]:
            p_o_s_with_f[i]['F']=Fitness(p_o_s[i])
    return p_o_s_with_f

# ## PSO

# ### Cleaning section with zeros

# In[15]:

def CleanZeros (Sol):
    Solution=deepcopy(Sol)
    j=0
    while j < len(Solution['G']):
        if max(Solution['G'][j])==0:
            Solution['G'].pop(j)
            Solution['P'].pop(j)
            continue
        j+=1

    #This is to make sure
    if len(Solution['G'])!=len(Solution['P']):
        raise ValueError('P and G lengths are not same')

    return Solution

# In[16]:

# CleanZeros(Sol_1)

# ## Velocity Update (Jarboui et al. 2008)

# Lets assume 1st sol as X, 2nd Sol as Pbest, and 3rd Sol as Gbest

# #### Now we have to calculate Y

# ##### Initial Velocity generator

# In[17]:

def initial_velocity(Range, Sol): #Range is a list
    a,b= Range
    m=len(Sol['G'])

    #generate a random uniform array  [-a,b] of the same size of the solutions

    v=(b-a) * np.random.random_sample(m) +a  #http://bit.ly/3To2OWe
    v=v.tolist()

    return {'V':v}

# In[18]:

def Get_Y(X,GBest,PBest): #(Jarboui et al., 2008, p. 302)
    y=[]
    lens=[len(i) for i in [X['G'],GBest['G'],PBest['G']]]
    min_len=min(lens)

    for i in range(min_len):
        if X['G'][i]==GBest['G'][i] and X['G'][i]==PBest['G'][i]:
            y.append(random.choice([-1,1]))
        elif X['G'][i]==GBest['G'][i]:
            y.append(1)
        elif X['G'][i]==PBest['G'][i]:
            y.append(-1)
        else:
            y.append(0)

    return {'Y':y}

# ### Now we have to calculate Velocity

# In[19]:

def New_V(YY,VV,c1=1,c2=1,w=.75): #Parameter setting: (Jarboui et al., 2008, p. 306)
    Y=deepcopy(YY)
    V=deepcopy(VV)

    lens=[len(i) for i in [Y['Y'],V['V']]]
    min_len=min(lens)

    for i in range(min_len):
        y=Y['Y'][i]
        v=V['V'][i]
        V['V'][i]= w*v+ np.random.rand()*c1*(-1-y)+np.random.rand()*c2*(1-y)

    return V

# ### Now we need to calculate 位

# In[20]:

def Get_位(YY,VV):
    Y=deepcopy(YY)
    V=deepcopy(VV)

    lens=[len(i) for i in [Y['Y'],V['V']]]
    min_len=min(lens)

    位=[]
    for i in range(min_len):
        位.append(Y['Y'][i]+V['V'][i])
    return {'位':位}
# 位=Get_位(Y,V)
# 位

# ### Update X with Eq-10 (Jarboui et al., 2008, p. 303)

# In[21]:

def Perturbation(xg,xp,R,p_rate):

    if np.random.rand()<p_rate:
        p1,p2=sorted([xp,min(P_max,max(P_min,max(R)))])
        xp= rng.integers(p1,p2+1)
        if xp<P_min:
            xp=P_min
    for j in range(len(xg)): #small purtubration (like mutaion)
        if np.random.rand()<p_rate:
            xg[j]=0
            temp= min(math.ceil(R[j]/xp),math.floor((l_max-Length(xg))/(Y_b[j]/U)))
            temp= max(0,temp)
            #xg[j]=max(0,temp)
            xg[j]=rng.integers(0,temp+1)

    return xg,xp

def Update_X(XX,GBest,PBest,位位, 蠒=0.5, p_rate=.05):
    X=deepcopy(XX)
    位=deepcopy(位位)

    lens=[len(i) for i in [X['G'],GBest['G'],PBest['G'],位['位']]]
    min_len=min(lens)
    XG=[]
    XP=[]
    R= Update_Res(R=Q_b,GG=XG,PP=XP)
    for i in range(min_len):
        if 位['位'][i] > 蠒:
            #print('Gbest')
            #xg,xp=Perturbation(xg=GBest['G'][i],xp=GBest['P'][i],R=R,p_rate=p_rate)
            xg=GBest['G'][i]
            xp=GBest['P'][i]

        elif 位['位'][i] < -蠒:
            #print('Pbest')
            #xg,xp=Perturbation(xg=GBest['G'][i],xp=GBest['P'][i],R=R,p_rate=p_rate)
            xg=PBest['G'][i]
            xp=PBest['P'][i]
        else:
            #print('X')
            xg,xp= Perturbation(xg=X['G'][i],xp=X['P'][i],R=R,p_rate=p_rate) #Perturbation function

        XG.append(xg)
        XP.append(xp)
        R= Update_Res(R=Q_b,GG=XG,PP=XP)
        if max(R)<=0:
            #print('break')
            return {'G':XG,'P':XP}

    for i in range(min_len, len(X['G'])):
        xg,xp= Perturbation(xg=X['G'][i],xp=X['P'][i],R=R,p_rate=p_rate) #Perturbation function
        XG.append(xg)
        XP.append(xp)
        R=Update_Res(R=Q_b,GG=XG,PP=XP)
        if max(R)<=0:
            #print('break')
            return {'G':XG,'P':XP}

    if max(R)>0:
        #print(R)
        #Use H1 or H3 algorithm to pack all sizes
        randint =rng.integers(2)
        if randint==0:
            #print('H1')
            h=H1(Q=R,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)
        else:
            #print('H3')
            h=H3(Q=R,Y=Y_b,Pm=[P_min,P_max],U=U,e=e,l_max=l_max)

        g,p = h.values()
        #print(g,p)
        XG=XG+g
        XP=XP+p

    return {'G':XG,'P':XP}
# newX= Update_X(X,Gbest,Pbest,newY)
# newX

# In[22]:

y=[1,2,3,4]
c=[1,2]
print(y[:len(c)])

# In[23]:

def Update_dimension(XX,VV, in_vel_range=[-0.5,0.5]):
    mm= len(XX['G'])
    m= len(VV['V'])

    if mm <= m:
        return {'V':VV['V'][:m]}
    else:
        a,b= in_vel_range
        v=(b-a) * np.random.random_sample(mm-m) +a  #http://bit.ly/3To2OWe
        v=v.tolist()
        V=VV['V']+v
        return {'V':V}

# In[24]:

def Get_Gbest(p_o_s):

    gbest=p_o_s[0]
    for i in range(len(p_o_s)):
        if Fitness(p_o_s[i])<Fitness(gbest):
            gbest= p_o_s[i]
    return gbest
# Gbest=Get_Gbest(Pool_of_Sol)
# Gbest

# In[25]:

# newX= Update_X(X,Gbest,Pbest,newY)
# newX

# In[26]:

# Fitness(newX)

# In[27]:

#Pool_of_Sol

# # Main PSO

# In[36]:

get_ipython().run_line_magic('matplotlib', 'inline')
def PSO(swarmsize,iteration,蠒=.7,c1=2,c2=2,w=1, in_vel_range=[-0.6,0.6],p_rate=.2):

    P_of_S= GeneratePopulation(swarmsize)
    P_of_Pbest=P_of_S
    P_of_Velocity= [initial_velocity(in_vel_range,P_of_S[i]) for i in range(len(P_of_S))]
    Gbest= P_of_S[rng.integers(0,swarmsize)]
    o= Gbest
    bests=[Fitness(Gbest)]
    for i in range(iteration):
        for j in range(len(P_of_S)):
            X=P_of_S[j]
            Pbest=P_of_Pbest[j]
            V= P_of_Velocity[j]
            Y= Get_Y(X=X,GBest=Gbest,PBest=Pbest)

            newV= New_V(YY=Y,VV=V,c1=c1,c2=c2,w=w)

            位= Get_位(YY=Y,VV=newV)

            newX= Update_X(XX=X,GBest=Gbest,PBest=Pbest,位位=位,蠒=蠒, p_rate=p_rate)

            P_of_S[j]=newX

            newV= Update_dimension(XX=newX,VV= newV, in_vel_range=in_vel_range)
            P_of_Velocity[j]= newV

            f=Fitness(newX)
            if f < Fitness(Pbest):
                P_of_Pbest[j]= newX
            if f < Fitness(Gbest):
                Gbest=newX
        #print(Gbest, Fitness(Gbest))
        bests.append(Fitness(Gbest))

    xx=[i for i in range(len(bests))]
    fig=px.line(x=xx,
                y=bests,
                title=f'swarmsize={swarmsize},iteration= {iteration},蠒={蠒},c1= {c1},c2={c2},w={w}, Gbest={bests[-1]}',
                labels=dict(x='iteration',y='fitness'))
    fig.show()
    #plt.plot(xx,bests)
    #plt.title(f'swarmsize={swarmsize},iteration= {iteration},蠒={蠒},c1= {c1},c2={c2},w={w}, Gbest={bests[-1]}')

    return CleanZeros(Gbest)
PSO(swarmsize=50,iteration=250)

# In[33]:

ObjectiveFunction(o)

# In[34]:

ObjectiveFunction(g)

# In[37]:

Dataset={
    'demands':[[872,1743,3486,2614,1743],
               [12,67,131,187,191,138,79,27],
               [990,1980,3961,2971,1980],
               [193,501,1018,1249,998,564,250,128]],
    'consumption':[[0.6119,0.6315,0.6499,0.6721,0.6921],
                   [0.7198,0.7352,0.7614,0.7878,0.8146,0.8423,0.8579,0.8985],
                   [1.3086,1.3671,1.4183,1.4538,1.5122],
                   [1.3350,1.3998,1.4356,1.4826,1.5440,1.5878,1.6313,1.6908]],
    'price':[1.51,2.43,1.95,2.9]
}
df = pd.DataFrame(columns=['蠒','c1','c2','w','p_rate','solution','fitness'])
for i in range(len(Dataset['demands'])):
    demand=Dataset['demands'][i]
    Q_b= Shortage_allowance(demand,d)
    Y_b=Dataset['consumption'][i]
    f=Dataset['price'][i]
    PSO(swarmsize=100,iteration=120,c1=1,c2=2,蠒 =.4,w=.75,p_rate=.2)

# In[ ]:

# In[29]:

from itertools import product

# In[30]:

蠒=[.4,.5,.6,.7]
c1=[1,1.5,2]
c2=[1,1.5,2]
ww=[.6,.75,1,1.25]
p_rate=[.05,.1,.2,.3]
iteration=product(蠒,c1,c2,ww,p_rate)
#print(list(iteration))

# In[31]:

df = pd.DataFrame(columns=['蠒','c1','c2','w','p_rate','solution','fitness'])
for 蠒,c1,c2,ww,p_rate in product(蠒,c1,c2,ww,p_rate):
    best=PSO(swarmsize=100,iteration=120,c1=c1,c2=c2,蠒=蠒,w=ww,p_rate=p_rate)
    fitness=Fitness(best)
    df = df.append({'蠒':蠒,'c1':c1,'c2': c2,'w': ww,'p_rate': p_rate,'solution':best,'fitness':fitness}, ignore_index=True)
    df.to_csv('PSO_GridSearch_from_Notebook8.csv')

# In[32]:

print(df[['蠒','c1','c2','w','p_rate','fitness']])

# import plotly
# import plotly.graph_objs as go
#
#
# #Read cars data from csv
#
#
# #Set marker properties
# markersize = df['c2']
# markercolor = df['w']
# markershape = df['c1'].replace(1,"square").replace(1.5,"circle").replace(2,'diamond')
#
#
# #Make Plotly figure
# fig1 = go.scater3d( x=df['伪'],
#                     y=df['p_rate'],
#                     z=df['fitness'],
#                     marker=dict(#size=markersize,
#                                 #color=markercolor,
#                                 #symbol=markershape,
#                                 opacity=0.9,
#                                 reversescale=True,
#                                 colorscale='dense'),
#                     line=dict (width=0.02),
#                     mode='markers')
#
# #Make Plot.ly Layout
# mylayout = go.Layout(scene=dict(xaxis=dict( title='伪'),
#                                 yaxis=dict( title='p_rate'),
#                                 zaxis=dict(title='fitness')),)
#
# #Plot and save html
# plotly.offline.plot({"data": [fig1],
#                      "layout": mylayout},
#                      auto_open=True,
#                      filename=("6DPlot.html"))
#

# In[33]:

import plotly.express as px #https://plotly.com/python/3d-scatter-plots/
fig = px.scatter_3d(df, x='蠒', y='p_rate', z='fitness',
                    color='c1', symbol='c2', size='w')
fig.show()

# In[34]:

df['c1+c2']=df['c1'].map(str)+','+df['c2'].map(str)
df

# In[35]:

fig = px.scatter_3d(df, x='蠒', y='p_rate', z='fitness',
                    color='c1+c2', symbol='w')
fig.show()

# In[36]:

fig = px.parallel_coordinates(df, color="fitness",
                              dimensions=['c1','蠒', 'c2','p_rate','w','fitness','c1+c2'],
                              #color_continuous_scale=px.colors.diverging.Tealrose,
                              #color_continuous_midpoint=0
                             )
fig.show()

# In[37]:

df.sort_values('fitness').head(10)

# In[38]:

type(df['c1'][1])

# In[39]:

Dataset={
    'demands':[[872,1743,3486,2614,1743],
               [12,67,131,187,191,138,79,27],
               [990,1980,3961,2971,1980],
               [193,501,1018,1249,998,564,250,128]],
    'consumption':[[0.6119,0.6315,0.6499,0.6721,0.6921],
                   [0.7198,0.7352,0.7614,0.7878,0.8146,0.8423,0.8579,0.8985],
                   [1.3086,1.3671,1.4183,1.4538,1.5122],
                   [1.3350,1.3998,1.4356,1.4826,1.5440,1.5878,1.6313,1.6908]],
    'price':[1.51,2.43,1.95,2.9]
}

# In[40]:

for i in range(len(Dataset['demands'])):
    demand=Dataset['demands'][i]
    Q_b= Shortage_allowance(demand,d)
    Y_b=Dataset['consumption'][i]
    f=Dataset['price'][i]
    best=PSO(swarmsize=100,iteration=120,c1=1,c2=2,蠒=.4,w=.75,p_rate=.2)
    fitness=Fitness(best)
    df = df.append({'蠒':蠒,'c1':c1,'c2': c2,'w': ww,'p_rate': p_rate,'solution':best,'fitness':fitness}, ignore_index=True)
    df.to_csv('PSO_GridSearch_from_Notebook7.csv')

# In[ ]:

# In[ ]:


import numpy as np
import matplotlib.pyplot as plt
import time

import numpy as np
import pandas as pd
import time
import gc
import random
import sklearn
import matplotlib.pyplot as plt
from sklearn.model_selection import cross_val_score, GridSearchCV, cross_validate, train_test_split
from sklearn.metrics import accuracy_score, classification_report
from sklearn.svm import SVC
from sklearn.linear_model import LinearRegression
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler, normalize
from sklearn.decomposition import PCA
from sklearn.impute import SimpleImputer
from sklearn import tree
from sklearn.metrics import plot_roc_curve
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from mpl_toolkits.mplot3d import Axes3D
from sklearn.cluster import KMeans
from kneed import KneeLocator
import seaborn as sb
from sklearn.metrics import silhouette_score
from sklearn import manifold
from sklearn.mixture import GaussianMixture

class Data():

    def dataAllocation(self, path):
        # Separate out the x_data and y_data and return each
        # args: string path for .csv file
        # return: pandas dataframe, pandas series
        # -------------------------------
        # ADD CODE HERE
        df = pd.read_csv(path)
        xcols = ['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8']
        ycol = ['y']
        x_data = df[xcols]
        y_data = df[ycol]
#        print(y_data[y_data.y == 1].shape[0])
 #       print(df.shape[0])
        # -------------------------------
        return x_data, y_data.values.ravel()

    def processed_data_Allocation(self, path):
    # Read the processed dataset
    # -------------------------------
        df = pd.read_csv(path)
        xcols = ["age","education","default","housing","loan","contact","month","day_of_week","campaign","previous","poutcome","emp.var.rate","cons.price.idx","cons.conf.idx","euribor3m","nr.employed","job_blue-collar","job_entrepreneur","job_housemaid","job_management","job_retired","job_self-employed","job_services","job_student","job_technician","job_unemployed","marital_married","marital_single"]
        ycol = ['y']
        x_data = df[xcols]
        y_data = df[ycol]

        return x_data, y_data.values.ravel()

    def trainSets(self, x_data, y_data):
        # Split 70% of the data into training and 30% into test sets. Call them x_train, x_test, y_train and y_test.
        # Use the train_test_split method in sklearn with the parameter 'shuffle' set to true and the 'random_state' set to 614.
        # args: pandas dataframe, pandas dataframe
        # return: pandas dataframe, pandas dataframe, pandas series, pandas series
        # -------------------------------
        # ADD CODE HERE
        x_train, x_test, y_train, y_test = train_test_split(
            x_data, y_data, test_size=0.2, shuffle=True, random_state=614)
        # -------------------------------
        return x_train, x_test, y_train, y_test

    def dataPreProcess(self, x_train, x_test):
        # Pre-process the data to standardize it, otherwise the grid search will take much longer.
        # args: pandas dataframe, pandas dataframe
        # return: pandas dataframe, pandas dataframe
        # -------------------------------
        # ADD CODE HERE
        scaler = StandardScaler()
        scaler.fit(x_train)
        scaled_x_train = scaler.transform(x_train)
        scaled_x_test = scaler.transform(x_test)
        # -------------------------------
        return scaled_x_train, scaled_x_test

fig, axs = plt.subplots(2, 3)
##################### Diabetes data #############################
dataset = Data()
data = 'data/pima-indians-diabetes.csv'
x_data, y_data = dataset.dataAllocation(data)
x_train, x_test, y_train, y_test = dataset.trainSets(x_data, y_data)
x_train_scaled, x_test_scaled = dataset.dataPreProcess(x_train, x_test)

# KM with DS1
kmeans_kwargs = {'init': 'random', 'n_init':10, 'max_iter':100, 'random_state':42, 'algorithm':'full',}
start = time.time()
kmeans = KMeans(n_clusters=4, **kmeans_kwargs)
label = kmeans.fit(x_train_scaled).labels_
end = time.time()
print("KM with DS1 time:", end-start)
tsne_transform = manifold.TSNE(n_components=2, perplexity=100, init='pca', random_state=42)
feature2D_DS1 = tsne_transform.fit_transform(x_train_scaled)
axs[0, 0].scatter(feature2D_DS1[:,0], feature2D_DS1[:,1], c=label, cmap=plt.cm.Spectral, s=5)
axs[0, 0].set_title('Clusters for DS1 with KM')

# EM with DS1
em_kwargs = {'covariance_type': 'full', 'n_init':10, 'max_iter':100, 'random_state':42, 'init_params':'kmeans'}
start = time.time()
em = GaussianMixture(n_components=5, **em_kwargs)
label = em.fit(x_train_scaled).predict(x_train_scaled)
end = time.time()
print("EM with DS1 time:", end-start)
tsne_transform = manifold.TSNE(n_components=2, perplexity=100, init='pca', random_state=42)
feature2D_DS1 = tsne_transform.fit_transform(x_train_scaled)
axs[0, 1].scatter(feature2D_DS1[:,0], feature2D_DS1[:,1], c=label, cmap=plt.cm.Spectral, s=5)
axs[0, 1].set_title('Clusters for DS1 with EM')

axs[0, 2].scatter(feature2D_DS1[:,0], feature2D_DS1[:,1], c=y_train, cmap=plt.cm.Spectral, s=5)
axs[0, 2].set_title('Clusters for DS1 - True Label')

dataset = Data()
data = 'data/bank_marketing.csv'
x_data, y_data = dataset.processed_data_Allocation(data)
x_train, x_test, y_train, y_test = dataset.trainSets(x_data, y_data)
x_train_scaled, x_test_scaled = dataset.dataPreProcess(x_train, x_test)

# KM with DS2
kmeans_kwargs = {'init': 'random', 'n_init':10, 'max_iter':100, 'random_state':42, 'algorithm':'full',}
start = time.time()
kmeans = KMeans(n_clusters=6, **kmeans_kwargs)
label = kmeans.fit(x_train_scaled).labels_
end = time.time()
print("KM with DS2 time:", end-start)
tsne_transform = manifold.TSNE(n_components=2, perplexity=100, init='pca', random_state=42)
feature2D_DS2 = tsne_transform.fit_transform(x_train_scaled)
axs[1, 0].scatter(feature2D_DS2[:,0], feature2D_DS2[:,1], c=label, cmap=plt.cm.Spectral, s=5)
axs[1, 0].set_title('Clusters for DS2 with KM')

# EM with DS2
em_kwargs = {'covariance_type': 'full', 'n_init':10, 'max_iter':100, 'random_state':42, 'init_params':'kmeans'}
start = time.time()
em = GaussianMixture(n_components=5, **em_kwargs)
label = em.fit(x_train_scaled).predict(x_train_scaled)
end = time.time()
print("EM with DS2 time:", end-start)
tsne_transform = manifold.TSNE(n_components=2, perplexity=100, init='pca', random_state=42)
feature2D_DS2 = tsne_transform.fit_transform(x_train_scaled)
axs[1, 1].scatter(feature2D_DS2[:,0], feature2D_DS2[:,1], c=label, cmap=plt.cm.Spectral, s=5)
axs[1, 1].set_title('Clusters for DS2 with EM')

axs[1, 2].scatter(feature2D_DS2[:,0], feature2D_DS2[:,1], c=y_train, cmap=plt.cm.Spectral, s=5)
axs[1, 2].set_title('Clusters for DS2 - True Label')

fig.tight_layout()

plt.show()


import numpy as np
import keras
from keras.datasets import mnist

class Dataset:

    def __init__(self, path, local):
        """
        Initialize the MNIST dataset.
        Parameters path and local are only included to fit the interface of Dataset
        :param path: Ignored
        :param local: Ignored
        """

        (x, y), (_, _) = mnist.load_data()

        # Configure input
        x = (x.astype(np.float32) - 127.5) / 127.5
        x = np.expand_dims(x, axis=3)
        x_padding = np.zeros((x.shape[0], 64, 64, 1)) - 1
        x_padding[:, :28, :28, :] = x
        x = x_padding
        y = keras.utils.np_utils.to_categorical(y, 10)

        self.x = x
        self.y = y

        print('Loaded dataset')
        print('X:', self.x.shape)
        print('Y:', self.y.shape)


import logging
import requests

from io import BytesIO
from django.core.management import BaseCommand

from places.models import Place, PlaceImage

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    help = 'Load information about places with media files'

    def add_arguments(self, parser):
        parser.add_argument('resource_url', type=str)

    def handle(self, *args, **options):
        resource_url = options['resource_url']

        logger.info(f'START LOADING DATA FROM RESOURCE {resource_url}')

        try:
            response = requests.get(resource_url)
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            logger.error(f'UNABLE TO LOAD DATA FROM RESOURCE {resource_url}, details: {e}')
            return

        place_data = response.json()
        place, created = Place.objects.get_or_create(
            title=place_data['title'],
            defaults={
                'short_title': place_data['title'],
                'short_description': place_data['description_short'],
                'long_description': place_data['description_long'],
                'lng': place_data['coordinates']['lng'],
                'lat': place_data['coordinates']['lat'],
                'place_id': place_data['title'],
            }
        )
        if created:
            for i, img_url in enumerate(place_data['imgs'], start=1):
                try:
                    img_response = requests.get(img_url)
                    img_response.raise_for_status()
                except requests.exceptions.HTTPError as e:
                    logger.error(f'UNABLE TO SAVE IMAGE FROM FROM RESOURCE {img_url}, details: {e}')
                    continue

                img = BytesIO(img_response.content)
                place_image, img_created = PlaceImage.objects.get_or_create(
                    place=place,
                    position=i
                )
                place_image.image.save(f'place-{place.id}-img-{i}', img, save=True)

        action = 'CREATED' if created else 'UPDATED'
        logger.info(f'{action} PLACE {place}')

        logger.info(f'END LOADING DATA FROM RESOURCE {resource_url}')