import sys, random, pickle, os, json, math, re, time

import numpy as np

import torch
import torch.nn.functional as F

class BaseAgent(torch.nn.Module):

    def __init__(self, tokenizer, model):

        super().__init__()

        self.model = model
        self.tokenizer = tokenizer
        self.optimizer = torch.optim.Adam(
            model.parameters(),
            lr=0.00005,
            betas=(0.9, 0.98),
            eps=1e-9,
        )

        self.num_gpus = torch.cuda.device_count()

        if self.num_gpus > 1:
            self.model.parallelize()
        elif self.num_gpus == 1:
            self.model = self.model.cuda()

    def get_response(self, prompt, max_len = 32):

        prompt_ids = self.tokenizer.encode(
            prompt,
            return_tensors="pt",
        )

        if self.num_gpus > 0:
            prompt_ids = prompt_ids.cuda()

        prompt_len = prompt_ids.shape[1]

        output_ids = self.model.generate(
            prompt_ids,
            max_length=prompt_len+max_len,
        )

        generated_text = self.tokenizer.batch_decode(output_ids)[0]

        return generated_text

    def memorize(self, prompt, num_epochs = 3):

        """ This is a rudimentary training loop
        that will train the agent to learn one
        sequence, the prompt. With enough epochs, this should
        result in memorizing the sequence, which is why this
        class method was names memorize.
        There is nothing returned because the model attribute is modified inplace.

        Args:
            prompt (string): the text to be learned
            num_epochs (int): the number of times we cycle though the training data, only 1 sample in this case

        """

        prompt_dic = self.tokenizer(prompt,return_tensors="pt")
        prompt_ids = prompt_dic.input_ids
        prompt_mask = prompt_dic.attention_mask
        prompt_len = prompt_ids.shape[1]

        if self.num_gpus > 0:
            prompt_ids = prompt_ids.cuda()
            prompt_mask = prompt_mask.cuda()

        source_ids = prompt_ids[:,:-1]
        target_ids = prompt_ids[:,1:]
        source_mask = prompt_mask[:,:-1]
        target_mask = prompt_mask[:,1:]

        # allow params to be updated
        self.model.train()

        for e in range(num_epochs):

            # Forward Pass
            output = self.model(
                input_ids = source_ids,
                attention_mask = source_mask,
            )

            # used logits and target tokens to calculate the loss
            logits = output.logits

            scalar_loss = cross_entropy_loss(
                logits,
                target_ids,
            )

            # backward pass
            self.optimizer.zero_grad()
            scalar_loss.backward()
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
            self.optimizer.step()

            print("epoch", e, "loss", scalar_loss.item())

def cross_entropy_loss(logits, target_ids):

    """
    For F.cross_entropy the Input is shape (N, C), where N = batch_size x sequence_length
    and C is the number of classes, in our case C is the number of tokens in the vocabulary
    Target is shape (N).

    https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html

    we flatten the batch dimension together with the max_seq length
    so that for the loss funstion, so afterwards, there is no batch dimension,
    just a vector sized C-dimensions for each of the seq_len tokens.
    If there had been 2 sampels with a batch size of 2, with 3 tokens in each sample
    then the predictions.shape would be torch.Size([6, 50257])

    Args:
        logits (torch.tensor, float): shape [batch_size, sequence_length, vocab_size]
        target_ids (torch.tensor, int): shape [batch_size, sequence_length]

    Returns:
        scalar_loss (torch.tensor, scalar float, grad_fn=<NllLossBackward0>)): no shape
            this is a loss you can backpropagate using:

            optimizer.zero_grad()
            scalar_loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()

    """

    predictions = logits.view(-1, logits.size(-1))
    target = target_ids.view(-1)

    scalar_loss = F.cross_entropy(
        predictions,
        target,
    )

    return scalar_loss


import os
import stat
import tempfile
import exiftool

def exiftool_is_executable():
    """Check whether exiftool is exif_executable if not truthy raises an PermissionError"""
    exif_executable = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "utils", "ExifTool", "exiftool"))
    is_executable = os.access(exif_executable, os.X_OK)
    if not is_executable:
        try:
            os.chmod(exif_executable, os.stat(exif_executable).st_mode | stat.S_IEXEC)
            return True
        except:
            raise PermissionError(f"Cannot execute nor set execution privileges for exiftool. Please run script as sudo or try running 'sudo chmod +x {exif_executable}'")

def get_metadata(response=None, path_to_local_file=None):
    """Returns metadata"""
    exif_executable = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "utils", "ExifTool", "exiftool"))

    if response:
        tmp = tempfile.NamedTemporaryFile()
        with open(tmp.name, 'wb') as f:
            f.write(response.content)
        with exiftool.ExifTool(executable=exif_executable) as et:
            result_dict = et.execute_json(tmp.name)[0]

    elif path_to_local_file:
        with exiftool.ExifTool(executable=exif_executable) as et:
            result_dict = et.execute_json(path_to_local_file)[0]

    blacklist = ["SourceFile", "ExifTool:ExifToolVersion", "File:FileName", "File:Directory", "File:FileSize", "File:FileModifyDate", "File:FileInodeChangeDate", "File:FilePermissions", "File:FileAccessDate", "File:FileType", "File:FileTypeExtension", "File:MIMEType"]
    result_dict = {k: v for k, v in result_dict.items() if k not in blacklist and v}
    return result_dict


from PyQt5 import QtCore, QtWidgets
import h5py
import os
import pyqtgraph as pg
import numpy as np

from results import ResultStoring
from filtering.filter_thread import FilterThread

# Here the tab widget, which will be added to the mea_file_view, is created. This class handles the visualization of
# what is going on in the background by a pyqtgraph plot widget which plots data live. Also, in this class a QThread
# is initialized, which will compute the filtering in the background, while the GUI will remain responsive.

# The QWidget class is used whenever you want to create a quite customized widget.
class FilterTab(QtWidgets.QWidget):
    # The FilterTab class receives the MeaFileView as a parent (where it will be embedded), the McsDataReader (another
    # class, which handles the reading of the h5 files of the recordings), the grid  (either all or a selection of grid
    # channels) and the filter settings.
    def __init__(self, parent, reader, grid_indices, grid_labels, settings):
        super().__init__(parent)
        # By setting variables to class variables (done with the 'self.' in front of them) it enables you to use them
        # outside this __init__() function. For example in functions you created yourself further down in the script.
        self.reader = reader
        self.grid_indices = grid_indices
        self.grid_labels = grid_labels
        self.settings = settings
        self.mea_file_view = parent

        # Line 28 and 29 set two class variables to None. This is convenient for the handling of the Thread. As long as
        # the filtering_thread is None, there is no Thread running.
        self.filtered_mat = None
        self.filtering_thread = None

        # Line 33 - 51 sets up different widgets (for detailed explanation see filter_settings_dialog.py and
        # filter_settings_widget.py)
        main_layout = QtWidgets.QVBoxLayout(self)
        main_layout.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignHCenter)

        operation_layout = QtWidgets.QVBoxLayout(self)
        operation_layout.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignLeft)
        self.operation_label = QtWidgets.QLabel(self)
        self.operation_label.setText('Nothing happens so far')
        operation_layout.addWidget(self.operation_label)
        self.progress_label = QtWidgets.QLabel(self)
        operation_layout.addWidget(self.progress_label)

        self.progress_bar = QtWidgets.QProgressBar(self)
        self.progress_bar.setMinimum(0)
        self.progress_bar.setMaximum(100)
        self.progress_bar.setTextVisible(True)
        operation_layout.addWidget(self.progress_bar)

        main_layout.addLayout(operation_layout)

        # Line 54 - 73 sets up the pyqtgraph plot widget. Pyqtgraph is a python library designed to enable its users
        # to plot data fast when using PyQt5. The look of the graphs is not as sophisticated as with matplotlib,
        # but the plotting works much faster. So, for any live plotting in MEAsure, the pyqtgraph library is used,
        # while for plotting only circumstances matplotlib is used
        self.plot_widget = pg.PlotWidget()
        # pyqtgraph uses the PyQt5 platform to plot, the setting up of plots works a little different. Happily, their
        # documentation is good (https://pyqtgraph.readthedocs.io/en/latest/index.html) and there are many examples to
        # be found there
        self.plot_widget.setBackground('w')
        styles = {'color': 'k', 'font-size': '10px'}
        self.plot_widget.setLabel('left', 'channel', **styles)
        self.plot_widget.setLabel('bottom', 'time [s]', **styles)
        main_layout.addWidget(self.plot_widget)

        # To be able to show the plot as soon as the tab is added to MeaFileView it needs some variables to plot. So
        # all the needed variables are set to zero at first.
        self.signal = [[0, 0, 0, 0]]
        self.filter = [[0, 0, 0, 0]]
        self.time_s = [[0, 0, 0, 0]]
        self.time_f = [[0, 0, 0, 0]]

        # With the pens in pyqtgraph you design the appearance of your plots
        pen_1 = pg.mkPen(color='#006e7d')
        self.fs = self.reader.sampling_frequency

        # the plot functionality is initially the same as with matplotlib. Later on (in the function on_data_updated in
        # this script), the function setData() will be used.
        self.unfiltered = self.plot_widget.plot(self.time_s[-1], self.signal[-1], pen=pen_1, name='unfiltered')

        pen_2 = pg.mkPen(color='#a7c9ba')
        self.filtered = self.plot_widget.plot(self.time_f[-1], self.filter[-1], pen=pen_2, name='filtered')
        self.plot_widget.addLegend()

    def initialize_filtering(self):
        """
        This function sets up the QThread for filtering
        :return: filtered_mat on triggering other functions, once the QThread is finnished.
        """
        # First we get the settings.
        filter_mode = self.settings.mode
        cutoff_1 = float(self.settings.lower_cutoff)
        cutoff_2 = None
        if self.settings.mode == 2:
            cutoff_2 = float(self.settings.upper_cutoff)
        # if self.settings.channel_selection == 1:

        # Here, it is checked if filtered_mat is None or if it already exists. If it exists, the thread has run before
        # or currently runs.
        if self.filtered_mat is None:
            # Line 103 - 105 are only GUI changes, to tell the user what is going on.
            self.progress_bar.setValue(0)
            self.progress_label.setText('')
            self.operation_label.setText('Filtering')
            # Here, the thread is initialized. We pass the McsDataReader and the setting variables to the Thread.
            self.filtering_thread = FilterThread(self, self.reader, filter_mode, cutoff_1, cutoff_2, self.grid_indices,
                                                 self.grid_labels)
            # The thread sends different signals to this FilterTab. So we have to connect the signals to defined
            # functions.
            self.filtering_thread.progress_made.connect(self.on_progress_made)
            self.filtering_thread.operation_changed.connect(self.on_operation_changed)
            self.filtering_thread.data_updated.connect(self.on_data_updated)
            self.filtering_thread.finished.connect(self.on_filter_thread_finished)

            # Line 116 - 122 are needed to actually start the thread, so in this case, the filtering of the traces.
            debug_mode = False  # set to 'True' in order to debug with embed
            if debug_mode:
                # synchronous filtering (runs in main thread and thus allows debugging)
                self.filtering_thread.run()
            else:
                # asynchronous filtering (default):
                self.filtering_thread.start()  # start will start thread (and run),
                # but main thread will continue immediately

    @QtCore.pyqtSlot(list)
    def on_data_updated(self, data):
        """
        This function handles the on_data_updated signal of the filtering thread. With this signal an update of the
        plot_widget (live plot) is triggered.
        :param data: list of unfiltered and filtered signal and the channel label.
        :return: updated plot widget with the unfiltered and filtered signal of the lastly filtered channel
        """
        signal, filterd, label = data[0], data[1], data[2]  # unpacking of the list got as signal
        # Line 135 - 137 sets the current channel label es ytick
        ay = self.plot_widget.getAxis('left')
        ticks = [0]
        ay.setTicks([[(v, label) for v in ticks]])

        # First, the current channel unfiltered signal has to be appended to the list
        self.signal.append(signal)
        # Normally, it would be enough to define this time list once, but I have not yet found out how to do so.
        self.time_s.append(list(np.arange(0, len(self.signal[-1])*(312/self.fs), (312/self.fs))))
        # unfiltered is the plot_widget with the first pen which was set and by
        # setData(self.time_s[-1], self.signal[-1]) the last entries of the individual lists are plotted
        self.unfiltered.setData(self.time_s[-1], self.signal[-1])

        # Same procedure as for unfiltered.
        self.filter.append(filterd)
        self.time_f.append(list(np.arange(0, len(self.filter[-1])*(312/self.fs), (312/self.fs))))
        self.filtered.setData(self.time_f[-1], self.filter[-1])

    # this function changes the label of the progress bar to inform the user what happens in the background
    @QtCore.pyqtSlot(str)
    def on_operation_changed(self, operation):
        self.operation_label.setText(operation)

    # this function updates the progress bar
    @QtCore.pyqtSlot(float)
    def on_progress_made(self, progress):
        self.progress_bar.setValue(int(progress))
        self.progress_label.setText(str(progress) + "%")

    def on_filter_thread_finished(self):
        """
        This function is triggered once the QThread is finnished. It shows this information in the GUI, but also
        initializes the saving of filtered traces to a .meae (basically .h5, but i just renamend the file format) file,
        if it is set in the settings to do so.
        :return: saved .meae file if it is set, self.filtering_thread is set to None again
        """
        self.progress_label.setText('Finished :)')
        if self.filtering_thread.filtered_mat:
            self.filtered_mat = self.filtering_thread.filtered_mat.copy()
            self.mea_file_view.results.set_filter_mat(self.filtering_thread.filtered_mat)
        self.filtering_thread = None

    def is_busy_filtering(self):
        return self.filtering_thread is not None

    def can_be_closed(self):
        # only allow closing if not busy
        return not self.is_busy_filtering()


import os
import sys
root_path = os.path.join(os.getenv("HOME"), "vqa2019")
sys.path.append(root_path)

import numpy as np
import pickle
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import confusion_matrix

from utils.data_provider import VQADataProvider

def train():
	# prepare the data
	# utils folder is in the root path, which is /home/leishi/vqa2019
	questions, labels = VQADataProvider.label_ques("train_val")
	vectorizer = TfidfVectorizer()
	ques_matrix = vectorizer.fit_transform(questions)

	# build classifier
	# svc
	classifier = SVC(gamma="scale", verbose=False)
	# # rf
	# classifier = RandomForestClassifier()
	# # gb
	# classifier = GradientBoostingClassifier()

	# train
	classifier.fit(ques_matrix, labels)

	# # training accuracy
	train_acc = classifier.score(ques_matrix, labels)
	print("training accuracy: ", train_acc)

	# save tf-idf
	with open(os.path.join(root_path, "models/pretrained/tfidf_train_val.pkl"), "wb") as f:
		pickle.dump(vectorizer, f)

	# save classifier
	with open(os.path.join(root_path, "models/pretrained/pretrained_svm_train_val.pkl"), "wb") as f:
		pickle.dump(classifier, f)

	# # debug
	# print("\nraw questions")
	# print(questions[:5])
	# print("question matrix")
	# print(vectorizer.transform(questions[:5]).todense())
	# # print("corresponding words of transformed questions")
	# # print(vectorizer.inverse_transform(vectorizer.transform(questions[:5])))
	# print("terms used")
	# print(vectorizer.get_feature_names())
	return vectorizer, classifier

# test load model
def test():
	# load tf-idf
	with open(os.path.join(root_path, "models/pretrained/tfidf.pkl"), "rb") as f:
		vectorizer = pickle.load(f)

	# load validation data
	val_ques, val_lab = VQADataProvider.label_ques("val")
	val_ques_matrix = vectorizer.transform(val_ques)

	# load model
	with open(os.path.join(root_path, "models/pretrained/pretrained_svm.pkl"), "rb") as f:
		clf = pickle.load(f)

	# compute validation accuracy
	val_acc = clf.score(val_ques_matrix, val_lab)
	print("validation accuracy")
	print(val_acc)

	# compute validation confusion matrix
	preds = clf.predict(val_ques_matrix)
	val_c_m = confusion_matrix(val_lab, preds)
	print("validation confusion matrix")
	print(val_c_m)

	# # for debug
	# print("\ntest on some validation question")
	# val_some_q = val_ques[5:10]
	# val_some_q_matrix = vectorizer.transform(val_some_q)
	# val_some_lab = val_lab[5:10]
	# print("question")
	# print(val_some_q)
	# print("prediction")
	# print(clf.predict(val_some_q_matrix))
	# print("ground truth")
	# print(val_some_lab)

# predict the category of a list of questions
def predict_q_category(questions):
	# load tf-idf
	with open(os.path.join(root_path, "models/pretrained/tfidf_train_val.pkl"), "rb") as f:
		vectorizer = pickle.load(f)
	# load model
	with open(os.path.join(root_path, "models/pretrained/pretrained_svm_train_val.pkl"), "rb") as f:
		clf = pickle.load(f)
	# create tf-idf matrix
	ques_matrix = vectorizer.transform(questions)
	# make predictions
	preds = clf.predict(ques_matrix)
	return preds

if __name__ == "__main__":
	train()
	# print("test")
	# test()


#!/usr/bin/env python

import setup_path
import airsim
import csv
import random
import numpy as np
from PIL import Image
from datetime import datetime
import matplotlib.pyplot as plt

# Demonstration script to show how the camera and GPU-LiDAR sensor can be used and their data retrieved.
# See docs/instance_segmentation.md,docs/gpu-lidar.md and docs/image_apis.md#segmentation for more information.
if __name__ == '__main__':

    # Make connection to AirSim API
    client = airsim.CarClient()
    client.confirmConnection()

    # Get an image from the main segmentation camera, show and save it as png
    print("Getting segmentation image from main camera...")
    responses = client.simGetImages([airsim.ImageRequest( "leftcamera", airsim.ImageType.Segmentation, False, False)])
    img_rgb_string = responses[0].image_data_uint8
    rgbarray = np.frombuffer(img_rgb_string, np.uint8)
    rgbarray_shaped = rgbarray.reshape((540,960,3))
    img = Image.fromarray(rgbarray_shaped, 'RGB')
    img.show()
    img.save("segmentation_sample.png", "PNG")
    print("Shown and saved segmentation image from main camera\n")

    # Get an image from the main segmentation camera, show and save it as png
    print("Getting rgb image from main camera...")
    responses = client.simGetImages([airsim.ImageRequest( "leftcamera", airsim.ImageType.Scene, False, False)])
    img_rgb_string = responses[0].image_data_uint8
    rgbarray = np.frombuffer(img_rgb_string, np.uint8)
    rgbarray_shaped = rgbarray.reshape((540,960,3))
    img = Image.fromarray(rgbarray_shaped, 'RGB')
    img.show()
    img.save("rgb_sample.png", "PNG")
    print("Shown and saved RGB image from main camera\n")

    # Get an image from the main segmentation camera, show and save it as png
    print("Getting depth image from main camera...")
    responses = client.simGetImages([airsim.ImageRequest( "leftcamera", airsim.ImageType.DepthPlanner, True, False)])
    img_depth_float = responses[0].image_data_float
    img_depth_float32 = np.asarray(img_depth_float, dtype=np.float32)
    rgbarray_shaped = img_depth_float32.reshape((540,960)).astype(np.float32)
    img = Image.fromarray(rgbarray_shaped)
    img.show()
    img.save("depth_sample.tif")
    print("Shown and saved depth image from main camera\n")

    print("Getting GPU LiDAR data...")
    lidar_data = client.getGPULidarData('lidar1', 'airsimvehicle')
    points = np.array(lidar_data.point_cloud, dtype=np.dtype('f4'))
    points = np.reshape(points, (int(points.shape[0] / 5), 5))

    rgb_values = points[:, 3].astype(np.uint32)
    rgb = np.zeros((np.shape(points)[0], 3))
    xyz = points[:, 0:3]
    for index, rgb_value in enumerate(rgb_values):
        rgb[index, 0] = (rgb_value >> 16) & 0xFF
        rgb[index, 1] = (rgb_value >> 8) & 0xFF
        rgb[index, 2] = rgb_value & 0xFF
    rgb = np.divide(rgb, 255)

    print("Parsing segmentation RGB8 values from float32 data values...")
    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')
    ax.scatter(xyz[:, 0], xyz[:, 1], xyz[:, 2], c=rgb)
    ax.set_xlabel('X Label')
    ax.set_ylabel('Y Label')
    ax.set_zlabel('Z Label')
    ax.set_xlim((-5, 5))
    ax.set_ylim((-5, 5))
    ax.set_zlim((-5, 5))
    plt.show()
    print("Shown and saved 3d segmented pointcloud\n")

    print("All instance segmentation tests completed")


from datetime import datetime

from abstract_currency import AbstractCurrency
from abstract_date import AbstractDate, START_DATE, END_DATE, OBSERVATIONS

class MonthEnd(AbstractDate):

    def __init__(self, date_to_lookup: datetime):
        super().__init__(date_to_lookup)

    def get_exchange_rate(self, currency: AbstractCurrency):
        year = str(self._date.year)
        bank_of_canada_response = currency.get_bank_of_canada_response(START_DATE + year + '-01-01' + END_DATE +
                                                                       year + '-12-31')
        first_available_year = bank_of_canada_response[OBSERVATIONS][0]['d'].split('-')[0]
        last_available_year = bank_of_canada_response[OBSERVATIONS][-1]['d'].split('-')[0]

        if first_available_year > year or last_available_year < year:
            print('No month end data available for ' + year)
        else:
            for i in range(1, 13):
                rates_for_months = list(filter(lambda x: x['d'].startswith(f'{year}-{i:02}'),
                                               bank_of_canada_response[OBSERVATIONS]))

                if len(rates_for_months):
                    AbstractDate._print_rate(rates_for_months[-1])
                else:
                    break


import matplotlib.pyplot as plt
import numpy as np
import torch
from scipy import special

class Resampler(torch.nn.Module):
    """
    Efficiently resample audio signals
    This module is much faster than resampling with librosa because it exploits pytorch's efficient conv1d operations
    This module is also faster than the existing pytorch resample function in
    https://github.com/pytorch/audio/blob/b6a61c3f7d0267c77f8626167cc1eda0335f2753/torchaudio/compliance/kaldi.py#L892

    Based on
    https://github.com/danpovey/filtering/blob/master/lilfilter/resampler2.py
    with improvements to include additional filter types and input parameters that align with the librosa api
    """

    def __init__(self, input_sr, output_sr, dtype, num_zeros=64, cutoff_ratio=0.95, filter='kaiser', beta=14.0):
        super().__init__()  # init the base class
        """This creates an object that can apply a symmetric FIR filter based
        on torch.nn.functional.conv1d.

        Args:
          input_sr:  The input sampling rate, AS AN INTEGER..
              does not have to be the real sampling rate but should
              have the correct ratio with output_sr.
          output_sr:  The output sampling rate, AS AN INTEGER.
              It is the ratio with the input sampling rate that is
              important here.
          dtype:  The torch dtype to use for computations (would be preferable to
               set things up so passing the dtype isn't necessary)
          num_zeros: The number of zeros per side in the (sinc*hanning-window)
              filter function.  More is more accurate, but 64 is already
              quite a lot. The kernel size is 2*num_zeros + 1.
          cutoff_ratio: The filter rolloff point as a fraction of the
             Nyquist frequency.
          filter: one of ['kaiser', 'kaiser_best', 'kaiser_fast', 'hann']
          beta: parameter for 'kaiser' filter

        You can think of this algorithm as dividing up the signals
        (input,output) into blocks where there are `input_sr` input
        samples and `output_sr` output samples.  Then we treat it
        using convolutional code, imagining there are `input_sr`
        input channels and `output_sr` output channels per time step.
        """
        assert isinstance(input_sr, int) and isinstance(output_sr, int)
        if input_sr == output_sr:
            self.resample_type = 'trivial'
            return

        def gcd(a, b):
            """Return the greatest common divisor of a and b."""
            assert isinstance(a, int) and isinstance(b, int)
            if b == 0:
                return a
            else:
                return gcd(b, a % b)

        d = gcd(input_sr, output_sr)
        input_sr, output_sr = input_sr // d, output_sr // d

        assert dtype in [torch.float32, torch.float64]
        assert num_zeros > 3  # a reasonable bare minimum
        np_dtype = np.float32 if dtype == torch.float32 else np.float64

        assert filter in ['hann', 'kaiser', 'kaiser_best', 'kaiser_fast']

        if filter == 'kaiser_best':
            num_zeros = 64
            beta = 14.769656459379492
            cutoff_ratio = 0.9475937167399596
            filter = 'kaiser'
        elif filter == 'kaiser_fast':
            num_zeros = 16
            beta = 8.555504641634386
            cutoff_ratio = 0.85
            filter = 'kaiser'

        # Define one 'block' of samples `input_sr` input samples
        # and `output_sr` output samples.  We can divide up
        # the samples into these blocks and have the blocks be
        # in correspondence.

        # The sinc function will have, on average, `zeros_per_block`
        # zeros per block.
        zeros_per_block = min(input_sr, output_sr) * cutoff_ratio

        # The convolutional kernel size will be n = (blocks_per_side*2 + 1),
        # i.e. we add that many blocks on each side of the central block.  The
        # window radius (defined as distance from center to edge)
        # is `blocks_per_side` blocks.  This ensures that each sample in the
        # central block can "see" all the samples in its window.
        #
        # Assuming the following division is not exact, adding 1
        # will have the same effect as rounding up.
        # blocks_per_side = 1 + int(num_zeros / zeros_per_block)
        blocks_per_side = int(np.ceil(num_zeros / zeros_per_block))

        kernel_width = 2 * blocks_per_side + 1

        # We want the weights as used by torch's conv1d code; format is
        #  (out_channels, in_channels, kernel_width)
        # https://pytorch.org/docs/stable/nn.functional.html
        weights = torch.tensor((output_sr, input_sr, kernel_width), dtype=dtype)

        # Computations involving time will be in units of 1 block.  Actually this
        # is the same as the `canonical` time axis since each block has input_sr
        # input samples, so it would be one of whatever time unit we are using
        window_radius_in_blocks = blocks_per_side

        # The `times` below will end up being the args to the sinc function.
        # For the shapes of the things below, look at the args to `view`.  The terms
        # below will get expanded to shape (output_sr, input_sr, kernel_width) through
        # broadcasting
        # We want it so that, assuming input_sr == output_sr, along the diagonal of
        # the central block we have t == 0.
        # The signs of the output_sr and input_sr terms need to be opposite.  The
        # sign that the kernel_width term needs to be will depend on whether it's
        # convolution or correlation, and the logic is tricky.. I will just find
        # which sign works.

        times = (
            np.arange(output_sr, dtype=np_dtype).reshape((output_sr, 1, 1)) / output_sr
            - np.arange(input_sr, dtype=np_dtype).reshape((1, input_sr, 1)) / input_sr
            - (np.arange(kernel_width, dtype=np_dtype).reshape((1, 1, kernel_width)) - blocks_per_side)
        )

        def hann_window(a):
            """hann_window returns the Hann window on [-1,1], which is zero if
            a < -1 or a > 1, and otherwise 0.5 + 0.5 cos(a*pi). This is applied
            elementwise to a, which should be a NumPy array.

            The heaviside function returns (a > 0 ? 1 : 0).
            """
            return np.heaviside(1 - np.abs(a), 0.0) * (0.5 + 0.5 * np.cos(a * np.pi))

        def kaiser_window(a, beta):
            w = special.i0(beta * np.sqrt(np.clip(1 - ((a - 0.0) / 1.0) ** 2.0, 0.0, 1.0))) / special.i0(beta)
            return np.heaviside(1 - np.abs(a), 0.0) * w

        # The weights below are a sinc function times a Hann-window function.
        #
        # Multiplication by zeros_per_block normalizes the sinc function
        # (to compensate for scaling on the x-axis), so that the integral is 1.
        #
        # Division by input_sr normalizes the input function. Think of the input
        # as a stream of dirac deltas passing through a low pass filter:
        # in order to have the same magnitude as the original input function,
        # we need to divide by the number of those deltas per unit time.
        if filter == 'hann':
            weights = (
                np.sinc(times * zeros_per_block)
                * hann_window(times / window_radius_in_blocks)
                * zeros_per_block
                / input_sr
            )
        else:
            weights = (
                np.sinc(times * zeros_per_block)
                * kaiser_window(times / window_radius_in_blocks, beta)
                * zeros_per_block
                / input_sr
            )

        self.input_sr = input_sr
        self.output_sr = output_sr

        # weights has dim (output_sr, input_sr, kernel_width).
        # If output_sr == 1, we can fold the input_sr into the
        # kernel_width (i.e. have just 1 input channel); this will make the
        # convolution faster and avoid unnecessary reshaping.

        assert weights.shape == (output_sr, input_sr, kernel_width)
        if output_sr == 1:
            self.resample_type = 'integer_downsample'
            self.padding = input_sr * blocks_per_side
            weights = torch.tensor(weights, dtype=dtype, requires_grad=False)
            self.weights = weights.transpose(1, 2).contiguous().view(1, 1, input_sr * kernel_width)

        elif input_sr == 1:
            # In this case we'll be doing conv_transpose, so we want the same weights that
            # we would have if we were *downsampling* by this factor-- i.e. as if input_sr,
            # output_sr had been swapped.
            self.resample_type = 'integer_upsample'
            self.padding = output_sr * blocks_per_side
            weights = torch.tensor(weights, dtype=dtype, requires_grad=False)
            self.weights = weights.flip(2).transpose(0, 2).contiguous().view(1, 1, output_sr * kernel_width)
        else:
            self.resample_type = 'general'
            self.reshaped = False
            self.padding = blocks_per_side
            self.weights = torch.tensor(weights, dtype=dtype, requires_grad=False)

        self.weights = torch.nn.Parameter(self.weights, requires_grad=False)

    @torch.no_grad()
    def forward(self, data):
        """Resample the data.

        Args:
         input: a torch.Tensor with the same dtype as was passed to the
           constructor.
         There must be 2 axes, interpreted as (minibatch_size, sequence_length)...
         the minibatch_size may in practice be the number of channels.

        Return:  Returns a torch.Tensor with the same dtype as the input, and
         dimension (minibatch_size, (sequence_length//input_sr)*output_sr),
         where input_sr and output_sr are the corresponding constructor args,
         modified to remove any common factors.
        """
        if self.resample_type == 'trivial':
            return data
        elif self.resample_type == 'integer_downsample':
            (minibatch_size, seq_len) = data.shape
            # will be shape (minibatch_size, in_channels, seq_len) with in_channels == 1
            data.unsqueeze_(1)
            data = torch.nn.functional.conv1d(data, self.weights, stride=self.input_sr, padding=self.padding)
            # shape will be (minibatch_size, out_channels = 1, seq_len);
            # return as (minibatch_size, seq_len)
            return data.squeeze_(1)

        elif self.resample_type == 'integer_upsample':
            data.unsqueeze_(1)
            data = torch.nn.functional.conv_transpose1d(data, self.weights, stride=self.output_sr, padding=self.padding)

            return data.squeeze_(1)
        else:
            assert self.resample_type == 'general'
            (minibatch_size, seq_len) = data.shape
            num_blocks = seq_len // self.input_sr
            if num_blocks == 0:
                # TODO: pad with zeros.
                raise RuntimeError('Signal is too short to resample')
            # data = data[:, 0:(num_blocks*self.input_sr)]  # Truncate input
            data = data[:, 0 : (num_blocks * self.input_sr)].view(minibatch_size, num_blocks, self.input_sr)

            # Torch's conv1d expects input data with shape (minibatch, in_channels, time_steps), so transpose
            data.transpose_(1, 2)

            data = torch.nn.functional.conv1d(data, self.weights, padding=self.padding)

            assert data.shape == (minibatch_size, self.output_sr, num_blocks)
            return data.transpose(1, 2).contiguous().view(minibatch_size, num_blocks * self.output_sr)

if __name__ == '__main__':
    fs = 100
    fs_ = 128
    R = Resampler(fs, fs_, dtype=torch.float32)
    f = 10
    t = torch.arange(-1 / 2, 1 / 2, 1 / fs)
    x = torch.sin(2 * np.pi * f * t) / (2 * np.pi * f * t)
    x[x.shape[0] // 2] = 1
    t_ = torch.arange(-1 / 2, 1 / 2, 1 / fs_)
    x = x.view(1, -1)
    y = R(x)
    # plt.subplot(2,1,1)
    plt.plot(t, x.view(-1), '.-')
    # plt.subplot(2,1,2)
    plt.plot(t_, y.view(-1))
    plt.grid()
    plt.show()


import os
import difflib
import subprocess

from web import app

@app.context_processor
def inject_grader():
    def run_grade(stale):
        dummy = open("sandbox/foo.py", "w+")
        dummy.write(stale)
        dummy.close()

        subprocess.call("python -m black sandbox/foo.py", shell=True)

        dummy = open("sandbox/foo.py", "r")
        f = dummy.read()
        diff = difflib.SequenceMatcher(None, stale, f)
        dummy.close()
        os.remove("sandbox/foo.py")

        r = round(diff.ratio(), 2) * 100

        if r >= 90:
            color = "#008000"
        elif r >= 80:
            color = "#ffff00"
        else:
            color = "#ff0000"

        return [str(r), color, str(f)]

    return dict(run_grade=run_grade)


import argparse
import os
import re
import requests  # Install with easy_install or pip install
import shutil
import subprocess
import sys
import tempfile
from contextlib import contextmanager
from updatecommon import get_release
from updatecommon import upload_release

BOTTLE_TARGET = 'yosemite_or_later'

@contextmanager
def os_closing(o):
    yield o
    os.close(o)

def parse_args(args):
    parser = argparse.ArgumentParser(description='Update homebrew-fb')
    parser.add_argument(
        'version_tag',
        help='The name of the tag to create a release for')
    parser.add_argument(
        'github_token',
        type=file,
        help='The authentication token to use to talk to GitHub')
    parser.add_argument(
        '--tap-repo-location',
        default='/usr/local/Library/Taps/facebook/homebrew-fb',
        help='The location of the homebrew-fb tap')
    return parser.parse_args(args)

def build_bottle(version_tag, tap_repo_location):
    print('Building bottle...')
    subprocess.check_call(
        [
            'brew',
            'unlink',
            'buck',
        ],
        cwd=tap_repo_location)
    subprocess.check_call(
        [
            'brew',
            'install',
            '--build-bottle',
            'buck',
        ],
        cwd=tap_repo_location)
    subprocess.check_call(
        [
            'brew',
            'bottle',
            'buck',
        ],
        cwd=tap_repo_location)
    dest_name = 'buck-{version_name}.{bottle_target}.bottle.tar.gz'.format(
        version_name=version_tag[1:],
        bottle_target=BOTTLE_TARGET)
    subprocess.check_call(
        [
            'mv',
            'buck-{version_name}.el_capitan.bottle.tar.gz'.format(
                version_name=version_tag[1:]),
            dest_name,
        ],
        cwd=tap_repo_location)
    return os.path.join(tap_repo_location, dest_name)

def fetch_tarball(url):
    print('Fetching tarball from `{url}`...'.format(url=url))
    r = requests.get(url, stream=True)
    handle, path = tempfile.mkstemp(suffix='.tar.gz')
    with os_closing(handle) as handle:
        for chunk in r.iter_content(chunk_size=4096):
            if chunk:
                os.write(handle, chunk)
    return path

def validate():
    print('Validating bottle...')
    subprocess.check_call([
        'brew',
        'uninstall',
        '--force',
        'buck',
    ])
    subprocess.check_call([
        'brew',
        'install',
        'buck',
    ])

def update_bottle(version_tag, github_token, tap_repo_location):
    release_data = get_release(version_tag)
    tarball_sha256 = sha256(fetch_tarball(release_data['tarball_url']))

    # First, update the bottle to have the new version and tarball sha.
    temp_handle, temp_path = tempfile.mkstemp(text=True)
    with os_closing(temp_handle):
        with open(os.path.join(tap_repo_location, 'buck.rb'), 'r') as orig:
            for line in orig:
                line = re.sub(
                    r'@@buck_version = .*$',
                    '@@buck_version = "{version_name}"'.format(
                        version_name=version_tag[1:]),
                    line)
                line = re.sub(
                    r'sha256 "[a-z0-9]{64}"$',
                    'sha256 "{sha}"'.format(sha=tarball_sha256),
                    line)
                line = re.sub(
                    r'  url "https://.+"$',
                    '  url "{url}"'.format(url=release_data['tarball_url']),
                    line)
                os.write(temp_handle, line)
        shutil.copyfile(temp_path, os.path.join(tap_repo_location, 'buck.rb'))

    # Now, build the bottle's binary, and update the file with the new sha.
    bottle_file = build_bottle(version_tag, tap_repo_location)
    bottle_sha256 = sha256(bottle_file)
    upload_release(
            bottle_file,
            release_data['upload_url'],
            github_token,
            {
                'Content-Type': 'application/x-tar',
            })
    os.remove(bottle_file)

    temp_handle, temp_path = tempfile.mkstemp(text=True)
    with os_closing(temp_handle):
        with open(os.path.join(tap_repo_location, 'buck.rb'), 'r') as orig:
            for line in orig:
                line = re.sub(
                    r'sha256 "[a-z0-9]{64}" => :.+$',
                    'sha256 "{sha}" => :{bottle_target}'.format(
                        sha=bottle_sha256,
                        bottle_target=BOTTLE_TARGET),
                    line)
                os.write(temp_handle, line)
        shutil.copyfile(temp_path, os.path.join(tap_repo_location, 'buck.rb'))

    validate()

    subprocess.check_call(
        [
            'git',
            'commit',
            '-m',
            'Update `buck.rb` to {version_tag}'.format(
                version_tag=version_tag),
            'buck.rb',
        ],
        cwd=tap_repo_location)
    print('Your commit is ready for testing!  Check it out:')
    print(tap_repo_location)

def sha256(file_name):
    return subprocess.check_output([
        'shasum',
        '-a',
        '256',
        file_name,
    ]).split()[0]

if __name__ == '__main__':
    args = parse_args(sys.argv[1:])
    github_token_lines = args.github_token.readlines()
    assert len(github_token_lines) == 1, (
        'Invalid number of lines in github_token file')
    update_bottle(
        args.version_tag,
        github_token_lines[0].strip(),
        args.tap_repo_location)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='home'),
    path('about/', views.about, name='about'),
    path('finches/', views.finches_index, name='index'),
    path('finches/<int:finch_id>/', views.finches_detail, name='detail'),
    path('finches/create/', views.FinchCreate.as_view(), name='finches_create'),
    path('finches/<int:pk>/update/', views.FinchUpdate.as_view(), name='finches_update'),
    path('finches/<int:pk>/delete/', views.FinchDelete.as_view(), name='finches_delete'),
    path('finches/<int:finch_id>/add_sighting/', views.add_sighting, name='add_sighting'),
    path('finches/<int:finch_id>/assoc_food/<int:food_id>/', views.assoc_food, name='assoc_food'),
    path('finches/<int:finch_id>/unassoc_food/<int:food_id>/', views.unassoc_food, name='unassoc_food'),
    path('food/create/', views.FoodCreate.as_view(), name='food_create'),
    path('food/<int:pk>/update/', views.FoodUpdate.as_view(), name='food_update'),
    path('food/<int:pk>/delete/', views.FoodDelete.as_view(), name='food_delete'),
    path('food/', views.food_index, name='food_index'),
    path('food/<int:food_id>/', views.food_detail, name='food_detail'),
]


import os
from flask import render_template, flash, redirect, url_for, request
from flask_login import current_user, login_user, logout_user, login_required
from werkzeug.urls import url_parse
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import BernoulliNB

from app import app, db
from app.forms import LoginForm, AnalyzeForm, RegistrationForm, SubmissionForm, TrainForm
from app.models import User, Submission

@app.route('/')
@app.route('/index')
def index():
    return render_template('index.html', title='Home')

@app.route('/login', methods=['GET', 'POST'])
def login():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = LoginForm()
    if form.validate_on_submit():
        user = User.query.filter_by(username=form.username.data).first()
        if user is None or not user.check_password(form.password.data):
            flash('Invalid username or password')
            return redirect(url_for('login'))
        login_user(user, remember=form.remember_me.data)
        next_page = request.args.get('next')
        if not next_page or url_parse(next_page).netloc != '':
            next_page = url_for('index')
        return redirect(next_page)
    return render_template('login.html', title='Sign In', form=form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('index'))

@app.route('/register', methods=['GET', 'POST'])
def register():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = RegistrationForm()
    if form.validate_on_submit():
        user = User(username=form.username.data, email=form.email.data)
        user.set_password(form.password.data)
        db.session.add(user)
        db.session.commit()
        flash('Congratulations, you are now a registered user!')
        return redirect(url_for('login'))
    return render_template('register.html', title='Register', form=form)

@app.route('/user/<username>')
@login_required
def user(username):
    user = User.query.filter_by(username=username).first_or_404()
    submissions = [
        {'author': user, 'body': 'Test post #1'},
        {'author': user, 'body': 'Test post #2'}
    ]
    return render_template('user.html', user=user, submissions=submissions)

@app.route('/analyze', methods=['GET', 'POST'])
def analyze():
    form = AnalyzeForm()
    if form.validate_on_submit():
        sentence = form.sentence.data
        model = joblib.load('./ml_backend/pickled_objects/bernoulli_model.pkl')
        countvectorizer = joblib.load('./ml_backend/pickled_objects/countvectorizer.pkl')
        prediction = model.predict(countvectorizer.transform([sentence]))[0]
        if prediction == 1:
            prediction = "Good"
        elif prediction == 0:
            prediction = "Bad"
        flash('Sentence: {} | Sentiment: {}.'.format(sentence, prediction))
        return redirect(url_for('analyze'))
    return render_template('analyze.html', title='Analyze', form=form)

@app.route('/train', methods=['GET', 'POST'])
@login_required
def train():
    form = TrainForm()
    if form.validate_on_submit():
        sentence_list = []
        is_good_list = []
        submissions = Submission.query.all()
        for submission in submissions:
            sentence_list.append(submission.sentence)
            is_good_list.append(int(submission.is_good))
        # ML Model
        countvectorizer = CountVectorizer(binary='true')
        sentence_list = countvectorizer.fit_transform(sentence_list)
        model = BernoulliNB().fit(sentence_list, is_good_list)
        # Pickle ML model and replace old model
        os.remove("./ml_backend/pickled_objects/countvectorizer.pkl")
        os.remove("./ml_backend/pickled_objects/bernoulli_model.pkl") # FIX 1
        joblib.dump(countvectorizer, "./ml_backend/pickled_objects/countvectorizer.pkl")
        joblib.dump(model, './ml_backend/pickled_objects/bernoulli_model.pkl')
        flash("Thank you for helping the machines! Your work won't go unnoticed in our dystopian future!")
        return redirect(url_for('train'))
    return render_template('train.html', title='Train', form=form)

@app.route('/add_submission', methods=['GET', 'POST'])
@login_required
def add_submission():
    form = SubmissionForm()
    if form.validate_on_submit():
        submission = Submission(sentence=form.sentence.data, is_good=form.is_good.data, author=current_user)
        db.session.add(submission)
        db.session.commit()
        flash('Thank you, add another or train our model!!')
        return redirect(url_for('add_submission'))
    return render_template('add_submission.html', title='Add Submission', form=form)


import json
import re

from django.http import JsonResponse
from django.views import View
from django.utils import timezone

from .models import MetroRegion, LocalRegion, PlaceType, Place

REGEX_ROAD_ADDRESS = '(([가-힣A-Za-z·\d~\-\.]{2,}(로|길).[\d]+)|([가-힣A-Za-z·\d~\-\.]+(읍|동)\s)[\d]+)'

class PlaceCreateView(View):
    def post(self, request):
        try:
            data = json.loads(request.body)
            place_type = PlaceType.objects.get(name = data['place_type'])
            metro_region = MetroRegion.objects.get(name=data['metro_region'])
            local_region = LocalRegion.objects.get(name=data['local_region'], metro_region = metro_region)
            road_address = data['road_address']
            name = data['name']

            assert re.match(REGEX_ROAD_ADDRESS, road_address), "INVALID_ROAD_ADDRESS_FORMAT"

            if Place.objects.filter(name = name, place_type = place_type, road_address = road_address, region = local_region, deleted_at = None).exists():
                return JsonResponse({"message": "EXIST_PLACE"}, status = 400)

            Place.objects.create(
                    name         = name,
                    place_type   = place_type,
                    road_address = road_address,
                    region       = local_region
            )
            return JsonResponse({"message": "PLACE_CREATED"}, status = 201)

        except json.JSONDecodeError as e:
            return JsonResponse({"message": f"{e}"}, status = 400)
        except KeyError:
            return JsonResponse({"message": "KEY_ERROR"}, status = 400)
        except AssertionError as e:
            return JsonResponse({"message": f"{e}"}, status = 400)
        except PlaceType.DoesNotExist:
            return JsonResponse({"message": "INVALID_PLACE_TYPE"}, status = 401)
        except MetroRegion.DoesNotExist:
            return JsonResponse({"message": "INVALID_REGION"}, status = 401)
        except LocalRegion.DoesNotExist:
            return JsonResponse({"message": "INVALID_REGION"}, status = 401)

    def get(self, request):
        places = Place.objects.select_related('place_type', 'region', 'region__metro_region').filter(deleted_at__isnull = True)

        result = [
                    {
                        "id"           : place.id,
                        "name"         : place.name,
                        "type"         : place.place_type.name,
                        "road_address" : place.road_address,
                        "local_region" : place.region.name,
                        "metro_region" : place.region.metro_region.name
                    } for place in places
                ]
        return JsonResponse({"result": result}, status = 200)

class PlaceView(View):
    def patch(self, request, place_pk):
        try:
            data = json.loads(request.body)

            data_keys = data.keys()

            place_type = PlaceType.objects.get(name = data['place_type'])
            metro_region = MetroRegion.objects.get(name=data['metro_region'])
            local_region = LocalRegion.objects.get(name=data['local_region'], metro_region = metro_region)
            road_address = data['road_address']
            name = data['name']

            assert re.match(REGEX_ROAD_ADDRESS, road_address), "INVALID_ROAD_ADDRESS_FORMAT"

            patch_object = Place.objects.get(id = place_pk)

            if patch_object.deleted_at == None:
                patch_object.place_type   = place_type
                patch_object.region       = local_region
                patch_object.road_address = road_address
                patch_object.name         = name
                patch_object.save()
                return JsonResponse({"message": "PLACE_UPDATED"}, status = 200)
            else:
                return JsonResponse({"message": "DELETED_PLACE"}, status = 400)

        except json.JSONDecodeError as e:
            return JsonResponse({"message": f"{e}"}, status = 400)
        except AssertionError as e:
            return JsonResponse({"message": f"{e}"}, status = 400)
        except Place.DoesNotExist:
            return JsonResponse({"message": "INVALID_PLACE"}, status = 401)
        except PlaceType.DoesNotExist:
            return JsonResponse({"message": "INVALID_PLACE_TYPE"}, status = 401)
        except MetroRegion.DoesNotExist:
            return JsonResponse({"message": "INVALID_REGION"}, status = 401)
        except LocalRegion.DoesNotExist:
            return JsonResponse({"message": "INVALID_REGION"}, status = 401)

    def delete(self, request, place_pk):
        try:
            delete_object = Place.objects.get(id = place_pk)

            if delete_object.deleted_at == None:
                delete_object.deleted_at = timezone.now()
                delete_object.save()
                return JsonResponse({"message": "PLACE_DELETED"}, status = 200)
            else:
                return JsonResponse({"message": "PLACE_IS_ALREADY_DELETED"}, status = 400)
        except Place.DoesNotExist:
            return JsonResponse({"message": "INVALID_PLACE"}, status = 401)


#!/usr/bin/env python3

import requests
import os
import hashlib
import io

def on_vocareum():
    return os.path.exists('.voc')

def get_url_base():
    return "https://cse6040.gatech.edu/datasets/us-flights/"

def get_data_path(filebase=""):
    if on_vocareum():
        DATA_PATH = "./resource/asnlib/publicdata/"
    else:
        DATA_PATH = ""
    return f"{DATA_PATH}{filebase}"

def download(file, local_dir=None, url_base=None, checksum=None):
    if local_dir is None: local_dir = get_data_path()
    if url_base is None: url_base = get_url_base()
    assert (local_dir is not None) and (url_base is not None)

    local_file = "{}{}".format(local_dir, file)
    if not os.path.exists(local_file):
        if url_base is None:
            url_base = "https://cse6040.gatech.edu/datasets/"
        url = "{}{}".format(url_base, file)
        print("Downloading: {} ...".format(url))
        r = requests.get(url)
        with open(local_file, 'wb') as f:
            f.write(r.content)

    if checksum is not None:
        with io.open(local_file, 'rb') as f:
            body = f.read()
            body_checksum = hashlib.md5(body).hexdigest()
            assert body_checksum == checksum, \
                "Downloaded file '{}' has incorrect checksum: '{}' instead of '{}'".format(local_file,
                                                                                body_checksum,
                                                                                checksum)
    print("'{}' is ready!".format(file))

def download_airport_dataset():
    datasets = {'L_AIRPORT_ID.csv': 'e9f250e3c93d625cce92d08648c4bbf0',
                'L_CITY_MARKET_ID.csv': 'f430a16a5fe4b9a849accb5d332b2bb8',
                'L_UNIQUE_CARRIERS.csv': 'bebe919e85e2cf72e7041dbf1ae5794e',
                'us-flights--2017-08.csv': 'eeb259c0cdd00ff1027261ca0a7c0332',
                'flights_atl_to_lax_soln.csv': '4591f6501411de90af72693cdbcc08bb',
                'origins_top10_soln.csv': 'de85c321c45c7bf65612754be4567086',
                'dests_soln.csv': '370f4c632623616b3bf26b6f79993fe4',
                'dests_top10_soln.csv': '4c7dd7edf48c4d62466964d6b8c14184',
                'segments_soln.csv': '516a78d2d9d768d78bfb012b77671f38',
                'segments_outdegree_soln.csv': 'b29d60151c617ebafd3a1c58541477c8'
               }
    for filename, checksum in datasets.items():
        download(filename, checksum=checksum)

# Pandas-based
def canonicalize_tibble(X):
    var_names = sorted(X.columns)
    Y = X[var_names].copy()
    Y.sort_values(by=var_names, inplace=True)
    Y.reset_index(drop=True, inplace=True)
    return Y

# Pandas-based
def tibbles_are_equivalent (A, B):
    A_canonical = canonicalize_tibble(A)
    B_canonical = canonicalize_tibble(B)
    cmp = A_canonical.eq(B_canonical)
    return cmp.all().all()

# eof


import platform, os, subprocess, shutil, argparse

def run_cmd(cmd):
    print(f'\n\n{cmd}')
    print('--------------------------------------------------------')
    subprocess.Popen(cmd, shell=True).communicate()

parser = argparse.ArgumentParser(description='args to Run SCM app')
parser.add_argument('--fresh_db', dest='fresh_db', type=bool, help='run app with fresh DB', default=False, action=argparse.BooleanOptionalAction)
args = parser.parse_args()
if(args.fresh_db):
    fresh_db_arg = '--fresh_db'
else:
    fresh_db_arg = ''

if __name__ == '__main__':
    try:
        src_dir = os.path.dirname(__file__)
        data_dir = os.path.join(src_dir,'storage')
        if (args.fresh_db):
            run_cmd('minikube delete')
            # cleanup and fresh create the database folder
            shutil.rmtree(data_dir, ignore_errors=True)
            os.makedirs(data_dir)
            run_cmd(f'minikube start --mount-string "{data_dir}:/scm-storage" --mount --driver=docker')
        else:
            run_cmd(f'minikube start --driver=docker')
        # navigate to setup directory
        os.chdir(os.path.join(src_dir, 'setup'))
        # set the environment variables for docker env in shell and setup SCM application
        if (platform.system() == 'Windows'):
            run_cmd(f'powershell -ExecutionPolicy bypass "& minikube -p minikube docker-env --shell powershell | Invoke-Expression; python setup.py {fresh_db_arg}"')
        else:
            run_cmd(f'eval $(minikube docker-env) && python setup.py {fresh_db_arg}')
    except KeyboardInterrupt:
        print('!!!!!!script run terminated from keyboard!!!!!!')
        exit()


import json

from django.contrib.auth.decorators import permission_required
from django.db import transaction, IntegrityError
from django.http import HttpResponse, HttpResponseBadRequest
from django.shortcuts import render, get_object_or_404
from django.utils.translation import ugettext_lazy as _
from django.views.generic import View

from bangoo.content.models import Content
from bangoo.decorators import class_view_decorator
from bangoo.navigation.menu.forms import MenuOrderForm, MenuRenameForm, MenuCreateForm
from bangoo.navigation.models import Menu

from .signals import menu_changed
from .utils import create_path

@permission_required('menu.add_menu')
def menu(request, template_name='navigation/menu/menu.html'):
    return render(request, template_name, {'nodes': Menu.objects.all(), 'form': MenuCreateForm()})

@class_view_decorator(permission_required('menu.add_menu'))
class ReorderMenuView(View):
    def post(self, request, *args, **kwargs):
        form = MenuOrderForm(request.POST)
        if form.is_valid():
            try:
                with transaction.atomic():
                    changed_paths = []

                    method = form.cleaned_data['method']
                    target_menu = form.cleaned_data['target']
                    source_menu = form.cleaned_data['source']

                    if method == 'move':
                        source_menu_parent = source_menu.parent
                        source_menu.move_to(target_menu, position='left')
                        source_menu.save()

                        menu_changed.send(sender=self.__class__,
                                          menu=source_menu,
                                          old_parent=source_menu_parent,
                                          new_parent=target_menu.parent)

                    if method == 'insert':
                        source_menu_parent = source_menu.parent
                        source_menu.move_to(target_menu)
                        source_menu.save()

                        menu_changed.send(sender=self.__class__,
                                          menu=source_menu,
                                          old_parent=source_menu_parent,
                                          new_parent=target_menu)

                    if method in {'move', 'insert'}:
                        for descendant in source_menu.get_descendants(include_self=True):
                            old_path = descendant.path

                            descendant.path = create_path(descendant)
                            descendant.save()

                            if descendant.plugin == 'bangoo.content':
                                content = Content.objects.language(descendant.language_code).filter(url=old_path).first()
                                content.url = descendant.path
                                content.save()

                            changed_paths.append({
                                'menu_id': descendant.id,
                                'path': descendant.path
                            })
                        return HttpResponse(json.dumps(changed_paths), content_type='application/json')
            except IntegrityError:
                return HttpResponseBadRequest(_("Failed to move the menu item because there is already one "
                                                "with the same name under the destination path."))
        else:
            reason = {
                'status': 'error',
                'reasons': dict([(k, form.error_class.as_text(v)) for k, v in list(form.errors.items())])
            }
            return HttpResponse(json.dumps(reason), content_type='application/json')

@class_view_decorator(permission_required('menu.add_menu'))
class RenameMenuView(View):
    def post(self, request, *args, **kwargs):
        menu_id = kwargs.pop('menu_id', None)
        menu = get_object_or_404(Menu, pk=menu_id)
        form = MenuRenameForm(request.POST, menu=menu)

        if form.is_valid():
            with transaction.atomic():
                changed_paths = []

                menu.title = form.cleaned_data['title']
                menu.path = create_path(menu)
                menu.save()

                changed_paths.append({
                    'menu_id': menu.id,
                    'path': menu.path
                })

                for descendant in menu.get_descendants():
                    descendant.path = create_path(descendant)
                    descendant.save()

                    changed_paths.append({
                        'menu_id': descendant.id,
                        'path': descendant.path
                    })

                return HttpResponse(json.dumps(changed_paths), content_type='application/json')
        else:
            reason = {
                'status': 'error',
                'menu_id': menu_id,
                'reasons': dict([(k, form.error_class.as_text(v)) for k, v in list(form.errors.items())])
            }
            return HttpResponse(json.dumps(reason), content_type='application/json')


import torch.nn as nn
import matplotlib.pyplot as plt
import torch
N, D_in, H, D_out = 64, 1000, 100, 10  # 一共有64个数据，每个数据输入10000维度，中间层100个神经元，输出10维
# 随机一些数据

x = torch.randn(N, D_in)  # 输入数据
y = torch.randn(N, D_out)  # 输出数据

class TwoLayerNet(torch.nn.Module):
    def __init__(self, D_in, H, D_out):
        super(TwoLayerNet, self).__init__()  # 继承了父类，就需要初始化一下父类
        self.linear1 = torch.nn.Linear(D_in, H)
        self.linear2 = torch.nn.Linear(H, D_out)

    def forward(self, x):
        x = self.linear1(x)
        x = torch.nn.functional.relu(x)
        x = self.linear2(x)
        return x

model = TwoLayerNet(D_in, H, D_out)

# if torch.cuda.is_available(): 有GPU的话
#     device = torch.device("cuda")          # a CUDA device object
#     y = torch.ones_like(x, device=device)  # directly create a tensor on GPU
#     x = x.to(device)                       # or just use strings ``.to("cuda")``
#     z = x + y
#     print(z)
#     print(z.to("cpu", torch.double))
learning_rate = 1e-6
loss_fn = nn.MSELoss(reduction='sum')
# optimizer = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.9)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
plt.ion()
for t in range(500):
    # forward pass
    y_pred = model(x)  # 直接做model.forward()
    loss = loss_fn(y_pred, y)
    #     print(t, loss.item())
    # backward pass
    model.zero_grad()
    loss.backward()
    # gradient
    # update weight of w1
    #     with torch.no_grad():
    #        for param in model.parameters():
    #             param-=learning_rate*param.grad
    optimizer.step()
    if t % 5 == 0:
        plt.cla()
        plt.scatter(y_pred.data.numpy(), y.data.numpy())
        # plt.scatter(x.data.numpy(),y_pred.data.numpy,c='red')
        plt.pause(0.1)
plt.ioff()
plt.show()


from django.core.management.base import BaseCommand
from django.contrib.gis.geos import Point
import os
import pandas as pd
from src.api.models import Address

COMMAND_BASE = os.path.dirname(os.path.abspath(__file__))
DATA_FILE = 'GeoLite2-City-Blocks-IPv4.csv'
DATA_FILE_PATH = os.path.join(COMMAND_BASE, 'data', DATA_FILE)

class Command(BaseCommand):
    def handle(self, *args, **options):
        """
        Ingests an IPv4 manifest file and writes it to the database.
        """
        df = pd.read_csv(DATA_FILE_PATH, usecols=['network', 'latitude', 'longitude'])
        start = 0
        size = len(df)
        batch_size = 1000
        while start < size:
            end = start + batch_size
            batch = df[start:end]
            addresses = []

            for _, row in batch.iterrows():
                coords = Point(float(row['longitude']), float(row['latitude']))
                new_address = Address(address=row['network'], coords=coords)
                addresses.append(new_address)

            Address.objects.bulk_create(addresses)
            start += batch_size


import os
import common
import string
import MyThread
from time import sleep
import requests
from urllib import request
import urllib

import socket  #
socket.setdefaulttimeout(10.0)   #设置全局的socket超时
import ssl
ssl._create_default_https_context = ssl._create_unverified_context

def downHtml(http,encodeStr="utf-8"):
    '''
    下载网址 ，获得HTML内容
    :param http:      网址
    :param encodeStr: 网址编码
    :return html:     返回获得的网址内容
    '''
    html = ""
    while html=="":
        try:
            re= requests.get(http,timeout=3)
            re.encoding = encodeStr    #直接从原网页获得编码信息  或者# re.encoding = re.apparent_encoding 这种方式自动分析，占用时间可能长
            html=re.text
        except:                          #请求不成功的话，循环 但sleep一段时间，保持频率不要太高，以免被拒绝
            sleep(5)
            continue
    return html

def bExist(httpPath):

    bExist=True
    while bExist:
        try:
           request.urlopen(httpPath, timeout=10)
           break
        except urllib.error.HTTPError as e:
            if e.code==404:
                bExist = False
        except urllib.error.URLError as e:
            print(e.reason)
        except socket.timeout as e:
            print(e)
    return bExist

    # request.urlretrieve(httpPath,localPath)  直接下载 出现10060错误
    # requests.get('https://www.zhihu.com/',verify=False)  verify=False 要不然会有SSL错误
def downImg(httpPath,localPath):
    html = ""
    num=0
    nImgType=0 #先尝试PNG，再尝试JPG
    while html=="":
         try:
             num+=1
             if num>1:
                print("--第%d次尝试--" %num)

             headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36"}

             req=request.Request(url=httpPath, headers=headers)
             #resp = request.urlopen(httpPath,timeout=60,headers=headers)
             resp = request.urlopen(req, timeout=60)
             html = resp.read()

         except urllib.error.HTTPError as e:
             if e.code==404:
                nImgType+=1
                httpPath='.'.join(httpPath.split(".")[0:-1])+".jpg"
                localPath=localPath.split(".")[0]+".jpg"
                if nImgType>=2:
                    return False #不存在
                else:
                    continue
         except (urllib.error.URLError, socket.timeout,Exception)as e:
             print(e)
             if num>50:
                 return False
             sleep(5)
             continue

    picFile = open(localPath, "wb")
    picFile.write(html)
    picFile.close()
    return True

#下载图片
def DownImg(domain,subDirList,subsubImgDir):
    '''
    :param subDirList:    文件夹列表
    :param subsubImgDir:  新创建的子文件夹，用以保存图像
    :return:
    '''
    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            imgName=img.split("/")[-1]
            print(imgName)
            common.downImg(domain+img,dir+"\\%s\\%s" %(subsubImgDir,imgName))

def GetRealImg(oriImg,ext=".png"):
    # img/flower2/s-flower5526.jpg
    # img/flower/flower5526.png
    splitlist = oriImg.split("/")
    s1=splitlist[0]                        # img
    s2=splitlist[1].rstrip(string.digits)  # flower
    s3=splitlist[2]                        # s-flower5526.jpg
    s3list=s3.split(".")
    s3=s3list[0]                           # s-flower5526
    subSplitList=s3.split("-")
    stemp="-".join(subSplitList[1:])        #flower5526
    return s1+"/"+s2+"/"+stemp+ext

#将图片链接，重新解析为新的链接地址，下载这个图片
def DownImgP(domain,subDirList,subsubImgDir,bParse=False):
    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            if bParse:
                img=GetRealImg(img,".png")
            name=img.split("/")[-1]
            print(name)
            common.downImg(domain+img,dir+"\\%s\\%s" %(subsubImgDir,name))

#多线程方式下载
def MultiDownImgP(domain,subDirList,subsubImgDir,bParse=False):

    mth=MyThread.CMultiDownThread(10)

    for dir in subDirList:
        print("-------%s------" %dir)
        imgList=[]
        common.readLinkFile(dir+"\\img.txt",imgList)

        if not os.path.exists(dir + "\\%s\\"%subsubImgDir):
            os.makedirs(dir + "\\%s\\"%subsubImgDir)
        for img in imgList:
            if bParse:
                img=GetRealImg(img,".png")
            name=img.split("/")[-1]
            print(name)
            th=MyThread.CDownThread(domain+img,dir+"\\%s\\%s" %(subsubImgDir,name))
            while not mth.AddThread(th):
                sleep(5)
    mth.join()

#多线程方式下载
def MultiDownImg(domain,imgLinkList,dir):

    mth=MyThread.CMultiDownThread(10)

    for img in imgLinkList:
        imglink=domain+img["src"]
        imgName = imglink.split("/")[-1]

        imgPath=dir + "\\"+imgName
        print(imgPath)

        th = MyThread.CDownThread(imglink, imgPath)
        while not mth.AddThread(th):
            sleep(5)

    mth.join()


from contextlib import asynccontextmanager
from datetime import datetime

RAPID_POSITION = 'G00'
LINEAR_MOVE = 'G01'
CIRCULAR_CLOCKWISE_MOVE = 'G02'
CIRCULAR_ANTICLOCKWISE_MOVE = 'G03'
PAUSE = 'G04'
SET_ORIGIN = 'G10'
USE_INCHES = 'G20'
USE_MM = 'G21'
RETURN_HOME = 'G28'
SERVO_CLOCKWISE_MOVE = 'M3'

class ScriptStage(object):
    reason = ''

    def __str__(self):
        return self.reason

    def as_json(self):
        return {
            'message': self.reason,
        }

class ChangePenStage(ScriptStage):
    def __init__(self,colour):
        self.reason = f'Change the pen to {colour}'
        self.colour = colour

    def as_json(self):
        data = super().as_json()
        data.update({
            'colour': self.colour,
        })
        return data

class SerialException(Exception):
    pass

class GCodeSender(object):

    transform_fn = lambda p: p
    # A transformation applied to all points

    speed = 2000
    # Default feed rate, in mm/minute

    RX_BUFFER_SIZE = 50
    # Size of the device's input buffer

    WAIT_TIME = 0.01

    PEN_UP_POSITION = 0
    PEN_DOWN_POSITION = 50

    MIN_MOVE = 0.1
    # Minimum distance to bother moving

    INTERRUPT_PERIOD = 1
    # The amount of time to spend running commands before interrupting and letting the rest of the event loop continue

    def __init__(self, device, flip_x = False, flip_y = False, flip_z = False, number_precision=3, immediate=False, debug=False):
        """
            Initialise a GCode sender.

            This sends GRBL-compatible commands by default, with the pen holder controlled by a servo.
            For other plotters, you should create a subclass of this and override the methods to produce different commands.

            Arguments:
                device: A pySerial Serial object
                flip_x: Should x coordinates be flipped? +x is mapped to -x
                flip_y: Should y coordinates be flipped? +y is mapped to -y
                flip_z: Should z coordinates be flipped? +z is mapped to -z
                number_precision: The number of decimal places to write numbers.
        """
        self.device = device
        self.flip_x = flip_x
        self.flip_y = flip_y
        self.flip_z = flip_z
        self.number_precision = number_precision

        #self.device.reset_input_buffer()

        self.staged_script = []

        self.immediate = immediate
        self.debug = debug

        self.current_pos = (0,0)

    def reset_connection(self, device):
        self.device = device

    def format_number(self, n):
        """
            Format a number for use in GCode.
            Rounds to `self.number_precision`.

            Argument:
                n - float

            Returns:
                string
        """

        return ('{:.' + str(self.number_precision) + 'f}').format(n)

    def format_coordinates(self, position, axes=('X','Y','Z')):
        """
            Format a set of 2D or 3D coordinates for GCode.

            Arguments:
                position - (float,float) or (float,float,float)
                axes - the names of the axes

            Returns
                list of strings
        """

        return [axis+self.format_number(n) for axis,n in zip(axes,position)]

    def transform_coordinates(self,coordinates):
        """
        Transform a pair of coordinates into the machine's coordinates.

        At the moment: apply flips.
        """
        return [-n if flip else n for n,flip in zip(self.transform_fn(coordinates), (self.flip_x, self.flip_y, self.flip_z))]

    def get_output(self):
        """
            Get a line of output from the plotter.

            Returns:
                string
        """

        out = ''
        while True:
            r = self.device.read_until()
            if r and r not in (b'\r\n',b'ok\r\n'):
                out += r.decode('ascii')
            else:
                break
        return out

    def send_command(self,command,arguments=[],immediate=False):
        """
        Send a command to the machine.

        Arguments:
            command: the g-code, for example 'G00'
            arguments: a list of string arguments.
            immediate: if true, run the command immediately. Otherwise, add it to the staged script.
        """

        line = ' '.join([command]+arguments)+'\n'
        if not immediate:
            self.staged_script.append(line)
        else:
            self.run_script([line])

    async def run_script(self,script=None, startline=0):
        """
            Run lines from the staged script until self.INTERRUPT_PERIOD seconds have elapsed, a `ScriptStage` object is reached, or the end of the script is reached.

            Arguments:
                script - a list of commands or `ScriptStage` objects
                startline - the index of the line to start from.

            Returns:
                (line, message) - the next line to evaluate, and a `ScriptStage` object or None
        """

        if script is None:
            script = self.staged_script

        if self.debug:
            print("running script")
            print(script)

        s = self.device

        if startline==0:
            try:
                s.reset_input_buffer()
            except Exception as e:
                raise SerialException()

            s.write(b'\r\n')
            s.readline()

        o = ''

        progress = 0

        t1 = datetime.now()

        i = 0

        for i,line in enumerate(script[startline:],startline):
            if isinstance(line,ScriptStage):
                return i+1, line
            # Send the command to the machine
            s.write(line.encode('ascii')) # Send g-code block to grbl
            oline = s.readline().decode('ascii').strip()
            o += oline
            t2 = datetime.now()
            if (t2-t1).total_seconds()> self.INTERRUPT_PERIOD:
                return i, None

        return i+1, None

    def before_script(self):
        """
            This is run by the `script` context manager before any other commands.
        """
        self.return_home()

    def after_script(self):
        """
            This is run by the `script` context manager at the end of the context.
        """
        self.return_home()

    @asynccontextmanager
    async def script(self,transform_fn=lambda p: p):
        """
            A context manager to stage a script for the plotter.

            Example:
                async with plotter.script():
                    plotter.move((0,0))
                    plotter.line((10,10))

            Argument:
                transform_fn - a function which applies a transformation to all points sent through the plotter.
        """
        try:
            self.transform_fn = transform_fn
            self.before_script()
            self.staged_script = []
            self.colour = None
            yield self
        finally:
            self.after_script()
            script = self.staged_script

    def move(self, position):
        """
            Move the pen up, then move the plotter to the given position, then move the pen down.
            Doesn't move if the distance from the current position is less than `self.MIN_MOVE`.

            Argument:
                position - (float,float)
        """
        x1,y1 = self.current_pos
        x2,y2 = position
        dx,dy = x2-x1,y2-y1
        d = dx*dx+dy*dy
        if d>self.MIN_MOVE**2:
            self.pen_up()
            self.send_command(RAPID_POSITION,self.format_coordinates(self.transform_coordinates(position)))
            self.pen_down()

        self.current_pos = position

    def line(self, position, speed=None):
        """
        Move to the given coordinates.

        Arguments:
            position: the coordinates to move to: a tuple of either two or three numbers.
            speed: optional speed to move, in mm/minute. If not given, the machine's most recent speed is used.
        """

        tposition = self.transform_coordinates(position)
        args = self.format_coordinates(tposition)

        if speed is None:
            speed = self.speed

        args.append('F'+self.format_number(speed))

        self.send_command(LINEAR_MOVE,args)

        self.current_pos = position

    def arcto(self, position, radius, speed=None, clockwise=True):
        """
            Draw a circular arc to the given position.

            Arguments:
                position - (float,float) - the position to move to.
                radius - float - the radius of the circle.
                speed - float - the speed to move, in mm/hour
                clockwise - if True, draw the clockwise arc between the two points, otherwise anti-clockwise.
        """
        tposition = self.transform_coordinates(position)

        if self.flip_x:
            clockwise = not clockwise
        if self.flip_y:
            clockwise = not clockwise

        args = self.format_coordinates(tposition)+['R'+self.format_number(radius)]

        command = CIRCULAR_CLOCKWISE_MOVE if clockwise else CIRCULAR_ANTICLOCKWISE_MOVE

        self.send_command(command, args)

        self.current_pos = position

    def move_pen(self,position):
        """
            Move the pen to the given position.

            Argument:
                position - float
        """
        self.send_command(SERVO_CLOCKWISE_MOVE,['S'+self.format_number(position)])

    def pen_up(self):
        """
            Move the pen up, to `self.PEN_UP_POSITION`.
        """
        self.move_pen(self.PEN_UP_POSITION)

    def pen_down(self):
        """
            Move the pen down, to `self.PEN_DOWN_POSITION`.
        """
        self.move_pen(self.PEN_DOWN_POSITION)

    def get_status(self):
        """
            Get the status output from the plotter.

            Returns:
                string
        """
        self.device.write(b'?')
        return self.get_output().strip()

    def return_home(self):
        """
            Move the pen up, then move the plotter to its home position.
            Unless you change it, home is (0,0).
        """
        if self.current_pos != (0,0):
            self.pen_up()
            self.send_command(RETURN_HOME)

        self.current_pos = (0,0)

    def change_pen(self, colour):
        """
            Get the plotter ready to change pen: move the plotter to its home position, and keep the pen up.
            Adds a `ChangePenStage` line to the script.

            Argument:
                colour - the colour to change to.
        """
        if self.colour != colour:
            self.return_home()
            self.pen_up()
            self.staged_script.append(ChangePenStage(colour))
            self.move(self.current_pos)
        self.colour = colour


import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from tqdm import tqdm
from time import time
from datetime import datetime

from model import AttentionModel
from baseline import RolloutBaseline
from data import generate_data, Generator,data_from_txt
from config import Config, load_pkl, train_parser

def train(cfg, log_path = None):
    #将会让程序在开始torch.save(model.state_dict(), '%s%s_epoch%s.pt' % (cfg.weight_dir, cfg.task, epoch))时花费一点额外时间，为整个网络的每个卷积层搜索最适合它的卷积实现算法，
    # 进而实现网络的加速
    torch.backends.cudnn.benchmark = True
    device = torch.device('cuda:2' if torch.cuda.is_available() else 'cpu')
    torch.cuda.set_device(device)
    log_path = '%s%s_%s.txt' % (cfg.log_dir, cfg.task, cfg.dump_date)  # cfg.log_dir = ./Csv/
    start_t=time()
    # open w就是覆盖，a就是在后面加 append
    with open(log_path, 'w') as f:
        f.write(datetime.now().strftime('%y%m%d_%H_%M'))
    with open(log_path, 'a') as f:
        f.write('\n start training \n')

    with open(log_path, 'a') as f:
        f.write(''.join('%s: %s\n' % item for item in vars(cfg).items()))
    #t1=time()
    #dataset = Generator(device, cfg.batch*cfg.batch_steps,
    #                    cfg.n_containers, cfg.max_stacks, cfg.max_tiers)

    #t2=time()
    #print('generate data: %dmin%dsec'%((t2-t1)//60,(t2-t1)%60)) 生成一次数据1分46s

    #print('233')

    model = AttentionModel(device=device,embed_dim=cfg.embed_dim,n_encode_layers= cfg.n_encode_layers,n_heads= cfg.n_heads,tanh_clipping= cfg.tanh_clipping
                           ,n_containers=cfg.n_containers,max_stacks=cfg.max_stacks,max_tiers=cfg.max_tiers)
    model.train()
    model=model.to(device)

    baseline = RolloutBaseline(model, cfg.task, cfg.weight_dir, cfg.n_rollout_samples,
                               cfg.embed_dim,cfg.n_containers, cfg.max_stacks,cfg.max_tiers , cfg.warmup_beta, cfg.wp_epochs, device,log_path)

    optimizer = optim.Adam(model.parameters(), lr=cfg.lr)

    #bs batch steps number of samples = batch * batch_steps
    def rein_loss(model, inputs, bs, t, device):
        # ~ inputs = list(map(lambda x: x.to(device), inputs))

        # decode_type是贪心找最大概率还是随机采样
        # L(batch) 就是返回的cost ll就是采样得到的路径的概率
        L, ll = model(inputs, decode_type='sampling')
        #b = bs[t] if bs is not None else baseline.eval(inputs, L)
        b=torch.FloatTensor([L.mean()]).to(device)
        return ((L - b) * ll).mean(), L.mean()

    tt1 = time()

    t1=time()
    for epoch in range(cfg.epochs):

        ave_loss, ave_L = 0., 0.

        datat1=time()
        dataset=Generator(device,cfg.batch*cfg.batch_steps,
                          cfg.n_containers,cfg.max_stacks,cfg.max_tiers)
        datat2=time()
        print('data_gen: %dmin%dsec' % ((datat2 - datat1) // 60, (datat2 - datat1) % 60))

        bs=baseline.eval_all(dataset)
        bs = bs.view(-1, cfg.batch) if bs is not None else None  # bs: (cfg.batch_steps, cfg.batch) or None

        model.train()
        dataloader=DataLoader(dataset, batch_size = cfg.batch, shuffle = True)
        for t, inputs in enumerate(dataloader):
            loss,L_mean=rein_loss(model,inputs,bs,t,device)
            optimizer.zero_grad()
            with torch.autograd.set_detect_anomaly(True):
                loss.backward()

            # print('grad: ', model.Decoder.Wk1.weight.grad[0][0])
            # https://github.com/wouterkool/attention-learn-to-route/blob/master/train.py
            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0, norm_type=2)
            optimizer.step()

            ave_loss += loss.item()
            ave_L += L_mean.item()

            if t % (cfg.batch_verbose) == 0:
                t2 = time()
                # //60是对小数取整
                print('Epoch %d (batch = %d): Loss: %1.3f L: %1.3f, %dmin%dsec' % (
                    epoch, t, ave_loss / (t + 1), ave_L / (t + 1), (t2 - t1) // 60, (t2 - t1) % 60))
                # 如果要把日志文件保存下来
                if cfg.islogger:
                    with open(log_path, 'a') as f:
                        f.write('Epoch %d (batch = %d): Loss: %1.3f L: %1.3f, %dmin%dsec \n' % (
                    epoch, t, ave_loss / (t + 1), ave_L / (t + 1), (t2 - t1) // 60, (t2 - t1) % 60))
                t1 = time()

        #print('after this epoch grad: ', model.Decoder.Wk1.weight.grad[0][0])

        # 看是不是要更新baseline
        #这里为了让baseline不变化给model加上eval
        model.eval()
        baseline.epoch_callback(model, epoch)
        #torch.save(model.state_dict(), '%s%s_epoch%s.pt' % (cfg.weight_dir, cfg.task, epoch))

        if epoch==cfg.epochs-1:
            #data = data_from_txt("data/test.txt")
            #data=data.to(device)
            #baseline.model.eval()
            torch.save(baseline.model.state_dict(), '%s%s_epoch%s_2.pt' % (cfg.weight_dir, cfg.task, epoch))
            #torch.save(baseline.model.Decoder.Encoder.state_dict(),'%s%s_encoder_epoch%s.pt' % (cfg.weight_dir, cfg.task, epoch))
            #with torch.no_grad():
                #cost=baseline.rollout(model=baseline.model,dataset=data,batch=40)
            #print('test baseline model')
            #print('test.txt:mean',cost.mean())

    tt2 = time()
    print('all time, %dmin%dsec' % (
        (tt2 - tt1) // 60, (tt2 - tt1) % 60))

if __name__ == '__main__':
	cfg = load_pkl(train_parser().path)

	train(cfg)
    #nohup python -u train.py -p Pkl/CRP_9_3_5_train.pkl >>./Csv/nohup.txt 2>&1 &


import numpy as np
import math
import random
from collections import deque
from shapely.geometry import Point, LineString
from trajectory.rrt import RRT

def is_saw(x, y, n):
    '''
    Checks if walk of length n is self-avoiding
    :param (x,y)(list,list): walk of length n
    :param n(int): length of the walk
    :return: True if the walk is self-avoiding
    '''
    # creating a set removes duplicates, so it suffices to check the size of the set
    return n+1 == len(set(zip(x, y)))

def in_map(x, y):
    '''
    Checks if walk is in map
    :param (x,y)(list,list): the walk
    :return judge: True if the walk is in map
    '''
    judge = False
    x, y = np.array(x), np.array(y)
    if (abs(x) < 9.5).all() and (abs(y) < 9.5).all():
        judge = True
    return judge

def myopic_saw(n, x_init, y_init):
    '''
    Tries to generate a SAW of length n using myopic algorithm
    :param n(int): length of walk
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :return (x,y)(list,list): the walk (length <= n)
    '''
    theta = [i * math.pi / 2 for i in range(0, 4)]
    r = 5    # original one: 0.75
    x, y = [x_init], [y_init]
    positions = set([(x_init,y_init)])    # positions is a set(no same element) that stores all sites visited
    for i in range(n):
        t = random.choice(theta)
        x_new = x[-1] + r * round(math.cos(t), 2)
        y_new = y[-1] + r * round(math.sin(t), 2)
        if (x_new, y_new) not in positions:
            x.append(x_new)
            y.append(y_new)
            positions.add((x_new, y_new))
        else:
            continue
    return x, y

def dimer(n, x_init, y_init):
    '''
    Generates a SAW by dimerization
    :param n(int): length of walk
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :return (x_concat,y_concat)(list,list): walk of length n
    '''
    if n <= 3:
        x, y = myopic_saw(n, x_init, y_init)
        return x, y
    else:
        not_saw = True
        while not_saw:
            x_1, y_1 = dimer(n//2, x_init, y_init)
            x_2, y_2 = dimer(n - n//2, x_init, y_init)
            x_2 = [(x_1[-1] - x_1[0] + x) for x in x_2]
            y_2 = [(y_1[-1] - y_1[0] + y) for y in y_2]
            x_concat, y_concat = x_1 + x_2[1:], y_1 + y_2[1:]
            if is_saw(x_concat, y_concat, n) and in_map(x_concat, y_concat):
                not_saw = False
        return x_concat, y_concat

def interpolation(x, y, n_trajectory):
    '''
    Generates continous trajectory by whittaker-shannon interpolation formula
    :param (x,y)(list,list): discrete self-avoiding walk
    :param n_trajectory(int): length of continous trajectory
    :return (x_contin,y_contin)(list,list): continous trajectory
    '''
    interval = 1
    N = len(x)
    t = np.linspace(0, (N - 1) * interval, n_trajectory)
    x_contin, y_contin = 0, 0
    for n in range(N):
        x_contin += x[n] * np.sinc(t / interval - n)
        y_contin += y[n] * np.sinc(t / interval - n)
    return x_contin, y_contin

def bezier_smooth(traj_points, n_trajectory):
    raw_trajectory = np.array(traj_points)
    raw_traj_x = raw_trajectory[:, 0]
    raw_traj_y = raw_trajectory[:, 1]

    N = raw_trajectory.shape[0] - 1
    t = np.linspace(0, 1, n_trajectory)
    polynomial = []
    for i in range(N+1):
        polynomial.append(math.comb(N, i) * (t ** (N - i)) * (1 - t) ** i)

    polynomial = np.array(polynomial)
    traj_x = np.dot(raw_traj_x, polynomial)
    traj_y = np.dot(raw_traj_y, polynomial)
    return list(zip(traj_x, traj_y))

def gen_saw_track(x_init, y_init, n_trajectory=1100, n_discre=69):
    '''
    Generates continous self-avoiding trajectory
    :param (x_init, y_init)(float, float): initial coordinate of saw
    :param n_discre(int): length of discrete self-avoiding walk
    :param n_trajectory(int): length of continous trajectory
    :return trajectory(list): continous trajectory
    '''
    x, y = dimer(n_discre, x_init, y_init)
    trajectory = bezier_smooth(list(zip(x, y)), n_trajectory)
    return trajectory

def gen_rose_track(x_init, y_init, n_trajectory=1200):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * np.pi / 400
        r = 5 * np.sin(4 * k)
        x.append(0.5 * x_init + r * np.cos(k))
        y.append(0.5 * y_init + r * np.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_spiral_track(x_init, y_init, n_trajectory=1200):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * math.pi / 90
        r = 0.12 * k
        x.append(0.5 * x_init + r * math.cos(k))
        y.append(0.5 * y_init + r * math.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_circle_track(x_init, y_init, r=10, n_trajectory=1010):
    x, y = [], []
    for i in range(n_trajectory):
        k = i * math.pi / 90
        x.append(x_init + r * math.cos(k))
        y.append(y_init + r * math.sin(k))
    trajectory = list(zip(x, y))
    return trajectory

def gen_square_track(x_init, y_init, a=19, r=6):
    '''
    number of trajectory points is 1012
    '''
    x, y = [], []
    i = 0
    center = [x_init+(a/2-r), y_init+(a/2-r)]
    while i < 150:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1] - (a-2*r)/100)
        y.append(y[-1])

    center = [x_init-(a/2-r), y_init+(a/2-r)]
    while i < 300:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1])
        y.append(y[-1] - (a-2*r)/100)

    center = [x_init-(a/2-r), y_init-(a/2-r)]
    while i < 450:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1] + (a-2*r)/100)
        y.append(y[-1])

    center = [x_init+(a/2-r), y_init-(a/2-r)]
    while i < 600:
        k = i * math.pi / 300
        x.append(center[0] + r * math.cos(k))
        y.append(center[1] + r * math.sin(k))
        i += 1
    for j in range(103):
        x.append(x[-1])
        y.append(y[-1] + (a-2*r)/100)

    trajectory = list(zip(x, y))
    return trajectory

def gen_random_rrt_track(x_init, y_init, n_trajectory, env_size, poly_list):
    p_start = [x_init, y_init]
    step_len = 0.5
    random_path_len = 11   # eval: 10-100-400    env3: 5-50-200
    self_avoid_poly = deque([Point(-10, -10), Point(-10, -10), Point(-10, -10)], maxlen=3)
    poly_list.extend(self_avoid_poly)
    overall_path = []
    while len(overall_path) < 400:
        rrt = RRT(p_start, (0, 0), step_len, random_path_len=random_path_len)
        random_path = rrt.random_planning(env_size, poly_list)
        if not random_path:
            return None
        new_path = bezier_smooth(random_path, 100)
        overall_path.extend(new_path)

        self_avoid_poly.append(LineString(new_path[:-10]))
        poly_list[-3:] = self_avoid_poly
        p_start = overall_path[-1]

    trajectory = bezier_smooth(overall_path, n_trajectory)
    trajectory.reverse()
    return trajectory

def gen_simple_rrt_track(multi_goal_list, n_trajectory, env_size, poly_list):
    step_len = 0.5
    rrt = RRT(multi_goal_list[0], multi_goal_list[1], step_len)
    path1 = rrt.path_planning(env_size, poly_list)
    rrt = RRT(multi_goal_list[1], multi_goal_list[2], step_len)
    path2 = rrt.path_planning(env_size, poly_list)
    if not path1 or not path2:
        return None
    overall_path = bezier_smooth(path1, 200)
    overall_path.extend(bezier_smooth(path2, 200))
    trajectory = bezier_smooth(overall_path, n_trajectory)
    trajectory.reverse()
    return trajectory