# Step 1. 필요한 모듈을 import합니다.
# -*- coding: utf-8 -*-
import common.common as cm
import pprint as pp
import matplotlib.pyplot as plt
from matplotlib import font_manager, rc
font_name = font_manager.FontProperties(fname="c:/Windows/Fonts/malgun.ttf").get_name()
rc('font', family=font_name)
import os
import numpy as np
import collections as clt
import pandas as pd
import random as rd

# Step 2. 경마정보조회를 위한 요청변수 및 요청주소 등을 구성합니다.
# Step 2-1. Looging을 위하여 공통 모듈에서 로거를 정의합니다.
f = cm.pre_logging() # 등록마 조회 결과 파일을 로딩

# Step 2-2. '4-1'에서 저장한 파일을 이용하여 해당 데이터를 로딩합니다.
c = []
fn = os.getcwd() + '/data/' + 'RegisteredHorse.da' # url = 'http://data.kra.co.kr/publicdata/service/hrReg/getHrReg' # 등록마 조회 (meet)
c.append(cm.open_file(fn, True))
cm.logF(f, c)

# Step 3. item의 구조를 분석하고, 분석하고자 하는 값들을 추출하기 위한 데이터를 전처리합니다.
pp = pp.PrettyPrinter(indent=4)
data = []
for i in range(0, len(c)):
    try:
        # 응답값이 XML일 경우
        _djson = cm.x2j(c[i])
        item = _djson['response']['body']['items']['item']

        # pp.pprint(item)
        cm.logF(f, item)
        if (isinstance(item, dict)):
            data.append(item)
        else:
            data.extend(item)
    except Exception as e:
        continue

cm.logF(f, data)

# Step 3-1. data에서 일부 데이터(전체 갯수의 10%수준)를 추출합니다.
data = rd.sample(data, int(len(data) * 0.1))
cm.logF(f, data)
rd.shuffle(data)

# Step 3-2. 일부 데이터를 기준으로 각 요소의 빈도를 집계합니다.
# color : 경주마 털색, sex : 경주마 성별
df = pd.DataFrame(data)
cm.logF(f, df)

color_counts = clt.Counter(df['color'])
stat = pd.DataFrame(
    {'color' : list(color_counts.keys()), 'count' : list(color_counts.values()), 'point' : [10, 5, 3, 2]}
)
cm.logF(f, stat)

# Step 4. 확률을 계산합니다. 확률변수 - 기대값, 분산(표준편차)
# Step 4-1. 등록마(표본전체) 중 털색이 ['갈색', '밤색', '흑갈색', '회색']인 말을 선택할 확률을 구합니다.
# 1. 경주마 털색, 2. 뽑을 때 점수, 3. 확률 : ['갈색', '밤색', '흑갈색', '회색'], [10, 5, 3, 2], [ ]
number_of_total_event_occurs = len(data)
cm.logF(f, '***** 확률을 계산합니다. *****')
cm.logF(f, ('표본갯수 : %d' %  number_of_total_event_occurs))
color = list(color_counts.keys()) # ['갈색', '밤색', '흑갈색', '회색']
cm.logF(f, ('경주마 털색 종류 : %s' % color))

i = 0
cp = {}
for key, value in color_counts.items():
    i += 1
    tv = float(value / number_of_total_event_occurs) # probability of color
    cp.update({key : tv})

df_color = pd.DataFrame(
    {'color' : list(cp.keys()), 'probability' : list(cp.values())}
)
cm.logF(f, df_color)

stat = pd.merge(stat, df_color, left_on='color', right_on='color')
cm.logF(f, stat)

# 기대값 : 각 요소별 '확률*포인트(사건이 발생할 때의 결과=이득)'의 합
expectation = (stat['point'] * stat['probability']).sum()
cm.logF(f, '경주마 털색 추출에 대한 기대값은? %f' % expectation)

# 분산 : 기대값과 확률변수들간의 거리(떨어진 정도)
variance = np.var(stat['point'] - expectation)
# 표준편차 : 기대값으로부터의 일반적 차이
stdev = np.std(stat['point'] - expectation)
cm.logF(f, '분산 : %f, 표준편차 : %f' % (variance, stdev))

# Step 5. 데이터 분석을 위한 표본에 대하여 시각화를 진행합니다.
x1 = stat['color']
y1 = stat['count']
x2 = ['기대값', '분산', '표준편차']
y2 = [expectation, variance, stdev]

fig, (ax1, ax2) = plt.subplots(2)
ax1.plot(x1, y1, 'ko-')
ax1.set(title='등록마 조회 서비스', ylabel='경주마 털색')
ax2.plot(x2, y2, 'r.-')
ax2.set(xlabel='%', ylabel='기대값,분산,표준편차')

plt.show()


import random
import pyperclip

def main(length: int):
    nums, upper, lower, chrs = "0123456789", "ABCDEFGHIJKLMNOPQRSTUVWXYZ", \
                               "abcdefghijklmnopqrstuvwxyz", r" !@#$%^&*()\")[]{}"
    while True:
        string = ''.join(random.choices(nums + upper + lower + chrs, k=length))
        string_as_set = set(string)
        if all((len(set(nums) & string_as_set),
                len(set(upper) & string_as_set),
                len(set(lower) & string_as_set),
                len(set(chrs) & string_as_set))):
            return string

if __name__ == "__main__":
    pyperclip.copy(main(64))


import os
import telebot
from dotenv import load_dotenv

from Handlers.add_test_task_by_level_handler import add_easy_test_task_function, add_middle_test_task_function, \
    add_hard_test_task_function
from Handlers.add_code_task_by_level_handler import add_easy_code_task_function, add_middle_code_task_function, \
    add_hard_code_task_function
from Handlers.add_view_comments import process_comments
from change_modes import BotProcessor

from Handlers.add_admin_handler import add_admin_function
from Handlers.add_lesson_handler import add_lesson_function
from Handlers.callback_query_handler import callback_query_handler
from Handlers.check_code_handler import check_code
from Handlers.documentation_handler import search_documentation
from Handlers.add_code_task_handler import add_code_task_function
from Handlers.change_code_task_handler import change_code_task_function
from Handlers.choose_code_task_handler import process_code_task_level
from Handlers.add_test_task_handler import add_test_task_function
from Handlers.change_test_task_handler import change_test_task_function
from Handlers.choose_test_task_handler import process_test_task_level
from Handlers.choose_lesson_handler import process_lesson_topic
from Handlers.request_help_handler import help_request_handler
from Handlers.csv_handler import handle_csv_lessons
from Handlers.csv_handler import handle_csv_test_tasks
from Handlers.csv_handler import handle_csv_code_tasks
from Handlers.add_level_handler import add_level_function
from Handlers.premium_status_handler import premium_status_function
from Handlers.account_handler import account_function

from utils.key_board import InlineKeyboard, ReplyKeyboard
from utils.modes import MODE_MAIN_MENU

class Bot:
    """
    This class implements the bot
    """

    def __init__(self, token):
        self.inline_keyboard = None
        self.bot = None
        self.chat_id = None
        self.token = token

        self.current_mode = MODE_MAIN_MENU
        self.reply_keyboard = None

        self.bot_processor = BotProcessor()

    def run(self):
        """
        This method runs the bot
        """
        load_dotenv()
        token = os.getenv('TOKEN')
        # create a telebot instance using the token
        self.bot = telebot.TeleBot(token)
        # create an instance of InlineKeyboardMarkup
        self.inline_keyboard = InlineKeyboard()
        # create an instance of ReplyKeyboardMarkup
        self.reply_keyboard = ReplyKeyboard()

    def get_bot(self):
        """
        This method returns the bot
        """
        return self.bot

    def delete_message(self, chat_id, message_id):
        self.bot.delete_message(chat_id, message_id)

    def start_handler(self, message):
        """
        This method sends the welcome message
        """
        from bot import bot_db
        # print(self.bot_processor.get_current_mode())
        if not bot_db.check_user_exists(message.chat.id):
            bot_db.add_user(message.chat.id, message.chat.first_name, message.chat.username)
            self.bot.send_message(message.chat.id, f"Welcome to my bot, {message.chat.first_name}!",
                                  reply_markup=self.inline_keyboard.get_keyboard())
        else:
            self.bot.send_message(message.chat.id, f"Welcome back, {message.chat.first_name}!",
                                  reply_markup=self.inline_keyboard.get_keyboard())

    def add_lesson_handler(self, message):
        """
        This method adds a lesson
        """
        if message.text.find("/add_lesson") != -1:
            print(self.bot_processor.get_current_mode())
            add_lesson_function(self.bot, message)

    def add_admin_handler(self, message):
        """
        This method adds an admin
        """
        if message.text.find("/add_admin") != -1:
            print(self.bot_processor.get_current_mode())
            add_admin_function(self.bot, message)

    def add_test_task_handler(self, message):
        """
        This method adds a test_task
        """
        if message.text.find("/add_test_task") != -1:
            add_test_task_function(self.bot, message)

    def add_test_task_by_level_handler(self, message):
        """
        This method adds a test_task based on the level.
        """

        if message.text.find("/add_easy_test_task") != -1:
            add_easy_test_task_function(self.bot, message)
        elif message.text.find("/add_middle_test_task") != -1:
            add_middle_test_task_function(self.bot, message)
        elif message.text.find("/add_hard_test_task") != -1:
            add_hard_test_task_function(self.bot, message)

    def change_test_task_handler(self, message):
        """
        This method changes a test_task
        """
        if message.text.find("/change_test_task") != -1:
            change_test_task_function(self.bot, message)

    def add_code_task_handler(self, message):
        """
        This method adds a code_task
        """
        if message.text.find("/add_test_task") != -1:
            add_code_task_function(self.bot, message)

    def add_code_task_by_level_handler(self, message):
        """
        This method adds a code_task based on the level.
        """

        if message.text.find("/add_easy_code_task") != -1:
            add_easy_code_task_function(self.bot, message)
        elif message.text.find("/add_middle_code_task") != -1:
            add_middle_code_task_function(self.bot, message)
        elif message.text.find("/add_hard_code_task") != -1:
            add_hard_code_task_function(self.bot, message)

    def change_code_task_handler(self, message):
        """
        This method changes a test_task
        """
        if message.text.find("/change_code_task") != -1:
            change_code_task_function(self.bot, message)

    def add_level_handler(self, message):
        """
        This method adds a lesson
        """
        if message.text.find("/add_level") != -1:
            add_level_function(self.bot, message)

    def send_error_message(self, error_message):
        self.bot.send_message(os.getenv('OWNER_CHAT_ID'), f"Error in bot:\n\n{error_message}")

    def csv_tables_names_lessons(self, message):
        if message.document.file_name == 'lessons.csv':
            handle_csv_lessons(self.bot, message, message.document)
        elif message.document.file_name == 'test_tasks.csv':
            handle_csv_test_tasks(self.bot, message, message.document)
        elif message.document.file_name == 'code_tasks.csv':
            handle_csv_code_tasks(self.bot, message, message.document)
        elif message.document.file_name != 'lessons.csv' or 'test_tasks.csv':
            self.bot.send_message(message.chat.id, "Please send the file in CSV format with the same name,\n"
                                                   "as a table name in the DATABASE.")

    def handle_callback_query(self, call):

        """This handler allows using callback query with pressing designated inline keyboard buttons"""
        self.bot_processor.message_handler(call)  # Set the current mode
        callback_query_handler(call, self.bot, self.inline_keyboard, self.reply_keyboard)

    def display_current_mode(self, message):
        """
        This method displays the current mode
        """
        if self.bot_processor.is_mode_main_menu():
            self.bot.send_message(message.chat.id, "You are in the main menu mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
        elif self.bot_processor.is_mode_documentation():
            self.bot.delete_message(message.chat.id, message.message_id)
            self.bot.send_message(message.chat.id, "You are in documentation mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            search_documentation(message, self.bot)
        elif self.bot_processor.is_mode_check_code():
            self.bot.send_message(message.chat.id, "You are in check code mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            check_code(message, self.bot)
        elif self.bot_processor.is_mode_lesson():
            self.bot.send_message(message.chat.id, "You are in lesson mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_lesson_topic(message, self.bot)
        elif self.bot_processor.is_mode_testing():
            self.bot.send_message(message.chat.id, "You are in testing mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_test_task_level(message, self.bot)
        elif self.bot_processor.is_mode_coding():
            self.bot.send_message(message.chat.id, "You are in coding mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_code_task_level(message, self.bot)
        elif self.bot_processor.is_mode_account():
            self.bot.delete_message(message.chat.id, message.message_id)
            self.bot.send_message(message.chat.id, "You are in account mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            account_function(message, self.bot)
        elif self.bot_processor.is_mode_comments():
            self.bot.send_message(message.chat.id, "You are in comments mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            process_comments(self.bot, message)
        elif self.bot_processor.is_mode_help():
            self.bot.send_message(message.chat.id, "You are in help mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            help_request_handler(message, self.bot)
        elif self.bot_processor.is_mode_premium():
            self.bot.send_message(message.chat.id, "You are in 'Premium' mode.",
                                  reply_markup=self.inline_keyboard.get_keyboard())
            premium_status_function(message, self.bot)

        # Handle the "HELP" button separately
        if message.text == "HELP":
            self.bot.send_message(message.chat.id, "This is the help message.",
                                  reply_markup=self.inline_keyboard.get_keyboard())


import numpy as np
import random
import cv2

def draw_rect(im, cords, color=None):
    im = im.copy()
    cords = cords.reshape(-1, 4)
    if not color:
        color = [255, 255, 255]
    for cord in cords:
        pt1, pt2 = (cord[0], cord[1]), (cord[2], cord[3])
        pt1 = int(pt1[0]), int(pt1[1])
        pt2 = int(pt2[0]), int(pt2[1])
        im = cv2.rectangle(im.copy(), pt1, pt2, color, int(max(im.shape[:2]) / 200))
    return im

class RandomHorizontalFlip(object):
    def __init__(self, prob=1):
        self.prob = prob

    def __call__(self, img, bboxes=None):
        if random.uniform(0, 1) < self.prob:
            return img, bboxes
        img_center = np.array(img.shape[:2])[::-1] / 2
        img_center = np.hstack((img_center, img_center))
        img = img.astype('float64')
        if not bboxes:
            return img,bboxes
        bboxes = bboxes.astype('float64')
        img = img[:, ::-1, :]
        img = img.astype('uint8')
        if len(bboxes) == 0:
            return img, bboxes
        bboxes[:, [0, 2]] += 2 * (img_center[[0, 2]] - bboxes[:, [0, 2]])
        box_w = abs(bboxes[:, 0] - bboxes[:, 2])
        bboxes[:, 0] -= box_w
        bboxes[:, 2] += box_w

        return img, bboxes


import os
import numpy as np
import netCDF4 as nc

import jigsawpy
import argparse

from reach.filter_hydrosheds import filter_hydrosheds

class base: pass

def ex1():

#-- filter the HydroSHEDS data-set based on a nonuniform
#-- resolution function, to be used in unstructured mesh
#-- generation

    tag_ = "filtered_o25_l25"

    args = base()
    args.shp_file = os.path.join(
        "dat",
        "HydroRIVERS_v10_shp", "HydroRIVERS_v10_shp",
        "HydroRIVERS_v10.shp")

    args.out_file = os.path.join(
        "out",
        "HydroRIVERS_v10_shp", tag_,
        "filtered_25p0km.shp")

    args.spc_file = os.path.join(
        "out",
        "HydroRIVERS_v10_shp", tag_,
        "resolution.nc")

    args.msh_tags = os.path.join(
        "msh",
        "HydroRIVERS_v10_msh", tag_)

    dir_ = os.path.dirname(
        os.path.abspath(args.spc_file))

    if (not os.path.exists(dir_)): os.makedirs(dir_)

   #spac = jigsawpy.jigsaw_msh_t()
   #name = "spac_o25_l25.msh"
   #jigsawpy.loadmsh(name, spac)  # to load existing

    nlon = 360; nlat = 180

    data = nc.Dataset(args.spc_file, "w")
    data.createDimension("nlon", nlon)
    data.createVariable("xlon", "f8", ("nlon"))
    data["xlon"][:] = np.linspace(-180., +180., nlon)
    data.createDimension("nlat", nlat)
    data.createVariable("ylat", "f8", ("nlat"))
    data["ylat"][:] = np.linspace(-90.0, +90.0, nlat)
    data.createVariable("vals", "f4", ("nlat", "nlon"))
    data["vals"][:, :] = 25000.0  # in [m]
    data.close()

    args.sph_size = 6371220.
    args.flt_endo = True  # strip no-outlet rivers??
    args.box_xmin = -180.
    args.box_ymin = -90.0
    args.box_xmax = +180.
    args.box_ymax = +90.0

    filter_hydrosheds(args)

   #build_jigsaw_mesh(args)

def build_jigsaw_mesh(args):

#-- Based on filtered reach output, build an aligned mesh
#-- using jigsaw

    opts = jigsawpy.jigsaw_jig_t()
    mesh = jigsawpy.jigsaw_msh_t()

    opts.geom_file = args.msh_tags + "_geom.msh"
    opts.hfun_file = args.msh_tags + "_spac.msh"
    opts.init_file = args.msh_tags + "_init.msh"
    opts.mesh_file = args.msh_tags + "_mesh.msh"

    opts.jcfg_file = args.msh_tags + "_opts.jig"

    opts.verbosity = +1

    opts.hfun_scal = "absolute"
    opts.hfun_hmax = float("inf")       # null HFUN limits
    opts.hfun_hmin = float(+0.00)

    opts.mesh_dims = +2
    opts.mesh_rad2 = +1.25              # relax edge-ratio
    opts.mesh_eps1 = +1.00
   #opts.mesh_top1 = True

    opts.optm_iter = +64                # tight optim. tol
    opts.optm_qtol = +5.E-05
    opts.optm_cost = "skew-cos"
   #opts.optm_dual = True

    jigsawpy.cmd.jigsaw(opts, mesh)

    jigsawpy.savevtk(args.msh_tags + "_mesh.vtk", mesh)

if (__name__ == "__main__"): ex1()


from abc import ABC, abstractmethod
from typing import Any

import numpy as np

from sportslabkit import Tracklet

class BaseMotionModel(ABC):
    """Abstract base class for motion models.

    This class defines a common interface for all motion models.
    Derived classes should implement the update, and predict methods. MotionModels are procedural and stateless. The state of tracklet is managed by the Tracklet class. The tracklet must have the required observations and states for the motion model to work. If the tracklet doesn't have the required observations or states, the motion model will raise an error and tell the user which observations or states are missing.
    """

    hparam_search_space: dict[str, type] = {}
    required_observation_types: list[str] = NotImplemented
    required_state_types: list[str] = NotImplemented

    def __init__(self, is_multi_target=False):
        """Initialize the MotionModel."""

        self.input_is_batched = False  # initialize the input_is_batched attribute
        self.name = self.__class__.__name__
        self.is_multi_target = is_multi_target

    def __call__(self, tracklet: Tracklet) -> Any:
        """Call the motion model to update its state and return the prediction.

        Args:
            tracklet (Tracklet): The single object tracker instance.

        Returns:
            Any: The predicted state after updating the motion model.
        """
        if self.is_multi_target:
            return self._multi_target_call(tracklet)

        self._check_required_observations(tracklet)
        self._check_required_states(tracklet)

        if isinstance(tracklet, Tracklet):
            _obs = tracklet.get_observations()
            observations = {t: _obs[t] for t in self.required_observation_types}
        else:
            observations = {t: tracklet[t] for t in self.required_observation_types}

        prediction, new_states = self.predict(observations, tracklet.states)
        tracklet.update_states(new_states)
        return prediction

    def _multi_target_call(self, tracklets: list[Tracklet]) -> list[Any]:
        """Call the motion model to update its state and return the prediction for multiple targets.

        Args:
            tracklets (List[Tracklet]): The list of tracklet instances.

        Returns:
            List[Any]: The list of predicted states after updating the motion model for each tracklet.
        """
        all_observations = []
        all_states = []
        for tracklet in tracklets:
            self._check_required_observations(tracklet)
            self._check_required_states(tracklet)

            if isinstance(tracklet, Tracklet):
                _obs = tracklet.get_observations()
                observations = {t: _obs[t] for t in self.required_observation_types}
            else:
                observations = {t: tracklet[t] for t in self.required_observation_types}
            all_observations.append(observations)
            all_states.append(tracklet.states)

        all_predictions, all_new_states = self.predict(all_observations, all_states)
        for i, tracklet in enumerate(tracklets):
            tracklet.update_states(all_new_states[i])
        return all_predictions

    @abstractmethod
    def predict(
        self,
        observations: float | np.ndarray,
        states: float | np.ndarray | None,
    ) -> tuple[float | np.ndarray | None, float | np.ndarray]:
        """Compute the next internal state and prediction based on the current observation and internal state.

        Args:
            observation (Union[float, np.ndarray]): The current observation.
            states (Union[float, np.ndarray, None]): The current internal state of the motion model.

        Returns:
            Tuple[Union[float, np.ndarray, None], Union[float, np.ndarray]]: The next internal state and the prediction.
        """
        pass

    @classmethod
    def from_config(cls: type["BaseMotionModel"], config: dict) -> "BaseMotionModel":
        """Initialize a motion model instance from a configuration dictionary.

        Args:
            config (Dict): The configuration dictionary containing the motion model's parameters.

        Returns:
            MotionModel: A new instance of the motion model initialized with the given configuration.
        """
        return cls(**config)

    def _check_required_observations(self, tracklet: Tracklet) -> None:
        """Check if the required observations are registered in the SingleObjectTracker instance.

        Args:
            sot (SingleObjectTracker): The single object tracker instance.

        Raises:
            KeyError: If a required observation is not registered in the SingleObjectTracker instance.
        """
        for obs_type in self.required_observation_types:
            if obs_type not in tracklet._observations:
                raise KeyError(f"{self.name} requires observation type `{obs_type}` but it is not registered.")
            if len(tracklet._observations[obs_type]) == 0:
                raise KeyError(f"{self.name} requires observation type `{obs_type}` but it is empty.")

    def _check_required_states(self, tracklet: Tracklet) -> None:
        """Check if the required states are registered in the SingleObjectTracker instance.

        Args:
            sot (SingleObjectTracker): The single object tracker instance.

        Raises:
            KeyError: If a required state is not registered in the SingleObjectTracker instance.
        """
        for state in self.required_state_types:
            if state not in tracklet._states:
                tracklet.register_state_type(state)


# https://leetcode.com/problems/flood-fill/
# ---------------------------------------------------
from collections import deque
from typing import List

# Runtime Complexity: O(N), where N is the amount of same-colored neighbors
# Space Complexity: O(N) in worst case, if same-colored neighbors are in positioned as continuous line.
class Solution:
    # __init_ is created, so that there is no warning that self.* vars are defined outside of __init__ initially
    def __init__(self):
        self.rows = self.cols = 0
        self.color = None
        self.image = None

    def floodFill(self, image: List[List[int]], sr: int, sc: int, new_color: int) -> List[List[int]]:
        self.rows, self.cols = len(image), len(image[0])
        self.color = image[sr][sc]
        self.image = image

        if self.color == new_color:
            return image

        stack = deque()
        stack.append((sr, sc))

        while stack:
            (r, c) = stack.pop()

            image[r][c] = new_color
            stack.extend(self.getSameColorNeighbors(r, c))

        return image

    def getSameColorNeighbors(self, r, c):
        neighbors = []
        for dr, dc in ((-1, 0), (0, 1), (1, 0), (0, -1)):
            if self.isValidCell(r + dr, c + dc) and self.image[r + dr][c + dc] == self.color:
                neighbors.append((r + dr, c + dc))

        return neighbors

    def isValidCell(self, r, c):
        return 0 <= r < self.rows and 0 <= c < self.cols

# ---------------------------------------------------
#                    Test Cases
# ---------------------------------------------------
solution = Solution()
#   [2,2,2],
#   [2,2,0],
#   [2,0,1]
print(solution.floodFill([
    [1,1,1],
    [1,1,0],
    [1,0,1]], 1, 1, 2))


import sys
import os
import utils.config
import argparse
import traceback
import importlib
from datetime import datetime
from datetime import timedelta
from random import shuffle
import syslog
from syslog import LOG_ERR
from cerebralcortex.cerebralcortex import CerebralCortex
from cerebralcortex.core.util.spark_helper import get_or_create_sc

cc_config_path = None
gps_key = None

def process_features(feature_list, all_users, all_days, num_cores=1):
    '''
    This method runs the processing pipeline for each of
    the features in the list.
    '''
    for module in feature_list:
        if num_cores > 1:
            #num_cores *= 4

            print('Driver: Spark job',module)
            spark_context = get_or_create_sc(type="sparkContext")
            if 'core.feature.gps.gps' == str(module) \
                or 'sleep_duration_analysis' in str(module) \
                or 'office_time' in str(module) \
                or 'phone_screen_touch_features' in str(module) \
                or 'socialjetlag' in str(module) \
                or 'gps_location_daywise' in str(module):
                '''
                # FIXME # TODO Currently only GPS feature computes features on a
                range of days. Need to find a better way if there are other
                modules that also works on range of days.
                '''
                print('-'*120)
                print('MODULE parallelized on only users',module)
                rdd = spark_context.parallelize(all_users,num_cores)
                results = rdd.map(
                    lambda user: process_feature_on_user(user, module, all_days,
                                                         cc_config_path))
                results.count()
            else:
                print('MODULE',module)
                parallelize_per_day = []
                for usr in all_users:
                    for day in all_days:
                        parallelize_per_day.append((usr,[day]))

                shuffle(parallelize_per_day)
                rdd = spark_context.parallelize(parallelize_per_day,
                                                len(parallelize_per_day))
                results = rdd.map(
                    lambda user_day: process_feature_on_user(user_day[0],
                                                             module, user_day[1],
                                                             cc_config_path))
                results.count()

            spark_context.stop()
        else:
            print('Driver: single threaded')
            for user in all_users:
                process_feature_on_user(user, module, all_days, cc_config_path)

def process_feature_on_user(user, module_name, all_days, cc_config_path):
    try:
        cc = CerebralCortex(cc_config_path)
        module = importlib.import_module(module_name)
        feature_class_name = getattr(module,'feature_class_name')
        feature_class = getattr(module,feature_class_name)
        feature_class_instance = feature_class(cc)

        if gps_key is not None:
            feature_class_instance.gps_api_key = gps_key

        f = feature_class_instance.process
        f(user,all_days)
    except Exception as e:
        err=str(e) + "\n" + str(traceback.format_exc())
        print(err)
        syslog.openlog(ident="CerebralCortex-Driver")
        syslog.syslog(LOG_ERR,err)
        syslog.closelog()

def discover_features(feature_list):
    '''
    This method discovers all the features that are present.
    '''
    feature_dir = os.path.join(utils.config.FEATURES_DIR_NAME)
    found_features = []
    if feature_list:
        feature_subdirs = feature_list
    else:
        feature_subdirs = os.listdir(feature_dir)

    for subdir in feature_subdirs:
        feature = os.path.join(feature_dir,subdir)
        if not os.path.exists(feature):
            syslog.openlog(ident="CerebralCortex-Driver")
            syslog.syslog(LOG_ERR,'Feature not found %s.' % subdir)
            syslog.closelog()
            continue
        if os.path.isdir(feature):
            mod_file_path = os.path.join(feature,subdir) + '.py'

            if not os.path.exists(mod_file_path):
                continue
            sys.path.append(feature)
            try:
                module_name = mod_file_path[:-3] # strip '.py'
                module_name_dotted = module_name.replace('/','.')
                found_features.append(module_name_dotted)
                syslog.openlog(ident="CerebralCortex-Driver")
                syslog.syslog('Added feature %s for importing' % feature)
                syslog.closelog()
            except Exception as exp:
                err = str(exp) + '\n' + str(traceback.format_exc())
                print(err)
                syslog.openlog(ident="CerebralCortex-Driver")
                syslog.syslog(LOG_ERR, str(exp) + err)
                syslog.closelog()

    return found_features

def generate_feature_processing_order(feature_list):
    '''
    This method returns the execution order of processing the features
    after resolving the inter dependencies.
    '''
    return feature_list

def main():
    global cc_config_path
    global metadata_dir
    global gps_key
    # Get the list of the features to process
    parser = argparse.ArgumentParser(description='CerebralCortex '
                                     'Feature Processing Driver')
    parser.add_argument("-f", "--feature-list", help="List of feature names "
                        "seperated by commas", nargs='?', default=None,
                        required=False)
    parser.add_argument("-c", "--cc-config", help="Path to file containing the "
                        "CerebralCortex configuration", required=True)
    parser.add_argument("-s", "--study-name", help="Study name.", required=True)
    parser.add_argument("-u", "--users", help="Comma separated user uuids",
                         nargs='?', default=None, required=False)
    parser.add_argument("-sd", "--start-date", help="Start date in "
                         "YYYYMMDD Format", required=True)
    parser.add_argument("-ed", "--end-date", help="End date in "
                         "YYYYMMDD Format", required=True)
    parser.add_argument("-p", "--num-cores", type=int, help="Set a number "
                        "greater than 1 to enable spark "
                        "parallel execution ", required=False)
    parser.add_argument("-k", "--gps-key", help="GPS API "
                         "key", required=False)

    args = vars(parser.parse_args())
    feature_list = None
    study_name = None
    users = None
    start_date = None
    end_date = None
    date_format = '%Y%m%d'
    num_cores = 1 # default single threaded

    if args['feature_list']:
        feature_list = args['feature_list'].split(',')
    if args['cc_config']:
        cc_config_path = args['cc_config']
    if args['study_name']:
        study_name = args['study_name']
    if args['users']:
        users = args['users'].split(',')
    if args['start_date']:
        start_date = datetime.strptime(args['start_date'], date_format)
    if args['end_date']:
        end_date = datetime.strptime(args['end_date'], date_format)
    if args['num_cores']:
        num_cores = args['num_cores']
    if args['gps_key']:
        gps_key = args['gps_key']

    all_days = []
    while True:
        all_days.append(start_date.strftime(date_format))
        start_date += timedelta(days = 1)
        if start_date > end_date : break

    CC = None
    all_users = None
    try:
        CC = CerebralCortex(cc_config_path)
        if not users:
            users = CC.get_all_users(study_name)
            if not users:
                print('No users found')
                return
            if not len(users):
                print('No users found')
                return # no point continuing
            all_users = [usr['identifier'] for usr in users]
        else:
            all_users = users
    except Exception as e:
        print(str(e))
        print( str(traceback.format_exc()))
    if not all_users:
        print('No users found for the study',study_name)
        return

    found_features = discover_features(feature_list)
    feature_to_process = generate_feature_processing_order(found_features)
    process_features(feature_to_process, all_users, all_days, num_cores)

if __name__ == '__main__':
    main()


from django.contrib import admin
from django.forms import TextInput, Textarea
from django.db import models

# Register your models here.

from .models import Product, Sku, Size, Spec, Material

class SkuInline(admin.StackedInline):
    model = Sku
    extra = 1

class SizeAdmin(admin.ModelAdmin):
    list_display = ['variant', 'sage_size', 'bed_size', 'imperial_width', 'imperial_length',
                    'metric_width', 'metric_length',
                    'amazon', 'description_long', ]

class ProductAdmin(admin.ModelAdmin):
    list_display = ['descriptive_content', 'id', 'product_type', 'product_code_root', 'bullet_point_1', 'bullet_point_2', 'bullet_point_3']
    inlines = [SkuInline]
    formfield_overrides = {
        models.CharField: {'widget': TextInput(attrs={'size':'100'})},
        models.TextField: {'widget': Textarea(attrs={'rows':4, 'cols':40})},
    }

    fieldsets = (
        (None,
            {'fields': (
                'descriptive_content',
                'product_type',
                'product_code_root',
                'material',
                'ebay_material',
                'ebay_features',
                'ebay_type',
                'bullet_point_1',
                'bullet_point_2',
                'bullet_point_3',
                'bullet_point_4',
                'bullet_point_5',)
            }
         ),
        ('Advanced options', {
            'classes': ('wide', 'collapse',),
            'fields': ( 'manufacturer_description',
              'factory_description',
              'sage_description',
              'sage_category',
              'in_the_box',
              'keywords')

        }),
    )

class SpecAdmin(admin.ModelAdmin):
    list_display = ['quilting_pattern', 'fixing']

class MaterialAdmin(admin.ModelAdmin):
    list_display = ['description', 'material', 'thread_count', 'weight']

admin.site.register(Product, ProductAdmin)
admin.site.register(Size, SizeAdmin)
admin.site.register(Spec, SpecAdmin)
admin.site.register(Material, MaterialAdmin)


import urllib2
import lxml.html as LH
from lxml import etree
import pandas as pd

def get_sp500_metadata(*args, **kwargs):
    wiki = urllib2.urlopen("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
    html = wiki.read()
    wiki_content = LH.fromstring(html)

    table = wiki_content.find_class("wikitable")[0]
    table_html = etree.tostring(table)
    sp500_wiki = pd.read_html(table_html, header=0)[0]
    sp500_wiki.index = range(1, sp500_wiki.shape[0]+1)

    sp500_wiki.drop(columns=["SEC filings", "GICS Sub Industry", "Address of Headquarters", "Date first added[3][4]", "CIK"], inplace=True)

    sp500_wiki_columns = dict(zip(
        sp500_wiki.columns,
        [
            "Symbol",
            "Company",
            "Sector"
        ]
    ))
    sp500_wiki.columns = [sp500_wiki_columns[colname] for colname in sp500_wiki.columns]
    return sp500_wiki


# -*- coding: utf-8 -*-
import scrapy
from  sport.items import footballItem

class FootballSpider(scrapy.Spider):
    name = 'football'
    allowed_domains = ['soccer.hupu.com']
    start_urls = ['http://soccer.hupu.com/']

    def parse(self, response):
        #获取联赛
        league_title = response.xpath('//li[@class="hp-dropDownMenu"][position()<=4]/a/@title').extract()
        league_url = response.xpath('//li[@class="hp-dropDownMenu"][position()<=4]/a/@href').extract()
        for title,url in zip(league_title,league_url):
            yield scrapy.Request(url=url, meta={'league': title},
                                    callback=self.team_detail,dont_filter=True)
    def team_detail(self,response):
        #获取球队
        league = response.meta.get('league')
        for team in response.xpath('//ul[@class="england-list-item"]/li'):
            team_url = team.xpath("a/@href").extract()
            team_name = team.xpath("a/@title").extract()
            team_url = team_url[0] if team_url else ''
            team_name = team_name[0] if team_name else ''
            #print(team_name,team_url)
            if not team_url:
                continue
            yield scrapy.Request(url=team_url, meta={'league': league,'team_name':team_name},
                                    callback=self.player_detail,dont_filter=True)

    def player_detail(self,response):
        #获取球员
        league = response.meta.get('league')
        team_name = response.meta.get('team_name')
        #team_name = response.xpath("//div[@class='team_info left']/h3/span/text()").extract_first("")
        team_logo = response.xpath("//li[@class=' left pic_logo']/img/@src").extract_first("")
        team_coach =response.xpath("//div[@class='team_info left']//dl[@class='clearfix']/dd[1]/text()").extract_first("")
        team_build = response.xpath("//div[@class='team_info left']//dl[@class='clearfix']/dd[2]/text()").extract_first("")
        team_city = response.xpath("//div[@class='team_info left']//dl[@class='clearfix']/dd[3]/text()").extract_first("")
        team_place = response.xpath("//div[@class='team_info left']/ul[2]/li[1]/text()").extract_first("")
        team_addr = response.xpath("//div[@class='team_info left']/ul[2]/li[2]/text()").extract_first("")
        team_href = response.xpath("//div[@class='team_info left']/ul[2]/li/a/@href").extract_first("")
        for playinfo in response.xpath('//table[@class="team_player"]'):
            #print(playinfo.xpath('//tr[not(@calss)]'))

            for player in playinfo.xpath('tr'):
                #print(player)
                player_number = player.xpath("td[1]/text()").extract()
                player_cha_name = player.xpath("td[2]/a/text()").extract()
                player_eng_name = player.xpath("td[3]/a/text()").extract()
                player_age = player.xpath("td[4]/text()").extract()
                player_courtry = player.xpath("td[5]/text()").extract()
                player_job = player.xpath("td[6]/text()").extract()
                #player_job = player.xpath("td[6]/text()").extract_first("")

                player_number = player_number[0] if player_number else ''
                player_cha_name = player_cha_name[0] if player_cha_name else ''
                player_eng_name = player_eng_name[0] if player_eng_name else ''
                player_age = player_age[0] if player_age else ''
                player_courtry = player_courtry[0] if player_courtry else ''
                player_job = player_job[0] if player_job else ''
                #print(player_number,player_cha_name,player_eng_name,player_age,player_courtry,player_job)
                if not player_number.isdigit():
                    continue
                item = footballItem()
                item['league'] = league #联赛名称
                item['team_name'] = team_name     #球队名称
                item['player_number'] = player_number   #球员号码
                item['player_cha_name'] = player_cha_name  #球员中文名
                item['player_eng_name'] = player_eng_name #球员英文名
                item['player_age'] = player_age       #球员年龄
                item['player_courtry'] = player_courtry  #球员所属国籍
                item['player_job'] = player_job #球员位置
                item['team_logo'] = team_logo   #球队logo
                item['team_coach'] = team_coach #球队教练
                item['team_build'] = team_build #成立时间
                item['team_city'] = team_city   #所在城市
                item['team_place'] = team_place #主球场
                item['team_addr'] = team_addr   #联系地址
                item['team_href'] = team_href   #官网
                yield item

if __name__ == '__main__':
    from lxml import etree

    path="E:/youbest/project/spider/sport/football.html"
    fd = open(path,'rb')
    html = fd.read()
    response = etree.HTML(html)
    #print(response.text)
    d=response.xpath('//table[@class="team_player"]')
    #print(d)


import os
import time
import json
import sqlite3

from threading import Thread, Lock
from plugin import *

class stalker(plugin):
    def __init__(self, bot):
        super().__init__(bot)
        self.db_name = self.bot.get_server_name()
        os.makedirs(os.path.dirname(os.path.realpath(self.config['db_location'])), exist_ok=True)

        self.db_connection = sqlite3.connect(self.config['db_location'], check_same_thread=False)
        self.db_cursor = self.db_connection.cursor()
        self.db_cursor.execute(f"CREATE TABLE IF NOT EXISTS '{self.db_name}' (host TEXT primary key not null, nicks TEXT)")  # host -> {nicknames}
        self.db_mutex = Lock()
        self.updating_thread = None

    def on_welcome(self, **kwargs):
        self.bot.names()

    def on_pubmsg(self, source, **kwargs):
        self.update_database(source.nick, source.host)

    def on_ctcp(self, source, **kwargs):
        self.update_database(source.nick, source.host)

    def on_whoisuser(self, nick, host, **kwargs):
        self.update_database(nick, host)

    def on_join(self, source, **kwargs):
        self.update_database(source.nick, source.host)

    def on_nick(self, source, new_nickname, **kwargs):
        self.update_database(new_nickname, source.host)

    def on_namreply(self, nicknames, **kwargs):
        if self.updating_thread is None or not self.updating_thread.is_alive():
            self.updating_thread = Thread(target=self.update_all, args=(nicknames,))
            self.updating_thread.start()

    def update_all(self, nicknames):
        self.logger.debug("updating whole stalker's database started...")
        for nick in nicknames:
            self.bot.whois(nick)
            time.sleep(1)  # to not get kicked because of Excess Flood

        self.logger.debug("updating whole stalker's database finished")

    def update_database(self, nick, host):
        result = self.get_nicknames_from_database(host)
        nick = irc_nickname(nick)

        if result:
            if nick not in result:
                result.extend([nick])
                with self.db_mutex:
                    self.db_cursor.execute(f"UPDATE '{self.db_name}' SET nicks = ? WHERE host = ?", (json.dumps(result), host))
                    self.db_connection.commit()

                self.logger.info(f'new database entry: {host} -> {nick}')
        else:
            with self.db_mutex:
                self.db_cursor.execute(f"INSERT INTO '{self.db_name}' VALUES (?, ?)", (host, json.dumps([nick])))
                self.db_connection.commit()

            self.logger.info(f'new database entry: {host} -> {nick}')

    def get_nicknames_from_database(self, host):
        with self.db_mutex:
            self.db_cursor.execute(f"SELECT nicks FROM '{self.db_name}' WHERE host = ?", (host,))
            result = self.db_cursor.fetchone()

        if result:
            result = json.loads(result[0])
            result = [irc_nickname(x) for x in result]

        return result

    def get_all_nicknames_from_database(self):
        with self.db_mutex:
            self.db_cursor.execute(f"SELECT nicks FROM '{self.db_name}'")
            result = self.db_cursor.fetchall()

        if result:
            result = [json.loads(x[0]) for x in result if x[0]]
            result = [[irc_nickname(y) for y in x] for x in result]

        return result

    def get_all_hosts_from_database(self):
        with self.db_mutex:
            self.db_cursor.execute(f"SELECT host FROM '{self.db_name}'")
            result = self.db_cursor.fetchall()

        if result:
            result = [x[0] for x in result]

        return result

    @command
    @doc('stalk_nick <nickname>: get all known hosts of <nickname> user')
    def stalk_nick(self, sender_nick, args, **kwargs):
        if not args: return
        nick = irc_nickname(args[0])
        all_hosts = self.get_all_hosts_from_database()
        result = set([host for host in all_hosts if nick in self.get_nicknames_from_database(host)])

        if result:
            response = f'known hosts of {nick}: {", ".join(result)}'
            if self.bot.is_msg_too_long(response):
                self.bot.say(f'{sender_nick}: too much data, check your privmsg')
                self.bot.say(response, sender_nick)
            else:
                self.bot.say(response)
        else:
            self.bot.say_err(nick)

        self.logger.info(f'{sender_nick} asks about hosts of {nick}: {result}')

    @command
    @doc("stalk <nickname>: get other <nickname> user's nicknames")
    def stalk(self, sender_nick, args, **kwargs):
        if not args: return
        nick = irc_nickname(args[0])
        result = set()
        all_nicknames = self.get_all_nicknames_from_database()
        for x in all_nicknames:
            if nick in x: result.update(x)

        self.logger.info(f'{sender_nick} stalks {nick}: {result}')

        if result == {nick}:
            self.bot.say(f"I don't know other {nick} nicks")
        else:
            if nick in result: result.remove(nick)

            if result:
                self.bot.say(f'other nicks of {nick}: {", ".join(result)}')
            else:
                self.bot.say_err(nick)

    @command
    @doc('stalk_host <host>: get all nicknames from <host>')
    def stalk_host(self, sender_nick, args, **kwargs):
        if not args: return
        host = args[0]
        nicks = self.get_nicknames_from_database(host)
        if nicks:
            response = f'known nicks from {host}: {", ".join(nicks)}'
            if self.bot.is_msg_too_long(response):
                self.bot.say(f'{sender_nick}: too much data, check your privmsg')
                self.bot.say(response, sender_nick)
            else:
                self.bot.say(response)
        else:
            self.bot.say_err(host)

        self.logger.info(f'{sender_nick} asks about nicks from {host}: {nicks}')


import pytest
import random
import skiplist
import multidict
import time

def setup_module(module):
    pass

def made_n_list(n: int):
    skip_list = skiplist.SkipList()
    for i in range(n):
        skip_list.update(True, i)
    return skip_list

def made_rand_n_list(n: int):
    skip_list = skiplist.SkipList()
    multiset = multidict.MultiDict()
    for _ in range(n):
        rand_num = random.randint(0, n * 2)
        multiset.add(str(rand_num), "")
        skip_list.update(True, rand_num)
    return [skip_list, multiset]

def test_skiplist_update_insert():
    skip_list = made_n_list(1000)
    for _ in range(100):
        rand_num = random.randint(0, 2000)
        assert skip_list.contains(rand_num) == (rand_num < 1000)

def test_skiplist_update_insert_random():
    skip_list, multiset = made_rand_n_list(1000)
    assert skip_list.__len__() == multiset.__len__()
    for _ in range(300):
        rand_num = random.randint(0, 1500)
        assert skip_list.contains(rand_num) == multiset.__contains__(str(rand_num))

def test_skiplist_update_delete():
    skip_list = made_n_list(1000)
    for i in range(500, 1000):
        skip_list.update(False, i)
    for _ in range(100):
        rand_num = random.randint(0, 1000)
        assert skip_list.contains(rand_num) == (rand_num < 500)

def test_skiplist_update_delete_random():
    skip_list, multiset = made_rand_n_list(1000)
    for i in range(500):
        rand_num = random.randint(0, 1500)
        skip_list.update(False, rand_num)
        if multiset.__contains__(str(rand_num)):
            multiset.pop(str(rand_num))
    for i in range(300):
        rand_num = random.randint(0, 1500)
        assert skip_list.contains(rand_num) == multiset.__contains__(str(rand_num))

def test_verify():
    skip_list = made_n_list(1500)
    validation = [random.randint(0, 3000) for _ in range(100)]
    for i in validation:
        response = skip_list.verify(i)
        assert response.validates_against()
        assert response.subject_contained() == (i < 1500)

def test_verify_random():
    skip_list, multiset = made_rand_n_list(1500)
    validation = [random.randint(0, 3000) for _ in range(150)]
    for i in validation:
        response = skip_list.verify(i)
        assert response.validates_against()
        assert response.subject_contained() == multiset.__contains__(str(i))

def common_test(n, m):
    skip_list, multiset = made_rand_n_list(n)
    argss = [(random.randint(0, 2), random.randint(0, 10000)) for _ in range(m)]
    lops = [lambda x: skip_list.verify(x).subject_contained(),
            lambda x: skip_list.update(True, x),
            lambda x: skip_list.update(False, x)]
    sops = [lambda x: str(x) in multiset, lambda x: multiset.add(str(x), x),
            lambda x: None if str(x) not in multiset else multiset.pop(str(x))]
    for op, c in argss:
        lres = lops[op](c)
        sres = sops[op](c)
        if op == 0:
            assert lres == sres
        assert skip_list.__len__() == multiset.__len__()

def test_light():
    common_test(2000, 4000)

def test_medium():
    common_test(4000, 10000)

def test_heavy():
    common_test(6000, 40000)


#!/usr/bin/env python

__libname__ = 'ldtsmash'
__description__ = 'LEGO Desktop Toy Smash Archive Tool'
__version__ = '1.0.1'
__copyright__ = 'Copyright (c) 2019 JrMasterModelBuilder'
__license__ = 'Licensed under the Mozilla Public License, v. 2.0'

import os
import sys
import errno
import struct
import json
import argparse

def mkdirp(path):
	try:
		os.makedirs(path)
	except OSError as ex:
		if ex.errno == errno.EEXIST and os.path.isdir(path):
			pass
		else:
			raise ex

def openp(path, mode):
	base = os.path.dirname(path)
	if base:
		mkdirp(base)
	return open(path, mode)

def read_write_buffered(reader, writter, total, chunk):
	if chunk is None:
		writter.write(reader.read())
		return
	read = 0
	while read < total:
		size = total - read
		if size > chunk:
			size = chunk
		writter.write(reader.read(size))
		read += size

def class_str(instance):
	return class_repr(instance)

def class_repr(instance):
	return '<%s: %s>' % (instance.__class__, instance.__dict__)

class Error(Exception):
	pass

class TypeError(Error):
	pass

class FileReadError(Error):
	pass

class FileWriteError(Error):
	pass

class File():
	def __str__(self):
		return class_str(self)

	def __repr__(self):
		return class_repr(self)

	def __init__(self, fio, offset=None, size=None):
		# If offset no set, use current input offset.
		if offset is None:
			offset = fio.tell()

		# If a size is not set, base it on file seek end.
		if size is None:
			before = fio.tell()
			fio.seek(0, os.SEEK_END)
			size = fio.tell() - offset
			fio.seek(before, os.SEEK_SET)

		self.fio = fio
		self.offset = offset
		self.size = size
		self.__pos = 0

	def tell(self):
		return self.__pos

	def seek(self, offset, from_what=os.SEEK_SET):
		pos = None
		if from_what == os.SEEK_CUR:
			pos = self.__pos + offset
		elif from_what == os.SEEK_END:
			pos = self.__pos - offset
		elif from_what == os.SEEK_SET:
			pos = offset
		else:
			raise TypeError('Invalid seek from type: %s' % (from_what))

		if pos < 0 or pos > self.size:
			raise FileReadError('Cannot seek to: %s' % (pos))

		self.__pos = pos

	def can_read(self, size):
		return self.__pos + size <= self.size

	def read(self, size=None):
		pos = self.__pos
		fio = self.fio
		before = fio.tell()

		if size is None:
			size = self.size - pos

		if pos + size > self.size:
			raise FileReadError('Cannot read past end of file')

		fio.seek(self.offset + pos, os.SEEK_SET)
		try:
			ret = fio.read(size)
		finally:
			fio.seek(before, os.SEEK_SET)
		read_size = len(ret)

		if read_size != size:
			raise FileReadError('Read an unexpected size %s' % (read_size))

		self.__pos = pos + read_size
		return ret

	def write(self, data):
		pos = self.__pos
		fio = self.fio
		before = fio.tell()
		seekto = self.offset + pos

		if seekto > self.size:
			raise BLKFileWriteError('Cannot seek past end of file')

		fio.seek(seekto, os.SEEK_SET)
		try:
			fio.write(data)
		finally:
			fio.seek(before, os.SEEK_SET)

		self.__pos = pos + len(data)
		if self.__pos > self.size:
			self.size = self.__pos

	def can_read_struct(self, structure):
		return self.can_read(structure.size)

	def read_struct(self, structure):
		return structure.unpack_from(self.read(structure.size))

	def write_struct(self, structure, *args):
		self.write(structure.pack(*args))

	def sub(self, offset=None, size=None):
		if offset is None:
			offset = self.tell()
		if size is None:
			size = self.size - offset
		return self.__class__(self.fio, self.offset + offset, size)

class Entry():
	def __str__(self):
		return class_str(self)

	def __repr__(self):
		return class_repr(self)

	structure = struct.Struct(''.join([
		'<',   # little endian
		'16s', # name
		'I',   # offset
		'I'    # size
	]))
	size = structure.size
	size_name = 16
	size_offset = 4
	size_size = 4

	def __init__(self, name=None, offset=None, size=None):
		self.name = name
		self.offset = offset
		self.size = size

	def read_from(self, io):
		(name_enc, offset, size) = io.read_struct(self.structure)
		name = name_enc.split(b'\x00')[0].decode('ascii')
		self.name = name
		self.offset = offset
		self.size = size

	def write_to(self, io, name_buffer=None):
		name = self.name
		offset = self.offset
		size = self.size

		name_enc = name.encode('ascii')
		name_enc_len = len(name_enc)
		size_name = self.size_name
		if not (name_enc_len < size_name):
			raise FileWriteError(
				'File name longer than %s characters: %s' % (size_name - 1, name)
			)

		# Copy name bytes into name buffer, if provided.
		name_write = None
		if name_buffer:
			for i in range(name_enc_len):
				name_buffer[i] = name_enc[i]
			name_buffer[name_enc_len] = 0
			name_write = bytes(name_buffer)
		else:
			name_write = name_enc

		io.write_struct(self.structure, name_write, offset, size)

	def format_string(self):
		return '0x%08X 0x%08X %s' % (self.offset, self.size, self.name)

class Reader():
	Entry = Entry

	def __str__(self):
		return class_str(self)

	def __repr__(self):
		return class_repr(self)

	def __init__(self, file_idx, file_wad):
		self.file_idx = file_idx
		self.file_wad = file_wad

	def count(self):
		return self.file_idx.size // self.Entry.size

	def read(self):
		file_idx = self.file_idx
		file_wad = self.file_wad
		if file_idx.can_read(self.Entry.size):
			entry = self.Entry()
			entry.read_from(file_idx)
			io = None
			if file_wad:
				io = file_wad.sub(entry.offset, entry.size)
			return [entry, io]
		return None

	def reader(self):
		while True:
			read = self.read()
			if not read:
				return
			yield read

class Writter():
	Entry = Entry

	def __str__(self):
		return class_str(self)

	def __repr__(self):
		return class_repr(self)

	def __init__(self, file_idx, file_wad):
		self.file_idx = file_idx
		self.file_wad = file_wad

		# Name buffer, reusable to mirror the original encoder.
		self.name_buffer = bytearray(self.Entry.size_name)

	def write(self, entry, io, buffer_size=None, use_buffer=True):
		file_idx = self.file_idx
		file_wad = self.file_wad
		entry.offset = file_wad.tell()
		entry.size = io.size
		buf = None
		if use_buffer:
			buf = self.name_buffer
		entry.write_to(file_idx, buf)
		read_write_buffered(io, file_wad, io.size, buffer_size)

class FilelistReader():
	def __init__(self, fio):
		self.fio = fio

	def read(self):
		fio = self.fio
		while True:
			line = fio.readline()
			if not line:
				break
			s = line.strip()
			if s.startswith('"') and s.endswith('"'):
				return json.loads(s)
		return None

	def reader(self):
		while True:
			d = self.read()
			if d is None:
				break
			yield d

class FilelistWritter():
	def __init__(self, fio):
		self.fio = fio

	def write(self, filename):
		fio = self.fio
		fio.write(json.dumps(filename))
		fio.write('\n')

	def comment(self, lines):
		fio = self.fio
		for line in lines.split('\n'):
			fio.write('# ')
			fio.write(line)
			fio.write('\n')

class Process():
	filelist_name = '_ldtsmashlist.txt'
	filelist_comment = 'ldtsmash ordered list of file (JSON encoded strings)'
	buffer_size = 0x4000

	File = File
	Entry = Entry
	Reader = Reader
	Writter = Writter
	FilelistReader = FilelistReader
	FilelistWritter = FilelistWritter

	def __init__(self, options):
		self.options = options

	def realp(self, path):
		return path.replace('\\', '/')

	def filelist_open_r(self, directory):
		return open(os.path.join(directory, self.filelist_name), 'r')

	def filelist_open_w(self, directory):
		return openp(os.path.join(directory, self.filelist_name), 'w')

	def filelist_reader_next(self, fio):
		while True:
			line = fio.readline()
			if not line:
				break
			s = line.strip()
			if s.startswith('"') and s.endswith('"'):
				return json.loads(s)
		return None

	def filelist_reader_itter(self, fio):
		while True:
			d = self.filelist_reader_next(fio)
			if d is None:
				break
			yield d

	def entry_to_file(self, directory, entry, io):
		filepath = os.path.join(
			directory,
			self.realp(entry.name)
		)
		with openp(filepath, 'wb') as fio:
			read_write_buffered(io, fio, io.size, self.buffer_size)
			fio.close()

	def entry_from_file(self, directory, filename, writter):
		entry = self.Entry(filename)
		filepath = os.path.join(
			directory,
			self.realp(entry.name)
		)
		with open(filepath, 'rb') as fio:
			io = self.File(fio)
			writter.write(entry, io, self.buffer_size)
			fio.close()
		return entry

	def run_l(self):
		ndx = self.options.ndx
		with open(ndx, 'rb') as fio_nxd:
			reader = self.Reader(self.File(fio_nxd), None)

			for [entry, _] in reader.reader():
				print(entry.format_string())

			fio_nxd.close()
		return 0

	def run_e(self):
		ndx = self.options.ndx
		wad = self.options.wad
		directory = self.options.dir
		mkdirp(directory)

		with self.filelist_open_w(directory) as fl:
			fl_writter = self.FilelistWritter(fl)
			fl_writter.comment(self.filelist_comment)

			with open(ndx, 'rb') as fio_nxd:
				with open(wad, 'rb') as fio_wad:
					reader = self.Reader(self.File(fio_nxd), self.File(fio_wad))

					for [entry, io] in reader.reader():
						print(entry.format_string())
						fl_writter.write(entry.name)
						self.entry_to_file(directory, entry, io)
					fio_wad.close()
				fio_nxd.close()
			fl.close()
		return 0

	def run_c(self):
		ndx = self.options.ndx
		wad = self.options.wad
		directory = self.options.dir

		with self.filelist_open_r(directory) as fl:
			fl_reader = self.FilelistReader(fl)

			with openp(ndx, 'wb') as fio_nxd:
				with openp(wad, 'wb') as fio_wad:
					writter = self.Writter(self.File(fio_nxd), self.File(fio_wad))

					for fn in fl_reader.reader():
						entry = self.entry_from_file(directory, fn, writter)
						print(entry.format_string())
					fio_wad.close()
				fio_nxd.close()
			fl.close()
		return 0

	def run(self):
		c = self.options.command
		return getattr(self, 'run_%s' % (c))()

def main():
	parser = argparse.ArgumentParser(
		description=os.linesep.join([
			'%s - %s' % (__libname__, __description__),
			'Version: %s' % (__version__)
		]),
		epilog=os.linesep.join([
			__copyright__,
			__license__
		]),
		formatter_class=argparse.RawTextHelpFormatter
	)
	parser.add_argument(
		'-v', '--version',
		action='version',
		version=__version__
	)
	subparsers = parser.add_subparsers(help='command', dest='command')

	parser_l = subparsers.add_parser('l', help='list')
	parser_l.add_argument('ndx', help='in nxd file')

	parser_e = subparsers.add_parser('e', help='extract')
	parser_e.add_argument('ndx', help='in nxd file')
	parser_e.add_argument('wad', help='in wad file')
	parser_e.add_argument('dir', help='out directory')

	parser_c = subparsers.add_parser('c', help='compile')
	parser_c.add_argument('dir', help='in directory')
	parser_c.add_argument('ndx', help='out nxd file')
	parser_c.add_argument('wad', help='out wad file')

	return Process(parser.parse_args()).run()

if __name__ == '__main__':
	sys.exit(main())


from turtle import forward
from typing import Optional, Union

from opendelta.utils.signature import get_arg_names, get_arg_names_inside_func
from opendelta.utils.name_based_addressing import *
from opendelta.basemodel import DeltaBase
from transformers.models.t5 import T5ForConditionalGeneration
import torch
import torch.nn as nn
from opendelta import BaseDeltaConfig
import math

class ReparameterizeFunction(nn.Module):
    """
    """
    def __init__(self, in_dim, mid_dim=512):
        super().__init__()
        self.in_dim = in_dim
        self.mid_dim = mid_dim

    def instantiate(self, module_list=[]):
        self.module_list = module_list
        self.record_parameters()
        self.define_reparameterization_network()

    def goto(self, module, path):
        n = module
        for p in path:
            n = getattr(n, p)
        return n

    def record_parameters(self):
        r""" Enumerate the parameters that need to be reparameterized.
        Then, delete the original parameters.
        """
        tot1 = 0
        tot2 = 0
        ith = 0

        self.path_list = []
        for module_idx, module in enumerate(self.module_list):
            self.path_list.append([])
            for n, parameters in module.named_parameters():
                ith ^= 1
                numel = parameters.numel()
                shape = parameters.size()

                names = n.split('.')
                path = names[:-1]
                name = names[-1]
                if(name=="bias" or name=="weight" or name=="lora_A"):
                    tot1 +=numel
                elif(name=="lora_B"):
                    tot2 +=numel
                self.path_list[-1].append((path, name, numel, shape))
            for path, name, _, _ in self.path_list[-1]:
                delattr(self.goto(module, path), name)
        self.total_parameters_num1 = tot1
        self.total_parameters_num2 = tot2

    def define_reparameterization_network(self) -> None:
        r""" Build the reparameterize module
        """
        in_embed1 = torch.empty(self.in_dim)
        nn.init.uniform_(in_embed1)
        self.in_embed1 = nn.Parameter(in_embed1)
        self.transform1 = nn.Sequential(
            nn.Linear(self.in_dim, self.mid_dim, bias=False),
            nn.Tanh(),
            nn.Linear(self.mid_dim, self.total_parameters_num1, bias=False)
        )

        in_embed2 = torch.empty(self.in_dim)
        nn.init.zeros_(in_embed2)
        self.in_embed2 = nn.Parameter(in_embed2)
        self.transform2 = nn.Sequential(
            nn.Linear(self.in_dim, self.mid_dim, bias=False),
            nn.Tanh(),
            nn.Linear(self.mid_dim, self.total_parameters_num2, bias=False)
        )

    def allocate_parameter(self):
        r""" At the beginning of each forward pass through the whole network(PLM),
        cacalulate the reparameterized params for later use in each layer.
        """
        params1 = self.transform1(self.in_embed1)
        params2 = self.transform2(self.in_embed2)

        cur1 = 0
        cur2 = 0
        ith = 0

        for module_idx, module in enumerate(self.module_list):
            for path, name, numel, shape in self.path_list[module_idx]:
                ith ^= 1
                if(name=="bias" or name=="weight" or name=="lora_A"):
                    setattr(self.goto(module, path), name, params1[cur1: cur1 + numel].view(shape))
                    cur1 +=numel
                elif(name=="lora_B"):
                    setattr(self.goto(module, path), name, params2[cur2: cur2 + numel].view(shape))
                    cur2 += numel

    def pre_forward(self, *args, **kwargs):
        r""" Firstly forward through the reparameterized network, and then go through normal forward pass of the PLM.
        """
        self.allocate_parameter()
        return args, kwargs

class LowRankLinear(nn.Module):
    #  ------------------------------------------------------------------------------------------
    #  Copyright (c) Microsoft Corporation. All rights reserved.
    #  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
    #  ------------------------------------------------------------------------------------------
    #  copy from loralib and do some refactor
    def __init__(self,
        in_features,
        out_features,
        weight,
        r=8,
        lora_alpha=16,
        lora_dropout=0.0,
    ):
        super().__init__()
        self.r = r
        self.lora_alpha = lora_alpha
        self.lora_dropout = lora_dropout
        # self.lin = nn.Linear(in_features, out_features) #
        if lora_dropout > 0.:
            self.lora_dropout = nn.Dropout(p=lora_dropout)
        else:
            self.lora_dropout = lambda x: x
        if r > 0:
            self.lora_A = nn.Parameter(weight.new_zeros((r, in_features)))
            self.lora_B = nn.Parameter(weight.new_zeros((out_features, r)))
            self.scaling = self.lora_alpha / self.r
            # self.lin.reset_parameters() #
            nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
            nn.init.zeros_(self.lora_B)

    def forward(self, x):
        return (self.lora_dropout(x) @ self.lora_A.T @ self.lora_B.T) * self.scaling

class LoraConfig(BaseDeltaConfig):
    r"""
    This is the configuration class to store the configuration of a :py:class:`~LoraModel`

    """
    def __init__(
        self,
        lora_r=8,
        lora_alpha=16,
        lora_dropout=0.0,
        in_dim =4,
        mid_dim=4,
        reparameterize=False,
        **kwargs
    ):
        super().__init__(**kwargs)
        arg_names = get_arg_names_inside_func(self.__init__)
        for arg_name in arg_names:
            if not hasattr(self, arg_name): # the arg has not been registered in parent config
                setattr(self, arg_name, locals()[arg_name])

class LoraModel(DeltaBase):

    config_class = LoraConfig
    delta_type = "lora"
    default_modified_modules = ['attn.q', 'attn.v']
    def __init__(self,
                 backbone_model: nn.Module,
                 lora_r=8,
                 lora_alpha=16,
                 lora_dropout=0.0,
                 in_dim=4,
                 mid_dim=4,
                 modified_modules: Optional[bool] = None,
                 unfrozen_modules: Optional[bool] = None,
                 common_structure: Optional[bool] = None,
                 interactive_modify: Optional[Union[bool, int]] = False,
                 reparameterize: Optional[bool] = False,
                 ):
        DeltaBase.__init__(self,
                           backbone_model,
                           modified_modules=modified_modules,
                           unfrozen_modules=unfrozen_modules,
                           common_structure=common_structure,
                           interactive_modify=interactive_modify,
                           )
        arg_names = get_arg_names_inside_func(self.__init__)
        for arg_name in arg_names:
            if not hasattr(self, arg_name): # not registered in parent class
                setattr(self, arg_name, locals()[arg_name])
        self.delta_modules = nn.ModuleList()

        if reparameterize:
            reparams = ReparameterizeFunction(
                in_dim = self.in_dim,
                mid_dim = self.mid_dim,
            )
        else:
            reparams = None
        self.delta_modules.append(self.backbone_model.classifier)
        self.add_all_delta_to_backbone(self.backbone_model,
                                   self.modified_modules,
                                   reparams = reparams,
                                   )

    def update_module(self, module: nn.Module, key: str):
        parent_ref, child_name, child_ref = self.find_module(module, key)
        parallel_module = self.new_module_like(child_module=child_ref)
        self.insert_parallel_module(child_ref, delta_module=parallel_module, delta_name="lora")

    def _pseudo_data_to_instantiate(self, module):
        # no need to pass pseudo input, so overwrite it
        pass

    def new_module_like(self, child_module):
        if isinstance(child_module, nn.Linear):
            in_features, out_features = child_module.in_features, child_module.out_features
            new_module = LowRankLinear(in_features = in_features,
                                     out_features = out_features,
                                     weight = child_module.weight,
                                     r = self.lora_r,
                                     lora_alpha = self.lora_alpha,
                                     lora_dropout = self.lora_dropout)
            self.delta_modules.append(new_module)
        else:
            raise NotImplementedError
        return new_module


from django.shortcuts import render
from django.http import HttpRequest

from . import graph_utils

from pandas_datareader import yahoo as yahoo
from yahoo_fin import stock_info as yf

import datetime
import requests as rq
# from io import BytesIO
# import base64

sesh = None
USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0'

def graph_view(request):

    global sesh

    if not sesh:
        sesh = rq.Session()
        sesh.headers.update({
            'User-Agent': USER_AGENT
        })

    product = request.GET.get('product', 'AMZN')
    strike = float(request.GET.get('strike', 130))
    expiry_str = request.GET.get('expiry', '2023-09-08')
    long = int(request.GET.get('long', 100))
    cover = int(request.GET.get('cover', 100))
    U = request.GET.get('spot', None)

    expiry = datetime.datetime.strptime(expiry_str, '%Y-%m-%d')

    context = {
        # 'graph_div': graph_div,
        'product': product,
        'strike': strike,
        'expiry': expiry_str,
        'long': long,
        'cover': cover,
    }

    try:
        U = U or round(yf.get_live_price(product), 2)
        U = float(U)
        context['spot'] = U

        tk = yahoo.options.Options(product, session=sesh)
        df = tk.get_call_data(expiry=expiry)
        df = df.reset_index()
        call = df[df['Strike'] == strike].iloc[0]

        K = strike
        P = call['Bid']

        # graph_div = get_payoff_graph(U, K, P, long, cover)

        op_list = [
            {'type': 'u',              'side': 'b', 'price': U, 'size': long},
            {'type': 'c', 'strike': K, 'side': 's', 'price': P, 'size': cover},
        ]

        x, (y_long, y_call), (label_long, label_call), combined = graph_utils.get_plots(U, op_list, spot_range=50)

        capital_required = U*long
        capital_withheld = capital_required - P*cover

        context.update({
            'x': x.tolist(),
            'y_long': y_long.tolist(),
            'y_call': y_call.tolist(),
            'label_long': label_long,
            'label_call': label_call,
            'y_combined': combined.tolist(),
            'capital_required': capital_required,
            'capital_withheld': capital_withheld,
        })

    except Exception as e:
        print(e)
        if 'spot' not in context:
            context['spot'] = 0
        context.update({
            'x': [],
            'y_long': [],
            'y_call': [],
            'label_long': 'null',
            'label_call': 'null',
            'y_combined': [],
        })

    return render(request, 'marketdata/covered_call.html', context=context)

# def get_payoff_graph(U, K, P, long_q, cover_q):

#     op_list = [
#         {'type': 'u',              'side': 'b', 'price': U, 'size': long_q},
#         {'type': 'c', 'strike': K, 'side': 's', 'price': P, 'size': cover_q},
#     ]

#     fig = graph_utils.multi_plotter(spot=U,spot_range=25, op_list=op_list)
#     tmp = BytesIO()
#     fig.savefig(tmp, format='png')
#     encoded = base64.b64encode(tmp.getvalue()).decode('utf-8')

#     graph_div = f'<img src=\'data:image\png;base64,{encoded}\'>'

#     return graph_div


import sys
from pathlib import Path

import click

from pinata.api_key import get_key_manager
from pinata.exceptions import PinataException
from pinata.sdk import Pinata
from pinata.utils import prettify_date

def profile_option():
    return click.option(
        "--profile",
        "-p",
        help="The profile to use",
        default=get_key_manager().default_profile_name,
    )

class ExceptionHandlingGroup(click.Group):
    def invoke(self, ctx):
        try:
            return super().invoke(ctx)
        except PinataException as err:
            click.echo(str(err), err=True)

@click.group(cls=ExceptionHandlingGroup)
def cli():
    pass

@cli.group("api-key")
def keys():
    """Manage your API key."""

@keys.command("list")
def list_keys():
    """List your API key profile names."""

    key_manager = get_key_manager()
    profile_names = key_manager.profile_names

    if not profile_names:
        _echo_no_profile()

    for name in profile_names:
        click.echo(name)

def _get_pinata(profile: str) -> Pinata:
    key_manager = get_key_manager()
    api_key, api_secret = key_manager.get_key_pair(profile)
    return Pinata.from_api_key(api_key, api_secret)

@keys.command("import")
@click.argument("profile_name")
@click.option("--api-key", help="The API key.", prompt=True)
@click.option("--api-secret", help="The API secret.", prompt=True)
def import_key(profile_name, api_key, api_secret):
    """Import an existing API key pair."""
    key_manager = get_key_manager()
    key_manager.set_key_pair(profile_name, api_key, api_secret)
    click.echo(f"Successfully added API key profile {profile_name}")

@keys.command("remove")
@click.argument("profile_name")
def remove_key(profile_name):
    """Remove an API key pair profile."""
    key_manager = get_key_manager()
    key_manager.delete_key_pair(profile_name)

@keys.command("rename")
@click.option("--old-name", help="The name of the profile to change.", required=True)
@click.option("--new-name", help="The new name of the profile.", required=True)
def rename_key(old_name, new_name):
    """Remove an API key pair profile."""
    key_manager = get_key_manager()
    key_manager.rename_key_pair(old_name, new_name)
    click.echo(f"Successfully renamed API key profile '{old_name}' to '{new_name}'.")

@cli.command()
@click.option("--status", default="pinned", type=click.Choice(["all", "pinned", "unpinned"]))
@profile_option()
def list_pins(status, profile):
    """List pins."""
    if not profile:
        _echo_no_profile()
        sys.exit(1)

    pinata = _get_pinata(profile)
    response = pinata.data.search_pins(status=status)
    pins = response["rows"]
    for pin in pins:
        name = pin["metadata"].get("name", "<Unnamed>")
        date_str = prettify_date(pin["date_pinned"])
        row = f"Name: {name}, CID (IPFS Pin Hash): {pin['ipfs_pin_hash']}, Pinned: {date_str}"
        click.echo(row)

@cli.command()
@click.argument("file_path", type=Path)
@profile_option()
def pin(file_path, profile):
    """Pin a new file."""
    pinata = _get_pinata(profile)
    cid = pinata.pin_file(file_path)
    click.echo(f"Successfully unpinned content. CID={cid}")

@cli.command()
@click.argument("content_hash")
@profile_option()
def unpin(content_hash, profile):
    """Remove a pin."""
    pinata = _get_pinata(profile)
    pinata.unpin(content_hash)
    click.echo("Successfully unpinned content.")

def _echo_no_profile():
    click.echo("There are no stored API keys.")


import numpy as np
from general_robotics_toolbox import *
from pandas import read_csv
import sys
from io import StringIO
from scipy.signal import find_peaks
# sys.path.append('../abb_motion_program_exec')
from abb_motion_program_exec_client import *
sys.path.append('../')
sys.path.append('../../toolbox')
from ilc_toolbox import *

from robots_def import *
from error_check import *
from MotionSend import *
from lambda_calc import *
from blending import *

def main():

	dataset='wood/'
	solution_dir='curve_pose_opt1/'
	data_dir="../../data/"+dataset+solution_dir
	cmd_dir='curve1_400_curve_pose_opt1/'

	curve = read_csv(data_dir+"Curve_in_base_frame.csv",header=None).values

	robot=abb6640(d=50)

	v=400
	s = speeddata(v,9999999,9999999,999999)
	z = z10

	ms = MotionSend()
	breakpoints,primitives,p_bp,q_bp=ms.extract_data_from_cmd(cmd_dir+'command.csv')

	###ilc toolbox def
	ilc=ilc_toolbox(robot,primitives)

	###TODO: extension fix start point, moveC support
	max_error=999
	inserted_points=[]
	iteration=50
	for i in range(iteration):

		ms = MotionSend()
		###execution with plant
		logged_data=ms.exec_motions(robot,primitives,breakpoints,p_bp,q_bp,s,z)
		# Write log csv to file
		with open("recorded_data/curve_exe_v"+str(v)+"_z10.csv","w") as f:
			f.write(logged_data)

		ms.write_data_to_cmd('recorded_data/command.csv',breakpoints,primitives, p_bp,q_bp)

		StringData=StringIO(logged_data)
		df = read_csv(StringData, sep =",")
		##############################data analysis#####################################
		lam, curve_exe, curve_exe_R,curve_exe_js, speed, timestamp=ms.logged_data_analysis(robot,df,realrobot=True)
		#############################chop extension off##################################
		lam, curve_exe, curve_exe_R,curve_exe_js, speed, timestamp=ms.chop_extension(curve_exe, curve_exe_R,curve_exe_js, speed, timestamp,curve[0,:3],curve[-1,:3])

		##############################calcualte error########################################
		error,angle_error=calc_all_error_w_normal(curve_exe,curve[:,:3],curve_exe_R[:,:,-1],curve[:,3:])
		print(max(error),min(speed))
		#############################error valley detection###############################
		valleys=[np.argmin(speed)]

		##############################plot error#####################################

		fig, ax1 = plt.subplots()
		ax2 = ax1.twinx()
		ax1.plot(lam, speed, 'g-', label='Speed')
		ax2.plot(lam, error, 'b-',label='Error')
		ax2.scatter(lam[valleys],error[valleys],label='valleys')
		ax2.plot(lam, np.degrees(angle_error), 'y-',label='Normal Error')
		ax2.axis(ymin=0,ymax=2)
		ax1.axis(ymin=0,ymax=1.2*v)

		ax1.set_xlabel('lambda (mm)')
		ax1.set_ylabel('Speed/lamdot (mm/s)', color='g')
		ax2.set_ylabel('Error/Normal Error (mm/deg)', color='b')
		plt.title("Speed and Error Plot")
		ax1.legend(loc="upper right")

		ax2.legend(loc="upper left")

		plt.legend()
		plt.savefig('recorded_data/iteration_'+str(i))
		plt.clf()
		# plt.show()

		##########################################calculate gradient for valleys######################################
		###restore trajectory from primitives
		curve_interp, curve_R_interp, curve_js_interp, breakpoints_blended=form_traj_from_bp(q_bp,primitives,robot)

		curve_js_blended,curve_blended,curve_R_blended=blend_js_from_primitive(curve_interp, curve_js_interp, breakpoints_blended, primitives,robot,zone=10)
		lam_blended=calc_lam_cs(curve_blended)
		print('estimating speed')
		speed_est=traj_speed_est(robot,curve_js_blended,lam_blended,v)
		plt.plot(lam_blended,speed_est)
		plt.savefig('recorded_data/speedest_iteration_'+str(i))
		plt.clf()

		for valley in valleys:
			######gradient calculation related to nearest 3 points from primitive blended trajectory, not actual one
			_,valley_speed_curve_idx=calc_error(curve_exe[valley],curve[:,:3])  # index of original curve closest to max error point

			###get closest to worst case point on blended trajectory
			_,valley_speed_curve_blended_idx=calc_error(curve_exe[valley],curve_blended)

			###############get numerical gradient#####
			###find closest 3 breakpoints
			order=np.argsort(np.abs(breakpoints_blended-valley_speed_curve_blended_idx))
			breakpoint_interp_2tweak_indices=order[:3]

			print('calculating gradient')
			dv_dq=ilc.get_speed_gradient_from_model(q_bp,breakpoints_blended,curve_js_blended,curve_blended,valley_speed_curve_blended_idx,breakpoint_interp_2tweak_indices,speed_est,v)
			p_bp, q_bp=ilc.update_bp_speed(p_bp,q_bp,dv_dq,speed[valley],breakpoint_interp_2tweak_indices,v)

if __name__ == "__main__":
	main()


# -*- coding: utf-8 -*-
import scrapy
from bs4 import BeautifulSoup
from baike.items import Crop
import re

class CropSpider(scrapy.Spider):
    name = 'crop'
    allowed_domains = ['baike.baidu.com']
    file_object = open('fenlei.txt', 'r', encoding='utf-8').read()
    wordList = file_object.split()  # 获取词表
    start_urls = []
    for i in wordList:  ##生成url列表
        cur = "https://baike.baidu.com/item/"
        cur = cur + str(i)
        start_urls.append(cur)

    # def __init__(self, type):
    #     self.type = type

    def parse(self, response):
        sketch = response.text
        exc = BeautifulSoup(sketch, "html.parser")
        crop_context = Crop()
        name = response.xpath(
            "/html/body[@class='wiki-lemma normal']/div[@class='body-wrapper']/div[@class='content-wrapper']/div[@class='content']/div[@class='main-content']/dl[@class='lemmaWgt-lemmaTitle lemmaWgt-lemmaTitle-']/dd[@class='lemmaWgt-lemmaTitle-title']/h1").extract_first()
        pattern = re.compile('<\w*>(.*)</\w*>')
        n = pattern.findall(name)
        p = exc.find_all(name='div', attrs='para')
        for i in range(len(p)):
            crop_context['context'] = p[i].get_text()
            crop_context['name'] = n[0]
            yield crop_context
            pass


# imports
import pandas as pd
import folium
import folium.features
import folium.map
import geopandas

# read in the data
borders = geopandas.read_file('states.json')
csvFile = pd.read_csv("tax.csv", thousands=',')
population = pd.read_csv("population.csv", thousands=',')

population['NAME'] = population['NAME'].str[1:]
population['Population'] = pd.to_numeric(population['Population'])

csvFile['Tax'] = csvFile['Tax'].astype('int')

csvFile = csvFile.map(lambda x: x.strip() if isinstance(x, str) else x)

# join csvfile and borders into combined
combined = borders.merge(population.merge(csvFile, on="NAME"), on="NAME")

# creates the default map with zoom to US
m = folium.Map(location=[48, -102], zoom_start=3)

# FOLIUM CREATING VISUALIZATION
# creates the choropleth and names it cp
cp = folium.Choropleth(
    # both geojson data and data come from new_borders
    geo_data=combined,
    data=combined,
    # key is fips, value and color is cases
    columns=["NAME", "Percent"],
    # eliminated as many grey counties as I could
    # for some reason there are still some left
    key_on="feature.properties.NAME",
    # color from colorbrewer
    fill_color="BuPu",
    fill_opacity=0.7,
    line_opacity=0.2,
    # change legend name
    legend_name="Percent of Personal Income Collected as Tax (%)",
    name="choropleth",
).add_to(m) #add to map

# # EXPANSION
# # hover tooltip
folium.features.GeoJsonTooltip(
    # fields displayed and their aliases
        fields=['NAME',
                'Population',
                'TotalTax',
                'Tax',
                'Percent'],
        aliases=['State Name: ',
                 'Population: ',
                 'Total Tax Revenue by State: $',
                 'Average Tax Revenue per Capita: $',
                 'Percent of Personal Income Collected as Tax: ',],
        localize=True,
                ).add_to(cp.geojson) # add to cholopleth's geojson

# add a layercontrol
folium.LayerControl().add_to(m)

# save it as an html
m.save("Tax.html")

print(combined)


from .protocol import Protocol
from .ip import IP
from .arp import ARP
from utils import b2mac, btable, color

frame_type = btable({
    0x0800: 'IP',
    0x0806: 'ARP',
    0x0835: 'RARP',
})

class Ethernet(Protocol):
    protocol = 'Ethernet'

    def parse(self):
        assert len(self._data) >= 14, 'invalid ethernet frame'
        self.src = self._read('6B')
        self.dst = self._read('6B')
        self.data_type = self._read('>H')

        if frame_type[self.data_type] == 'IP':
            ip = IP(self, self._data[self._pos:])
            ip.parse()
            self.child.append(ip)
        elif frame_type[self.data_type] in ('ARP', 'RARP'):
            arp = ARP(self, self._data[self._pos:])
            arp.parse()
            self.child.append(arp)
        else:
            raise Exception('Unrecognize frame type %04X' % self.data_type)

    def __str__(self):
        return '[%s] [%s -> %s]' % (
                color(self.protocol, 'cyan'),
                color(b2mac(self.src), 'yellow'),
                color(b2mac(self.dst), 'yellow')
        )


from gpiozero import LED
from time import sleep

class Lights:
	def __init__(self):
		self.red = LED(7)
		self.amber = LED(25)
		self.green = LED(8)

	def set(self, level):
		self.red.off()
		self.amber.off()
		self.green.off()

		if level == 0:
			print("level 1 lights")
			self.green.blink()
			sleep(2)
			self.green.on()
		elif level == 1:
			print("level 2 lights")
			self.amber.blink()
			sleep(2)
			self.amber.on()
		elif level == 2:
			print("level 3 lights")
			self.red.blink()
			sleep(2)
			self.red.on()
		else:
			self.red.on()
			self.amber.on()
			self.green.on()