"""Decoding functions using multiple attentions for RNN decoders.

See http://arxiv.org/abs/1606.07481

The attention mechanisms used in Neural Monkey are inherited from the
``BaseAttention`` class defined in this module.

Each attention object has the ``attention`` function which operates on the
``attention_states`` tensor.  The attention function receives the query tensor,
the decoder previous state and input, and its inner state, which could bear an
arbitrary structure of information. The default structure for this is the
``AttentionLoopState``, which contains a growing array of attention
distributions and context vectors in time. That's why there is the
``initial_loop_state`` function in the ``BaseAttention`` class.

Mainly for illustration purposes, the attention objects can keep their
*histories*, which is a dictionary populated with attention distributions in
time for every decoder, that used this attention object. This is because for
example the recurrent decoder is can be run twice for each sentence - once in
the *training* mode, in which the decoder gets the reference tokens on the
input, and once in the *running* mode, in which it gets its own outputs. The
histories object is constructed *after* the decoding and its construction
should be triggered manually from the decoder by calling the ``finalize_loop``
method.
"""
from typing import NamedTuple, Dict, Optional, Any, Tuple, Union

import tensorflow as tf

from neuralmonkey.model.stateful import TemporalStateful, SpatialStateful
from neuralmonkey.model.model_part import ModelPart, FeedDict
from neuralmonkey.dataset import Dataset

# pylint: disable=invalid-name
Attendable = Union[TemporalStateful, SpatialStateful]
AttentionLoopState = NamedTuple("AttentionLoopState",
                                [("contexts", tf.TensorArray),
                                 ("weights", tf.TensorArray)])
# pylint: enable=invalid-name

def empty_attention_loop_state() -> AttentionLoopState:
    """Create an empty attention loop state.

    The attention loop state is a technical object for storing the attention
    distributions and the context vectors in time. It is used with the
    ``tf.while_loop`` dynamic implementation of the decoder.

    This function returns an empty attention loop state which means there are
    two empty arrays, one for attention distributions in time, and one for
    the attention context vectors in time.
    """
    return AttentionLoopState(
        contexts=tf.TensorArray(
            dtype=tf.float32, size=0, dynamic_size=True,
            name="contexts"),
        weights=tf.TensorArray(
            dtype=tf.float32, size=0, dynamic_size=True,
            name="distributions", clear_after_read=False))

def get_attention_states(encoder: Attendable) -> tf.Tensor:
    if isinstance(encoder, TemporalStateful):
        return encoder.temporal_states

    elif isinstance(encoder, SpatialStateful):
        # pylint: disable=no-member
        shape = [s.value for s in encoder.spatial_states.get_shape()[1:]]
        # pylint: enable=no-member
        return tf.reshape(encoder.spatial_states,
                          [-1, shape[0] * shape[1], shape[2]])
    else:
        raise AssertionError("Unknown encoder type")

def get_attention_mask(encoder: Attendable) -> Optional[tf.Tensor]:
    if isinstance(encoder, TemporalStateful):
        if encoder.temporal_mask is None:
            raise ValueError("The encoder temporal mask should not be none")
        return encoder.temporal_mask

    elif isinstance(encoder, SpatialStateful):
        if encoder.spatial_mask is None:
            return None

        # pylint: disable=no-member
        shape = [s.value for s in encoder.spatial_mask.get_shape()[1:]]
        # pylint: enable=no-member
        return tf.reshape(encoder.spatial_mask, [-1, shape[0] * shape[1]])
    else:
        raise AssertionError("Unknown encoder type")

class BaseAttention(ModelPart):
    def __init__(self,
                 name: str,
                 save_checkpoint: str = None,
                 load_checkpoint: str = None) -> None:
        ModelPart.__init__(self, name, save_checkpoint, load_checkpoint)

        self.query_state_size = None  # type: tf.Tensor
        self._histories = {}  # type: Dict[str, tf.Tensor]

        self.train_mode = tf.placeholder(tf.bool, [], "train_mode")

    @property
    def histories(self) -> Dict[str, tf.Tensor]:
        return self._histories

    def attention(self,
                  query: tf.Tensor,
                  decoder_prev_state: tf.Tensor,
                  decoder_input: tf.Tensor,
                  loop_state: Any,
                  step: tf.Tensor) -> Tuple[tf.Tensor, Any]:
        """Get context vector for a given query."""
        raise NotImplementedError("Abstract method")

    def initial_loop_state(self) -> Any:
        """Get initial loop state for the attention object."""
        raise NotImplementedError("Abstract method")

    def finalize_loop(self, key: str, last_loop_state: Any) -> None:
        raise NotImplementedError("Abstract method")

    def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
        return {self.train_mode: train}

    @property
    def context_vector_size(self) -> int:
        raise NotImplementedError("Abstract property")

    def visualize_attention(self, key: str) -> None:
        if key not in self.histories:
            raise ValueError(
                "Key {} not among attention histories".format(key))

        alignments = tf.expand_dims(
            tf.transpose(self.histories[key], perm=[1, 2, 0]), -1)

        tf.summary.image(self.name, alignments,
                         collections=["summary_att_plots"], max_outputs=256)


import random

from textual.reactive import Reactive
from textual.views import GridView

from swept.cell import BOMB_CHAR, Cell, CellStatus
from swept.config import BOMB_PERCENT, N_COLS, N_ROWS
from swept.enums import GameStatus
from swept.exceptions import CantToggleFlag, CantUncover

class CellGrid(GridView):
    """
    Minesweeper-like game.
    """

    game_status = Reactive(GameStatus.IN_PROGRESS)

    def watch_game_status(self, game_status: GameStatus) -> None:
        """Control grid based on status."""
        if game_status == GameStatus.LOST:
            self.uncover_bombs()
        elif game_status == GameStatus.WON:
            for btn in self.cell_btns:
                if btn.status == CellStatus.FLAGGED:
                    btn.status = CellStatus.FLAGGED_WON

    def on_mount(self) -> None:
        """
        Event when widget is first mounted.
        """
        self.cell_btns = [
            Cell("", style=Cell.COVERED_CELL, name=f"{idx}")
            for idx in range(N_COLS * N_ROWS)
        ]
        self.reset_game()

        # set basic grid settings
        self.grid.set_gap(1, 1)
        self.grid.set_gutter(2)
        self.grid.set_align("center", "center")

        # create rows / columns / areas
        self.grid.add_column("col", max_size=6, repeat=N_COLS)
        self.grid.add_row("row", max_size=3, repeat=N_ROWS)

        self.grid.place(*self.cell_btns)

    def reset_game(self) -> None:
        """Reset game."""
        n_bombs = len(self.cell_btns) * BOMB_PERCENT // 100
        bombs = [True] * n_bombs + [False] * (len(self.cell_btns) - n_bombs)
        random.shuffle(bombs)

        for btn, bomb in zip(self.cell_btns, bombs):
            btn.value = BOMB_CHAR if bomb else ""

        for idx, btn in enumerate(self.cell_btns):
            if btn.value != BOMB_CHAR:
                btn.value = str(self.count_adjacent_bombs(idx))
            self.log(btn.value)
            btn.status = CellStatus.COVERED

    def uncover_cell(self, cell_idx: int) -> None:
        """Flip a cell."""
        cell = self.cell_btns[cell_idx]

        try:
            cell.uncover()
        except CantUncover:
            return

        # propagate 0 cell uncovering
        if cell.value == "0":
            for adj_idx in self._adjacent_idxs(cell_idx):
                self.uncover_cell(adj_idx)

    def uncover_bombs(self) -> None:
        """Uncover all bombs."""
        for btn in self.cell_btns:
            if btn.value == BOMB_CHAR and btn.status != CellStatus.EXPLODED:
                btn.status = CellStatus.UNCOVERED

    def uncover_adjacent(self, cell_idx: int) -> None:
        """Uncover adjacent cells if all bombs around a specific cell have been identified."""
        try:
            expected_bombs = int(self.cell_btns[cell_idx].label)
        except ValueError:
            return

        seen_bombs = sum(
            1
            for adj_idx in self._adjacent_idxs(cell_idx)
            if self.cell_btns[adj_idx].status == CellStatus.FLAGGED
        )

        if expected_bombs == seen_bombs:
            for adj_idx in self._adjacent_idxs(cell_idx):
                self.uncover_cell(adj_idx)

    def flag_cell(self, cell_idx: int) -> None:
        """Flag a cell where the player thinks is a bomb."""
        cell = self.cell_btns[cell_idx]

        try:
            cell.toggle_flag()
        except CantToggleFlag:
            return

    def win_condition(self) -> bool:
        """Compute if the game has been won."""
        return all(
            cell.status in [CellStatus.UNCOVERED, CellStatus.FLAGGED]
            for cell in self.cell_btns
        )

    def lose_condition(self) -> bool:
        """Compute if the game has been lost."""
        return any(cell.status == CellStatus.EXPLODED for cell in self.cell_btns)

    def _adjacent_idxs(self, idx: int) -> list[int]:
        """Compute all adjacent valid indices of a cell."""
        adjacents = []

        col = idx % N_COLS
        row = idx // N_COLS

        if col != 0 and row != 0:
            adjacents.append(idx - N_COLS - 1)
        if col != 0:
            adjacents.append(idx - 1)
        if col != 0 and row != N_ROWS - 1:
            adjacents.append(idx + N_COLS - 1)
        if row != 0:
            adjacents.append(idx - N_COLS)
        if row != N_ROWS - 1:
            adjacents.append(idx + N_COLS)
        if col != N_COLS - 1 and row != 0:
            adjacents.append(idx - N_COLS + 1)
        if col != N_COLS - 1:
            adjacents.append(idx + 1)
        if col != N_COLS - 1 and row != N_ROWS - 1:
            adjacents.append(idx + N_COLS + 1)

        return adjacents

    def count_adjacent_bombs(self, idx: int) -> int:
        """
        Compute the number of adjacent bombs to a cell.
        """
        return sum(
            1
            for adj_idx in self._adjacent_idxs(idx)
            if self.cell_btns[adj_idx].value == BOMB_CHAR
        )


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
import time

FORM_LINK = 'https://docs.google.com/forms/d/e/1FAIpQLScOkQbo9fi5ea9ib2Pfs5BrLh8oGKvTnnGk3ekrVkTM0cxR0Q/viewform?usp=sf_link'

class GoogleForms:
    def __init__(self) -> None:
        self.driver = self.create_driver()

    def max_screen(self):
        self.driver.maximize_window()

    def create_driver(self):
        service = Service(executable_path='C:\Development\chromedriver.exe')
        option = webdriver.ChromeOptions()
        option.add_experimental_option('detach', True)
        option.add_experimental_option('excludeSwitches', ['enable-logging'])
        return webdriver.Chrome(service=service, options=option)

    def update_google_form(self, property_data: dict):
        for key in property_data:
            self.driver.get(url=FORM_LINK)
            time.sleep(2)
            addr = self.driver.find_element(
                By.XPATH, '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[1]/div/div/div[2]/div/div[1]/div/div[1]/input')
            addr.send_keys(property_data[key]['address'])

            price = self.driver.find_element(
                By.XPATH, '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[2]/div/div/div[2]/div/div[1]/div/div[1]/input')
            price.send_keys(property_data[key]['rent_price'])

            lnk = self.driver.find_element(
                By.XPATH, '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[3]/div/div/div[2]/div/div[1]/div/div[1]/input')
            lnk.send_keys(property_data[key]['link_to_property'])

            self.driver.find_element(
                By.XPATH, '//*[@id="mG61Hd"]/div[2]/div/div[3]/div[1]/div[1]/div/span').click()
        self.driver.quit()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

ON = 255
OFF = 0

def randomGrid(N):
    return np.random.choice([ON, OFF] , N*N, p=[0.25, 0.75]).reshape(N, N)

def update(frameNum, img, grid, N):

    newGrid = grid.copy()
    for i in range(N): #for row
        total = 0
        for j in range(N): #for column

            #No edge cases
            if total == 0: #if first time through
                left_sum =    grid[(i-1)%N, (j-1)%N] + grid[i, (j-1)%N] +  grid[(i+1)%N, (j-1)%N] #j-1 column
                central_sum = grid[(i-1)%N, j]                          +  grid[(i+1)%N, j]      #j column
                right_sum =   grid[(i-1)%N, (j+1)%N] + grid[i, (j+1)%N] +  grid[(i+1)%N, (j+1)%N] #j+1 column
                total = int((left_sum + central_sum + right_sum)/255)
            else:
                left_sum =    central_sum + grid[i, (j-1)%N]
                central_sum = right_sum - grid[i, j]
                right_sum =   grid[(i-1)%N, (j+1)%N] + grid[i, (j+1)%N] +  grid[(i+1)%N, (j+1)%N]
                total = int((left_sum + central_sum + right_sum)/255)

            # apply Conway's rules
            if grid[i, j]  == ON:
                if (total < 2) or (total > 3):
                    newGrid[i, j] = OFF
            else:
                if total == 3:
                    newGrid[i, j] = ON

    # update data
    img.set_data(newGrid)
    grid[:] = newGrid[:]
    return img

def main():

    N = 150
    grid = randomGrid(N)

    updateInterval = 16 #60FPS baby
    fig, ax = plt.subplots()
    img = ax.imshow(grid, interpolation='nearest')
    ani = animation.FuncAnimation(fig, update, fargs=(img, grid, N ), interval=updateInterval)
    plt.show()

main()


import json
from collections import defaultdict
import os
import csv

filters = ['C0303', 'C0304', 'C0305', 'C0103', 'C0112', 'C0114', 'C0115', 'C0116', 'W0611', 'C0415', 'W0404',
           'C0413', 'C0411', 'C0412', 'W0401', 'W0614', 'C0410', 'C0413', 'C0414', 'R0402', 'C2403', 'E0401', 'E0001']

smellList = []
smellDict = defaultdict()
smellDictName = defaultdict()

dirPath = './pylint_data/'
for file in os.listdir(dirPath):
    # check only json files
    if file.endswith('.json'):
        # print(file)
        with open(dirPath+file) as f:
            data = json.load(f)
            # print(data)
            for items in data:
                for item in items:
                    smellList.append(item)
                    found = False
                    for filter in filters:
                        if item["message-id"] == filter:
                            found = True
                        if item["message-id"].startswith('F'):
                            found = True
                    if found:
                        continue
                    try:
                        # smellDict[item["message-id"]] += 1
                        smellDictName[item["message-id"] +
                                      "-"+item["symbol"]] += 1
                    except KeyError:
                        # smellDict[item["message-id"]] = 1
                        smellDictName[item["message-id"] +
                                      "-"+item["symbol"]] = 1

data = []
for key in smellDictName:
    print(key)
    row = []
    row.append(key)
    row.append(smellDictName[key])
    data.append(row)
header = ['Message', 'Count']

with open('dataset.csv', 'w',  newline='', encoding='UTF8') as f:
    writer = csv.writer(f)

    # write the header
    writer.writerow(header)

    # write the data
    writer.writerows(data)


a=open("File.txt", "w")
text = input("add text here...write quit to exit")

while True:
    if text!="quit":
        a.writelines(text+"\n")
        text =input()
    elif text =="quit":
        break

print('Exiting program....Bye')
a.close()

class LowAgeError(Exception):
    def __init__(self):
        super(LowAgeError,self).__init__("Age should be greater than 18")

age = int(input("Enter the age"))
try:
    if age<18:
        raise LowAgeError
    else:
        print("Accepted")
except LowAgeError as e:
    print(e)

list = [x for x in range(2, 2500) if not [y for y in range(2, x) if (x % y == 0)]]
c = 0
for a in list:
    c = c + 1
    print(c,":",a)


class PeakRegion(object):
    """a rectangle region for a single peak"""
    def __init__(self, frame_start, frame_end, scan_start, scan_end):
        super(PeakRegion, self).__init__()
        self.frame_start = frame_start
        self.frame_end = frame_end
        self.scan_start = scan_start
        self.scan_end = scan_end

class Peak(object):
    '''MS peak
    '''
    def __init__(self, mz, z, frame_start, frame_end, scan_start, scan_end):
        self.mz = mz
        self.z = z
        self.peak_region = PeakRegion(frame_start, frame_end, scan_start, scan_end)


import numpy as np
import math as m
import matplotlib.pyplot as plt
import queue
import pandas as pd

NOISE = 0
UNASSIGNED = -1

def load_data():
    """
    导入数据
    :return: 数据
    """
    points = []
    fileIn = open("network_point.csv")
    for line in fileIn.readlines():
        temp=[]
        #line.strip()把末尾的'\n'去掉
        lineArr = line.strip().split(',')
        temp.append(float(lineArr[1]))
        temp.append(float(lineArr[2]))
        temp.append(float(lineArr[0]))
        points.append(temp)
        #dataSet.append([float(lineArr[0]), float(lineArr[1])])
    fileIn.close()
    points = np.array(points)
    print(points)
    return points

def dist(a, b):
    """
    计算两个向量的距离
    :param a: 向量1
    :param b: 向量2
    :return: 距离
    """
    return m.sqrt(np.power(a-b, 2).sum())

def neighbor_points(data, pointId, radius):
    """
    得到邻域内所有样本点的Id
    :param data: 样本点
    :param pointId: 核心点
    :param radius: 半径
    :return: 邻域内所用样本Id
    """
    points = []
    for i in range(len(data)):
        if dist(data[i, 0: 2], data[pointId, 0: 2]) < radius:
            points.append(i)
    return np.asarray(points)

def to_cluster(data, clusterRes, pointId, clusterId, radius, minPts):
    """
    判断一个点是否是核心点，若是则将它和它邻域内的所用未分配的样本点分配给一个新类
    若邻域内有其他核心点，重复上一个步骤，但只处理邻域内未分配的点，并且仍然是上一个步骤的类。
    :param data: 样本集合
    :param clusterRes: 聚类结果
    :param pointId:  样本Id
    :param clusterId: 类Id
    :param radius: 半径
    :param minPts: 最小局部密度
    :return:  返回是否能将点PointId分配给一个类
    """
    points = neighbor_points(data, pointId, radius)
    points = points.tolist()

    q = queue.Queue()

    if len(points) < minPts:
        clusterRes[pointId] = NOISE
        return False
    else:
        clusterRes[pointId] = clusterId
    for point in points:
        if clusterRes[point] == UNASSIGNED:
            q.put(point)
            clusterRes[point] = clusterId

    while not q.empty():
        neighborRes = neighbor_points(data, q.get(), radius)
        if len(neighborRes) >= minPts:                      # 核心点
            for i in range(len(neighborRes)):
                resultPoint = neighborRes[i]
                if clusterRes[resultPoint] == UNASSIGNED:
                    q.put(resultPoint)
                    clusterRes[resultPoint] = clusterId
                elif clusterRes[clusterId] == NOISE:
                    clusterRes[resultPoint] = clusterId
    return True

def dbscan(data, radius, minPts):
    """
    扫描整个数据集，为每个数据集打上核心点，边界点和噪声点标签的同时为
    样本集聚类
    :param data: 样本集
    :param radius: 半径
    :param minPts:  最小局部密度
    :return: 返回聚类结果， 类id集合
    """
    clusterId = 1
    nPoints = len(data)
    clusterRes = [UNASSIGNED] * nPoints
    for pointId in range(nPoints):
        if clusterRes[pointId] == UNASSIGNED:
            if to_cluster(data, clusterRes, pointId, clusterId, radius, minPts):
                clusterId = clusterId + 1
    return np.asarray(clusterRes), clusterId

def plotRes(data, clusterRes, clusterNum):
    nPoints = len(data)
    scatterColors = ['black', 'blue', 'green', 'yellow', 'red', 'purple', 'orange', 'brown']
    for i in range(clusterNum):
        color = scatterColors[i % len(scatterColors)]
        x1 = [];  y1 = []
        for j in range(nPoints):
            if clusterRes[j] == i:
                x1.append(data[j, 0])
                y1.append(data[j, 1])
        plt.scatter(x1, y1, c=color, alpha=1, marker='+')

if __name__ == '__main__':
    data = load_data()
    cluster = np.asarray(data[:, 2])
    clusterRes, clusterNum = dbscan(data, 0.5 ,5)#这里改变DBSCAN的超参数
#    clusterRes, clusterNum = dbscan(data, 1 ,5)
#    clusterRes, clusterNum = dbscan(data, 1 ,2.5)
    cluster_DBSCAN=list(clusterRes)
    print(len(cluster_DBSCAN))
    print('clusterNumber=',clusterNum)

    #把聚类结果写入Clust_DBSCAN.csv中
    b = []
    for i in range(len(cluster_DBSCAN)):
        markIndex = cluster_DBSCAN[i]
        b.append(markIndex)
    a = []
    for i in range(len(cluster_DBSCAN)):
        a.append(i+1)

    #字典中的key值即为csv中列名
    dataframe = pd.DataFrame({'row':a,'type':b})

    #将DataFrame存储为csv,index表示是否显示行名，default=True
    dataframe.to_csv("Clust_DBSCAN.csv",index=False,sep=',')

    plotRes(data, clusterRes, clusterNum)
    plt.show()


import cv2
import numpy as np

# Loading the Image
image = cv2.imread('19.BMP')
cv2.waitKey(0)

# Grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# Denoising the image
denoised = cv2.fastNlMeansDenoising(gray, None,40,13,31)

# Canny edge detection
canny = cv2.Canny(denoised, 30, 200)

# Thresholding the image.
_, threshold = cv2.threshold(canny, 127, 255, cv2.THRESH_BINARY)

# Finding Contours on the Binary image
contours, hierarchy = cv2.findContours(threshold,
	cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.imshow('Threshold image: ', threshold)
cv2.waitKey(0)

print("Number of Contours found = " + str(len(contours)))

# Finding the max length in the Contours found
cnt = [cv2.arcLength(cnt,True) for cnt in contours]
a = contours[cnt.index(max(cnt))]

# Drawing the contour on the image
cv2.drawContours(image, [a], 0, (0, 255, 0), 3)
cv2.imshow('Contours', image)
cv2.waitKey(0)
cv2.destroyAllWindows()


#!/usr/bin/env python3

# https://github.com/colinmccormick/CarND-Advanced-Lane-Lines

import os
import numpy as np
import cv2
import matplotlib.pyplot as plt
import pickle
import random
#from IPython import get_ipython

# set parameters

#get_ipython().run_line_magic('matplotlib', 'inline')

# processing mode
TEST_MODE = False
RECORD = False

# image size
ROWS = 720
COLS = 1280
ROI_TOP = 0.65
ROI_BOTTOM = 0.90
ROI_LEFT = 0.10
ROI_RIGHT = 0.95
INSET_ROWS = int(0.4 * ROWS)
INSET_COLS = int(0.4 * COLS)

# colors and scales
RED_COLOR = (0,0,255)
GREEN_COLOR = (0,255,0)
XM_PER_PIX = 3.7 / 700
YM_PER_PIX = 15 / 720   # based on estimate of 5 white lane dashes fitting vertically in warped image; 3m each

# camera calibration
CHESS_NX = 9
CHESS_NY = 6

# lane pixel finding
L_THRESHOLD = [225,255]              # range for L channel in LUV color space (for white)
B_THRESHOLD = [190,255]              # range for B channel in LAB color space (for yellow)

# perspective transform
GB_KERNEL = 3                       # size of gaussian blur kernel
CANNY_LOW = 50                      # canny edge detection threshold (low)
CANNY_HIGH = 400                    # canny edge detection threshold (high)
HOUGH_DIST_RES = 0.5                # hough line finder distance resolution (pixels)
HOUGH_ANGLE_RES = 3.14159/180       # hough line finder angle resolution (rads)
HOUGH_THRESHOLD = 20                # hough line finder threshold
HOUGH_MIN_LINE = 60                 # hough line finder min line length (pixels)
HOUGH_MAX_GAP = 120                 # hough line finder max line gap (pixels)

# lane fitting
NUM_WINS = 8                        # number of sliding windows to use
WIN_WIDTH = 64                      # half total width of sliding window
WIN_HEIGHT = int(ROWS/NUM_WINS)     # total height of sliding window
FRAME_MEMORY = 4                    # number of past frames to remember
LANE_WIDTH_PIXELS = 810             # width of lane at bottom of image in pixels
FRAME_MARKER_THRESHOLD = 4          # frames to wait until new lane marker can reasonably appear

# speed measurement
METERS_PER_MARKER = 15.75           # meters between white lane markers (40 feet)

# directories and filenames
TEST_DIR = 'test_images'
CALIBRATION_DIR = 'camera_cal'
CALIBRATION_FILENAME = 'camera_calibration.pickle'
PERSPECTIVE_FILENAME = 'perspective_transform.pickle'

OUTPUT_DIR_CALIBRATION = 'output_images/calibration'
OUTPUT_DIR_S_CHANNEL = 'output_images/s_channel'
OUTPUT_DIR_X_GRAD = 'output_images/x_gradient'
OUTPUT_DIR_VP = 'output_images/vanishing_point'

OUTPUT_DIR_UNDISTORT = 'output_images/undistort'
OUTPUT_DIR_WARPED = 'output_images/warped'
OUTPUT_DIR_MARKER_PIXELS = 'output_images/marker_pixels'
OUTPUT_DIR_SLIDING_WINDOW = 'output_images/sliding_window'
OUTPUT_DIR_LANE_MARK = 'output_images/lane_mark'

class LaneLine(object):
    # class to hold info about each lane line

    def __init__(self,side):
        # side: 'left' or 'right'
        self.side = side
        self.goodFitLastFrame = False
        self.recentFits = np.zeros((FRAME_MEMORY,3),dtype=np.float32)      # second-order polynomial: 3 coefficients
        self.curvatureRadius = np.zeros((FRAME_MEMORY,),dtype=np.float32)
        self.framesSeenSinceMarker = 0
        self.estimatedSpeed = 0

    def calculateCurvature(self,f):
        # calculate radius of curvature; use some algebra to scale pixels to meters

        y0 = ROWS * YM_PER_PIX
        A = f[0] *  XM_PER_PIX / YM_PER_PIX**2
        B = f[1] * XM_PER_PIX / YM_PER_PIX

        curvature = ((1 + (2*A*y0 + B)**2)**1.5)/float(np.absolute(2*A))
        self.curvatureRadius = np.roll(self.curvatureRadius,1)
        self.curvatureRadius[0] = curvature

        return np.mean(self.curvatureRadius)
        # define camera calibration

def calibrateCamera():
    # use chessboard photos to calibrate camera

    # set termination criteria (for use with cv2.cornerSubPix)
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    # make object points
    objp = np.zeros((CHESS_NX*CHESS_NY,3), np.float32)
    objp[:,:2] = np.mgrid[0:CHESS_NX,0:CHESS_NY].T.reshape(-1,2)
    imagePoints = []
    objectPoints = []

    # loop through calibration images
    for imageFile in os.listdir(CALIBRATION_DIR):
        if imageFile.endswith('.jpg'):

            # read image, grayscale, find corners
            im = cv2.imread(os.path.join(CALIBRATION_DIR,imageFile))
            gray = cv2.cvtColor(im,cv2.COLOR_RGB2GRAY)
            found, corners = cv2.findChessboardCorners(gray,(CHESS_NX,CHESS_NY),None)

            if found:
                # draw corners on image
                cv2.drawChessboardCorners(im,(CHESS_NX,CHESS_NY),corners,found)
                # find corners with higher (sub-pixel) precision
                corners2 = cv2.cornerSubPix(gray,corners,(5,5),(-1,-1),criteria)
                imagePoints.append(corners2)
                objectPoints.append(objp)

                # save image with corners marked
                cv2.imwrite(os.path.join(OUTPUT_DIR_CALIBRATION,imageFile),im)

    # do the actual calibration and save
    ret, mtx, dst, rvecs, tvecs = cv2.calibrateCamera(objectPoints,imagePoints,gray.shape[::-1],None,None)
    imSize = im.shape
    calibrationData = {'cam_matrix':mtx, 'dist_coeffs':dst, 'im_size':imSize}
    with open(CALIBRATION_FILENAME,'wb') as f:
        pickle.dump(calibrationData,f)

    return mtx,dst

# do camera calibration and display example images

print("Calibrating camera...")
mtx,dst = calibrateCamera()
filenameList = [fn for fn in os.listdir(CALIBRATION_DIR) if fn.endswith('.jpg')]
fn = random.choice(filenameList)
imOriginal = cv2.imread(os.path.join(CALIBRATION_DIR,fn))
imCorners = cv2.imread(os.path.join(OUTPUT_DIR_CALIBRATION,fn))

f = plt.figure(figsize=(30,40))
ax = f.add_subplot(1,2,1)
ax.imshow(imOriginal)
ax = f.add_subplot(1,2,2)
ax.imshow(imCorners)
plt.show()

# display undistorted image example

print("Undistorting image...")
fn = os.path.join(CALIBRATION_DIR,'calibration1.jpg')
imCB = cv2.imread(fn)
imCB = cv2.cvtColor(imCB,cv2.COLOR_BGR2RGB)
imCBUndistorted = cv2.undistort(imCB,mtx,dst)
f2 = plt.figure(figsize=(30,40))
ax = f2.add_subplot(1,2,1)
ax.imshow(imCB)
ax = f2.add_subplot(1,2,2)
ax.imshow(imCBUndistorted)
plt.show()

filenameList = [fn for fn in os.listdir(TEST_DIR) if fn.endswith('.jpg')]
fn = random.choice(filenameList)
imOriginal = cv2.imread(os.path.join(TEST_DIR,fn))
imOriginalRGB = cv2.cvtColor(imOriginal,cv2.COLOR_BGR2RGB)
imUndistorted = cv2.undistort(imOriginalRGB,mtx,dst)
f3 = plt.figure(figsize=(30,40))
ax = f3.add_subplot(1,2,1)
ax.imshow(imOriginalRGB)
ax = f3.add_subplot(1,2,2)
ax.imshow(imUndistorted)
plt.show()

# define lane pixel finder

def selectLanePixels(im, fn='frame.jpg'):
    # select pixels that are lane markers; assume RGB image; use LUV, LAB color spaces

    luv = cv2.cvtColor(im,cv2.COLOR_RGB2LUV)
    lab = cv2.cvtColor(im,cv2.COLOR_RGB2Lab)

    # find yellow lane pixels using LUV color space
    lChannel = luv[:,:,0]
    lBinary = np.zeros_like(lChannel)
    lBinary[ (lChannel > L_THRESHOLD[0]) & (lChannel <= L_THRESHOLD[1]) ] = 255

    # find white lane pixels using LAB color space
    # treat lab b-channel differently: normalize up to 255
    # however, skip this if max value is low (implying no yellow in the image)
    # thanks to jeremy shannon for this idea: https://github.com/jeremy-shannon/CarND-Advanced-Lane-Lines
    bChannel = lab[:,:,2]
    bMax = np.max(bChannel)
    if bMax > 175:
        bChannel = 255 * ( bChannel / bMax )
    bBinary = np.zeros_like(bChannel)
    bBinary[ (bChannel > B_THRESHOLD[0]) & (bChannel <= B_THRESHOLD[1]) ] = 255

    # make combined mask
    lanePixels = np.zeros_like(lChannel)
    lanePixels[ (lBinary > 0) | (bBinary > 0) ] = 255

    return lanePixels, lBinary, bBinary

# display example lane pixel finder

print("Finding example lane pixels...")

filenameList = [fn for fn in os.listdir(TEST_DIR) if fn.endswith('.jpg')]
fn = random.choice(filenameList)
imBGR = cv2.imread(os.path.join(TEST_DIR,fn))
imRGB = cv2.cvtColor(imBGR,cv2.COLOR_BGR2RGB)

lanePixels, lBinary, bBinary = selectLanePixels(imRGB,fn)

f = plt.figure(figsize=(30,40))
ax = f.add_subplot(1,2,1)
ax.imshow(imRGB)
ax = f.add_subplot(1,2,2)
ax.imshow(lanePixels,cmap='gray')
plt.show()

# define utility function for selecting roi

def selectRoi(im):
    # mask out any pixels outside of the roi

    #rows,cols = im.shape[:2]
    bottomLeft = [0,ROWS*ROI_BOTTOM]
    topLeft = [COLS*ROI_LEFT,ROWS*ROI_TOP]
    topRight = [COLS*ROI_RIGHT,ROWS*ROI_TOP]
    bottomRight = [COLS,ROWS*ROI_BOTTOM]
    vertices = np.array([[bottomLeft,topLeft,topRight,bottomRight]],dtype=np.int32)
    mask = np.zeros_like(im)
    cv2.fillPoly(mask,vertices,255)

    return cv2.bitwise_and(im,mask)

# define utility function for determining line intersection

def findIntersection(lines):
    # find the point minimizing lsq distance from all lines
    # if the lines all intersect, this is the intersection point

    numLines = len(lines)
    a = np.zeros((numLines,2),dtype=np.float32)
    b = np.zeros((numLines,),dtype=np.float32)
    for n,line in enumerate(lines):
        for x1,y1,x2,y2 in line:
            slope = (y2-y1) / float(x2-x1)
            a[n] = np.array([slope,-1],dtype=np.float32)
            b[n] = slope * x1 - y1     # this is -1 times the intercept
    return np.linalg.lstsq(a,b)[0]

# define utility function for finding vanishing point

def findVanishingPoint(mtx,dst):
    # determine vanishing point in image

    # analyze straight-road images used to find vanishing point
    straightImages = [os.path.join(TEST_DIR,f) for f in os.listdir(TEST_DIR) if 'straight' in f]
    vps = []
    for imFile in straightImages:
        im = cv2.imread(imFile)
        im = cv2.undistort(im,mtx,dst)

        # smooth with gaussian blur
        imSmoothed = cv2.GaussianBlur(im,(GB_KERNEL,GB_KERNEL),0)

        # find edges with canny
        edges = cv2.Canny(imSmoothed,CANNY_LOW,CANNY_HIGH)

        # apply roi mask
        edgesRoi = selectRoi(edges)

        # find lines with hough; use probabilistic version to increase speed
        lines = cv2.HoughLinesP(edgesRoi,HOUGH_DIST_RES,HOUGH_ANGLE_RES,HOUGH_THRESHOLD,None,HOUGH_MIN_LINE,HOUGH_MAX_GAP)

        # identify best overlap point of lines - this is the vanishing point
        vp = findIntersection(lines)
        vps.append(vp)

        # save image
        for line in lines:
            for x1,y1,x2,y2 in line:
                cv2.line(im, (x1,y1), (x2,y2), GREEN_COLOR, thickness=2)
        cv2.circle(im,(vp[0],vp[1]),8,GREEN_COLOR,2)
        savepath = os.path.join(OUTPUT_DIR_VP,os.path.split(imFile)[1])
        cv2.imwrite(savepath,im)

    # return arithmetic mean of vp coordinates
    return np.mean(vps,axis=0)

# display example of vanishing point

print("Calculating vanishing point...")
vp = findVanishingPoint(mtx,dst)
print("Vanishing point: {0}".format(vp))
filenameList = [fn for fn in os.listdir(OUTPUT_DIR_VP) if fn.endswith('.jpg')]
fn = random.choice(filenameList)
imBGR = cv2.imread(os.path.join(OUTPUT_DIR_VP,fn))
imRGB = cv2.cvtColor(imBGR,cv2.COLOR_BGR2RGB)
plt.imshow(imRGB)
plt.show()

# define function for calculating perspective transform

def calculatePerspectiveTransform():
    # calculate perspective transform matrix

    # read in camera calibration
    with open(CALIBRATION_FILENAME,'rb') as f:
        calibrationData = pickle.load(f)
    mtx = calibrationData['cam_matrix']
    dst = calibrationData['dist_coeffs']

    # find vanishing point; note this is returned as (x,y)
    vp = findVanishingPoint(mtx,dst)

    xVP,yVP = int(vp[0]),int(vp[1])
    xBottomLeft = int(ROI_LEFT * COLS)
    xBottomRight = int(ROI_RIGHT * COLS)
    yTop = int(ROWS * ROI_TOP)
    yBottom = int(ROWS * ROI_BOTTOM)

    # calculate x positions of upper ROI corners
    # x = my + b; m = (x1-x0)/(y1-y0); b = x0 - m * y0
    leftM = (xVP - xBottomLeft) / float(yVP - yBottom)
    leftB = xBottomLeft - leftM * yBottom
    rightM = (xVP - xBottomRight) / float(yVP - yBottom)
    rightB = xBottomRight - rightM * yBottom
    xTopLeft = int(leftM * yTop + leftB)
    xTopRight = int(rightM * yTop + rightB)

    corners = [(xTopLeft,yTop),(xBottomLeft,yBottom),(xBottomRight,yBottom),(xTopRight,yTop)]
    pts1 = np.float32(corners)
    pts2 = np.float32( [[0,0],[0,ROWS],[COLS,ROWS],[COLS,0]] )

    perspectiveTrans = cv2.getPerspectiveTransform(pts1,pts2)
    inversePerspectiveTrans = cv2.getPerspectiveTransform(pts2,pts1)

    transformDict = {'forwardWarp': perspectiveTrans, 'backwardWarp': inversePerspectiveTrans,'corners': corners}

    with open(PERSPECTIVE_FILENAME,'wb') as f:
        pickle.dump(transformDict,f)

    return transformDict

    # display example perspective warp

print("Warping perspective forward and backward...")

td = calculatePerspectiveTransform()
p = td['corners']

filenameList = [fn for fn in os.listdir(TEST_DIR) if fn.endswith('.jpg') and 'straight' in fn]
fn = random.choice(filenameList)
im = cv2.imread(os.path.join(TEST_DIR,fn))
im = cv2.cvtColor(im,cv2.COLOR_BGR2RGB)
im = cv2.undistort(im,mtx,dst)

cv2.line(im,p[0],p[1],color=(255,0,0),thickness=4)
cv2.line(im,p[1],p[2],color=(255,0,0),thickness=4)
cv2.line(im,p[2],p[3],color=(255,0,0),thickness=4)
cv2.line(im,p[3],p[0],color=(255,0,0),thickness=4)

imWarped = cv2.warpPerspective(im,td['forwardWarp'],(COLS,ROWS))
imWarpedBack = cv2.warpPerspective(imWarped,td['backwardWarp'],(COLS,ROWS))

f = plt.figure(figsize=(30,40))
ax = f.add_subplot(1,3,1)
ax.imshow(im)
ax = f.add_subplot(1,3,2)
ax.imshow(imWarped)
ax = f.add_subplot(1,3,3)
ax.imshow(imWarpedBack)
plt.show()

def findCentroid(im):
    # utility function to find horizontal centroid of distribution

    hist = np.sum(im,axis=0)
    nonzero = np.flatnonzero(hist)
    if len(nonzero) > 0:
        centroid = int(nonzero.mean())
        found = True
    else:
        centroid = int(0.5 * im.shape[1])
        found = False

    return found, centroid

    # define speed estimation routine
# watches for lane markers to pass and counts frames since last one

def estimateSpeed(roi):
    # check for lane marker
    hist = np.sum(roi,axis=1)
    nonzero = np.flatnonzero(hist)
    if len(nonzero) == len(hist):
        if rightLane.framesSeenSinceMarker > FRAME_MARKER_THRESHOLD:
            # we found a new lane marker
            metersTraveled = METERS_PER_MARKER
            secondsTaken = rightLane.framesSeenSinceMarker / float(FPS)
            rightLane.estimatedSpeed = (metersTraveled / secondsTaken) * (0.001) * 3600
            rightLane.framesSeenSinceMarker = 0

            # define main routine to fit lane lines for left and right

def findAndFit(im,yVals):

    # set up image for drawing
    imFit = np.copy(im)
    imFit = cv2.cvtColor(imFit,cv2.COLOR_GRAY2BGR)
    proposedFits = []

    # increment frame count
    rightLane.framesSeenSinceMarker += 1

    for lane in ([leftLane,rightLane]):
        xCentroids = []
        if lane.goodFitLastFrame:
            # case where we got a good fit last frame
            fit = np.poly1d(lane.recentFits[0])
            for i in range(len(yVals)):
                y = yVals[i]
                currentXValue = int(fit(y))
                xLeft,xRight = currentXValue - WIN_WIDTH, currentXValue + WIN_WIDTH
                yBottom,yTop = y, y - WIN_HEIGHT
                roi = im[yTop:yBottom,xLeft:xRight]
                found, centroid = findCentroid(roi)
                xCentroids.append(centroid + xLeft)
                cv2.rectangle(imFit,(xLeft,yTop),(xRight,yBottom),GREEN_COLOR,2)
                imFit[yTop:yBottom,currentXValue] = RED_COLOR

                # do speed estimation
                if (i == 0) & (lane == rightLane):
                    estimateSpeed(roi)

        else:
            # case where we didn't get a good fit last frame
            # start by finding x position of lane at bottom of image
            if lane.side == 'left':
                xLeft,xRight = 0,int(np.floor(COLS/2))
            elif lane.side == 'right':
                xLeft,xRight = int(np.ceil(COLS/2)),COLS
            roi = im[int(ROWS/2):ROWS,xLeft:xRight]
            found, centroid = findCentroid(roi)
            if found:
                currentXValue = centroid + xLeft
            else:
                # worst case: can't find it, so go with mid-point
                currentXValue = int(xLeft + 0.25 * ROWS)
            for i in range(len(yVals)):
                y = yVals[i]
                xLeft, xRight = currentXValue - WIN_WIDTH, currentXValue + WIN_WIDTH
                yBottom, yTop = y, y-WIN_HEIGHT
                roi = im[yTop:yBottom,xLeft:xRight]
                found, centroid = findCentroid(roi)
                if found:
                    currentXValue = centroid + xLeft
                xCentroids.append(currentXValue)
                cv2.rectangle(imFit,(xLeft,yTop),(xRight,yBottom),GREEN_COLOR,2)
                imFit[yTop:yBottom,currentXValue] = RED_COLOR

        # do polynomial fit
        xCentroidsArray = np.array(xCentroids)
        proposedFit = np.polyfit(yVals,xCentroidsArray,2)     # 2nd-order polynomial fit
        proposedFits.append(proposedFit)

    # do sanity check
    fitLeft = np.poly1d(proposedFits[0])
    fitRight = np.poly1d(proposedFits[1])
    xLeft = fitLeft(ROWS)
    xRight = fitRight(ROWS)
    if (np.absolute(xRight - xLeft - LANE_WIDTH_PIXELS) < WIN_WIDTH):
        leftLane.goodFitLastFrame = True
        rightLane.goodFitLastFrame = True
        leftLane.recentFits = np.roll(leftLane.recentFits,1,axis=0)
        rightLane.recentFits = np.roll(rightLane.recentFits,1,axis=0)
        leftLane.recentFits[0] = proposedFits[0]
        rightLane.recentFits[0] = proposedFits[1]

    return imFit

    # define function to draw lane on image

def drawLane(im):

    # get pixels in lane from fits (in warped space)
    yPixelVals = range(ROWS)

    if TEST_MODE:
        # don't average
        leftFit = leftLane.recentFits[0]
        rightFit = rightLane.recentFits[0]
    else:
        # video mode; do average
        leftFit = np.mean(leftLane.recentFits,axis=0)
        rightFit = np.mean(rightLane.recentFits,axis=0)

    fitLeft = np.poly1d(leftFit)
    fitRight = np.poly1d(rightFit)
    xPixelValsLeft = fitLeft(yPixelVals)
    xPixelValsRight = fitRight(yPixelVals)

    pts = zip(yPixelVals,xPixelValsLeft,xPixelValsRight)

    # draw lane on warped image
    imLaneWarped = np.zeros((ROWS,COLS,3),np.uint8)
    for p in pts:
        imLaneWarped[p[0],int(p[1]):int(p[2]),:] = GREEN_COLOR

    # warp image back and combine with original image
    imLane = cv2.warpPerspective(imLaneWarped,inversePerspectiveTrans,(COLS,ROWS))
    imLane = cv2.addWeighted(im,1,imLane,0.5,0)

    # calculate curvature
    curveLeft = leftLane.calculateCurvature(leftFit)
    curveRight = rightLane.calculateCurvature(rightFit)
    curveMean = 0.5 * (curveLeft + curveRight)
    curveStr = 'Lane curvature: {:06.2f} m'.format(curveMean)
    cv2.putText(imLane,curveStr,(100,90),cv2.FONT_HERSHEY_SIMPLEX,1.5,(255,255,255),thickness=2)

    # calculate lane center
    centerOffset = 0.5*(COLS-fitLeft(ROWS)-fitRight(ROWS)) * XM_PER_PIX
    centerStr = 'Lane offset: {:3.2f} m'.format(centerOffset)
    cv2.putText(imLane,centerStr,(100,140),cv2.FONT_HERSHEY_SIMPLEX,1.5,(255,255,255),thickness=2)

    # write estimated speed on image
    speedStr = 'Est. speed: {:3.2f} km/h'.format(rightLane.estimatedSpeed)
    cv2.putText(imLane,speedStr,(100,190),cv2.FONT_HERSHEY_SIMPLEX,1.5,(255,255,255),thickness=2)

    return imLane

    # define main routine for lane finding

def findLane(im):

    # read file and undistort
    imUndistorted = cv2.undistort(im,mtx,dst)

    # find lane pixels
    imBinary, lBinary, bBinary = selectLanePixels(imUndistorted)

    # transform perspective
    imWarped = cv2.warpPerspective(imBinary,perspectiveTrans,(COLS,ROWS))

    # find lanes and fit to polynomial
    yVals = range(ROWS,0,-WIN_HEIGHT)
    imFit = findAndFit(imWarped,yVals)

    # draw lane on image
    imLane = drawLane(imUndistorted)

    # write sliding window inset onto image
    imFitSmall = cv2.resize(imFit,(INSET_COLS,INSET_ROWS))
    imLane[32:32+INSET_ROWS,COLS-INSET_COLS-1:-1] = imFitSmall

    if RECORD:
        # save images at various processing stages
        cv2.imwrite(os.path.join(OUTPUT_DIR_UNDISTORT,imageFile),imUndistorted)
        cv2.imwrite(os.path.join(OUTPUT_DIR_MARKER_PIXELS,imageFile),imBinary)
        cv2.imwrite(os.path.join(OUTPUT_DIR_WARPED,imageFile),imWarped)
        cv2.imwrite(os.path.join(OUTPUT_DIR_SLIDING_WINDOW,imageFile),imFit)
        cv2.imwrite(os.path.join(OUTPUT_DIR_LANE_MARK,imageFile),imLane)

    return imLane

    # do video processing

# read in camera calibration or perform calibration if pickled file is unavailable
try:
    with open(CALIBRATION_FILENAME,'rb') as f:
        calibrationData = pickle.load(f)
        mtx = calibrationData['cam_matrix']
        dst = calibrationData['dist_coeffs']
        print("Read camera calibration from file...")
except:
    print("Performing camera calibration...")
    mtx,dst = calibrateCamera()

# read in perspective transform or perform transform matrix calculation if pickled file is unavailable
try:
    with open(PERSPECTIVE_FILENAME,'rb') as f:
        transformDict = pickle.load(f)
        perspectiveTrans = transformDict['forwardWarp']
        inversePerspectiveTrans = transformDict['backwardWarp']
    print("Read perspective transform from file...")
except:
    print("Performing perspective transform...")
    transformDict = calculatePerspectiveTransform()
    perspectiveTrans = transformDict['forwardWarp']
    inversePerspectiveTrans = transformDict['backwardWarp']

# set up lane line objects
leftLane = LaneLine('left')
rightLane = LaneLine('right')

# if we're in test mode, analyze test images
if TEST_MODE:
    for imageFile in os.listdir(TEST_DIR):
        if imageFile.endswith('.jpg') and 'straight' not in imageFile:
            print('Loading image {0}...'.format(imageFile))
            fn = os.path.join(TEST_DIR,imageFile)
            im = cv2.imread(fn)
            im = cv2.cvtColor(im,cv2.COLOR_BGR2RGB)
            findLane(im)

else:
    # we're in video mode
    video_output = 'output_video/video_output.mp4'
    clip = VideoFileClip('./project_video.mp4')
    FPS = clip.fps
    clip_out = clip.fl_image(findLane)
    clip_out.write_videofile(video_output, audio=False)


#!/usr/bin/python

__author__="marcus"
__date__ ="$Aug 20, 2011 12:14:18 PM$"

import web
import StringIO
from string import split
from os.path import basename
import eigenfaces
import time
import os

urls = (
    '/(.*)', 'index'
)

web.config.debug = True    # Debug true causa problema quando usado junto com sessoes.

gallery_dir = os.environ['HOME'] + "/robot_/gallery/"
erase_cache_file = True
recurso_nome = "FaceRecognition"
recurso = { \
    "status" : "ready", \
    "local_face" : "", \
    "threshold" : "", \
    "id" : 0 \
    }

class index:

    # ========== Inicializacao unica por classe =================
    # Variaveis declaradas aqui sao usadas na classe usando self.

#    testimg = ""
#    imgsdir = ""
#    threshold = 3.0
#    egfnum = 6
    # ============ Final da Inicializacao unica por classe ======

    def __init__(self):
        global erase_cache_file
        # Apagando o arquivo na primeira execucao.
        if erase_cache_file == True:
            cachefile=os.path.join(gallery_dir,"saveddata.cache")
            try:
                os.remove(cachefile)
            except:
                erase_cache_file = False
        erase_cache_file = False

    def GET(self, name):
        web.header('Content-Type', 'application/xml; charset=utf-8')
        global recurso

        return self.to_xml(recurso)

    def POST(self, name):
        t0 = int(time.time() * 1000)

        global recurso
        # Atualizando recurso com dados do post.
        self.from_xml(web.data())

        # ToDo: Sinalizador. Permite apenas uma transferencia em andamento por vez.
#       # Processamento
        result = self.recognize(recurso.copy())

        web.debug("POST time: " + str(int(time.time() * 1000) - t0) + "ms")

        self.reset_resource()

        return result

    def reset_resource(self):
        global recurso
        recurso['status'] = 'ready'
        recurso['local_face'] = ""
        recurso['threshold'] = ""
        recurso['id'] = 0

    def recognize(self, resource):
        # ToDo: Validacao dos parametros.

        global gallery_dir

        self.testimg = resource['local_face']
        self.imgsdir = gallery_dir
        self.threshold = float(resource['threshold'])
#        self.egfnum = int(resource['engenfaces'])
        parts = split(basename(self.testimg),'.')
        extn = parts[len(parts) - 1]
        web.debug("to match:" + str(self.testimg) + " to all " \
            + str(extn) + " images in directory:" + self.imgsdir)
        self.facet = eigenfaces.FaceRec()
#        self.egfnum = self.set_selected_eigenfaces_count(self.egfnum,extn)
        self.imgnamelist=self.facet.parsefolder(self.imgsdir,extn)
#        web.debug("number of eigenfaces used:" + str(self.egfnum))
#        self.facet.checkCache(self.imgsdir,extn,self.imgnamelist,self.egfnum,self.threshold)
        self.facet.checkCache(self.imgsdir,extn,self.imgnamelist,self.threshold)
#        web.debug("number of eigenfaces used:" + str(self.facet.eigenfaces_calc))
#        mindist,matchfile = self.facet.findmatchingimage(self.testimg,self.egfnum,self.threshold)
        mindist,matchfile = self.facet.findmatchingimage(self.testimg,self.threshold)
        if mindist < 1e-10:
            mindist=0
        if not matchfile:
            result = "<matches></matches><dist></dist>"
        else:
            result = "<matches>"+basename(matchfile)+"</matches><dist>"+str(mindist)+"</dist>"

        web.debug(result)
        return '<result>' + result + '</result>'

#    def set_selected_eigenfaces_count(self,selected_eigenfaces_count,ext):
#        #call eigenfaces.parsefolder() and get imagenamelist
#        self.imgnamelist=self.facet.parsefolder(self.imgsdir,ext)
#        numimgs=len(self.imgnamelist)
#        if(selected_eigenfaces_count >= numimgs  or selected_eigenfaces_count == 0):
#            selected_eigenfaces_count=numimgs/2
#        return selected_eigenfaces_count

    def from_xml(self, xml):
        #print xml
        # ToDo: Remover tudo que nao estiver dentro da tag do recurso
        global recurso
        recurso['status'] = 'reading xml data'
        for item in recurso.items():
            key, value = item
            # Ignorando atributos 'status' e 'id'.
            if key != 'status' and key != 'id':
                inicio = xml.find("<" + key + ">")
                if inicio >= 0:
                    final = xml.find("</" + key + ">")
                    if inicio >= 0 and final > inicio:
                        stringXml = xml[inicio + len(key) + 2 : final]
                        if len(stringXml) > 0: # Ignorando chaves vazias.
                            # Diferenciando valor (string ou int)
                            if isinstance(recurso[key], str):
                                recurso[key] = stringXml
                            else:
                                recurso[key] = int(stringXml)

#                            web.debug("key:value = " + key + " : " + str(recurso[key]))

    def to_xml(self, resource):
        """ Transform a dict into a XML, writing to a stream """
        stream = StringIO.StringIO()
#        stream.write("<?xml version='1.0' encoding='UTF-8'?>")
        stream.write('<%s>' %  recurso_nome)

        for item in resource.items():
            key, value = item
            if isinstance(value, str) or isinstance(value, unicode):
                stream.write('\n<%s>%s</%s>' % (key, value, key))
            else:
                stream.write('\n<%s>%d</%s>' % (key, value, key))

        stream.write('\n</%s>' % recurso_nome)
        stream.seek(0)
        return stream.read()

app = web.application(urls, globals())

if __name__ == "__main__":
    app.run()


def mk_fib(n):
    fib = [0, 1]

    for i in range(n - 2):
        fib.append(fib[-1] + fib[-2])

    return fib

r1 = mk_fib(10)
print(r1)
a = int(input("长度: "))
r2 = mk_fib(a)
print(r2)


import os, sys

# Configuration
BUILD_DIR        = 'build'
PROJECT_DIR      = 'src'
PLATFORM_TOOLSET = 'v140'
F4SE_REVISION    = 'tags/v0.6.18'

# Run build tools
buildOK = os.system('python tools/build-tools/build_plugin.py "{}" "{}" "{}" "{}"'
    .format(BUILD_DIR, PROJECT_DIR, PLATFORM_TOOLSET, F4SE_REVISION))

# Report result
sys.exit(buildOK)


from scrapytest.tests import Equal, Required
from scrapytest.spec import StatsSpec
from tests.spiders import *

"""
Stats tests are defined here for testing stats of finished crawl
"""

class TestStats(StatsSpec):
    spider_cls = TestHackernewsSpider
    validate = {
        **StatsSpec.validate,  # include defaults
        # should be as many results as there are urls scheduled
        'item_scraped_count': Equal(len(TestHackernewsSpider.test_urls)),
    }

class TestStats2(StatsSpec):
    # test just defaults for this one
    spider_cls = TestHackernewsSpider2
    validate = {
        'item_scraped_count': (Equal("I'm here!"), Equal('meow')),
        # 'some_missing_stat': Equal("I'm here!"),
    }
    required = [
        'some_missing_stat',
    ]


#
# @lc app=leetcode.cn id=1482 lang=python3
#
# [1482] 制作 m 束花所需的最少天数
#

# @lc code=start
class Solution:
    def minDays(self, bloomDay, m: int, k: int) -> int:
        def condition(threshold):
            bouquet = 0
            flowers = 0
            for day in bloomDay:
                if day <= threshold:
                    bouquet += (flowers + 1) // k
                    flowers = (flowers + 1) % k
                else:
                    flowers = 0
            return bouquet >= m

        if m *k > len(bloomDay):
            return -1
        left, right = min(bloomDay), max(bloomDay)
        while left < right:
            mid = left + (right - left)//2
            if condition(mid):
                right = mid
            else:
                left = mid + 1
        return left

# @lc code=end

if __name__ == '__main__':
    solution = Solution()
    ans = solution.minDays([1,10,3,10,2], 3, 1)
    print(ans)

1482.制作-m-束花所需的最少天数.py


class Solution:
    grid: List[List[str]]

    def dfs(self, i: int, j: int):
        if i < 0 or i >= len(self.grid) or j < 0 or j >= len(self.grid[0]) or self.grid[i][j] == '0':
            return
        self.grid[i][j] = '0'
        self.dfs(i + 1, j)
        self.dfs(i - 1, j)
        self.dfs(i, j + 1)
        self.dfs(i, j - 1)

    def numIslands(self, grid: List[List[str]]) -> int:
        self.grid = grid
        ret = 0
        for i in range(len(grid)):
            for j in range(len(grid[0])):
                if grid[i][j] == '1':
                    self.dfs(i, j)
                    ret += 1
        return ret

Graph/[Leet] 200_Number of Islands/200_taeyoung.py


'''
@author: Liu Fan
'''
import tensorflow as tf
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL']='0'
from tensorflow.python.ops.sparse_ops import KeywordRequired
from utility.helper import *
from utility.batch_test import *

class IMP_GCN(object):
    def __init__(self, data_config, pretrain_data):
        # argument settings
        self.model_type = 'IMP_GCN'
        self.adj_type = args.adj_type
        self.alg_type = args.alg_type
        self.group = args.groups

        self.pretrain_data = pretrain_data

        self.n_users = data_config['n_users']
        self.n_items = data_config['n_items']

        self.n_fold = 20

        self.norm_adj = data_config['norm_adj']
        self.n_nonzero_elems = self.norm_adj.count_nonzero()

        self.lr = args.lr

        self.emb_dim = args.embed_size
        self.batch_size = args.batch_size

        self.dropout = args.mlp_dropout

        self.weight_size = eval(args.layer_size)
        self.n_layers = len(self.weight_size)

        self.model_type += '_%s_%s_l%d' % (self.adj_type, self.alg_type, self.n_layers)

        self.regs = eval(args.regs)
        self.decay = self.regs[0]

        self.verbose = args.verbose

        '''
        *********************************************************
        Create Placeholder for Input Data & Dropout.
        '''
        # placeholder definition
        self.users = tf.placeholder(tf.int32, shape=(None,))
        self.pos_items = tf.placeholder(tf.int32, shape=(None,))
        self.neg_items = tf.placeholder(tf.int32, shape=(None,))

        # dropout: node dropout (adopted on the ego-networks);
        #          ... since the usage of node dropout have higher computational cost,
        #          ... please use the 'node_dropout_flag' to indicate whether use such technique.
        #          message dropout (adopted on the convolution operations).
        self.node_dropout_flag = args.node_dropout_flag
        self.node_dropout = tf.placeholder(tf.float32, shape=[None])
        self.mess_dropout = tf.placeholder(tf.float32, shape=[None])
        """
        *********************************************************
        Create Model Parameters (i.e., Initialize Weights).
        """
        # initialization of model parameters
        self.weights = self._init_weights()

        self.ua_embeddings, self.ia_embeddings, self.A_fold_hat_group_filter, self.user_group_embeddings = self._create_imp_gcn_embed()
        """
        *********************************************************
        Establish the final representations for user-item pairs in batch.
        """
        self.u_g_embeddings = tf.nn.embedding_lookup(self.ua_embeddings, self.users)
        self.pos_i_g_embeddings = tf.nn.embedding_lookup(self.ia_embeddings, self.pos_items)
        self.neg_i_g_embeddings = tf.nn.embedding_lookup(self.ia_embeddings, self.neg_items)
        self.u_g_embeddings_pre = tf.nn.embedding_lookup(self.weights['user_embedding'], self.users)
        self.pos_i_g_embeddings_pre = tf.nn.embedding_lookup(self.weights['item_embedding'], self.pos_items)
        self.neg_i_g_embeddings_pre = tf.nn.embedding_lookup(self.weights['item_embedding'], self.neg_items)
        """
        *********************************************************
        Inference for the testing phase.
        """
        self.batch_ratings = tf.matmul(self.u_g_embeddings, self.pos_i_g_embeddings, transpose_a=False, transpose_b=True)

        """
        *********************************************************
        Generate Predictions & Optimize via BPR loss.
        """
        self.mf_loss, self.emb_loss, self.reg_loss = self.create_bpr_loss(self.u_g_embeddings,
                                                                          self.pos_i_g_embeddings,
                                                                          self.neg_i_g_embeddings)
        self.loss = self.mf_loss + self.emb_loss + self.reg_loss

        self.opt = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)

    def _init_weights(self):
        all_weights = dict()
        initializer = tf.contrib.layers.xavier_initializer()

        if self.pretrain_data is None:
            all_weights['user_embedding'] = tf.Variable(initializer([self.n_users, self.emb_dim]),
                                                        name='user_embedding')
            all_weights['item_embedding'] = tf.Variable(initializer([self.n_items, self.emb_dim]),
                                                        name='item_embedding')
            print('using xavier initialization')
        else:
            all_weights['user_embedding'] = tf.Variable(initial_value=self.pretrain_data['user_embeded'], trainable=True,
                                                        name='user_embedding', dtype=tf.float32)
            all_weights['item_embedding'] = tf.Variable(initial_value=self.pretrain_data['item_embeded'], trainable=True,
                                                        name='item_embedding', dtype=tf.float32)
            print('using pretrained initialization')

        all_weights['W_gc_1'] = tf.Variable(initializer([self.emb_dim, self.emb_dim]), name='W_gc_1')
        all_weights['b_gc_1'] = tf.Variable(initializer([1, self.emb_dim]), name='b_gc_1')

        all_weights['W_gc_2'] = tf.Variable(initializer([self.emb_dim, self.emb_dim]), name='W_gc_2')
        all_weights['b_gc_2'] = tf.Variable(initializer([1, self.emb_dim]), name='b_gc_2')

        all_weights['W_gc'] = tf.Variable(initializer([self.emb_dim, self.group]), name='W_gc')
        all_weights['b_gc'] = tf.Variable(initializer([1, self.group]), name='b_gc')

        self.weight_size_list = [self.emb_dim] + self.weight_size

        return all_weights

    def _split_A_hat(self, X):
        A_fold_hat = []
        fold_len = (self.n_users + self.n_items) // self.n_fold
        for i_fold in range(self.n_fold):
            start = i_fold * fold_len
            if i_fold == self.n_fold -1:
                end = self.n_users + self.n_items
            else:
                end = (i_fold + 1) * fold_len
            A_fold_hat.append(self._convert_sp_mat_to_sp_tensor(X[start:end]))
        return A_fold_hat

    def _split_A_hat_group(self, X, group_embedding):
        group_embedding = tf.transpose(group_embedding)
        A_fold_hat_group = []
        A_fold_hat_group_filter = []
        A_fold_hat = []

        # split L in fold
        fold_len = (self.n_users + self.n_items) // self.n_fold
        for i_fold in range(self.n_fold):
            start = i_fold * fold_len
            if i_fold == self.n_fold - 1:
                end = self.n_users + self.n_items
            else:
                end = (i_fold + 1) * fold_len
            A_fold_hat.append(self._convert_sp_mat_to_sp_tensor(X[start:end]))

        # k groups
        for k in range(0, self.group):
            A_fold_item_filter = []
            A_fold_hat_item = []

            # n folds in per group (filter user)
            for i_fold in range(self.n_fold):
                start = i_fold * fold_len
                if i_fold == self.n_fold - 1:
                    end = self.n_users + self.n_items
                else:
                    end = (i_fold + 1) * fold_len

                A_fold_hat_item.append(A_fold_hat[i_fold].__mul__(group_embedding[k]).__mul__(
                    tf.expand_dims(group_embedding[k][start:end], axis=1)))
                item_filter = tf.sparse_reduce_sum(A_fold_hat_item[i_fold], axis=1)
                item_filter = tf.where(item_filter > 0., x=tf.ones_like(item_filter), y=tf.zeros_like(item_filter))
                A_fold_item_filter.append(item_filter)

            A_fold_item = tf.concat(A_fold_item_filter, axis=0)
            A_fold_hat_group_filter.append(A_fold_item)
            A_fold_hat_group.append(A_fold_hat_item)

        return A_fold_hat_group, A_fold_hat_group_filter

    def _split_A_hat_node_dropout(self, X):
        A_fold_hat = []
        fold_len = (self.n_users + self.n_items) // self.n_fold
        for i_fold in range(self.n_fold):
            start = i_fold * fold_len
            if i_fold == self.n_fold - 1:
                end = self.n_users + self.n_items
            else:
                end = (i_fold + 1) * fold_len
            # A_fold_hat.append(self._convert_sp_mat_to_sp_tensor(X[start:end]))
            temp = self._convert_sp_mat_to_sp_tensor(X[start:end])
            n_nonzero_temp = X[start:end].count_nonzero()
            A_fold_hat.append(self._dropout_sparse(temp, 1 - self.node_dropout[0], n_nonzero_temp))
        return A_fold_hat

    def _create_imp_gcn_embed(self):
        if self.node_dropout_flag:
            A_fold_hat = self._split_A_hat_node_dropout(self.norm_adj)
        else:
            A_fold_hat = self._split_A_hat(self.norm_adj)

        ego_embeddings = tf.concat([self.weights['user_embedding'], self.weights['item_embedding']], axis=0)

        # group users
        temp_embed = []
        for f in range(self.n_fold):
            temp_embed.append(tf.sparse_tensor_dense_matmul(A_fold_hat[f], ego_embeddings))
        user_group_embeddings_side = tf.concat(temp_embed, 0) + ego_embeddings

        user_group_embeddings_hidden_1 = tf.nn.leaky_relu(tf.matmul(user_group_embeddings_side, self.weights['W_gc_1']) + self.weights['b_gc_1'])

        user_group_embeddings_hidden_d1 = tf.nn.dropout(user_group_embeddings_hidden_1, 0.6)

        #user_group_embeddings_hidden_2 = tf.nn.leaky_relu(tf.matmul(user_group_embeddings_hidden_1, self.weights['W_gc_2']) + self.weights[
        #    'b_gc_2'])

        #user_group_embeddings_hidden_2 = tf.matmul(user_group_embeddings_hidden_d1, self.weights['W_gc_2']) + self.weights[
        #            'b_gc_2']

        #user_group_embeddings_hidden_d2 = tf.nn.dropout(user_group_embeddings_hidden_2, 0.8)

        user_group_embeddings_sum = tf.matmul(user_group_embeddings_hidden_d1, self.weights['W_gc']) + self.weights['b_gc']

        # user 0-1
        a_top, a_top_idx = tf.nn.top_k(user_group_embeddings_sum, 1, sorted=False)
        user_group_embeddings = tf.cast(tf.equal(user_group_embeddings_sum,a_top), dtype=tf.float32)
        u_group_embeddings, i_group_embeddings = tf.split(user_group_embeddings, [self.n_users, self.n_items], 0)
        i_group_embeddings = tf.ones(tf.shape(i_group_embeddings))
        user_group_embeddings = tf.concat([u_group_embeddings, i_group_embeddings], axis = 0)
        # Matrix mask
        A_fold_hat_group, A_fold_hat_group_filter = self._split_A_hat_group(self.norm_adj, user_group_embeddings)

        # embedding transformation
        all_embeddings = [ego_embeddings]
        temp_embed = []
        for f in range(self.n_fold):
            temp_embed.append(tf.sparse_tensor_dense_matmul(A_fold_hat[f], ego_embeddings))

        side_embeddings = tf.concat(temp_embed, 0)
        all_embeddings += [side_embeddings]

        ego_embeddings_g = []
        for g in range(0,self.group):
            ego_embeddings_g.append(ego_embeddings)

        ego_embeddings_f = []
        for k in range(1, self.n_layers):
            for g in range(0,self.group):
                temp_embed = []
                for f in range(self.n_fold):
                    temp_embed.append(tf.sparse_tensor_dense_matmul(A_fold_hat_group[g][f], ego_embeddings_g[g]))
                side_embeddings = tf.concat(temp_embed, 0)
                ego_embeddings_g[g]=ego_embeddings_g[g] + side_embeddings
                temp_embed = []
                for f in range(self.n_fold):
                    temp_embed.append(tf.sparse_tensor_dense_matmul(A_fold_hat[f], side_embeddings))
                if k == 1:
                    ego_embeddings_f.append(tf.concat(temp_embed, 0))
                else:
                    ego_embeddings_f[g] = tf.concat(temp_embed, 0)
            ego_embeddings = tf.reduce_sum(ego_embeddings_f, axis=0, keepdims=False)
            all_embeddings += [ego_embeddings]
        all_embeddings = tf.stack(all_embeddings, 1)
        all_embeddings = tf.reduce_mean(all_embeddings, axis=1, keepdims=False)
        u_g_embeddings, i_g_embeddings = tf.split(all_embeddings, [self.n_users, self.n_items], 0)
        return u_g_embeddings, i_g_embeddings, A_fold_hat_group_filter, user_group_embeddings_sum

    def _create_lightgcn_embed(self):
        if self.node_dropout_flag:
            A_fold_hat = self._split_A_hat_node_dropout(self.norm_adj)
        else:
            A_fold_hat = self._split_A_hat(self.norm_adj)

        ego_embeddings = tf.concat([self.weights['user_embedding'], self.weights['item_embedding']], axis=0)
        all_embeddings = [ego_embeddings]

        for k in range(0, self.n_layers):

            temp_embed = []
            for f in range(self.n_fold):
                temp_embed.append(tf.sparse_tensor_dense_matmul(A_fold_hat[f], ego_embeddings))
            side_embeddings = tf.concat(temp_embed, 0)
            ego_embeddings = side_embeddings
            all_embeddings += [ego_embeddings]
        all_embeddings=tf.stack(all_embeddings,1)
        all_embeddings=tf.reduce_mean(all_embeddings,axis=1,keepdims=False)
        u_g_embeddings, i_g_embeddings = tf.split(all_embeddings, [self.n_users, self.n_items], 0)
        return u_g_embeddings, i_g_embeddings

    def create_bpr_loss(self, users, pos_items, neg_items):
        pos_scores = tf.reduce_sum(tf.multiply(users, pos_items), axis=1)
        neg_scores = tf.reduce_sum(tf.multiply(users, neg_items), axis=1)

        regularizer = tf.nn.l2_loss(self.u_g_embeddings_pre) + tf.nn.l2_loss(
                self.pos_i_g_embeddings_pre) + tf.nn.l2_loss(self.neg_i_g_embeddings_pre)
        regularizer = regularizer/self.batch_size

        ## In the first version, we implement the bpr loss via the following codes:
        # maxi = tf.log(tf.nn.sigmoid(pos_scores - neg_scores))
        # mf_loss = tf.negative(tf.reduce_mean(maxi))

        # In the second version, we implement the bpr loss via the following codes to aviod 'NAN' loss during training:
        mf_loss = tf.reduce_mean(tf.nn.softplus(-(pos_scores - neg_scores)))

        emb_loss = self.decay * regularizer

        reg_loss = tf.constant(0.0, tf.float32, [1])

        return mf_loss, emb_loss, reg_loss

    def _convert_sp_mat_to_sp_tensor(self, X):
        coo = X.tocoo().astype(np.float32)
        indices = np.mat([coo.row, coo.col]).transpose()
        return tf.SparseTensor(indices, coo.data, coo.shape)

    def _dropout_sparse(self, X, keep_prob, n_nonzero_elems):
        """
        Dropout for sparse tensors.
        """
        noise_shape = [n_nonzero_elems]
        random_tensor = keep_prob
        random_tensor += tf.random_uniform(noise_shape)
        dropout_mask = tf.cast(tf.floor(random_tensor), dtype=tf.bool)
        pre_out = tf.sparse_retain(X, dropout_mask)
        return pre_out * tf.div(1., keep_prob)

def load_pretrained_data():
    #pretrain_path = '%spretrain/%s/%s.npz' % (args.proj_path, args.dataset, 'embedding')
    pretrain_path = args.dataset+'.npz'
    try:
        pretrain_data = np.load(pretrain_path)
        print(pretrain_data)
        print('load the pretrained embeddings.')
    except Exception:
        pretrain_data = None
    return pretrain_data

if __name__ == '__main__':

    os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)

    config = dict()
    config['n_users'] = data_generator.n_users
    config['n_items'] = data_generator.n_items

    """
    *********************************************************
    Generate the Laplacian matrix, where each entry defines the decay factor (e.g., p_ui) between two connected nodes.
    """
    plain_adj, norm_adj, mean_adj,pre_adj = data_generator.get_adj_mat()

    if args.adj_type == 'plain':
        config['norm_adj'] = plain_adj
        print('use the plain adjacency matrix')

    elif args.adj_type == 'norm':
        config['norm_adj'] = norm_adj
        print('use the normalized adjacency matrix')

    elif args.adj_type == 'gcmc':
        config['norm_adj'] = mean_adj
        print('use the gcmc adjacency matrix')

    elif args.adj_type=='pre':
        config['norm_adj']=pre_adj
        print('use the pre adjcency matrix')

    else:
        config['norm_adj'] = mean_adj + sp.eye(mean_adj.shape[0])
        print('use the mean adjacency matrix')

    t0 = time()
    if args.pretrain == -1:
        pretrain_data = load_pretrained_data()
    else:
        pretrain_data = None

    model = IMP_GCN(data_config=config, pretrain_data=pretrain_data)

    """
    *********************************************************
    Save the model parameters.
    """
    saver = tf.train.Saver()

    if args.save_flag == 1:
        layer = '-'.join([str(l) for l in eval(args.layer_size)])
        weights_save_path = '%sweights/%s/%s/%s/l%s_r%s' % (args.weights_path, args.dataset, model.model_type, layer,
                                                            str(args.lr), '-'.join([str(r) for r in eval(args.regs)]))
        ensureDir(weights_save_path)
        save_saver = tf.train.Saver(max_to_keep=1)

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)

    """
    *********************************************************
    Reload the pretrained model parameters.
    """

    sess.run(tf.global_variables_initializer())
    cur_best_pre_0 = 0.

    """
    Train.
    """
    loss_loger, pre_loger, rec_loger, ndcg_loger, hit_loger = [], [], [], [], []
    stopping_step = 0
    should_stop = False

    for epoch in range(args.epoch):
        t1 = time()
        loss, mf_loss, emb_loss, reg_loss = 0., 0., 0., 0.
        n_batch = data_generator.n_train // args.batch_size + 1

        for idx in range(n_batch):
            users, pos_items, neg_items = data_generator.sample()
            _, batch_loss, batch_mf_loss, batch_emb_loss, batch_reg_loss, group_filter, user_group_embeddings = sess.run([model.opt, model.loss, model.mf_loss, model.emb_loss, model.reg_loss, model.A_fold_hat_group_filter, model.user_group_embeddings],
                               feed_dict={model.users: users, model.pos_items: pos_items,
                                          model.node_dropout: eval(args.node_dropout),
                                          model.mess_dropout: eval(args.mess_dropout),
                                          model.neg_items: neg_items})
            loss += batch_loss/n_batch
            mf_loss += batch_mf_loss/n_batch
            emb_loss += batch_emb_loss/n_batch
            reg_loss += batch_reg_loss/n_batch

        if np.isnan(loss) == True:
            print('ERROR: loss is nan.')
            sys.exit()

        # print the test evaluation metrics each 10 epochs; pos:neg = 1:10.
        if (epoch + 1) % 10 != 0:
            if args.verbose > 0 and epoch % args.verbose == 0:
                perf_str = 'Epoch %d [%.1fs]: train==[%.5f=%.5f + %.5f]' % (
                    epoch, time() - t1, loss, mf_loss, reg_loss)
                print(perf_str)
            continue

        t2 = time()
        users_to_test = list(data_generator.test_set.keys())
        ret = test(sess, model, users_to_test, drop_flag=True)

        t3 = time()

        loss_loger.append(loss)
        rec_loger.append(ret['recall'])
        pre_loger.append(ret['precision'])
        ndcg_loger.append(ret['ndcg'])
        hit_loger.append(ret['hit_ratio'])

        if args.verbose > 0:
            perf_str = 'Epoch %d [%.1fs + %.1fs]: train==[%.5f=%.5f + %.5f + %.5f], recall=[%.5f, %.5f], ' \
                       'precision=[%.5f, %.5f], hit=[%.5f, %.5f], ndcg=[%.5f, %.5f]' % \
                       (epoch, t2 - t1, t3 - t2, loss, mf_loss, emb_loss, reg_loss, ret['recall'][0], ret['recall'][-1],
                        ret['precision'][0], ret['precision'][-1], ret['hit_ratio'][0], ret['hit_ratio'][-1],
                        ret['ndcg'][0], ret['ndcg'][-1])
            print(perf_str)

        cur_best_pre_0, stopping_step, should_stop = early_stopping(ret['recall'][0], cur_best_pre_0,
                                                                    stopping_step, expected_order='acc', flag_step=5)

        # *********************************************************
        # early stopping when cur_best_pre_0 is decreasing for ten successive steps.
        if should_stop == True:
            break

        # *********************************************************
        # save the user & item embeddings for pretraining.
        if ret['recall'][0] == cur_best_pre_0 and args.save_flag == 1:
            save_saver.save(sess, weights_save_path + '/weights', global_step=epoch)
            print('save the weights in path: ', weights_save_path)

    recs = np.array(rec_loger)
    pres = np.array(pre_loger)
    ndcgs = np.array(ndcg_loger)
    hit = np.array(hit_loger)

    best_rec_0 = max(recs[:, 0])
    idx = list(recs[:, 0]).index(best_rec_0)

    final_perf = "Best Iter=[%d]@[%.1f]\trecall=[%s], precision=[%s], hit=[%s], ndcg=[%s]" % \
                 (idx, time() - t0, '\t'.join(['%.5f' % r for r in recs[idx]]),
                  '\t'.join(['%.5f' % r for r in pres[idx]]),
                  '\t'.join(['%.5f' % r for r in hit[idx]]),
                  '\t'.join(['%.5f' % r for r in ndcgs[idx]]))
    print(final_perf)

    save_path = '%soutput/%s/%s.result' % (args.proj_path, args.dataset, model.model_type)
    ensureDir(save_path)
    f = open(save_path, 'a')

    f.write(
        'embed_size=%d, lr=%.4f, layer_size=%s, node_dropout=%s, mess_dropout=%s, regs=%s, adj_type=%s\n\t%s\n'
        % (args.embed_size, args.lr, args.layer_size, args.node_dropout, args.mess_dropout, args.regs,
           args.adj_type, final_perf))
    f.close()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
fetched German hospital occupancy data from DIVI https://www.intensivregister.de
lk_id sind https://de.wikipedia.org/wiki/Amtlicher_Gemeindeschl%C3%BCssel
"""

__author__ = "Dr. Torben Menke"
__email__ = "https://entorb.net"
__license__ = "GPL"

# Built-in/Generic Imports
import os
import os.path
import re
import csv
import glob
import requests  # for read_url_or_cachefile
import pandas as pd
import numpy as np
import datetime

# my helper modules
import helper

col_date='Datum'
col_infectionsperweek='CoV-Infektionen pro Woche'
col_infections='Cases'
col_intensiv='Intensiv-CoV-Patienten'
col_beatmet='Beatmete CoV-Patienten'
col_tod='Todesfälle mit CoV pro Woche'
col_erstimpfung='Personen mit Erstimpfung'
col_vollschutz='Personen mit Vollschutz'
col_auffrischung='Personen mit Auffrischung'
col_ratio_intensiv='Verhältnis Intensivpatienten/Infektionen (mit deltaT)'
col_ratio_beatmet='Verhältnis beatmete Patienten/Infektionen (mit deltaT)'
col_ratio_tod='Verhältnis Todesfälle/Infektionen (mit deltaT)'

all_deltat={}

def extractLinkList(cont: str) -> list:
    # myPattern = '<a href="(/divi-intensivregister-tagesreport-archiv-csv/divi-intensivregister-[^"]+/download)"'
    myPattern = r' 10, "Error: no csv download links found"
    return myMatches

# '01' -> 'SH' etc, see https://de.wikipedia.org/wiki/Amtlicher_Gemeindeschl%C3%BCssel -> Aktuelle Länderschlüssel

d_bl_id2code = {'01': 'SH', '02': 'HH', '03': 'NI', '04': 'HB', '05': 'NW', '06': 'HE', '07': 'RP',
                '08': 'BW', '09': 'BY', '10': 'SL', '11': 'BE', '12': 'BB', '13': 'MV', '14': 'SN', '15': 'ST', '16': 'TH', 'DE-total': 'DE-total'}

def fetch_latest_csv():
    """
    fetches the latest (top) file from https://www.divi.de/divi-intensivregister-tagesreport-archiv-csv?layout=table
    output: latest.csv (overwrites old file)
    """

    url = 'https://www.divi.de/divi-intensivregister-tagesreport-archiv-csv?layout=table'
    file = 'cache/de-divi/list-csv-page-1.html'
    payload = {"filter_order_Dir": "DESC",
               "filter_order": "tbl.ordering",
               "start": 0}
    # "cid[]": "0", "category_id": "54", "task": "", "8ba87835776d29f4e379a261512319f1": "1"

    cont = helper.read_url_or_cachefile(
        url=url, cachefile=file, request_type='post', payload=payload, cache_max_age=3600, verbose=True)

    # extract link of from  5:
    #     l_csv_urls.pop()

    d_csvs_in_table = {}

    # loop over urls to replace outdated files by latest file per day
    # '/divi-intensivregister-tagesreport-archiv-csv/divi-intensivregister-2020-06-25-12-15/download'
    # '/divi-intensivregister-tagesreport-archiv-csv/divi-intensivregister-2020-06-25-12-15-2/download'
    for url in l_csv_urls:
        url = f"https://www.divi.de{url}"
        # '/divi-intensivregister-tagesreport-archiv-csv/viewdocument/5330/divi-intensivregister-2020-12-21-12-15'
        filename = re.search(
            r'/divi-intensivregister-tagesreport-archiv-csv/viewdocument/\d+?/divi-intensivregister-(\d{4}\-\d{2}\-\d{2})[^/]', url).group(1)
        d_csvs_in_table[filename] = url
    del l_csv_urls, filename, url

    l = sorted(d_csvs_in_table.keys())
    latest_filename = l[-1]
    latest_url = d_csvs_in_table[latest_filename]
    file = f"data/source/de-divi/latest.csv"

    if not os.path.isfile(file):
        cont = helper.read_url_or_cachefile(
            url=latest_url, cachefile=file, request_type='get', payload={}, cache_max_age=3600, verbose=True)

def generate_database() -> dict:
    """ from 2021-10-29 on Divi publisheds all data in the latest file"""
    d_database = {}
    d_database_states = {'01': {}, '02': {}, '03': {}, '04': {}, '05': {}, '06': {}, '07': {
    }, '08': {}, '09': {}, '10': {}, '11': {}, '12': {}, '13': {}, '14': {}, '15': {}, '16': {}, 'DE-total': {}}
    csv_file = 'data/source/de-divi/latest.csv'

    with open(csv_file, mode='r', encoding='utf-8') as f:
        csv_reader = csv.DictReader(f, delimiter=",")
        for row in csv_reader:
            date = row["date"]
            assert len(row) >= 8, "Error: too few rows found"
            bl_id = row["bundesland"]
            divi_col_beatmet="faelle_covid_aktuell_invasiv_beatmet"
            datetime_date = datetime.datetime.strptime(date, "%Y-%m-%d")
            d = {
                "Date": date,
                "anzahl_meldebereiche": int(row["anzahl_meldebereiche"]),
                "faelle_covid_aktuell": int(row["faelle_covid_aktuell"]),
                "faelle_covid_aktuell_beatmet": int(row[divi_col_beatmet]),
                "anzahl_standorte": int(row["anzahl_standorte"]),
                "betten_frei": int(float(row["betten_frei"])),
                "betten_belegt": int(float(row["betten_belegt"]))
            }
            if "faelle_covid_aktuell_beatmet" in row:
                d["faelle_covid_aktuell_beatmet"] = int(
                    row["faelle_covid_aktuell_beatmet"])
            elif "faelle_covid_aktuell_invasiv_beatmet" in row:
                d["faelle_covid_aktuell_beatmet"] = int(
                    row["faelle_covid_aktuell_invasiv_beatmet"])

            d["betten_ges"] = d["betten_frei"] + d["betten_belegt"]

            # calc de_states_sum
            d2 = dict(d)
            del d2['Date']
            if date not in d_database_states[bl_id]:
                d_database_states[bl_id][date] = d2
            else:
                for k in d2.keys():
                    d_database_states[bl_id][date][k] += d2[k]
            # 'DE-total'
            if date not in d_database_states['DE-total']:
                d_database_states['DE-total'][date] = d2
            else:
                for k in d2.keys():
                    d_database_states['DE-total'][date][k] += d2[k]
            d2['Date']=date
            # print(d_database_states[bl_id][date])

    return d_database_states

def export_csv(d_database):
    for bl_id, l_time_series in d_database.items():
        fileOut = f"data/source/de-divi-bl/{bl_id}"
        with open(fileOut+'.csv', mode='w', encoding='utf-8', newline='\n') as fh:
            csvwriter = csv.DictWriter(fh, delimiter=',', extrasaction='ignore', fieldnames=[
                'Date',
                'betten_ges',
                'betten_belegt',
                'faelle_covid_aktuell',
                'faelle_covid_aktuell_beatmet'])
            csvwriter.writeheader()
            for d in l_time_series:
                csvwriter.writerow(l_time_series[d])
    pass

def df_addcol_peakrefrelatedratio(df,col_a, col_b, col_deltaratio):
  dfrelevant=df[(df[col_date] <= datetime.datetime.strptime("2021-05-01", "%Y-%m-%d"))].copy()
  peakdate_a = dfrelevant[dfrelevant[col_a]==dfrelevant[col_a].max()][col_date].min()
  peakdate_b = dfrelevant[dfrelevant[col_b]==dfrelevant[col_b].max()][col_date].min()
  deltat = peakdate_b - peakdate_a

  if col_b==col_intensiv:
      deltat = datetime.timedelta(days = 10)
  if col_b==col_beatmet:
      deltat = datetime.timedelta(days = 11)
  if col_b==col_tod:
      deltat = datetime.timedelta(days = 15)

  col_refval = f'{col_b} -{deltat}d (peak matched)'
  df_deltat = df[[col_date, col_b]].copy()
  df_deltat[col_date] = df_deltat[col_date]-deltat
  df_deltat = df_deltat.rename(columns={col_b:col_refval})
  df = pd.merge(df, df_deltat, on=col_date, how='left')

  df[col_deltaratio] = df[col_refval] / df[col_a]

  all_deltat[col_b] = deltat.days

  return df

def calc_efficiency(df):
   df = df_addcol_peakrefrelatedratio(df, col_infectionsperweek, col_intensiv, col_ratio_intensiv)
   df = df_addcol_peakrefrelatedratio(df, col_infectionsperweek, col_beatmet, col_ratio_beatmet)
   df = df_addcol_peakrefrelatedratio(df, col_infectionsperweek, col_tod, col_ratio_tod)
   # Pre 2020-04-15 not enough data has been gathered => set to nan
   df[col_ratio_tod][df[col_date] < pd.to_datetime('2020-04-15')] = np.nan

   fileOut = "data/peak_deltat.txt"
   with open(fileOut, mode='w', encoding='utf-8', newline='\n') as fh:
      fh.write(
         f"deltaT: Infektion-Intensiv:{all_deltat[col_intensiv]}d, "
         f"Infektion-Beatmung: {all_deltat[col_beatmet]}d, "
         f"Infektion-Tod: {all_deltat[col_tod]}d"
      )

   return df

def retrieve_ageincidents():

    helper.download_as_browser("https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Projekte_RKI/COVID-19_Todesfaelle.xlsx?__blob=publicationFile", "data/source/COVID-19_Todesfaelle.xlsx")
    helper.download_as_browser("https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Daten/Impfquotenmonitoring.xlsx?__blob=publicationFile", "data/source/Impfquotenmonitoring.xlsx")
    try:
    	helper.download_as_browser("https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Daten/Altersverteilung.xlsx?__blob=publicationFile", "data/source/Altersverteilung.xlsx")
    	df_age_original = pd.read_excel("data/source/Altersverteilung.xlsx", sheet_name="7-Tages-Inzidenz", engine='openpyxl')
    except KeyError:
    	df_age_original = pd.read_excel("data/source/Altersverteilung.xlsx", sheet_name="7-Tage-Inzidenz", engine='openpyxl')
    except ValueError:
    	try:
    	    df_age_original = pd.read_excel("data/source/Altersverteilung.xlsx", sheet_name="7-Tage-Inzidenz", engine='openpyxl')
    	except ValueError:
 	    df_age_original = pd.read_excel("data/source/Altersverteilung.xlsx", sheet_name="7-Tages-Inzidenzen", engine='openpyxl')
    except:
    	helper.download_as_browser("https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/Daten/Altersverteilung.xls?__blob=publicationFile", "data/source/Altersverteilung.xls")
    	df_age_original = pd.read_excel("data/source/Altersverteilung.xls", sheet_name="Fälle", engine='xlrd')

    df_age = df_age_original.transpose()
    df_age = df_age.set_axis(
        [
        "Gesamt",
        "90+",
        "85 - 89",
        "80 - 84",
        "75 - 79",
        "70 - 74",
        "65 - 69",
        "60 - 64",
        "55 - 59",
        "50 - 54",
        "45 - 49",
        "40 - 44",
        "35 - 39",
        "30 - 34",
        "25 - 29",
        "20 - 24",
        "15 - 19",
        "10 - 14",
        "5 - 9",
        "0 - 4"
        ], axis="columns")
    df_age.reset_index(inplace=True)
    df_age = df_age.drop(labels=0, axis=0)
    df_age = df_age.rename(columns = {'index':'Date'})
    df_age["Datum"] = pd.to_datetime(df_age["Date"] + "-1", format="%G_%V-%u")
    df_age.pop("Date")
    df_age_pyramid = pd.read_excel('data/source/Age_structure_germany.xlsx', engine='openpyxl')
    return df_age

df_age = retrieve_ageincidents()
df_age.to_csv('data/df_age.csv', index=False)

fetch_latest_csv()
d_database = generate_database()
export_csv(d_database)

df_divi = pd.read_csv('data/source/de-divi-bl/DE-total.csv')
df_vaccine = pd.read_csv('data/source/all-vaccine-region_DE.csv')

df_vaccine = df_vaccine.rename(columns={"date":'Date'})
df_vaccine_personen_erst = df_vaccine[["Date","personen_erst_kumulativ"]].copy().reset_index(drop=True)
df_vaccine_personen_voll = df_vaccine[["Date","personen_voll_kumulativ"]].copy().reset_index(drop=True)
df_vaccine_personen_auffr = df_vaccine[["Date","personen_auffr_kumulativ"]].copy().reset_index(drop=True)
df_infections = pd.read_csv('data/source/de-state-DE-total-infections.tsv', sep='\t')

df_all = pd.merge(df_infections, df_divi, on='Date', how='left')
df_vaccine_personen_erst = df_vaccine_personen_erst.rename(columns={"personen_erst_kumulativ":col_erstimpfung})
df_all = pd.merge(df_all, df_vaccine_personen_erst, on='Date', how='left')
df_vaccine_personen_voll = df_vaccine_personen_voll.rename(columns={"personen_voll_kumulativ":col_vollschutz})
df_all = pd.merge(df_all, df_vaccine_personen_voll, on='Date', how='left')
df_vaccine_personen_auffr = df_vaccine_personen_auffr.rename(columns={"personen_auffr_kumulativ":col_auffrischung})
df_all = pd.merge(df_all, df_vaccine_personen_auffr, on='Date', how='left')

df_all = df_all.rename(columns={"Date":col_date})
df_all = df_all.rename(columns={"Cases_Last_Week":col_infectionsperweek})
df_all = df_all.rename(columns={"Deaths_Last_Week":col_tod})
df_all = df_all.rename(columns={"faelle_covid_aktuell":col_intensiv})
df_all = df_all.rename(columns={"faelle_covid_aktuell_beatmet":col_beatmet})

df_all[col_date] = pd.to_datetime(df_all[col_date])

df_all = calc_efficiency(df_all)

df_all = df_all[[col_date,
  col_infectionsperweek,
  col_infections,
  col_intensiv,
  col_beatmet,
  col_tod,
  col_erstimpfung,
  col_vollschutz,
  col_auffrischung,
  col_ratio_intensiv,
  col_ratio_beatmet,
  col_ratio_tod]]

df_all = df_all.sort_values(by = col_date)

df_all.to_csv('data/df_all.csv', index=False)

df_min_max_scaled = df_all.copy()
# apply normalization techniques by Column 1
for column in [col_infectionsperweek,
  col_infections,
  col_intensiv,
  col_beatmet,
  col_tod,
  col_erstimpfung,
  col_vollschutz,
  col_auffrischung]:
    df_min_max_scaled[column] = (df_min_max_scaled[column] - df_min_max_scaled[column].min()) / (df_min_max_scaled[column].max() - df_min_max_scaled[column].min())

df_min_max_scaled = calc_efficiency(df_min_max_scaled)

for column in [col_ratio_intensiv,
  col_ratio_beatmet,
  col_ratio_tod]:
    df_min_max_scaled[column] = (df_min_max_scaled[column] - df_min_max_scaled[column].min()) / (df_min_max_scaled[column].max() - df_min_max_scaled[column].min())

df_min_max_scaled.to_csv('data/df_all_min_max_scaled.csv', index=False)

pass


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb 24 03:23:30 2020

@author: angellica
"""

from sklearn.model_selection import train_test_split

def train_dummy(dummy, features, target_feature, df):
    input_x = df[features]
    input_y = df[target_feature]

    X_train, X_test, y_train, y_test = train_test_split(input_x, input_y, test_size=0.033, random_state = 0, stratify = input_y, shuffle = True)

    # First round: running dummy classifier for non injected features
    y_score = dummy.fit(X_train, y_train).predict_proba(X_test)
    y_pred = dummy.predict(X_test)

    return dummy, y_score, y_pred, y_train, X_train, X_test, y_train, y_test

def split_test_train(features, target_feature, df):
    input_x = df[features]
    input_y = df[target_feature]

    X_train, X_test, y_train, y_test = train_test_split(input_x, input_y, test_size=0.033, random_state = 0, stratify = input_y, shuffle = True)

    return X_train, X_test, y_train, y_test


import unittest
from Advisor import Advisor
from Student import Student

class StudentTest(unittest.TestCase):
    def testConstruction(self):
        sampleAdvisor  = Advisor("109", "Paul", "Walker", "deneme@gmail.com", "M2-9")
        sampleStudent  = Student("150120000", "Jonny", "Deep", 4, sampleAdvisor )

        expectedAdvisorName  = "Paul"
        AdvisorName  = sampleStudent.getAdvisor().getFirstName()

        self.assertEqual(expectedAdvisorName , AdvisorName)

if __name__ == '__main__':
    unittest.main()


#!/usr/local/bin/python3

from sys import platform as _platform
from ctypes import cdll
from ctypes.util import find_library

lib_m = cdll.LoadLibrary(find_library('m'))

if _platform == "darwin":  # macOS
    ROUND_DOWN = 0x400
    ROUND_UP = 0x800
    ROUND_NEAR = 0

else:
    raise ImportError("Specify rounding mode constants for you platform")

def setRoundDown():
    lib_m.fesetround(ROUND_DOWN)

def setRoundUp():
    lib_m.fesetround(ROUND_UP)

def setRoundNear():
    lib_m.fesetround(ROUND_NEAR)


from django.shortcuts import get_object_or_404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from .models import Topup
from .serializers import TopupsSerializer
from vouchers.models import Voucher
from users.models import User
from topups.serializers import TopupsSerializer

class TopupsList(APIView):
    def get(self, request):
        topups = Topup.objects.all()
        serializer = TopupsSerializer(topups, many=True)
        return Response(serializer.data)

    def post(self, request):
        print(request.data['voucher_code'])
        voucher_code = request.data['voucher_code']
        user_id = request.data['user_id']

        try:
            voucher = Voucher.objects.get(voucher_code=voucher_code, is_active=1)
        except Voucher.DoesNotExist:
            response_data = {}
            response_data['error'] = "Error"
            response_data['message'] = "Incorrect Voucher Entered"
            return Response(response_data, status=status.HTTP_404_NOT_FOUND)

        print(voucher.amount)
        amount_purchased = voucher.amount

        try:
            user = User.objects.get(id=user_id)
        except User.DoesNotExist:
            response_data = {}
            response_data['error'] = "Error"
            response_data['message'] = "User doesn't exist in database"
            return Response(response_data, status=status.HTTP_404_NOT_FOUND)

        user.amount +=amount_purchased
        user.save()

        voucher.is_active = 0
        voucher.save()

        topup_data = {}
        topup_data['user_id'] = user_id
        topup_data['amount'] = amount_purchased
        topup_data['payment_type'] = "VCH"
        topup_data['voucher_id'] = voucher.id

        topup_serializer = TopupsSerializer(data=topup_data)
        if topup_serializer.is_valid():
            topup_serializer.save()
            return Response(topup_serializer.data, status=status.HTTP_201_CREATED)
        return Response(topup_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        print(topup_data)

        response_data = {}
        response_data['success'] = "Success"
        response_data['message'] = "Topup Successful"
        return Response(response_data, status=status.HTTP_200_OK)


import os.path

from records import Record
from tabulate import tabulate

class Directory:
    def __init__(self):
        """
        Конструктор для создания справочника.
        """
        self.records = []
        self.next_pk = 1

    def add_record(self, record: Record):
        """
        Добавление записи в справочник.
        :param record: Запись
        """
        while any(r.pk == self.next_pk for r in self.records):
            self.next_pk += 1
        record.pk = self.next_pk
        self.records.append(record)
        self.next_pk += 1

    def edit_record(self, pk, **kwargs):
        """
        Редактирование существующей записи.
        :param pk: Первичный ключ записи
        :param kwargs: Словарь с новыми значениями атрибутов
        """
        record = self.get_record_by_pk(pk)
        if record:
            record.edit(**kwargs)
        else:
            print("Запись с указанным PK не найдена.")

    def get_record_by_pk(self, pk):
        """
        Получение записи по её PK.
        :param pk: Первичный ключ записи
        :return: Объект записи или None, если запись не найдена
        """
        for record in self.records:
            if record.pk == pk:
                return record
        return None

    def search_records(self, **kwargs):
        """
        Поиск записей по характеристикам.
        :param kwargs: Словарь с характеристиками для поиска
        :return: Список кортежей (PK, запись) найденных записей
        """
        matching_records = []
        for record in self.records:
            match = False
            for key, value in kwargs.items():
                record_value = getattr(record, key, None)
                if record_value and value.lower() in record_value.lower():
                    match = True
                    break
            if match:
                matching_records.append((record.pk, record))
        return matching_records

    def print_page(self, page_number, page_size=10):
        """
        Вывод страницы с записями на экран.
        :param page_number: Номер страницы
        :param page_size: Количество записей на странице
        """
        start_index = (page_number - 1) * page_size
        end_index = start_index + page_size
        records_to_display = self.records[start_index:end_index]

        table_data = []
        for record in records_to_display:
            table_data.append([record.pk, f"{record.surname} {record.name} {record.patronymic}",
                               record.organization, record.work_phone, record.cell_phone])

        headers = ["PK", "ФИО", "Организация", "Рабочий телефон", "Сотовый телефон"]

        print(tabulate(table_data, headers=headers, tablefmt="grid"))

    def save_to_file(self, filename='yellow_pages.txt', ):
        """
        Сохранение данных справочника в текстовый файл.
        :param filename: Имя файла, по умолчанию в корень папки с программой в ффайл yellow_pages.txt
        """
        with open(filename, "w", encoding="utf-8") as file:
            for record in self.records:
                file.write(f"{record.pk}; {record.surname}; {record.name}; {record.patronymic}; {record.organization}; {record.work_phone}; {record.cell_phone}\n")

    @classmethod
    def load_from_file(cls, filename='yellow_pages.txt'):
        """
        Загрузка данных справочника из текстового файла.
        :param filename: Имя файла
        :return: Экземпляр справочника с загруженными данными
        """
        directory = cls()
        if os.path.exists(filename):
            with open(filename, "r", encoding="utf-8") as file:
                for line in file:
                    pk, surname, name, patronymic, organization, work_phone, cell_phone = line.strip().split("; ")
                    record = Record(surname, name, patronymic, organization, work_phone, cell_phone)
                    directory.add_record(record)
        return directory