# https://music.douban.com/tag/
# 5-6个标签
# 歌曲名 歌手 日期 评分 评价人数
import time

import requests, csv
from bs4 import BeautifulSoup
from pyecharts.charts import Pie, Bar, Scatter3D, Scatter, Line
from pyecharts import options as opts

tags = ['OST', '民谣', '流行', 'pop', 'indie', 'Electronic', 'Folk', '摇滚', 'J-POP', 'rock', '电影原声', 'R&B','中国摇滚','纯音乐'][:9]
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE'
}
url_demo = 'https://music.douban.com/tag/{}?start={}&type=T'

# 获取html源代码
def get_html(url):
    r = requests.get(url, headers=header)
    html = r.text
    return html

# 找class=pl2下的a的内容
def get_song_name(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    a = div.find('a')
    if a:
        return a.text.replace(' ', '').replace('\n', '')
    return '空标题'

# 找class=pl2下的p的内容/分割第一个
def get_song_author(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    p = div.find('p')
    if p:
        texts = p.text.split('/')
        if len(texts) != 0:
            return texts[0].strip()
    return '空作者'

# 找class=pl2下p的/分割第2个
def get_song_date(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    p = div.find('p')
    if p:
        texts = p.text.split('/')
        if len(texts) >= 2:
            return texts[1].strip()
    return '空时间'

# 找class=rating_nums 的span的内容
def get_song_rank(soup):
    span = soup.find('span', attrs={'class': "rating_nums"})
    if span:
        return span.text
    return '空评分'

# 找class=pl2下的class=pl的span
def get_song_comment_count(soup):
    div = soup.find('div', attrs={'class': "pl2"})
    if div:
        span = div.find('span', attrs={'class': 'pl'})
        if span:
            # 替换无效的字符
            return span.text.replace(' ', '').replace('(', '').replace(')', '').replace('\n', '')
    return '空评论人数'

# 根据偏移量获取数据
def get_songs_by_offset(tag, page):
    songs = []
    # 设置完整url
    url = url_demo.format(tag, page)
    print(url)
    # 获取html源码
    html = get_html(url)
    soup = BeautifulSoup(html, 'lxml')
    # 找class=item 的tr数组
    items = soup.find_all('tr', attrs={'class': 'item'})
    # 没找到返回空
    if len(items) == 0:
        return songs
    # 遍历每一个tr
    for item in items:
        # 传tr，让函数获取其中的内容
        name = get_song_name(item)
        author = get_song_author(item)
        date = get_song_date(item)
        rank = get_song_rank(item)
        comment_count = get_song_comment_count(item)
        # 组合成一个数据
        song = (name, author, date, rank, comment_count)
        print(song)
        # 添加每一条，到一页数据的数组
        songs.append(song)
    return songs

# 根据tag采集内容
def get_songs_by_tag(tag):
    offset = 0 #初始偏移量0
    page = 1 #初始页数
    while True:
        print("正在采集", page, '页')
        #获取每页数据
        songs_ = get_songs_by_offset(tag, offset)
        if len(songs_) == 0:
            print('tag采集结束')
            break
        print("第{}页采集成功".format(page))
        # 将数据写入文件
        write_many_to_csv(songs_, tag)
        # 下一页的偏差
        offset += 20
        page += 1

        if page == 2: #采集两页测试
            break
        #程序暂停5秒
        time.sleep(5)

def write_many_to_csv(songs, tag):
    #打开csv文件
    with open(tag + '_songs.csv', 'a', encoding='utf8') as f:
        writer = csv.writer(f)
        #将数组写入csv
        writer.writerows(songs)

#获取全部数据
def get_songs():
    for tag in tags:
        print("采集", tag)
        get_songs_by_tag(tag)

#根据tag获取文件内容
def read(tag):
    data = []
    with open(tag + '_songs.csv', 'r', encoding='utf-8-sig') as f:
        reader = csv.reader(f)
        for item in reader:
            if item:
                data.append(item)
    return data

def draw1():
    counts = {}
    # 遍历每一个tag
    for tag in tags:
        #获取每一个tag的数据
        data = read(tag)
        #获取数据的长度
        count = len(data)
        #记录每个tag有多少数据
        counts[tag] = count
    #生成图
    Pie().add('', list(counts.items())).render('1.html')

def filt(x):
    try:
        #将x转为数字，失败则返回0
        return float(x)
    except:
        return 0

def draw2():
    data_ = []
    for tag in tags:
        #根据tag获取数据
        data = read(tag)
        #提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    #按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    #生成图
    bar = Bar()
    bar.add_xaxis([item[0] for item in data])
    bar.add_yaxis("评论数", [item[1] for item in data])
    bar.set_global_opts(title_opts=opts.TitleOpts(title="歌曲评论"))
    bar.render('2.html')

def draw3():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data = read(tag)
        # 提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    # 按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    # 生成图
    Scatter().add_xaxis(xaxis_data=[item[0] for item in data]).add_yaxis(
        series_name="",
        y_axis=[item[1] for item in data],
        symbol_size=20,
        label_opts=opts.LabelOpts(is_show=False),
    ).render('3.html')

def draw4():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data = read(tag)
        # 提取数据中的名字和评论数，组成新数据
        data_.extend([(x[0], filt(x[-1][:-3])) for x in data])
    # 按评论数排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)[:30]
    # 生成图
    Line().add_xaxis(xaxis_data=[item[0] for item in data]).add_yaxis(
        series_name="",
        y_axis=[item[1] for item in data],
        symbol="emptyCircle",
        is_symbol_show=True,
        label_opts=opts.LabelOpts(is_show=False),
        areastyle_opts=opts.AreaStyleOpts(opacity=1, color="#C67570"),
    ).render('4.html')

def draw5():
    data_ = []
    for tag in tags:
        # 根据tag获取数据
        data =read(tag)
        # 提取数据中的名字和时间，组成新数据
        data_.extend([(x[0] ,x[2]) for x in data])
    #根据时间排序
    data = sorted(data_, key=lambda x: x[-1], reverse=True)
    #日期计数
    dic = ['2000-01-01', '2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01',
           '2005-01-01', '2006-01-01','2007-01-01', '2008-01-01', '2009-01-01',
           '2010-01-01', '2011-01-01','2012-01-01', '2013-01-01', '2014-01-01',
           '2015-01-01', '2016-01-01','2017-01-01', '2018-01-01', '2019-01-01','2020-01-01','2021-01-01']
    count_ = {}
    for item in data:
        for di in dic:
            #获取每一个日期
            if item[-1] < di:
                #日期计数
                if di not in count_:
                    count_[di] = 0
                else:
                    count_[di] += 1
                break
    #生成图
    Pie().add('', list(count_.items())).render('5.html')

if __name__ == '__main__':
    get_songs()
    draw1()
    draw2()
    draw3()
    draw4()
    draw5()


"""
pyspark.sql.DataFrame.freqItems¶
DataFrame.freqItems(cols, support=None)[source]
Finding frequent items for columns, possibly with false positives. Using the frequent element count algorithm described
 in “https://doi.org/10.1145/762471.762473, proposed by Karp, Schenker, and Papadimitriou”. DataFrame.freqItems() and
 DataFrameStatFunctions.freqItems() are aliases.

New in version 1.4.0.

Parameters
cols: list or tuple
Names of the columns to calculate frequent items for as a list or tuple of strings.

support: float, optional
The frequency with which to consider an item ‘frequent’. Default is 1%. The support must be greater than 1e-4.
"""
import findspark
findspark.init()

"""
The freqItems function in PySpark is used to compute frequent items for selected columns in a DataFrame.
It returns a DataFrame containing the frequent items for each specified column.

Here's an example to illustrate the usage of freqItems with different ways of applying it:
"""
from pyspark.sql import SparkSession

# Create a SparkSession
spark = SparkSession.builder.getOrCreate()

# Create a DataFrame
data = [
    (1, "Apple", "Red"),
    (2, "Orange", "Orange"),
    (3, "Banana", "Yellow"),
    (4, "Apple", "Green"),
    (5, "Orange", "Yellow"),
    (6, "Banana", "Green")
]
df = spark.createDataFrame(data, ["ID", "Fruit", "Color"])

# Example 1: Compute frequent items for a single column
freq_items_1 = df.freqItems(["Fruit"], support=0.5)
freq_items_1.show(truncate=False)

# Example 2: Compute frequent items for multiple columns
freq_items_2 = df.freqItems(["Fruit", "Color"], support=0.5)
freq_items_2.show(truncate=False)

# Example 3: Set parameters for frequent items computation
freq_items_3 = df.freqItems(["Fruit"], support=0.5, numPartitions=2)
freq_items_3.show(truncate=False)

"""
# Example 1:
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Orange]  |
+-----------------+

# Example 2:
+-----------------+----------------------+
|Fruit_freqItems  |Color_freqItems       |
+-----------------+----------------------+
|[Apple, Orange]  |[Green, Yellow, Orange]|
+-----------------+----------------------+

# Example 3:
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Orange]  |
+-----------------+

In the first example, we compute the frequent items for the "Fruit" column. The result shows the list of frequent items in that column.

In the second example, we compute the frequent items for both the "Fruit" and "Color" columns. The result shows the frequent items for each column separately.

In the third example, we set additional parameters for the frequent items computation. We specify the number of partitions to control the parallelism of the
computation.

Note that the support parameter represents the minimum fraction of rows that should contain an item to be considered frequent.
It should be a value between 0 and 1.
"""

##########################################################################################
##########################################################################################
"""
Let's go through the concepts of support and numPartitions, and also address the question about the first example.

Support:

The support parameter in freqItems represents the minimum fraction of rows that should contain an item for it to be considered frequent.
It is specified as a value between 0 and 1. For example, a support value of 0.5 means that an item must appear in at least 50% of the rows to be considered frequent.
By setting a higher support value, you can filter out less frequent items and focus on the most common ones.
numPartitions:

The numPartitions parameter in freqItems determines the number of partitions used for parallel computation of frequent items.
It specifies the degree of parallelism and can be useful for improving performance when working with large datasets.
By default, the value is set to None, which means that the number of partitions is determined automatically based on the cluster configuration.
You can set numPartitions to a specific integer value to control the number of partitions used.
Regarding the first example and the absence of "Banana" in the frequent items:

In the first example, the support parameter was not explicitly specified, so the default value is used.
The default value of support is 0.1, which means that an item must appear in at least 10% of the rows to be considered frequent.
If the frequency of "Banana" is less than 10% in the "Fruit" column, it will not be considered a frequent item and will not appear in the result.
To include "Banana" as a frequent item, you can either reduce the support threshold or increase the frequency of "Banana" in the DataFrame.
"""
freq_items_1 = df.freqItems(["Fruit"], support=0.2)
freq_items_1.show(truncate=False)

"""
+-----------------+
|Fruit_freqItems  |
+-----------------+
|[Apple, Banana, Orange]  |
+-----------------+

"""


import numpy as np

from typing import List, Union, Dict
from pathlib import Path

import os
import torch
import logging
import flair
import gensim

from flair.data import Sentence
from flair.embeddings.token import TokenEmbeddings
from flair.embeddings import StackedEmbeddings
from flair.embeddings import WordEmbeddings, CharacterEmbeddings
from flair.file_utils import cached_path

log = logging.getLogger("flair")

def get_embedding(embedding, finetune_bert=False):
    embeddings = embedding.split('+')
    result = [CaseEmbedding()]
    # skip updating to new flair version
    old_base_path = "https://flair.informatik.hu-berlin.de/resources/embeddings/token/"
    cache_dir = Path("embeddings")
    cached_path(f"{old_base_path}glove.gensim.vectors.npy", cache_dir=cache_dir)
    cached_path(
        f"{old_base_path}glove.gensim", cache_dir=cache_dir
    )

    cached_path(f"https://flair.informatik.hu-berlin.de/resources/characters/common_characters", cache_dir="datasets")

    for embedding in embeddings:
        if embedding == 'char':
            result.append(CustomCharacterEmbeddings())
        if embedding == 'glove':
            result.append(LargeGloveEmbeddings('./data/glove'))

    return StackedEmbeddings(embeddings=result)

class LargeGloveEmbeddings(WordEmbeddings):
    """Standard static word embeddings, such as GloVe or FastText."""

    def __init__(self, glove_dir):
        """
        Initializes classic word embeddings - made for large glove embedding
        """

        super().__init__('glove')
        embeddings = '840b-300d-glove'
        self.field = ""
        self.embeddings = embeddings
        self.static_embeddings = True

        # Large Glove embeddings
        embeddings = os.path.join(glove_dir, 'glove.bin')
        self.name: str = str(embeddings)
        self.precomputed_word_embeddings = gensim.models.KeyedVectors.load(embeddings)
        self.__embedding_length: int = self.precomputed_word_embeddings.vector_size

    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:

        for i, sentence in enumerate(sentences):
            for token, token_idx in zip(sentence.tokens, range(len(sentence.tokens))):
                word_embedding = self.get_cached_vec(word=token.text)
                token.set_embedding(self.name, word_embedding)

        return sentences

    @property
    def embedding_length(self) -> int:
        return 300

class CaseEmbedding(TokenEmbeddings):
    """Static Case Embedding"""

    def __init__(self):
        self.name: str = 'case-embedding-shun'
        self.static_embeddings = False
        self.__embedding_length: int = 3
        super().__init__()

    @property
    def embedding_length(self) -> int:
        return self.__embedding_length

    def embed(self, sentences: Union[Sentence, List[Sentence]]) -> List[Sentence]:
        for sentence in sentences:
            for token in sentence:
                text = token.text
                is_lower = 1 if text == text.lower() else 0
                is_upper = 1 if text == text.upper() else 0
                is_mix = 1 if is_lower + is_upper == 0 else 0
                word_embedding = torch.tensor(
                    np.array([is_lower, is_upper, is_mix]), device=flair.device, dtype=torch.float
                )
                token.set_embedding('case-embedding-shun', word_embedding)

        return sentences

    def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
        return sentences

    def __str__(self):
        return self.name

class CustomCharacterEmbeddings(CharacterEmbeddings):
    """Batched-version of CharacterEmbeddings. """

    def _add_embeddings_internal(self, sentences: List[Sentence]):

        token_to_embeddings = {}

        for sentence in sentences:
            for token in sentence.tokens:
                token_to_embeddings[token.text] = None

        tokens_char_indices = []
        for token in token_to_embeddings:
            char_indices = [
                self.char_dictionary.get_idx_for_item(char) for char in token
            ]
            tokens_char_indices.append(char_indices)

        # sort words by length, for batching and masking
        tokens_sorted_by_length = sorted(
            tokens_char_indices, key=lambda p: len(p), reverse=True
        )
        d = {}
        for i, ci in enumerate(tokens_char_indices):
            for j, cj in enumerate(tokens_sorted_by_length):
                if ci == cj:
                    d[j] = i
                    continue
        chars2_length = [len(c) for c in tokens_sorted_by_length]
        longest_token_in_sentence = max(chars2_length)
        tokens_mask = torch.zeros(
            (len(tokens_sorted_by_length), longest_token_in_sentence),
            dtype=torch.long,
            device=flair.device,
        )

        for i, c in enumerate(tokens_sorted_by_length):
            tokens_mask[i, : chars2_length[i]] = torch.tensor(
                c, dtype=torch.long, device=flair.device
            )

        # chars for rnn processing
        chars = tokens_mask

        character_embeddings = self.char_embedding(chars).transpose(0, 1)

        packed = torch.nn.utils.rnn.pack_padded_sequence(
            character_embeddings, chars2_length
        )

        lstm_out, self.hidden = self.char_rnn(packed)

        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out)
        outputs = outputs.transpose(0, 1)
        chars_embeds_temp = torch.zeros(
            (outputs.size(0), outputs.size(2)),
            dtype=torch.float,
            device=flair.device,
        )
        for i, index in enumerate(output_lengths):
            chars_embeds_temp[i] = outputs[i, index - 1]
        character_embeddings = chars_embeds_temp.clone()
        for i in range(character_embeddings.size(0)):
            character_embeddings[d[i]] = chars_embeds_temp[i]

        for token_number, token in enumerate(token_to_embeddings.keys()):
            token_to_embeddings[token] = character_embeddings[token_number]

        for sentence in sentences:
            for token in sentence.tokens:
                token.set_embedding(self.name, token_to_embeddings[token.text])


"""
@brief      test log(time=14s)
"""

import sys
import os
import unittest
import warnings
import numpy as np

from pyquickhelper.loghelper import fLOG
from pyquickhelper.ipythonhelper import StaticInteract, RangeWidget, RadioWidget, DropDownWidget
from pyquickhelper.pycode import fix_tkinter_issues_virtualenv, ExtTestCase

class TestInteractive(ExtTestCase):

    def test_interactive_StaticInteract(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        def show_fib(N):
            sequence = ""
            a, b = 0, 1
            for i in range(N):
                sequence += f"{a} "
                a, b = b, a + b
            return sequence

        res = StaticInteract(show_fib,
                             N=RangeWidget(1, 100, default=10))
        self.assertNotEmpty(res)
        ht = res.html()
        self.assertNotEmpty(ht)

    def test_interactive2_RadioWidget(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        fix_tkinter_issues_virtualenv(fLOG=fLOG)
        import matplotlib.pyplot as plt

        def plot(amplitude, color, sele):
            fig, ax = plt.subplots(figsize=(4, 3),
                                   subplot_kw={'axisbelow': True})
            ax.grid(color='w', linewidth=2, linestyle='solid')
            x = np.linspace(0, 10, 1000)
            ax.plot(x, amplitude * np.sin(x), color=color,
                    lw=5, alpha=0.4)
            try:
                ax.set_xlim(0, 10)
                ax.set_ylim(-1.1, 1.1)
            except TypeError as e:
                warnings.warn(e)
            return fig

        res = StaticInteract(plot,
                             amplitude=RangeWidget(0.1, 0.3, 0.1, default=0.2),
                             color=RadioWidget(
                                 ['blue', 'green'], default='blue'),
                             sele=DropDownWidget(['a', 'b']))
        self.assertNotEmpty(res)
        ht = res.html()
        self.assertNotEmpty(ht)
        plt.close('all')
        fLOG("end")

if __name__ == "__main__":
    unittest.main()


import math
def sumSmallSquares(n):
    """
    :param n:
    :return:sum of small squares
    """
    result=0
    for i in range(1,n):
        if i%2==0:
            continue
        else:
            result=result+math.pow(i,2)

    return result

print(sumSmallSquares(6))

2-basics/Study basics/4-sum small of n squares.py


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 23 17:43:32 2022
SSAI Interview Data Comparison
@author: Mitch Creelman
"""

#%% Load packages

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy as sp
import seaborn as sns
from string import ascii_letters

#%% Adjustables

# Set File Path
path = "C:/Users/Owner/Documents/School/Spring 2022/Interview Questions/SSAI_Task_Data/"

# use subsample( 1 = yes, 0 = no)
# use this to use a smaller subset until we are ready to do the whole thing
sub = 0

# turn on seaborn
sns.set()

#%% Read in the csv's

# data
test = pd.read_csv(path + "testing.csv")  # testing Data
train = pd.read_csv(path + "training.csv") # training data
val = pd.read_csv(path + "validation.csv") # validation Data

if sub == 1:
    train = train.head(300)

# individual indexing without label
L = list(val.columns[:-1]) # wavelengths of samples
Chl = val.columns[-1:]

out = 'C:/Users/Owner/Documents/School/Spring 2022/Interview Questions/SSAI_Task_Data/Output/Figures/'

#%% create Graph of each band vs. Chl
d = val # set the data

# Normalize the data
column_maxes = d[L].max() # find the max of each column
# d_max = column_maxes.max()
normalized_d = d[L] / column_maxes # express the dependent variables as a %

d[L] = normalized_d
#%%
fig, axes = plt.subplots(nrows=4,
                         ncols=4,
                         sharex = True,
                         sharey = True,
                         figsize=(14,8))

i = 0
n = 0

for row in axes:
    if i == 0:
        p = 0
    else:
        p = p + 1

    i = 0
    for col in row:
        # col.plot(x[L[i]], y)
        f = sns.regplot(x = d[Chl] ,
                         y = L[n],
                         data = d,
                         scatter_kws={"s":1},
                         ax = axes[p,i])

        # Calculate and print r^2 and p values
        x = d[Chl].to_numpy().squeeze()
        y = d[L[n]].to_numpy().squeeze()
        r, k = sp.stats.pearsonr(x, y)
        ax = axes[p,i]
        ax.text(.05, .8, 'r={:.2f}, p={:.2g}'.format(r, k),
        transform=ax.transAxes)

        # hide the labels and axis
        f.set(xlabel=None, ylabel = None)

        # set the graph name to the individual
        col.set_title(str(L[n]), pad = 0.5)
        print(str(p) + str ( i ))
        n = n + 1
        i = i + 1

plt.suptitle('Chlorophyll-a vs. Reflectance by Wavelength',size = 25)
fig.supylabel('Reflectance (% of maximum)', size = 20)
fig.supxlabel('Chlorophyll-a Levels', size = 20)

plt.show()
ig.savefig(out + 'Correlation_By_Band.png', dpi = 900)
#%% Correlation Matrix Plot
d = val

# Compute the correlation matrix
corr = d.corr()

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5)

plt.suptitle('Correlation Matrix for Chlorophyll Concentrationv \n' +
             'and Band Reflectance',
             size = 20 )

fig.savefig(out + 'Correlation_Matrix.png', dpi = 900)


__author__ = 'halley'
from constants import *
from chunk import *
import copy
import probabilityhelpers as ph

def inChord(note, chord):
    if chord == []:
        return True
    if note == None:
        return True
    else:
        return (note % 7) in [i%7 for i in chord]

#check if an entire chunk matches
def chunkMatches(chunk1, chunk2):
    if chunk1.sub_chunks == [] or chunk1.sub_chunks == None:
        if len(chunk1.pits) != len(chunk1.durs) or len(chunk2.pits) != len(chunk2.durs):
            return False
        tot_durs1 = [sum(chunk1.durs[:i]) for i in range(0, len(chunk1.durs))]
        pits1 = chunk1.pits
        tot_durs2 = [sum(chunk2.durs[:i]) for i in range(0, len(chunk2.durs))]
        pits2 = chunk2.pits
        j_index = 0
        for i in range(0, len(tot_durs1)):
            for j in range(j_index, len(tot_durs2)):
                if tot_durs1[i] == tot_durs2[j]:
                    if not matches(pits1[i], pits2[j]):
                        return False
                    j_index = j
        return True
    else:
        print('error - larger chunks not implemented yet')

#check if a chunk is in a chord
def chunkInChord(chunk, chord):
    pits = chunk.pits
    #print(pits)
    if inChord(pits[0], chord):
        for i in range(1, len(pits)):
            if abs(pits[i] - pits[i - 1]) > 1:
                if not (inChord(pits[i], chord) and inChord(pits[i - 1], chord)):
                    return False
        return True
    else:
        return False

#get closest note in chord above note
def getClosestAbove(note, chord):
    #print('note ' + str(note) + 'chord ' + str(chord))
    i = 1
    while (not inChord(note + i, chord)):
        i += 1
    return note + i

#get closest note in chord below note
def getClosestBelow(note, chord):
    #print('note ' + str(note) + 'chord ' + str(chord))
    i = 1
    while (not inChord(note - i, chord)):
        i += 1
    return note - i

def matches(n1, n2):
    #print('matching ' + str(n1) + ' and ' + str(n2))
    if n1 == None or n2 == None:
        return True
    diff = abs((n1%7) - (n2%7))
    if diff == 2 or diff == 5 or diff == 4 or diff == 0 or diff == 3:
        return True
    return False

#find what chord matches a given cell
def getChord(cell):
    pits = cell.pits
    diffs = [abs(pits[i] - pits[i - 1]) for i in range(1, len(pits))]
    important = []
    important.append(pits[0])
    for i in range(0, len(diffs)):
        if diffs[i] > 2:
            important.append(pits[i])
            important.append(pits[i + 1])
    return (-7,-5,-3)

#transpose chunk so that it fits a new chord
def transposeToChord(chunk, chord, prev_note = 0):
    if chunk.depth == 0:
        #get closest above with chord
        distance = getClosestAbove(chunk.pits[0], chord) - chunk.pits[0]
        new_pits = [i + distance for i in chunk.pits]
        chunk_above = Chunk(None, new_pits, chunk.durs, chunk.ctype)

        distance = chunk.pits[0] - getClosestBelow(chunk.pits[0], chord)
        new_pits = [i - distance for i in chunk.pits]
        chunk_below = Chunk(None, new_pits, chunk.durs, chunk.ctype)

        new_chunks = [chunk_below, chunk_above]
        if chunkInChord(chunk, chord):
            new_chunks.append(chunk)
        return new_chunks
    elif chunk.depth == 1:
        distance = getClosestAbove(chunk.pits[0], chord) - chunk.pits[0]
        chunk_above = copy.deepcopy(chunk)
        for i in range(0, len(chunk_above.sub_chunks)):
            chunk_above.sub_chunks[i].setPitsDurs([j + distance for j in chunk_above.sub_chunks[i].pits], chunk_above.sub_chunks[i].durs)

        distance = getClosestBelow(chunk.pits[0], chord) - chunk.pits[0]
        chunk_below = copy.deepcopy(chunk)
        for i in range(0, len(chunk_below.sub_chunks)):
            chunk_below.sub_chunks[i].setPitsDurs([j + distance for j in chunk_below.sub_chunks[i].pits], chunk_below.sub_chunks[i].durs)
        new_chunks = [chunk_above, chunk_below]
        if inChord(chunk.pits[0], chord):
            new_chunks.append(chunk)
        return new_chunks
    else:
        return [chunk]

#convert from root of chord to chord
def rootToChord(base):
    return [base, base + 2, base + 4]

#convert from chord to root
def chordToRoot(chord):
    return chord[0]

#get all pos chords fitting between prev_chord and end_Chord
def getPosChords(prev_chord = None, end_chord = None):
    if prev_chord != None:
        pos_prev_chord = chord_movements[chordToRoot(prev_chord)]
    else:
        pos_prev_chord = range(0,6)
    if end_chord != None:
        pos_prev_chord = filter(lambda i: end_chord in chord_movements[i], pos_prev_chord)
    return [rootToChord(root) for root in pos_prev_chord]

#get all pos chord roots fitting between prev_chord and end_Chord
def getPosChordRoots(prev_chord = None, end_chord = None):
    if prev_chord != None:
        pos_prev_chord = chord_movements[chordToRoot(prev_chord)]
    else:
        pos_prev_chord = range(0,6)
    if end_chord != None:
        pos_prev_chord = filter(lambda i: end_chord in chord_movements[i], pos_prev_chord)
    return pos_prev_chord

chord_prob_dict = {0:0.5, 1:0.1, 2:0.01, 3:0.2, 4:0.3, 5:0.1, 6:0.05}

def getHighestProbChord(prev_chord, end_chord):
    roots = getPosChordRoots(prev_chord, end_chord)
    pdict = chord_prob_dict.values()
    pdict = dict(filter(lambda i: i[0] in roots, pdict))
    return rootToChord(ph.highestProb(pdict))


import os
import numpy as np
import torch
import math
from scipy import io
from construct import get_Pearson_fc,get_BOLD_feature,get_Degree_fc,get_sc,get_sc_feature,get_dataloader, get_fc_edgeWeight, get_sc_dataloader, get_fc_dataloader
from torch_geometric.utils import dense_to_sparse
from torch_geometric.data import Data
from scipy import sparse as sp
import hashlib

os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'

def decide_dataset(root):  # 返回标签
    if 'ABIDE' in root:
        class_dict = {
            "HC": 0,
            "ASD": 1,
        }
    elif 'Multi' in root or "xinxiang" in root or "zhongda" or "multi_SCFC" in root:
        class_dict = {
            "HC": 0,
            "MDD": 1,
        }
    elif "HCP" in root:
        class_dict = {
            "female": 0,
            "male": 1,
        }
    return class_dict

def read_dataset_fc_regionSeries(root, label_files, files):  # 返回时间序列
    FC_dir = "RegionSeries.mat"
    if 'ABIDE' in root:
        subj_fc_dir = os.path.join(root, label_files, files)
        subj_mat_fc = np.loadtxt(subj_fc_dir)[:176, :90]
    elif 'Multi' in root:
        subj_fc_dir = os.path.join(root, label_files, files)
        subj_mat_fc = io.loadmat(subj_fc_dir)['ROISignals_AAL'][:170, :]
    else:  # zhonda、xinxiang、ASD的读法都一样
        subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
        subj_mat_fc = io.loadmat(subj_fc_dir)['RegionSeries']
    return subj_mat_fc

def read_dataset_fc_edgeWeight(root, label_files, files):  # 返回时间序列
    FC_dir = "EdgeWeight.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['EdgeWeight']
    return subj_mat_fc

def read_dataset_fc_region_features(root, label_files, files):  # 返回时间序列
    FC_dir = "region_features_norm.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['region_features']
    return subj_mat_fc

def read_dataset_sc_connectivity(root, label_files, files):  # 返回sc connectivity
    SC_connectivity_dir = "dti_FACT_45_02_1_1_Matrix_FA_AAL_Contract_90_2MM_90.txt"
    subj_sc_connectivity_dir = os.path.join(root, label_files, files, SC_connectivity_dir)
    subj_sc_connectivity = np.loadtxt(subj_sc_connectivity_dir)
    # subj_sc_connectivity = io.loadmat(subj_sc_connectivity_dir)['connectivity']
    # subj_sc_connectivity = subj_sc_connectivity[0:90, 0:90]
    return subj_sc_connectivity

def read_dataset_sc_features(root, label_files, files):  # 返回sc特征
    SC_features_dir = "region_features_norm.mat"
    subj_sc_feature_dir = os.path.join(root, label_files, files, SC_features_dir)
    subj_sc_feature = io.loadmat(subj_sc_feature_dir)['region_features']
    return subj_sc_feature

def read_dataset_fc_region_features(root, label_files, files):  # 返回时间序列
    FC_dir = "region_features_norm.mat"
    subj_fc_dir = os.path.join(root, label_files, files, FC_dir)
    subj_mat_fc = io.loadmat(subj_fc_dir)['region_features']
    return subj_mat_fc

def d_constraint(X, D, a):
    n, d = X.shape
    sum_dist = 0

    for i in range(n):
        for j in range(i+1, n):
            if D[i, j] == 1:
                d_ij = X[i] - X[j]
                dist_ij = distance1(a, d_ij)
                sum_dist += dist_ij
    fD = gf(sum_dist)
    return fD

def gf(sum_dist):
    fD = np.log(sum_dist)
    return fD

def distance1(a, d_ij):
    fudge = 0.000001
    dist_ij = np.sqrt(np.dot(d_ij**2, a))  # distance between X[i] and X[j]
    return dist_ij

class FSDataset:
    def __init__(self, args):
        self.sc = []
        self.fc = []
        self.scfc = []
        self.class_dict = decide_dataset(args.path)
        self.y = []

        root = args.path
        label_list = os.listdir(root)
        label_list.sort()

        for label_files in label_list:
            list = os.listdir(os.path.join(root, label_files))
            list.sort()
            label = torch.LongTensor([self.class_dict[label_files]])
            for files in list:
                ############
                sc_connectivity = read_dataset_sc_connectivity(root, label_files, files)
                sc_feature = read_dataset_sc_connectivity(root, label_files, files)
                subj_sc_adj = get_sc(sc_connectivity, args)  # 构造邻接矩阵
                subj_sc_feature = get_sc_feature(sc_feature, args)  # 构造特征
                # 构造sc Data
                # sc_data = get_sc_dataloader(subj_sc_adj, subj_sc_feature)
                # sc_data.y = label
                # self.sc.append(sc_data)
                ############
                fc_timeseries = read_dataset_fc_regionSeries(root, label_files, files)  # 这里得到的已经是节点数*长度的时间序列
                subj_fc_adj = get_Pearson_fc(fc_timeseries,
                                                   args)
                # fc_edgeWeight = read_dataset_fc_edgeWeight(root, label_files, files)
                d = np.ones((90, 90)) - np.tril(np.ones((90, 90)))
                a = np.ones(90)
                re = d_constraint( subj_fc_adj, d, a)
                re = math.exp(-(re/2*args.bandwidth*args.bandwidth))
                # subj_fc_adj = get_Pearson_fc(fc_timeseries,
                #                                    args)  # 这里传参是时间序列，返回的是邻接矩阵，以方法命名，因为可能有不同的构造方法，例如度矩阵等construct_Degree_fc;threshold等参数通过args传递
                fc_region_features = read_dataset_fc_region_features(root, label_files, files)
                subj_fc_feature = re * get_fc_edgeWeight(subj_fc_adj ,
                                                        args)  # 同样传参时间序列，返回特征，注意这里特征和上面邻接矩阵都返回的是矩阵，构造Data放在后面，以防有的会直接使用这两个矩阵，可以将Data构造部分注释掉
                # subj_fc_feature = re * subj_fc_adj
                # 构造fc Data
                # fc_data = get_dataloader(subj_fc_adj, subj_fc_feature)
                # fc_data.y = label
                # self.fc.append(fc_data)
                ############
                # 构造sc fc Data

                # self.fuse_weight_1 = torch.nn.Parameter(torch.FloatTensor(1), requires_grad=True)
                # self.fuse_weight_2 = torch.nn.Parameter(torch.FloatTensor(1), requires_grad=True)
                # self.fuse_weight_1.data.fill_(1)
                # self.fuse_weight_2.data.fill_(1)
                feature = np.identity(90) +  subj_fc_feature + subj_sc_feature
                adj = np.identity(90)

                data = get_dataloader(adj, feature)
                data.y = label
                # data.fc_x = get_dataloader(subj_fc_adj, subj_fc_feature).x
                # data.fc_edge_index = get_dataloader(subj_fc_adj, subj_fc_feature).edge_index
                self.scfc.append(data)
                self.y.append(label)

        self.choose_data = self.scfc


import sys

sys.setrecursionlimit(10 ** 5)
v, e = map(int, sys.stdin.readline().split())
graph = [list([0] * (v + 1)) for _ in range(v + 1)]
visited = list([0] * (v + 1))
for _ in range(e):
    a, b = map(int, sys.stdin.readline().split())
    graph[a][b] = graph[b][a] = 1

def dfs(x):
    visited[x] = 1
    for i in range(1, v + 1):
        if visited[i] == 0 and graph[x][i] == 1:
            dfs(i)

count = 0
for i in range(1, v + 1):
    dx = sum(visited)
    if dx == v:
        print(count)
        exit(0)
    dx_ = sum(visited)
    if dx != dx_:
        count += 1

print(count)


import socket
import threading
import socketserver
import uuid

class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):

    def handle(self):
        id = uuid.uuid4()
        data = str(self.request.recv(1024), 'ascii')
        cur_thread = threading.current_thread()
        mensaje = data.split(" ")
        operacion = mensaje[0]
        print(mensaje)
        response = bytes("Default", 'ascii')
        if operacion == "nocompra":
            #print(str(mensaje[0]) + " " +  str(mensaje[1]))
            respuesta = "nocompra_exitoso " + str(mensaje[1])
            response = bytes(str(respuesta), 'ascii')
        if operacion == "noventa":
            #print(str(mensaje[0]) + " " +  str(mensaje[1]))
            respuesta = "noventa_exitoso " + str(mensaje[1])
            response = bytes(str(respuesta), 'ascii')
        if operacion == "compra":
            response = bytes(str(id), 'ascii')
        if operacion == "venta":
            response = bytes(str(id), 'ascii')
        self.request.sendall(response)

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    pass

def client(ip, port, message):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((ip, port))
    try:
        sock.sendall(bytes(message, 'ascii'))
        response = str(sock.recv(1024), 'ascii')
        print("Received: {}".format(response))
    finally:
        sock.close()


from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.translation import gettext_lazy as _

from feincms.admin.item_editor import FeinCMSInline
from feincms.module.medialibrary.fields import ContentWithMediaFile
from feincms.utils.tuple import AutoRenderTuple

class MediaFileContentInline(FeinCMSInline):
    raw_id_fields = ("mediafile",)
    radio_fields = {"type": admin.VERTICAL}

class MediaFileContent(ContentWithMediaFile):
    """
    Rehashed, backwards-incompatible media file content which does not contain
    the problems from v1 anymore.

    Create a media file content as follows::

        from feincms.content.medialibrary.models import MediaFileContent
        Page.create_content_type(MediaFileContent, TYPE_CHOICES=(
            ('default', _('Default')),
            ('lightbox', _('Lightbox')),
            ('whatever', _('Whatever')),
            ))

    For a media file of type 'image' and type 'lightbox', the following
    templates are tried in order:

    * content/mediafile/image_lightbox.html
    * content/mediafile/image.html
    * content/mediafile/lightbox.html
    * content/mediafile/default.html

    The context contains ``content`` and ``request`` (if available).
    """

    feincms_item_editor_inline = MediaFileContentInline

    class Meta:
        abstract = True
        verbose_name = _("media file")
        verbose_name_plural = _("media files")

    @classmethod
    def initialize_type(cls, TYPE_CHOICES=None):
        if TYPE_CHOICES is None:
            raise ImproperlyConfigured(
                "You have to set TYPE_CHOICES when" " creating a %s" % cls.__name__
            )

        cls.add_to_class(
            "type",
            models.CharField(
                _("type"),
                max_length=20,
                choices=TYPE_CHOICES,
                default=TYPE_CHOICES[0][0],
            ),
        )

    def render(self, **kwargs):
        return AutoRenderTuple(
            (
                [
                    f"content/mediafile/{self.mediafile.type}_{self.type}.html",
                    "content/mediafile/%s.html" % self.mediafile.type,
                    "content/mediafile/%s.html" % self.type,
                    "content/mediafile/default.html",
                ],
                {"content": self},
            )
        )


import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold, GridSearchCV

from ._utils_boot import boot_manual, draw_weights

def fit_nuisance_pliv_partial_x(y, x, d, z, ml_m, ml_g, ml_r, smpls, g_params=None, m_params=None, r_params=None):
    assert z.ndim == 2
    g_hat = []
    for idx, (train_index, test_index) in enumerate(smpls):
        if g_params is not None:
            ml_g.set_params(**g_params[idx])
        g_hat.append(ml_g.fit(x[train_index], y[train_index]).predict(x[test_index]))

    m_hat = []
    m_hat_array = np.zeros_like(z, dtype='float64')
    for i_instr in range(z.shape[1]):
        this_instr_m_hat = []
        for idx, (train_index, test_index) in enumerate(smpls):
            if m_params is not None:
                ml_m.set_params(**m_params[i_instr][idx])
            this_instr_m_hat.append(ml_m.fit(x[train_index], z[train_index, i_instr]).predict(x[test_index]))
            m_hat_array[test_index, i_instr] = this_instr_m_hat[idx]
        m_hat.append(this_instr_m_hat)

    r_hat = []
    r_hat_array = np.zeros_like(d, dtype='float64')
    for idx, (train_index, test_index) in enumerate(smpls):
        if r_params is not None:
            ml_r.set_params(**r_params[idx])
        r_hat.append(ml_r.fit(x[train_index], d[train_index]).predict(x[test_index]))
        r_hat_array[test_index] = r_hat[idx]

    r_hat_tilde = LinearRegression(fit_intercept=True).fit(z - m_hat_array, d - r_hat_array).predict(z - m_hat_array)

    return g_hat, r_hat, r_hat_tilde

def tune_nuisance_pliv_partial_x(y, x, d, z, ml_m, ml_g, ml_r, smpls, n_folds_tune, param_grid_g, param_grid_m, param_grid_r):
    g_tune_res = [None] * len(smpls)
    for idx, (train_index, _) in enumerate(smpls):
        g_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
        g_grid_search = GridSearchCV(ml_g, param_grid_g,
                                     cv=g_tune_resampling)
        g_tune_res[idx] = g_grid_search.fit(x[train_index, :], y[train_index])

    m_tune_res = [[None] * len(smpls) for i in range(z.shape[1])]
    for i_instr in range(z.shape[1]):
        for idx, (train_index, _) in enumerate(smpls):
            m_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
            m_grid_search = GridSearchCV(ml_m, param_grid_m,
                                         cv=m_tune_resampling)
            m_tune_res[i_instr][idx] = m_grid_search.fit(x[train_index, :], z[train_index, i_instr])

    r_tune_res = [None] * len(smpls)
    for idx, (train_index, _) in enumerate(smpls):
        r_tune_resampling = KFold(n_splits=n_folds_tune, shuffle=True)
        r_grid_search = GridSearchCV(ml_r, param_grid_r,
                                     cv=r_tune_resampling)
        r_tune_res[idx] = r_grid_search.fit(x[train_index, :], d[train_index])

    g_best_params = [xx.best_params_ for xx in g_tune_res]
    m_best_params = [[xx.best_params_ for xx in m_tune_res[i_instr]] for i_instr in range(z.shape[1])]
    r_best_params = [xx.best_params_ for xx in r_tune_res]

    return g_best_params, m_best_params, r_best_params

def pliv_partial_x_dml1(y, x, d, z, g_hat, r_hat, r_hat_tilde, smpls, score):
    thetas = np.zeros(len(smpls))
    n_obs = len(y)

    for idx, (_, test_index) in enumerate(smpls):
        u_hat = y[test_index] - g_hat[idx]
        w_hat = d[test_index] - r_hat[idx]
        thetas[idx] = pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde[test_index], d[test_index], score)
    theta_hat = np.mean(thetas)

    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]
    se = np.sqrt(var_pliv_partial_x(theta_hat, d, u_hat, w_hat, r_hat_tilde, score, n_obs))

    return theta_hat, se

def pliv_partial_x_dml2(y, x, d, z, g_hat, r_hat, r_hat_tilde, smpls, score):
    n_obs = len(y)
    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]
    theta_hat = pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde, d, score)
    se = np.sqrt(var_pliv_partial_x(theta_hat, d, u_hat, w_hat, r_hat_tilde, score, n_obs))

    return theta_hat, se

def var_pliv_partial_x(theta, d, u_hat, w_hat, r_hat_tilde, score, n_obs):
    assert score == 'partialling out'
    var = 1/n_obs * 1/np.power(np.mean(np.multiply(r_hat_tilde, w_hat)), 2) * \
        np.mean(np.power(np.multiply(u_hat - w_hat*theta, r_hat_tilde), 2))

    return var

def pliv_partial_x_orth(u_hat, w_hat, r_hat_tilde, d, score):
    assert score == 'partialling out'
    res = np.mean(np.multiply(r_hat_tilde, u_hat))/np.mean(np.multiply(r_hat_tilde, w_hat))

    return res

def boot_pliv_partial_x(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                        smpls, score, se, bootstrap, n_rep):
    n_obs = len(y)
    weights = draw_weights(bootstrap, n_rep, n_obs)
    assert np.isscalar(theta)
    boot_theta, boot_t_stat = boot_pliv_partial_x_single_treat(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                                                               smpls, score, se, weights, n_rep)
    return boot_theta, boot_t_stat

def boot_pliv_partial_x_single_treat(theta, y, d, z, g_hat, r_hat, r_hat_tilde,
                                     smpls, score, se, weights, n_rep):
    assert score == 'partialling out'
    u_hat = np.zeros_like(y, dtype='float64')
    w_hat = np.zeros_like(d, dtype='float64')
    for idx, (_, test_index) in enumerate(smpls):
        u_hat[test_index] = y[test_index] - g_hat[idx]
        w_hat[test_index] = d[test_index] - r_hat[idx]

    J = np.mean(-np.multiply(r_hat_tilde, w_hat))

    psi = np.multiply(u_hat - w_hat*theta, r_hat_tilde)

    boot_theta, boot_t_stat = boot_manual(psi, J, smpls, se, weights, n_rep)

    return boot_theta, boot_t_stat


from flask import Flask, render_template, request, jsonify

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def main():
    return render_template('index.html')

@app.route('/create-ticket', methods=['GET', 'POST'])
def createTicket():
    if request.method == 'POST':
        content = request.args.get('content')
        position = request.args.get('position')
        time = request.args.get('time')
        name = request.args.get('name')
        if content and position:
            print(f"{content} {position} {time} {name}")
        else:
            pass
    return jsonify({
        "status": "success"
    })

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port="80")


from random import randint
number = randint(1000000000000, 9999999999999)

def get_max_digit(number):  # returns int
    """Using string and method sorted """
    number_str = str(number)
    sorted_numbers = sorted(number_str, reverse=True)
    return int(sorted_numbers[0])

def get_max_digit_list(number):
    """Using list and iterations"""
    number_list = []
    while number != 0:
        number_list.insert(0, number % 10)  # used insert here to make  elements order  according to number
        number = number // 10
    max_digit = None
    for e in number_list:
        if max_digit is None or e > max_digit:
            max_digit = e

    return max_digit

print(get_max_digit(number))
print(get_max_digit_list(number))


from email.message import EmailMessage
import smtplib

class MailSender:
    def __init__(self):
        self.msg = EmailMessage()

    def form_message_success(self, destination, device_name):

        self.msg['From'] = 'stalozaidimudalinimasis@outlook.com'
        self.msg['To'] = destination
        self.msg['Subject'] = 'The program was tested successfully!'
        self.msg.set_content('''
        <!DOCTYPE html>

                    The tested device {0} finished without interruptions.

        '''.format(device_name), subtype='html')

    def form_message_failed(self, destination, device_name):
        self.msg['From'] = 'stalozaidimudalinimasis@outlook.com'
        self.msg['To'] = destination
        self.msg['Subject'] = 'The program has failed!'
        self.msg.set_content('''
        <!DOCTYPE html>

                    The tested device {0} failed while testing.

        '''.format(device_name), subtype='html')

    def send_mail(self):
        try:

            mailserver = smtplib.SMTP('smtp.office365.com', 587)
            mailserver.ehlo()
            mailserver.starttls()

            try:
                mailserver.login('stalozaidimudalinimasis@outlook.com',
                                 'gqjcfxntqitshslo')
                mailserver.send_message(self.msg)
            finally:
                mailserver.quit()
        except:
            pass


import argparse
import matplotlib.pyplot as plt
from matplotlib.pyplot import MultipleLocator
import numpy as np
from collections.abc import Iterable

color_tuple = ('#7f7f7f', '#2ca02c', '#e377c2', '#1f77b4')

color_tuple_targeted = ('#7f7f7f', '#2ca02c', '#e377c2', '#9467bd', '#8c564b','#1f77b4')

style_tuple = ('-', '-', '-', '-', '-', '-', '-', '-')
marker_tuple = ('s', '^', '^', '^', '^', '*', '*', 'h')
ms_tuple = (8, 9, 9, 9, 9, 9, 9, 9)

def plot_curve(curve_arr, curve_label, title='', color=None, style=None, curve_type='pr', targeted=False):
    filter_label = ('Original',) if targeted else ('',)
    if not isinstance(curve_arr, Iterable):
        curve_arr = (curve_arr,)
        curve_label = (curve_label,)

    if color is None:
        color = color_tuple_targeted if targeted else color_tuple

    if style is None:
        style = style_tuple

    fig, ax = plt.subplots()
    plt.figure(1)

    for i, curve in enumerate(curve_arr):
        if curve_label[i] in filter_label:
            continue
        x = curve[:, 0]
        y = curve[:, 1]
        plt.plot(x, y, label=curve_label[i], c=color[i], ls=style[i], lw=2.5)

    x_major_locator = MultipleLocator(0.2 if curve_type == 'pr' else 200)
    y_major_locator = MultipleLocator(0.1)
    loc = (0.05, 0.05) if targeted else (0.05, 0.65)
    ax.xaxis.set_major_locator(x_major_locator)
    ax.yaxis.set_major_locator(y_major_locator)
    plt.ylim(0.05, 0.7)
    plt.xlabel('Recall' if curve_type == 'pr' else 'Number of top ranked samples')
    plt.ylabel('Precision')
    plt.title(title)
    plt.legend(loc=loc, framealpha=0.7)
    # plt.grid(linestyle='--')
    fig.subplots_adjust(left=0.09, right=0.99)
    fig.set_size_inches(6, 5.5)
    filename = curve_type + ('_targeted' if targeted else '')
    plt.savefig('../documents/{}-{}.pdf'.format(title, filename), dpi=600, format='pdf', transparent=True)
    # plt.show()

def plot_ablation(row, para_type='lambda'):
    import pandas as pd
    data = pd.DataFrame(pd.read_excel('../documents/AttackMAP.xlsx', sheet_name=0))
    curve_label = ['Original', 'HAG', 'SDHA', 'Ours']

    parameters = []
    map_value = []
    for i in range(row[0], row[1]):
        value = data.loc[i].values
        if i == row[0]:
            parameters = [float(v) for v in value[1:7]]
        else:
            map_value.append([float(v) for v in value[1:7]])
    curve_arr = np.array([list(zip(parameters, val)) for val in map_value])

    color, style = color_tuple, style_tuple
    fig, ax = plt.subplots()
    plt.figure(1)

    for i, curve in enumerate(curve_arr):
        x = range(len(curve[:, 0]))
        y = curve[:, 1] * 100
        plt.plot(x, y, label=curve_label[i], c=color[i], ls=style[i], lw=2.5, markerfacecolor='none'
                 , markeredgewidth=2, marker=marker_tuple[i], markersize=ms_tuple[i])
    plt.xticks(range(curve_arr.shape[1]), labels=[str(i) for i in curve_arr[0, :, 0]])
    plt.xlabel('weighting factor ' + ('$\lambda$' if para_type == 'lambda' else '$\mu$'))
    plt.ylabel('MAP(%)')
    plt.legend(loc=(0.05, 0.65))
    plt.grid(linestyle='--')
    fig.subplots_adjust(left=0.1, right=0.99)
    plt.savefig('../documents/{}-{}.pdf'.format('ablation', para_type), dpi=600, format='pdf', transparent=True)
    # plt.show()

def parser_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset_name', dest='dataset', default='NUS-WIDE',
                        choices=['CIFAR-10', 'ImageNet', 'FLICKR-25K', 'NUS-WIDE', 'MS-COCO'],
                        help='name of the dataset')
    parser.add_argument('--hash_method', dest='hash_method', default='DPH',
                        choices=['DPH', 'DPSH', 'HashNet', 'CSQ'],
                        help='deep hashing methods')
    parser.add_argument('--backbone', dest='backbone', default='AlexNet',
                        choices=['AlexNet', 'VGG11', 'VGG16', 'VGG19', 'ResNet18', 'ResNet50'],
                        help='backbone network')
    parser.add_argument('--code_length', dest='bit', type=int, default=32, help='length of the hashing code')
    parser.add_argument('--type', dest='type', type=str, default='pr', help='curve type')
    parser.add_argument('--targeted', dest='targeted', action="store_true", default=False, help='targeted attack')
    return parser.parse_args()

if __name__ == '__main__':
    args = parser_arguments()
    attack_model = '{}_{}_{}_{}'.format(args.dataset, args.hash_method, args.backbone, args.bit)
    filename = args.type + ('_targeted' if args.targeted else '')
    arr = np.load('../log/{}/{}.npy'.format(attack_model, filename))
    label = np.loadtxt('../log/{}/{}.txt'.format(attack_model, filename), dtype=str)
    label = [l if 'Ours' not in l else 'Ours' for l in label]
    plot_curve(arr, label, title=args.dataset, curve_type=args.type, targeted=args.targeted)
    # plot_ablation(row=(89, 94), para_type='lambda')
    # plot_ablation(row=(97, 102), para_type='mu')


#!/usr/bin/env python
# coding: utf-8

# # カーブフィッティング

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd
import scipy as sp
from scipy.optimize import curve_fit
from scipy.stats import bernoulli, norm, poisson, expon

# 日本語フォントの設定（Mac:'Hiragino Sans', Windows:'MS Gothic'）
plt.rcParams['font.family'] = 'Hiragino Sans'

# ## カーブフィッティングとは？

# 手元にあるデータをある理論曲線に当てはめることを**カーブフィッティング**と呼ぶ．
# データ分析では，以下の場面でカーブフィッティングが用いられる：
#
# 1. 回帰分析
#    - 複数の変数から成るデータに対して説明変数と目的変数の間の関数関係（理論曲線）を仮定し，パラメータを調整して当てはめる
# 2. 確率分布のパラメータ推定
#    - 手元のデータから求めたヒストグラムに対して理論的な確率分布を仮定し，そのパラメータを推定する
#    - 推定したパラメータを用いてヒストグラムに理論分布を当てはめる
#
# ※ 統計モデルでは，回帰分析における関数関係や理論的な確率分布など，データを当てはめる理論曲線そのものをモデルと呼ぶ．
# 一方，物理モデルでは，理論曲線のことはモデルと呼ばずに，理論曲線を導くための単純化した基礎方程式などをモデルと呼ぶ．

# ## 回帰分析

# ### 回帰モデル
#
# データ $ X, Y $ が与えられたとき，変数 $ X,\ Y $ の間に
#
# $$
# 	y = f(x; \boldsymbol{\theta}) + \varepsilon
# $$(eq:regression_model)
#
# という関数関係が成り立つとする．
# ここで，$ f(x; \boldsymbol{\theta}) $ は変数 $ X $ の関数で $ \boldsymbol{\theta} = (\theta_1, \theta_2, \ldots) $ はパラメータである．
# また，$ \varepsilon $ は確率的な誤差を表し，特定の確率分布に従うものとする．
# この式は，与えられたデータ $ x $ に対して $ y $ を生成する数理モデルであり，**回帰モデル**と呼ばれる．
# また，$ X $ を**説明変数**（独立変数），$ Y $ を**目的変数**（従属変数，被説明変数）と呼ぶ．
# **回帰分析**とは，回帰モデルを用いて説明変数と目的変数の間の関係を明らかにする手法である．

# 回帰モデルには様々な種類が存在するが，式{eq}`eq:regression_model`のように説明変数が1つだけの場合を**単回帰モデル**，説明変数が複数ある場合を**重回帰モデル**と呼ぶ．
# また，説明変数の１次関数で表されるモデルを**線形回帰モデル**，非線形な関数の場合を**非線形回帰モデル**と呼ぶ．
# 以下では，単回帰モデルのみを扱う．

# ### 最小二乗法
#
# １つの説明変数 $ X $ と目的変数 $ Y $ について $ n $ 組のデータ $ (x_{1}, y_{1}), \ldots, (x_{n}, y_{n}) $ を取得したとする．
# このとき，横軸に変数 $ X $，縦軸に変数 $ Y $ を取ったグラフを**散布図**と呼ぶ．
# 散布図に対し，回帰モデル{eq}`eq:regression_model`のパラメータを変化させて理論曲線をうまく当てはめる作業がカーブフィッティングである．
# カーブフィッティングによって得られた式 $ f(x; \boldsymbol{\theta}) $ （**回帰曲線**）にデータの $ x_{i} $ を代入すると回帰モデルによる予測値が得られる．
# 以下では，この予測値をハット $\hat{}$ を付けて次のように表す：
#
# $$
# 	\hat{y}_{i} = f(x_{i}; \boldsymbol{\theta})
# $$(eq:regression_curve)
#

# 回帰分析では，最適な曲線を求める手法がいくつか存在するが，最も基本的な手法が**最小二乗法**である．
# 最小二乗法では，回帰モデルによる予測値 $ \hat{y}_{i}=f(x_{i}; \boldsymbol{\theta}) $ と実データ $ y_{i} $ の差（**残差**）の二乗和
#
# $$
# 	E(\boldsymbol{\theta}) = \sum_{i=1}^{n}(\hat{y}-y_{i})^{2} = \sum_{i=1}^{n}(f(x_{i}; \boldsymbol{\theta})-y_{i})^{2}
# $$(eq:rss)
#
# が最小となるようなパラメータ $ \boldsymbol{\theta} = (\theta_1, \theta_2, \ldots) $ を選ぶ．
# このとき，$ E $ のことを**残差変動**（残差二乗和）と呼ぶ．
# 残差変動 $ E $ を最小にすることは，$ \theta_1,\ \theta_2,\ \ldots $ を変数とする関数 $ E(\boldsymbol{\theta}) $ の最小値を求めることと言い換えられる．
# このための必要条件は，残差変動 $ E(\boldsymbol{\theta}) $ の $ \theta_1,\ \theta_2,\ \ldots $ による偏微分がゼロになることである：
#
# $$
# 	\frac{\partial E}{\partial \theta_1} = 0, \quad \frac{\partial E}{\partial \theta_2} = 0,\ \ldots
# $$(eq:rss_diff)

# ### 線形単回帰モデル
#
# #### 理論
#
# 線形単回帰モデルは
#
# \begin{align}
#  y = ax + b + \varepsilon
# \end{align}
#
# で与えられ，残差変動は
#
# $$
# 	E(a, b) = \sum_{i=1}^{n} (ax_{i}+b - y_{i})^{2}
# $$
#
# と表される．
# この場合には，最小二乗法による最適なパラメータ $ a,\ b $ を理論的に求めることができる．
# 実際に条件を適用すると，$ a,\ b $ を変数とする以下の連立方程式が得られる：
#
# \begin{align*}
# 	\sum_{i=1}^{n} (ax_{i}+b-y_{i})x_{i} &= 0 \\
# 	\sum_{i=1}^{n} (ax_{i}+b-y_{i}) &= 0
# \end{align*}
#
# この連立方程式を解くと，最適解$ a,\ b $ は
#
# \begin{align}
# 	a &=\frac{\displaystyle\left(\frac{1}{n}\sum_{i=1}^{n} x_{i}y_{i}\right) - \left(\frac{1}{n} \sum_{i=1}^{n}x_{i}\right)\left(\frac{1}{n}\sum_{i=1}^{n}y_{i}\right)}{\displaystyle \left(\frac{1}{n}\sum_{i=1}^{n} x_{i}^{2}\right) -  \left(\frac{1}{n}\sum_{i=1}^{n}x_{i}\right)^{2}}
# 	   = \frac{\overline{xy}-\bar{x}\bar{y}}{\overline{x^2}-\bar{x}^2} \\[10pt]
# 	b &= \left(\frac{1}{n} \sum_{i=1}^{n}y_{i}\right) - a\left(\frac{1}{n} \sum_{i=1}^{n}x_{i}\right) = \bar{y} - a \bar{x}
# \end{align}
#
# と求まる．
# なお，傾き $ a $ の式の分子は $ X, Y $ の共分散，分母は $ X $ の分散となっている（相関係数の式に似ているが違う）．
# よって，傾き $ a $ が正なら正の相関，負なら負の相関，ゼロなら無相関であることが分かる．
#
# ```{admonition} 分散公式と共分散
# :class: dropdown
# $ X $ の分散は以下のように表すことができる：
#
# $$
# \frac{1}{n} \sum_{i=1}^{n} (x_{i} - \bar{x})^{2}
#     = \frac{1}{n} \sum_{i=1}^{n} x_{i}^{2} - \left(\frac{1}{n} \sum_{i=1}^{n}x_{i}\right)^{2}
# 	= \overline{x^2}-\bar{x}^2
# $$
#
# これを**分散公式**と呼ぶ．
# また，$ X, Y $ の共分散は分散公式を用いると以下のように表される：
#
# $$
# 	\frac{1}{n} \sum_{i=1}^{n}(x_{i}-\bar{x})(y_{i}-\bar{y}) = \overline{xy} - \bar{x}\bar{y}
# $$
#
# ```

# #### 決定係数
#
# $ n $ 組のデータ $ (x_{1}, y_{1}), \ldots (x_{n}, y_{n}) $ に対して，データ $ Y $ のばらつきを
#
# $$
#   S_{y}^{2} = \sum_{i=1}^{n} (y_{i} - \bar{y})^{2}
# $$
#
# と定義する．
# これを**全変動**と呼ぶ．
# また，実データと予測値のズレは残差変動{eq}`eq:rss`によって表される．
# ここでは，残差変動を
#
# $$
#   \displaystyle S_{e}^{2} = \sum_{i=1}^{n} (y_{i} - \hat{y}_{i})^{2}
# $$
#
# と表す．
#
# このとき，回帰直線のデータへの当てはまりの良さを
#
# $$
# 	R^{2} = 1 - \frac{S_{e}^{2}}{S_{y}^{2}}
# $$(eq:R2)
#
# で定義する．
# これを**決定係数**と呼ぶ（他の定義もあるので注意）．
#
# 最小二乗法による線形単回帰分析の場合，決定係数は $ X,\ Y $ の相関係数の２乗に等しく，その定義域は $ 0 \leq R^{2} \leq 1 $ である．
# よって，以下が成り立つ
# - 残差変動 $ S_{e}^{2} $ が全変動 $ S_{y}^{2} $ に比べて小さくなると $ R^{2} $ は $ 1 $ に近づく（データへの当てはまりが良い）
# - 残差変動 $ S_{e}^{2} $ が全変動 $ S_{y}^{2} $ に比べて大きくなると $ R^{2} $ は $ 0 $ に近づく（データへの当てはまりが悪い）
#
# ※ 予測値 $ \hat{y} $ が最小二乗法以外の方法によって決められた場合，決定係数が負になることもあり，さらに相関係数の２乗にはならない．

# #### Pythonによる実装

# `scipy`には最小二乗法を用いてカーブフィッティングを実行するための `scipy.optimize.curve_fit` 関数が用意されている．
# まずはこの関数をインポートしておく．

# In[3]:

from scipy.optimize import curve_fit

# `curve_fit` 関数を用いたカーブフィッティングは以下の手順に従って行う．

# **1. フィッティングに用いる関数を定義する**

# In[4]:

# フィッティング関数の定義
def f_linear(x, a, b):
    return a*x + b

# この関数は，第１引数に $ x $ 座標のデータ，第２引数以降にパラメータ $ a, b $ を入力し，出力として $ y $ 座標のデータが得られる．

# **2. フィッティング対象となる実データを用意する**
#
# 実データはNumPy配列やPandasのデータフレーム形式で準備する．
# ここでは，以下のように乱数を用いて生成したデータをcsv形式で一旦保存する．
# その上で，保存したcsvをPandasのDataFrame形式で読み込み，解析する．

# In[6]:

# データの作成
np.random.seed(1234)
x_data = np.linspace(-10, 10, num=100)
y_data = f_linear(x_data, 2, 5) + 5.*np.random.randn(x_data.size)
data = np.vstack([x_data, y_data]).T

# データをcsv形式で保存
np.savetxt('./data_linear.csv', data, delimiter=',')

# In[7]:

# データをDataFrame形式で読み込む
data = pd.read_csv('./data_linear.csv', header=None, names=['x', 'y'])

# 散布図の描画
fig, ax = plt.subplots()
ax.plot(data['x'], data['y'], 'ko', mfc='None')
ax.set_xlabel('$x$', fontsize=15); ax.set_ylabel('$y$', fontsize=15);

# **3. フィッティングを実行する**
#
# `curve_fit` 関数には，第１引数にフィッティング関数，第２，３引数にデータ，それ以降にオプションを指定する．
# オプションにはパラメータの初期値 `p0` などを指定することができる．
# `curve_fit` 関数を実行すると，最小二乗法によって得られた最適なパラメータ（`p_opt`） と共分散（`p_cov`）が戻り値として得られる．

# In[9]:

# フィッティングの実行
p_opt, p_cov = curve_fit(f_linear, data['x'], data['y'], p0=[1, 1])
print(p_opt)

# 以下は公式から得られた最適解を求めた結果である．
# 確かに，`curve_fit` 関数から求めた値と同じ値が得られていることが分かる．

# In[10]:

# 公式から
nume = np.mean(data['x'] * data['y']) - np.mean(data['x']) * np.mean(data['y'])
denom = np.mean(data['x']**2) - np.mean(data['x'])**2
a = nume / denom
b = np.mean(data['y'] - a*data['x'])
print(a, b)

# **4. フィッティング結果を可視化する**

# In[11]:

fig, ax = plt.subplots()
ax.plot(data['x'], data['y'], 'ko', mfc='None')
ax.plot(data['x'], f_linear(data['x'], p_opt[0], p_opt[1]), 'r-', mfc='None')
ax.set_xlabel('$x$', fontsize=15); ax.set_ylabel('$y$', fontsize=15);

# **5. 決定係数を求める**

# In[12]:

# 決定係数
y_reg = f_linear(data['x'], p_opt[0], p_opt[1]) # 回帰直線の値
R2 = 1 - np.var(data['y'] - y_reg) / np.var(data['y']) # 決定係数
R2

# In[13]:

# 相関係数の２乗
r_xy = pd.DataFrame(np.c_[data['x'], data['y']]).corr()[0][1] # pandasのcorr()関数で相関係数を計算
r_xy**2

# ### 非線形単回帰モデル

# 次に，非線形単回帰モデルを考える．
# この場合はフィッティング関数に任意の非線形関数 $ f(x; \boldsymbol{\theta}) $ を指定することができる．
# フィッティングの手順は線形単回帰モデルの場合と同様である．

# **1. フィッティングに用いる関数を定義する**

# In[16]:

def f_exp(x, a, b, c):
    return a * np.exp(-b * x) + c

# **2. フィッティング対象となる実データを用意する**

# In[17]:

# データの作成
np.random.seed(4321)
x_data = np.linspace(0, 4, 50)
y_data = f_exp(x_data, 2.5, 1.3, 0.5) + 0.2 * np.random.normal(size=len(x_data))
data = np.vstack([x_data, y_data]).T

# データをcsv形式で保存
np.savetxt('./data_exp.csv', data, delimiter=',')

# In[18]:

# データをDataFrame形式で読み込む
data = pd.read_csv('./data_exp.csv', header=None, names=['x', 'y'])

# 散布図の描画
fig, ax = plt.subplots()
ax.plot(data['x'], data['y'], 'x')
ax.set_xlabel('$x$', fontsize=15); ax.set_ylabel('$y$', fontsize=15);

# **3. フィッティングを実行する**

# In[21]:

# フィッティングの実行
p_opt = curve_fit(f_exp, data['x'], data['y'])[0]
print(p_opt)

# **4. フィッティング結果を可視化する**

# In[22]:

fig, ax = plt.subplots()
ax.plot(data['x'], data['y'], 'x')
ax.plot(data['x'], f_exp(data['x'], p_opt[0], p_opt[1], p_opt[2]), 'r-', mfc='None')

ax.set_xlabel('$x$', fontsize=15); ax.set_ylabel('$y$', fontsize=15);

# **5. 決定係数を求める**

# In[23]:

# 決定係数
y_reg = f_exp(data['x'], p_opt[0], p_opt[1], p_opt[2]) # 回帰直線の値
R2 = 1 - np.var(data['y']-y_reg) / np.var(data['y']) # 決定係数
R2

# ### 演習問題
#
# 1. 適当な非線形関数からデータを生成し，手順１〜５に従ってカーブフィッティングを実行せよ．
# 2. [soccer_player_europe_2017.csv](https://drive.google.com/uc?export=download&id=13NU87F430KkYJGJZrY44aQECBLjSjKnf)は2017シーズンにサッカーのヨーロッパ5大リーグに所属していた選手のプロフィールである．これをPandasのDataFrameに読み込み，体重（`weight`）が0の選手を削除せよ．体重（`weight`）と身長（`height`）の散布図を描き，線形単回帰モデルによるカーブフィッティングを実行せよ．
# 3. [covid19_korea.csv](https://drive.google.com/uc?export=download&id=14l9chvX4PqHMQQl2yTQTPm7J7S5Us6Xz)は，韓国における新型コロナウイルス感染者数の推移データである．このデータを読み込み，横軸に2020年1月22日を0日とした経過日数，縦軸に感染者数を取った散布図を描け．50日目までと100日目までの散布図に対して，以下のシグモイド関数によるフィッティングを実行せよ．
#
#    $$
#    f(t) = \frac{a}{1+b\mathrm{e}^{-ct}}
#    $$
#
#    ※ 本データの出典：[John Hopkins CSSE](https://github.com/CSSEGISandData/COVID-19/tree/master/csse_covid_19_data/csse_covid_19_time_series)

# ## 確率分布のパラメータ推定
#

# 手元にあるデータから作成したヒストグラムがある確率分布に従うと仮定し，データに合うように確率分布のパラメータを推定することもカーブフィッティングの一つである．

# 確率分布のパラメータ推定では，以下の方法がよく用いられる：
#
# - 最小二乗法
# - 最尤推定
#
# このうち，最小二乗法によるパラメータ推定は回帰分析と手続きは同じである．
# そこで，以下では最尤推定を扱う．

# ### ヒストグラムの作成とフィッティング

# 準備として`scipy`から主要な確率分布をインポートしておく：

# In[2]:

from scipy.stats import bernoulli, norm, poisson, expon

# また，与えられた観測データからヒストグラムを描画し，適当な確率分布をフィッティングするプログラムを作成しておく．

# In[3]:

# データの作成（正規分布に従う1000個のデータ）
np.random.seed(20)
x_data = sp.stats.norm.rvs(loc=170, scale=10, size=1000)

# ヒストグラムの描画
fig, ax = plt.subplots()
ret = ax.hist(x_data, bins=10, density=1, color='c', edgecolor='w')
ax.set_xlabel('$x$', fontsize=12)
ax.set_ylabel('相対度数', fontsize=12)
ax.set_xticks(np.arange(130, 210, 10));

# フィッティング
x = np.arange(140, 200)
ax.plot(x, norm.pdf(x, loc=170, scale=10), 'r--')

# ### 最尤推定

# これまで，パラメータ $ \boldsymbol{\theta} $ を持つ１次元確率分布を $ f(x; \boldsymbol{\theta}) $ と表記していたが，変数と定数の区別を強調するために，これ以降は $ f(x|\boldsymbol{\theta}) $ と表記する．
# 例えば，$ f(x|\boldsymbol{\theta}) $ と書いたときは，$ \boldsymbol{\theta} $ は与えられた定数（前提条件），$ x $ は変数を表しており，$ f(x|\boldsymbol{\theta}) $ は $ x $ の関数であると解釈できる．

# #### 尤度関数
#
# 以上を踏まえ，ある確率分布 $ f(x|\boldsymbol{\theta}) $ に従う母集団から $ n $ 個のデータ $ (x_{1}, x_{2}, \ldots, x_{n}) $ を独立に生成することを考える．
# このようなデータの生成は，データの生成元から見た場合とデータの観測者から見た場合の２通りの視点が考えられる．

# まず，データの生成元から見た場合，確率分布のパラメータ $ \boldsymbol{\theta} $ は分かっており，毎回確率的にデータ $ x_{i} $ が生成される．
# 具体的に，$ x_{i} $ というデータが生成される確率は $ f(x_{i}|\boldsymbol{\theta}) $ であるので，$ n $ 個のデータが得られる確率は
#
# $$
#     f(x_{1}, x_{2},\ldots, x_{n}|\boldsymbol{\theta}) = \prod_{i=1}^{n} f(x_{i}|\boldsymbol{\theta})
# $$
#
# と表される（独立なので，同時確率が個々の確率の積で書ける）．
# このように，パラメータ $ \boldsymbol{\theta} $ が固定されており，生成されるデータ $ (x_{1}, x_{2}, \ldots, x_{n}) $ を変数と捉えた場合は，$ f(x_{1}, x_{2},\ldots, x_{n}|\boldsymbol{\theta}) $ という表記になる．

# 一方，この状況をデータの観測者から見ると，手元のデータ $ (x_{1},\ x_{2},\ \ldots, x_{n}) $ に対して，このデータの生成元となった確率分布のパラメータ $ \boldsymbol{\theta} $ はどのような値だったのだろうか？という見方になる（母集団分布の関数は既知とする）．
# この場合，手元のデータ $ (x_{1},\ x_{2},\ \ldots, x_{n}) $ は定数で，パラメータ $ \boldsymbol{\theta} $ は変数と考えていることになる．
# そこで，同時確率 $ f(x_{1}, x_{2},\ldots, x_{n}|\boldsymbol{\theta}) $ について，$ (x_{1},\ x_{2},\ \ldots, x_{n}) $ を定数，パラメータ $ \boldsymbol{\theta} $ を変数と捉えた場合には
#
# $$
#     L(\boldsymbol{\theta}|x_{1},\ldots, x_{n}) = \prod_{i=1}^{n} f(x_{i}|\boldsymbol{\theta})
# $$
#
# と表記し，これを**尤度関数**と呼ぶことにする．

# #### 最尤推定法
#
# さて，尤度関数というのは，手元のデータが得られる確率を，生成元となった確率分布のパラメータ $ \boldsymbol{\theta} $ の関数と捉えたものである．
# いま，手元にデータが与えられた状態で生成元の確率分布のパラメータがどのような値なら尤もらしいか？と考えてみる．
# すると，尤度が小さくなるようなパラメータだと，生起確率が小さいあり得ないことが起こっていることになるので不適切と考えられる．
# そこで，手元に $ n $ 個のデータが与えられたとき，尤度関数 $ L(\boldsymbol{\theta}|x_{1}, x_{2},\ldots, x_{n}) $ を最大にするように生成元の確率分布のパラメータ $ \boldsymbol{\theta} $ を最も尤もらしいパラメータと考えることにする．
# これは，確率分布のパラメータ推定手法の一つであり，**最尤推定法**と呼ばれている．

# #### 対数尤度方程式
#
# 尤度関数を最大にするようなパラメータを求めるには，尤度関数を各パラメータで偏微分して0になるという条件を課せば良いが，通常は尤度関数の対数を取った**対数尤度関数**に対して偏微分を実行する：
#
# $$
#     \frac{\partial \log L}{\partial \theta_{1}} = 0, \quad \frac{\partial \log L}{\partial \theta_{2}} = 0, \quad \cdots
# $$
#
# この条件式は**対数尤度方程式**と呼ばれている．
# このように対数尤度関数を考えると，得られる解は尤度関数を偏微分した場合と変わらないが，
#
# - 尤度関数の積が和の形に分解できるので計算が簡単になる
# - 尤度関数が極端に小さい値になることを防ぐ
#
# というメリットがある．
# ただし，最尤推定値が実際に計算可能かどうかは対数尤度関数の形によって異なる．
# 例えば，対数尤度関数が凸関数の場合には対数尤度方程式を解くことで大域解が求まるが，非凸関数の場合にはその限りではない．
# また，対数尤度関数は非常に複雑な形をしていることが多いため，数値解を求めるための様々なアルゴリズム（例えばEMアルゴリズムなど）が考案されている．

# ### ベルヌーイ分布の最尤推定

# 最尤推定の具体例として，まずはデータ生成元の確率分布がベルヌーイ分布の場合を考える．
# ベルヌーイ分布とは，以下の式で与えられる離散型確率分布である：
#
# $$
#     f(x|p) = p^{x}(1-p)^{1-x}
# $$
#
# ここで，$ x $ の取りうる値は $ 0, 1 $ の2値である．
# ベルヌーイ分布はコイン投げのように確率変数 $ X $ が2値を取るような試行（ベルヌーイ試行）を1回行ったときの確率分布であり，$ p $ は $ x=1 $ となる確率（表が出る確率）を表すパラメータである．

# #### 解析解
#
# 観測データ $ (x_{1}, \ldots, x_{n}) $ に対する尤度関数は
#
# $$
#     L(p|x_{1}, \ldots, x_{n}) = \prod_{i=1}^{n} p^{x_{i}}(1-p)^{1-x_{i}}=p^{\sum_{i}x_{i}}(1-p)^{n-\sum_{i}x_{i}}
# $$
#
# となるので，対数尤度関数は
#
# $$
#     \log L(p|x_{1}, \ldots, x_{n}) = \left(\sum_{i} x_{i}\right) \log p + \left(n-\sum_{i}x_{i}\right)\log(1-p)
# $$
#
# と表される．
#
# これより，対数尤度方程式は
#
# $$
#     \frac{dL}{dp} = \frac{\sum_{i}x_{i}}{p} - \frac{n-\sum_{i}x_{i}}{1-p} = 0
# $$
#
# となり，これを解けば
#
# $$
#     p = \frac{1}{n} \sum_{i=1}^{n} x_{i}
# $$
#
# が得られる．
# すなわち，$ p $ の最尤推定値は観測データの標本平均である．
# 例えば，コインを5回投げて最初に表が連続して3回出たとすると，データは $ (1,1,1,0,0) $ である．
# よって，この場合の最尤推定値は $ p=3/5 $ となる．

# #### Pythonによる実装

# **1. データの生成**
#
# まずは，パラメータ $ p $ のベルヌーイ分布からサイズ100の標本を発生させる．
# 例えば，以下では $ p=0.3 $ を真の値としている．

# In[3]:

# データの作成
np.random.seed(seed=35)
x = bernoulli.rvs(p=0.3, size=100)
x

# **2. 尤度関数の計算**
#
# 次に，得られた標本（観測データ）から尤度関数を計算する．
# 以下では，0から1まで0.01刻みでパラメータpの値を発生させ，配列`P`に格納する．
# また，同じサイズの配列`L`を作成し，配列`P`の各値に対応する尤度を求めて格納する．
# 同様に，対数尤度を配列`log_L`に格納する．

# In[4]:

L, log_L = [], []
P = np.arange(0, 1, 0.01)

# 様々なPについて尤度と対数尤度を計算する
for p in P:
    l = np.prod(bernoulli.pmf(x, p))
    log_l = np.sum(np.log(np.maximum(bernoulli.pmf(x, p), 1e-100)))
    L.append(l)
    log_L.append(log_l)

L = np.array(L)
log_L = np.array(log_L)

# **3. 最尤推定値を求める**
#
# 2.の時点で様々なパラメータ $ p $ の値に対して尤度 $ L $ と対数尤度 $ \log L $ が求まっている．
# このうち，尤度の値はサンプルサイズが100程度でも非常に小さい値になっていることに注意する．
# サンプルサイズをこれ以上増やすと尤度の値はさらに小さくなるため，数値計算の際には対数尤度を用いた方が良い．

# In[37]:

pd.DataFrame({'P': P, 'L': L, 'log_L': log_L}).head()

# `np.argmax`関数を用いると，ある配列について最大値に対応するインデックスを取得できる．
# これを用いれば，尤度および対数尤度が最大となるパラメータを取得できる．
# いずれの場合も同じ最尤推定値が得られ，解析解（標本平均）と等しいことが分かる．

# In[38]:

# 尤度の最大値を与えるパラメータ
p_mle_1 = P[np.argmax(L)]
p_mle_1

# In[39]:

# 対数尤度の最大値を与えるパラメータ
p_mle_2 = P[np.argmax(log_L)]
p_mle_2

# In[40]:

# 最尤推定値の解析解（標本平均）
np.mean(x)

# **4. 尤度関数と最尤推定値を描画する**
#
# 最後に，尤度関数を描画し，さらに尤度関数が最大となる $ p $ の値をグラフ中にプロットする．

# In[41]:

fig, ax = plt.subplots(figsize=(5, 4))

# 尤度関数の描画
ax.plot(P, L, '-') # 尤度関数
ax.vlines(p_mle_1, 0, np.max(L), color='r', linestyles='dashed') # Lの最大値を与えるパラメータを赤線で表示
ax.set_xlabel('$p$', fontsize=12)
ax.set_ylabel('尤度$L(p)$', fontsize=12)
ax.set_xlim(0, 1); ax.set_ylim(ymin=0);

# 対数尤度関数の描画
ax2 = ax.twinx() # 新しいy軸を作成
ax2.plot(P, log_L, 'b-') # 対数尤度関数
ax2.set_xlabel('$p$', fontsize=12)
ax2.set_ylabel('対数尤度 $\log L(p) $', fontsize=12)
ax2.set_xlim(0, 1); ax2.set_ylim(-200, 0);

# ### 正規分布の最尤推定

# 次に，最尤推定値の解析解が求まるもう少し複雑な例として，生成元の確率分布が正規分布の場合を考える．
# 正規分布の確率密度関数は以下で与えられる：
#
# $$
#     f(x|\mu, \sigma) = \frac{1}{\sqrt{2\pi\sigma^2}} \mathrm{e}^{-\frac{(x-\mu)^2}{2\sigma^2}}
# $$
#
# ここで，$ \mu, \sigma $ がパラメータでそれぞれ期待値と標準偏差に対応する．

# #### 解析解

# 観測データ $ x_{1}, \ldots, x_{n} $ に対する尤度関数は
#
# $$
#     L(\mu, \sigma|x_{1}, \ldots, x_{n}) = \prod_{i=1}^{n} \frac{1}{\sqrt{2\pi\sigma^2}} \mathrm{e}^{-\frac{(x_{i}-\mu)^2}{2\sigma^2}}
#     = \frac{1}{(2\pi\sigma^2)^{n/2}} \exp\left[-\frac{1}{2\sigma^2}\sum_{i=1}^{n}(x_{i}-\mu)^2\right]
# $$
#
# となるので，対数尤度関数は
#
# $$
#     \log L = -\frac{n}{2} \log(2\pi) -\frac{n}{2} \log(\sigma^2) - \frac{1}{2\sigma^2} \sum_{i=1}^{n} (x_{i}-\mu)^2
# $$
#
# と表される．

# これより，対数尤度方程式は
#
# \begin{align*}
#     \frac{\partial \log L}{\partial \mu} &= -\frac{1}{2\sigma^2} \sum_{i=1}^{n}2(x_{i}-\mu)\times (-1)
#     = \frac{1}{\sigma^2}\sum_{i}^{n}(x_{i}-\mu) = 0 \\
#     \frac{\partial \log L}{\partial \sigma^2} &= -\frac{n}{2\sigma^2} + \frac{1}{2\sigma^4} \sum_{i=1}^{n} (x_{i} - \mu)^2 = 0
# \end{align*}
#
# となり，これらを解けば，
#
# \begin{align*}
#     \mu &= \frac{1}{n}\sum_{i=1}^{n}x_{i}\\
#     \sigma^2 &= \frac{1}{n}\sum_{i=1}^{n}(x_{i}-\mu)^2
# \end{align*}
#
# が得られる．
# すなわち，$ \mu,\ \sigma^2 $ の最尤推定値はそれぞれ標本平均と標本分散であることが分かる．

# #### Pythonによる実装

# **1. データの生成**
#
# まずは平均0，標準偏差1の正規分布（標準正規分布）からサイズ100の標本を生成する．

# In[42]:

# 標準正規分布からサイズ1000の標本を生成する
np.random.seed(seed=32)
x = norm.rvs(loc=0, scale=1, size=100)

# **2. 尤度関数を計算する**
#
# 次の，得られた標本（観測データ）から尤度関数を計算する．
# 以下では，$ \mu $ の値を-0.1から0.1の範囲，標準偏差 $ \sigma $ の値を0.9から1.1の範囲でそれぞれ100個発生させ，$ \mu $ と $ \sigma $ のパラメータの組を作る．
# その上で，すべてのパラメータの組について尤度と対数尤度を計算し，それぞれ配列 `L` と `log_L` に格納する．

# In[51]:

mean_range = np.linspace(-0.1, 0.1, 100)
std_dev_range = np.linspace(0.9, 1.1, 100)
M, S = np.meshgrid(mean_range, std_dev_range)
L, log_L = [], []

for m, s in zip(M.ravel(), S.ravel()):
    L.append(np.prod(norm.pdf(x, loc=m, scale=s))) # 尤度の計算
    log_L.append(np.sum(np.log(np.maximum(norm.pdf(x, loc=m, scale=s), 1e-100)))) # 対数尤度の計算

L = np.array(L).reshape(M.shape)
log_L = np.array(log_L).reshape(M.shape)

# **3. 最尤推定値を求める**
#
# 今回は `L` および `log_L` の最大値に対応する行番号と列番号を取得する必要があるので，`np.argmax`関数ではなく`np.where`関数を用いる．

# In[68]:

# Lとlog_Lの最大値に対応するインデックス（行番号，列番号）を取得
ind_max = np.where(L==L.max())
ind_max2 = np.where(log_L==log_L.max())
ind_max, ind_max

# In[69]:

# 尤度の最大値を与えるパラメータ
M[ind_max], S[ind_max]

# In[70]:

# 対数尤度の最大値を与えるパラメータ
M[ind_max2], S[ind_max2]

# In[71]:

# 解析解
np.mean(x), np.std(x, ddof=0)

# **4. 尤度関数を描画する**
#
# 今回はパラメータが2つあるので，尤度関数と対数尤度をそれぞれお3次元プロットする必要がある．

# In[72]:

# 尤度関数の描画
fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))
ax.plot_wireframe(M, S, L)
ax.set_xlabel('$\mu$')
ax.set_ylabel('$\sigma$')
ax.set_zlabel('Likelihood')

# In[73]:

# 対数尤度関数の描画
fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))
ax.plot_wireframe(M, S, log_L)
ax.set_xlabel('$\mu$')
ax.set_ylabel('$\sigma$')
ax.set_zlabel('Log Likelihood')

# **5. 最尤推定値でカーブフィッティング**
#
# 最後に最尤推定したパラメータを用いて，正規分布の確率密度関数を描画する．
# 実データのヒストグラムと重ねて描画すると，上手くフィッティングできていることが分かる．

# In[74]:

# ヒストグラムの描画
fig, ax = plt.subplots()
ret = ax.hist(x, bins=10, density=1, color='c', edgecolor='w')

x2 = np.linspace(-3, 3, 100)
ax.plot(x2, norm.pdf(x2, loc=np.mean(x), scale=np.std(x)), 'r-');

# ### 演習問題

# **1. 指数分布の最尤推定**
#
# 指数分布は以下で与えられる：
#
# $$
#     f(x|\lambda) = \frac{1}{\lambda}\mathrm{e}^{-\frac{x}{\lambda}}
# $$
#
# - 最尤推定値の解析解を計算せよ．
# - ベルヌーイ分布の場合を参考に，指数分布の最尤推定をPythonで実装せよ．
#
# **2. ポアソン分布の最尤推定**
#
# ポアソン分布は以下で与えられる：
#
# $$
#     f(x|\lambda) = \frac{\lambda^{x}\mathrm{e}^{-\lambda}}{k!}
# $$
#
# - 最尤推定値の解析解を計算せよ．
# - ベルヌーイ分布の場合を参考に，ポアソン分布の最尤推定をPythonで実装せよ．
#
#
# **3. Scipyによる最尤推定の実装**
#
# - `scipy.optimize.minimize`関数を用いて，任意の確率分布の最尤推定を行うプログラムを作成せよ．
# - ChatGPTなどを駆使して良い


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import uiautomator

try:
    from setuptools import setup
except ImportError:
    from distutils.core import setup

requires = ["urllib3>=1.7.1"]

setup(name='uiautomator',
      version=uiautomator.__version__,
      description='Python Wrapper for Android UiAutomator test tool',
      long_description=uiautomator.__doc__,
      author=uiautomator.__author__,
      author_email='xiaocong@gmail.com',
      url='https://github.com/xiaocong/uiautomator',
      install_requires=requires,
      py_modules=['uiautomator'],
      scripts=['uiautomator.py'],
      setup_requires=['nose>=1.0', 'mock>=1.0.1', 'coverage>=3.6'] + requires,
      license='MIT',
      platforms='any',
      classifiers=(
            'Development Status :: 2 - Pre-Alpha',
            'Environment :: Console',
            'Intended Audience :: Developers',
            'Operating System :: POSIX :: Linux',
            'Programming Language :: Python',
            'Topic :: Software Development :: Libraries',
            'Topic :: Software Development :: Testing',
            )
      )


## 5014 스타트링크 (DFS/BFS)

import sys
input = sys.stdin.readline
from collections import deque

f, start, end, up, down = map(int, input().split())
## 백만층까지 ㄱㄴ

## start => end 까지의 최소값
visit = [0]*(f+1)
queue = deque()
ud = [up, (-down)]

def circle(start):
    global visit
    queue.append(start)
    while queue:
        idx = queue.popleft()
        if idx == end:
            break

        for i in range(2):
            next = idx + ud[i]
            if i==0 and up==0:
                continue
            if i==1 and down==0:
                continue
            if 0 < next <= f and visit[next] == 0:
                visit[next] = visit[idx]+1
                queue.append(next)

circle(start)

if start == end:
    print(0)
elif visit[end] == 0:
    print("use the stairs")
else:
    print(visit[end])


import os
import sys
import pickle
import pandas as pd
import dask.dataframe as dd
import numpy as np
import xgboost as xgb

from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer

DATA_PATH = '../../data/balanced-data.csv'
TFIDF_PATH = './tfidf.pkl'
THEMES = [5, 6, 33, 163, 181, 188, 232, 313, 339, 350, 406, 409, 424, 555, 589,
          597, 634, 660, 695, 766, 773, 793, 800, 810, 852, 895, 951]

def groupby_process(df):
    new_df = df.sort_values(['process_id', 'page'])
    new_df = new_df.groupby(
                ['process_id', 'themes'],
                group_keys=False
            ).apply(lambda x: x.body.str.cat(sep=' ')).reset_index()
    new_df = new_df.rename(index=str, columns={0: "body"})
    return new_df

def get_data(path):
    data = pd.read_csv(path)
    data = data.rename(columns={'document_id': 'process_id', 'pages': 'page'})
    #print(data.head())
    return data

def predict(dtest, clf):
    pred = clf.predict(dtest)
    return pred

def split_data(data):
    data.themes = data.themes.apply(lambda x: 'Theme ' + str(x))
    train_body, test_body, train_labels, test_labels = train_test_split(
        data.body,
        data.themes,
        test_size=0.33,
        random_state=42,
        stratify=data.themes
    )

    return train_body, test_body, train_labels, test_labels

def tfidf(train_body, test_body):
    tfidf = TfidfVectorizer(
        min_df=0.2,
        max_df=0.9,
        lowercase=False,
        use_idf=1,
        smooth_idf=1,
        #  sublinear_tf=1,
        # max_features=20000
    )

    X_train = tfidf.fit_transform(train_body)
    X_test = tfidf.transform(test_body)

    print('\tX_train', X_train.shape)
    print('\t\tX_test', X_test.shape)

    return X_train, X_test

def tfidf_from_file(path, body):
    with open(path, 'rb') as f:
        tfidf = pickle.load(f)

    transformed = tfidf.transform(body)
    return transformed

def transform_y(train_labels, test_labels):
    le = LabelEncoder()
    le.fit(train_labels)

    le_train = le.transform(train_labels)
    le_test = le.transform(test_labels)

    print(le.classes_)

    return le_train, le_test, le

def report(y_true, pred):
    try:
        print(classification_report(y_true, pred))
        # print(confusion_matrix(y_true, pred))
    except Exception as e:
        print(e)

if __name__ == '__main__':
    #  dtrain, dtest = get_data('./dtrain', './dtest')

    CHUNKS = 30000

    data = get_data(DATA_PATH)
    #print(data.head())

    print('Values counts: \n', data.themes.value_counts())

    train_body, test_body, train_labels, test_labels = split_data(data)
    #print(train_labels.head(100))

    print('Chunk size: {}'.format(CHUNKS))

    try:
        X_train, X_test = tfidf(train_body, test_body)
    except Exception as e:
        print(e)

    y_train, y_test, le = transform_y(train_labels, test_labels)

    print('We\'re classifying {} themes!'.format(len(le.classes_)))

    print('Started Training!')
    svc = OneVsRestClassifier(
            LinearSVC(),
            n_jobs=-1
            )

    svc.fit(X_train, y_train)
    print("Finished training")

    print('deleting obsolette vars')
    del X_train, y_train

    print('Starting predictions on dtest!')
    pred = svc.predict(X_test)

    print(y_test[:5], pred[:5])
    print(pred.max(), pred.min(), pred.mean(), pred.sum())
    #print(le.inverse_transform(y_test))
    #print(le.inverse_transform(pred.astype(int)))
    report(le.inverse_transform(y_test), le.inverse_transform(pred.astype(int)))


# -*- coding: utf-8 -*-
#
#
# Author: alex
# Created Time: 2017年12月01日 星期五 22时14分24秒
import jieba
import logging
import gensim
from sklearn.cluster import KMeans
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                    level=logging.INFO)

# get input file,text format
source_filename = 'source.txt'
seq_filename = 'output.seq'
if False:
    with open(source_filename, 'r') as f, open(seq_filename, 'w') as output:
        input_lines = f.readlines()
        count = len(input_lines)
        print('源文件行数为：' + str(count))

        # read file and seperate words
        for line in input_lines:
            line = line.strip('\n')
            seg_list = jieba.cut(line)
            output.write(' '.join(seg_list)+'\n')

# 训练并保存模型
model_filename = "output.model"
sentences = gensim.models.doc2vec.TaggedLineDocument(seq_filename)
model = gensim.models.Doc2Vec(sentences, size=100, window=3)
model.train(sentences)
model.save(model_filename)

# k-means预测
num_clusters = 20    # 设定聚类个数
km = KMeans(n_clusters=num_clusters)  # 设置模型参数
result = km.fit_predict(model.docvecs)  # 用doc2vec训练的模型产生的句向量作为k-means的输入

# 将聚类预测结果和具体文档处理对应上
output_filename = 'output.txt'
with open(source_filename, 'r') as fr, open(output_filename, 'w') as fw:
    for i in range(0, num_clusters-1):
        for linenumber, eachline in enumerate(fr.readlines()):
            if linenumber >= len(result):
                break
            if result[linenumber] == i:
                fw.write(eachline+'\n')

        fw.write('------------------ %d -----------------------\n' % i)


import random, math
from utils import *

class Tile:

	def __init__(self, passable, symbol, stair=False):
		self.passable = passable
		assert len(symbol) == 1, "Symbol must be exactly one character"
		self.symbol = symbol
		self.revealed = False
		self.walked = False
		self.stair = stair
		self.items = []

class Board:

	def __init__(self, g, cols, rows):
		self.g = g
		self.cols = cols
		self.rows = rows
		self.data = [[Tile(True, " ") for x in range(cols)] for y in range(rows)]
		self.clear_cache()

	def clear_cache(self):
		self.mons_cache = [[None for x in range(self.cols)] for y in range(self.cols)]

	def line_between(self, pos1, pos2, skipfirst=False, skiplast=False):
		x1, y1 = pos1
		x2, y2 = pos2
		dx = abs(x2 - x1)
		sx = 1 if x1 < x2 else -1
		dy = -abs(y2 - y1)
		sy = 1 if y1 < y2 else -1
		error = dx + dy
		while True:
			doyield = True
			if (skipfirst and (x1, y1) == pos1) or (skiplast and (x1, y1) == pos2):
				doyield = False
			if doyield:
				yield (x1, y1)
			if (x1, y1) == (x2, y2):
				return
			e2 = 2 * error
			if e2 >= dy:
				if x1 == x2:
					return
				error += dy
				x1 += sx
			if e2 <= dx:
				if y1 == y2:
					return
				error += dx
				y1 += sy

	def in_bounds(self, x, y):
		return 0 <= x < self.cols and 0 <= y < self.rows

	def line_of_sight(self, pos1, pos2):
		for point in self.line_between(pos1, pos2, skiplast=True):
			if self.blocks_sight(*point):
				return False
		return True

	def is_clear_path(self, pos1, pos2):
		for point in self.line_between(pos1, pos2, skipfirst=True, skiplast=True):
			if not self.is_passable(*point):
				return False
		return True

	def get_in_radius(self, pos, radius):
		x, y = pos
		for px in range(x-radius, x+radius+1):
			for py in range(y-radius, y+radius+1):
				if (px, py) != pos and self.in_bounds(x, y):
					yield (px, py)

	def get_in_circle(self, pos, radius):
		cx, cy = pos
		for x, y in self.get_in_radius(pos, radius):
			dx = x - cx
			dy = y - cy
			dist = math.sqrt(dx**2 + dy**2)
			if round(dist) > radius or not self.in_bounds(x, y):
				continue
			yield x, y

	def get_in_cone(self, pos, radius, angle, widthdeg=45):
		cx, cy = pos
		angle %= 360
		for x, y in self.get_in_radius(pos, radius):
			dx = x - cx
			dy = y - cy
			dist = math.sqrt(dx**2 + dy**2)
			if round(dist) > radius or not self.in_bounds(x, y):
				continue
			dir = math.degrees(math.atan2(dy, dx))
			half = widthdeg/2
			if dir < 0:
				dir += 360

			if abs(angle - dir) <= half:
				yield x, y
			elif angle + half >= 360 and dir <= (angle + half) % 360:
				yield x, y
			elif angle - half < 0 and dir >= angle-half+360:
				yield x, y

	#A monster collision cache is used to improve the performance of detecting collisions with monsters
	#This way, checking if there's a monster at a position can be O(1) instead of O(m)

	def set_cache(self, x, y, mon):
		self.mons_cache[y][x] = mon

	def unset_cache(self, x, y):
		self.mons_cache[y][x] = None

	def get_mon_cache(self, x, y):
		return self.mons_cache[y][x]

	def swap_cache(self, pos1, pos2):
		if pos1 == pos2:
			return
		x1, y1 = pos1
		x2, y2 = pos2
		tmp = self.mons_cache[y1][x1]
		self.mons_cache[y1][x1] = self.mons_cache[y2][x2]
		self.mons_cache[y2][x2] = tmp

	def blocks_sight(self, col, row):
		if (col, row) == (self.g.player.x, self.g.player.y):
			return False
		return not self.get(col, row).passable

	def is_passable(self, col, row):
		if self.blocks_sight(col, row):
			return False
		return not self.mons_cache[row][col]

	def generate(self):
		self.data = [[Tile(False, "#") for x in range(self.cols)] for y in range(self.rows)]
		self.clear_cache()
		WIDTH_RANGE = (5, 10)
		HEIGHT_RANGE = (3, 5)
		ATTEMPTS = 100
		NUM = random.randint(5, 8)
		rooms = []
		randchance = dice(2, 10)
		if one_in(7):
			randchance = 100
		for i in range(NUM):
			for _ in range(ATTEMPTS):
				width = random.randint(*WIDTH_RANGE)
				height = random.randint(*HEIGHT_RANGE)
				xpos = random.randint(1, self.cols - width - 1)
				ypos = random.randint(1, self.rows - height - 1)
				for x, y, w, h in rooms:
					flag = True
					if x + w < xpos or xpos + width < x:
						flag = False
					elif y + h < ypos or ypos + height < y:
						flag = False
					if flag:
						break
				else:
					for x in range(width):
						for y in range(height):
							self.carve_at(xpos + x, ypos + y)
					if i > 0:
						prev = rooms[-1]
						if random.randint(1, randchance) == 1:
							prev = random.choice(rooms)
						x, y, w, h = prev
						pos1_x = x + random.randint(1, w - 2)
						pos1_y = y + random.randint(1, h - 2)
						pos2_x = xpos + random.randint(1, width - 2)
						pos2_y = ypos + random.randint(1, height - 2)
						dx = 1 if pos1_x < pos2_x else -1
						dy = 1 if pos1_y < pos2_y else -1
						if one_in(2):
							x = pos1_x
							while x != pos2_x:
								self.carve_at(x, pos1_y)
								x += dx
							y = pos1_y
							while y != pos2_y:
								self.carve_at(pos2_x, y)
								y += dy
						else:
							y = pos1_y
							while y != pos2_y:
								self.carve_at(pos1_x, y)
								y += dy
							x = pos1_x
							while x != pos2_x:
								self.carve_at(x, pos2_y)
								x += dx

					rooms.append((xpos, ypos, width, height))
					break

	def carve_at(self, col, row):
		if not (0 <= col < self.cols and 0 <= row < self.rows):
			raise ValueError(f"carve_at coordinate out of range: ({col}, {row})")
		self.data[row][col] = Tile(True, " ")

	def get(self, col, row):
		return self.data[row][col]

###############
#Pathfinding
#Algorithm used is A* Search
from collections import defaultdict

class OpenSet:

	def __init__(self, key=None):
		self._data = []
		self._dup = set()
		self.key = key or (lambda v: v)

	def add(self, value):
		if value in self._dup:
			return
		self._dup.add(value)
		a = self._data
		key = self.key
		i = len(a)
		a.append(value)
		while i > 0:
			parent = i // 2
			if key(a[parent]) < key(a[i]):
				break
			a[parent], a[i] = a[i], a[parent]
			i = parent

	def pop(self):
		if len(self._data) == 0:
			raise IndexError("pop from an empty heap")
		a = self._data
		val = a[0]
		a[0] = a[-1]
		a.pop()
		key = self.key
		i = 0
		while True:
			left = 2 * i + 1
			right = 2 * i + 2
			if left >= len(a):
				break
			node = left
			if right < len(a) and key(a[right]) < key(a[left]):
				node = right
			if key(a[i]) > key(a[node]):
				a[i], a[node] = a[node], a[i]
				i = node
			else:
				break
		self._dup.remove(val)
		return val

	def __contains__(self, value):
		return value in self._dup

	def __bool__(self):
		return len(self._data) > 0

def pathfind(board, start, end, *, rand=False):
	#Actual A* Search algorithm
	def h(a, b):
		return abs(a[0] - b[0]) + abs(a[1] - b[1])
	gScore = defaultdict(lambda: float("inf"))
	gScore[start] = 0
	fScore = defaultdict(lambda: float("inf"))
	fScore[start] = h(start, end)
	open_set = OpenSet(fScore.__getitem__)
	open_set.add(start)
	came_from = {}
	rows = board.rows
	cols = board.cols
	def can_pass(x, y):
		if (x, y) == end:
			return not board.blocks_sight(x, y)
		return board.is_passable(x, y)
	while open_set:
		curr = open_set.pop()
		if curr == end:
			path = [curr]
			while curr in came_from:
				curr = came_from[curr]
				path.append(curr)
			path.reverse()
			return path
		neighbors = []
		x, y = curr
		if x + 1 < cols and can_pass(x + 1, y):
			neighbors.append((x + 1, y))
		if x - 1 >= 0 and can_pass(x - 1, y):
			neighbors.append((x - 1, y))
		if y + 1 < rows and can_pass(x, y + 1):
			neighbors.append((x, y + 1))
		if y - 1 >= 0 and can_pass(x, y - 1):
			neighbors.append((x, y - 1))
		if rand:
			random.shuffle(neighbors)

		for n in neighbors:
			cost = 1
			t = gScore[curr] + cost
			if t < gScore[n]:
				came_from[n] = curr
				gScore[n] = t
				fScore[n] = t + h(n, end)

				if n not in open_set:
					open_set.add(n)
	return []

#End pathfinding
###############