import warnings
import sys

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sqlalchemy import create_engine

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords, wordnet
from nltk.stem.wordnet import WordNetLemmatizer
import re

from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.metrics import classification_report

from sklearn.externals import joblib
from flask import Flask
from flask import render_template, request, jsonify
from plotly.graph_objs import Bar

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('wordnet')
nltk.download('words')

app = Flask(__name__)

def tokenize_fct(text):

    '''
    Tokenize by word, remove stop words, lemmatize all phrases in the dataframe

    INPUT:
    text

    OUTPUT:
    df - df with three new columns: one normalised, one tokenized, another with stop words removed in addition, and the
    last one lemmatized in addition to previous steps
    '''

    new_text = re.sub(r"[^a-zA-Z]", " ", text.lower())
    new_text = word_tokenize(new_text)

    sw_list = stopwords.words("english")
    eng_words = set(nltk.corpus.words.words())
    new_text = [w for w in new_text if w not in sw_list]
    new_text = [w for w in new_text if w in eng_words]

    for postag in ['a','s','r','n','v']:
        new_text = [WordNetLemmatizer().lemmatize(w, pos = postag) for w in new_text]

    return new_text

class TfidfEmbeddingVectorizer(BaseEstimator, TransformerMixin):

    def fit(self, X, y = None, col_name = ['message'], vect_size = 50):
        '''
        Tokenize sentences, then fit with w2v model (for word vectors) and tfidf
        for weighting
        '''
        self.vect_size = vect_size

        self.tok_X = []
        for line in X[col_name].values:
            self.tok_X.append(tokenize_fct(line[0]))

        # Fit w2v model
        model = Word2Vec(self.tok_X, size=vect_size)
        self.word_vector = dict(zip(model.wv.index2word, model.wv.vectors))

        # Fit tfidf for weighting
        tfidf = TfidfVectorizer(analyzer=lambda x: x, lowercase = False)
        tfidf.fit(self.tok_X)
        self.word_list = list(tfidf.vocabulary_.keys())
        self.idf_list = list(tfidf.idf_[list(tfidf.vocabulary_.values())])
        self.word_weight = dict(zip(word_list, idf_list))

        print('Embedding fitted!')
        return self

    def transform(self, X):
        '''
        return weighted mean vector of vect size as specified above
        '''
        self.weighted_vector = np.empty((0, self.vect_size), float)
        for line in self.tok_X:
            if len(line)>0:
                new_weight = [self.word_vector[w] * self.word_weight[w] for w in line \
                              if (w in self.word_vector) and (w in self.word_weight)]\
                            + [np.zeros(self.vect_size) for w in line \
                               if (w not in self.word_vector) or (w not in self.word_weight)]
                new_weight_mean = np.mean(new_weight, axis = 0).reshape(1, -1)
                self.weighted_vector = np.concatenate((self.weighted_vector, new_weight_mean), axis = 0)
            else:
                self.weighted_vector = np.concatenate((self.weighted_vector,
                                                       np.zeros(self.vect_size).reshape(1,-1)), axis = 0)

        return pd.DataFrame(self.weighted_vector)

class tfidfExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'message'):
        self.col_name = col_name
        self.countvect = CountVectorizer(tokenizer = tokenize_fct)
        X_cvect = self.countvect.fit_transform(X[col_name].values.ravel())
        self.tfidftrans = TfidfTransformer(smooth_idf=False)
        self.tfidftrans.fit(X_cvect)

        print('tfidf fitted!')
        return self

    def transform(self, X):
        # apply tfidf
        X_cvect = self.countvect.transform(X[self.col_name].values.ravel())
        X_tfidf = self.tfidftrans.transform(X_cvect)
        return X_tfidf

class GenreExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'genre'):
        self.col_name = col_name
        print('Genre fitted!')
        return self

    def transform(self, X):
        # apply one hot encoding
        X_one_hot = pd.get_dummies(X[self.col_name], drop_first = True)
        return X_one_hot

class LengthExtractor(BaseEstimator, TransformerMixin):

    def fit(self, X, y=None, col_name = 'original'):
        self.col_name = col_name
        return self

    def transform(self, X):
        new_col = pd.DataFrame(X[self.col_name].fillna('').apply(lambda x:len(x)))
        return new_col.values

# load data
def load_data(data_filepath = '../data/cleaned_data.db'):
    '''
    Load in data from designated filepath
    '''
    engine = create_engine('sqlite:///' + data_filepath)
    df = pd.read_sql_table('cleaned_data', engine)
    return df

df = load_data()

# load model
model = joblib.load("../models/trained_model.pkl")

# index webpage displays cool visuals and receives user input text for model
@app.route('/')
@app.route('/index')
def index():

    # extract data needed for visuals
    genre_counts = df.groupby('genre').count()['message']
    genre_names = list(genre_counts.index)

    Y = df.drop(['id', 'genre', 'message', 'original'], axis = 1)
    top5_cat = (Y.sum()/len(Y)).sort_values(ascending = False)[:5]
    top5_cat_val = top5_cat.values * 100
    top5_cat_names = [i.replace('_', ' ') for i in list(top5_cat.index)]

    # create visuals
    graphs = [
        {
            'data': [
                Bar(
                    x=genre_names,
                    y=genre_counts
                )
            ],

            'layout': {
                'title': 'Distribution of Message Genres',
                'yaxis': {
                    'title': "Count"
                },
                'xaxis': {
                    'title': "Genre"
                }
            }
        },

        {
            'data': [
                Bar(
                    x=top5_cat_names,
                    y=top5_cat_val
                )
            ],

            'layout': {
                'title': 'Top 5 message categories',
                'yaxis': {
                    'title': "Proportion of messages in the category (%)"
                },
                'xaxis': {
                    'title': "Category"
                }
            }
        }
    ]

    # encode plotly graphs in JSON
    ids = ["graph-{}".format(i) for i, _ in enumerate(graphs)]
    graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)

    # render web page with plotly graphs
    return render_template('master.html', ids=ids, graphJSON=graphJSON)

# web page that handles user query and displays model results
@app.route('/go')
def go():
    # save user input in query
    query = request.args.get('query', '')

    # use model to predict classification for query
    classification_labels = model.predict([query])[0]
    classification_results = dict(zip(df.columns[4:], classification_labels))

    # This will render the go.html Please see that file.
    return render_template(
        'go.html',
        query=query,
        classification_result=classification_results
    )

def main():
    app.run(host='0.0.0.0', port=3001, debug=True)

if __name__ == '__main__':
    main()


'''折线图条形图组合:补贴总额和平均补贴'''
import pandas as pd
from pyecharts import options as opts
from pyecharts.charts import Line
from pyecharts.charts import Bar
from pyecharts.charts import Grid
from pyecharts.render import make_snapshot
from snapshot_selenium import snapshot

# 导入数据:
datafile = r'E:\学术论文\博士论文\论文图表\政府补贴现状数据.xlsx'
data = pd.read_excel(datafile)
X = list(data.年份) # 或者attr = list(data.['年份'])
Z1 = list(data['补贴金额（亿元）'])
Y1 = []
for i in Z1:
    Y1.append(float('%.2f'%i)) # 总补贴金额（亿元） 条形图数据
Z2 = list(data['平均每家公司补贴'])
Y2 = []
for i in Z2:
    Y2.append(float('%.2f'%i)) # 平均每家公司补贴（亿元） 折线图数据

def grid_multiy():
    bar=(
        Bar()
        .add_xaxis(xaxis_data=X)
        .add_yaxis(
            series_name='上市公司获得总补贴(亿元)',
            yaxis_data=Y1,
            color='#5793f3', #蓝色
            label_opts=opts.LabelOpts(is_show=False),
            )
        .extend_axis(
            yaxis=opts.AxisOpts(
                name='平均补贴',
                type_='value',
                min_=0.2,
                max_=0.8,
                position='right',
                name_textstyle_opts=opts.TextStyleOpts(
                    font_size=20,
                    font_weight='bold',
                    color='#d14a61',
                ),
                #is_scale='True',
                axislabel_opts=opts.LabelOpts(
                    font_size=15,
                    color='#d14a61'
                ),
                axisline_opts=opts.AxisLineOpts(
                    linestyle_opts=opts.LineStyleOpts(
                        color='#d14a61',
                    ),
                ),
            )
        )
        .set_global_opts(
            legend_opts=opts.LegendOpts(
                orient='vertical',
                textstyle_opts=opts.TextStyleOpts(
                    font_size=20,
                ),
            ),
            title_opts=opts.TitleOpts(
                #title='图2.1 上市公司获得政府补贴及每家公司平均获得补贴金额',
                title_textstyle_opts=opts.TextStyleOpts(
                    font_size=25,
                ),
                pos_left='center',
                pos_bottom='bottom',
            ),
            yaxis_opts=opts.AxisOpts(
                name='总补贴',
                position='left',
                #offset=30,
                name_textstyle_opts=opts.TextStyleOpts(
                    font_size=20,
                    font_weight='bold',
                    color='#5793f3',
                ),
                #is_scale='True',
                axislabel_opts=opts.LabelOpts(
                    font_size=15,
                    color='#5793f3',
                ),
                axisline_opts=opts.AxisLineOpts(
                    linestyle_opts=opts.LineStyleOpts(
                        color='#5793f3',
                    ),
                ),
            ),
        ) # 此处不能多加逗号，否则会结果对象会改变，由一个Bar对象变为多个Bar构成的tuple集合；
    )

    line=(
        Line()
        .add_xaxis(xaxis_data=X)
        .add_yaxis(
            series_name='每家上市公司获得的平均补贴（亿元）',
            y_axis=Y2,
            yaxis_index=1,  #多Y轴时，要设置此项；
            label_opts=opts.LabelOpts(is_show=False),
        )
        .set_global_opts(
            legend_opts=opts.LegendOpts(
                orient='vertical',
                textstyle_opts=opts.TextStyleOpts(
                    font_size=20,
                ),
            ),
        )
    )

    bar.overlap(line)
    return bar
                                # 函数不能多一个括号
grid_multiy().render()
grid_multiy().render(r"E:\学术论文\博士论文\论文图表\图2-1上市公司获得补贴数据.html")
make_snapshot(snapshot,grid_multiy().render(),r"E:\学术论文\博士论文\论文图表\图2-1上市公司获得补贴数据.png")


import traceback
import os
import sys
import time
import datetime
current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)
from cosmoscan_api.cosmosWsApi import get_validators

sys.path.append(os.path.dirname(current_dir))
from influxdb_client.influxdb_client_v1 import InfluxClient
from utility.error_logger_writer import logger

db = InfluxClient()

measurement = "cosmoscan_validators"

def subscribe_validators(measurement):
    data = get_validators()['validators']
    for d in data:
        fields = {k:d[k] for k in d if k != "moniker" and k != "__typename" and k != "details"}
        fields.update({"tokens":int(d["tokens"])})
        fields.update({"commissionRate":float(d["commissionRate"])})
        #fields.update({"bondedHeight":int(d["bondedHeight"])})
        fields.update({"is_api_return_timestamp": False})
        tags = {}
        tags.update({"moniker":d["moniker"]})
        dbtime = False
        db.write_points_to_measurement(measurement,dbtime,tags,fields)

if __name__ == "__main__":
    try:
        subscribe_validators(measurement)
    except:
        error = traceback.format_exc()
        #logger(measurement,error)
    while True:
        time.sleep(55)
        try:
            subscribe_validators(measurement)
        except:
            error = traceback.format_exc()
            #logger(measurement, error)


import requests
from pprint import pprint
class DataManager:
    def __init__(self):
        self.getresponse = requests.get(url="https://api.sheety.co/5843be2185b02f97b1f6fc8d2a584dfa/flightDeals/prices")
        self.length = len(self.getresponse.json()['prices'])
        self.prices = self.getresponse.json()['prices']
        self.putresponse = ""
        self.body = {}

    def put(self, sheet_data):
        for i in range(len(sheet_data)):
            self.body = {
                'price': {
                    "iataCode": f"{sheet_data[i]['iataCode']}"
                }
            }
            self.putresponse = requests.put(
                url=f"https://api.sheety.co/5843be2185b02f97b1f6fc8d2a584dfa/flightDeals/prices/{sheet_data[i]['id']}",
                json=self.body)


from geoalchemy2 import Geometry, Geography
from sqlalchemy import Column, Integer, Float, DateTime, ForeignKey, String
from sqlalchemy.orm import composite, relationship

from src.configs.database import Base
from src.rate_policies.domain.models import Location

class Area(Base):
    __tablename__ = "areas"

    area_id = Column(Integer, primary_key=True, autoincrement=True)
    area_boundary = Column(Geometry(geometry_type='POLYGON'))
    area_lat = Column(Float)
    area_lng = Column(Float)

    area_center = composite(Location, area_lat, area_lng)

class ParkingZone(Base):
    __tablename__ = "parking_zones"

    parkingzone_id = Column(Integer, primary_key=True, autoincrement=True)
    parkingzone_center = Column(Geometry(geometry_type='POINT'))
    parkingzone_radius = Column(Float)

class ForbiddenArea(Base):
    __tablename__ = "forbidden_areas"

    forbidden_area_id = Column(Integer, primary_key=True, autoincrement=True)
    forbidden_area_boundary = Column(Geometry(geometry_type='POLYGON'))

class Deer(Base):
    __tablename__ = "deers"

    deer_name = Column(Integer, primary_key=True, autoincrement=True)
    deer_area_id = Column(Integer, ForeignKey('areas.area_id'))

    deer_area = relationship("Area")

class Usage(Base):
    __tablename__ = "usages"

    usage_id = Column(Integer, primary_key=True, autoincrement=True)
    user_id = Column(Integer)
    use_deer_name = Column(Integer, ForeignKey('deers.deer_name'))
    use_end = Column(Geometry(geometry_type='POINT'))
    use_start_at = Column(DateTime)
    use_end_at = Column(DateTime)

    use_deer = relationship("Deer", uselist=True)

class AreaFee(Base):
    __tablename__ = "fees"

    area_fee_id = Column(Integer, primary_key=True, autoincrement=True)
    area_id = Column("Area", ForeignKey('areas.area_id'))
    base = Column(Integer)
    rate_per_minute = Column(Integer)
    currency = Column(String)


from datetime import datetime
from numbers import Number

from tqdm import tqdm

from clefts.catmaid_interface import get_catmaid
from clefts.manual_label.constants import Circuit, TABLES_DIR

HEADERS = ("Circuit", "Skeletons", "Nodes", "Presynaptic", "Postsynaptic", "Time")

def fmt_mins(total_mins):
    hrs, mins = divmod(total_mins, 60)
    return f"${hrs}$hr ${mins}$min"

def skids_to_row(*skids):
    contrib = catmaid.get_contributor_statistics_single(*skids)
    assert contrib["n_nodes"] > 0, "no nodes found"
    ret = (
        len(skids),
        contrib["n_nodes"],
        contrib["n_pre"],
        contrib["n_post"],
        fmt_mins(contrib["construction_minutes"]),
    )
    assert len(ret) == len(HEADERS) - 1
    return ret

def join_row(label, *values):
    fmt = "${}$"
    val_strs = [fmt.format(v) if isinstance(v, Number) else v for v in values]
    return " & ".join([label] + val_strs)

catmaid = get_catmaid()

pre_rows = (
    "\\begin{tabular}{l|" + "l" * len(HEADERS) + "}",
    " & ".join(HEADERS) + " \\\\ \\hline",
)

post_rows = ("\\end{tabular}",)

rows = list(pre_rows)

all_skids = set()

for circuit in tqdm(Circuit):
    annotation = circuit.annotation()

    skels = catmaid.get_skeletons_by_annotation(annotation)
    skids = [skel["skeleton_id"] for skel in skels]
    all_skids.update(skids)

    rows.append(join_row(circuit.tex_short(), *skids_to_row(*skids)) + " \\\\")

rows[-1] = rows[-1] + " \\hline"

total_label = "\\textbf{TOTAL}"
total_row = join_row(total_label, *skids_to_row(*all_skids))

rows.append(total_row)
rows.extend(post_rows)

s = "\n".join(rows) + "\n"

timestamp = datetime.now().isoformat()

fpath = TABLES_DIR / f"circuit_stats_{timestamp}.tex"
with open(fpath, "w") as f:
    f.write(s)


def MRL(sample, alpha): #MRL function
    import numpy as np
    import matplotlib.pyplot as plt
    import numpy as np
    import matplotlib.pyplot as plt
    from scipy.stats import norm
    import seaborn as sns; sns.set(style = 'whitegrid')
    from scipy.stats import genpareto
    import pandas as pd
    import math as mt
    import scipy.special as sm
    #Defining the threshold array and its step
    step = np.quantile(sample, .995)/60
    threshold = np.arange(0, max(sample), step=step)
    z_inverse = norm.ppf(1-(alpha/2))

    #Initialization of arrays
    mrl_array = [] #mean of excesses intialization
    CImrl = [] #confidence interval for the excesses initialization

    #First Loop for getting the mean residual life for each threshold value and the
    #second one getting the confidence intervals for the plot
    for u in threshold:
        excess = [] #initialization of the excesses array for each loop
        for data in sample:
            if data > u:
                excess.append(data - u) #adding excesses to the excesses array
        mrl_array.append(np.mean(excess)) #adding the mean of the excesses in the mean excesses array
        std_loop = np.std(excess) #getting standard deviation in the loop
        CImrl.append(z_inverse*std_loop/(len(excess)**0.5)) #getting confidence interval

    CI_Low = [] #initialization of the low confidence interval array
    CI_High = [] #initialization of the high confidence interval array

    #Loop to add in the confidence interval to the plot arrays
    for i in range(0, len(mrl_array)):
        CI_Low.append(mrl_array[i] - CImrl[i])
        CI_High.append(mrl_array[i] + CImrl[i])

    #Plot MRL
    plt.figure(1)
    sns.lineplot(x = threshold, y = mrl_array)
    plt.fill_between(threshold, CI_Low, CI_High, alpha = 0.4)
    plt.xlabel('u')
    plt.ylabel('Mean Excesses')
    plt.title('Mean Residual Life Plot')
    plt.show()


from svgwrite import Drawing
from svgwrite.shapes import Line, Rect
from svgwrite.text import Text
from svgwrite.container import Group
from svgwrite.image import Image

from .logo_lookup import get_logo
from .draw_runners import DrawRunners
from .constants import (ORIGIN_X, ORIGIN_Y, ATBAT_W, ATBAT_HT, NAME_W,
                        TEXT_HOP, SCORE_W, SEPARATION,
                        AWAY_NAME_X, AWAY_SCORING_X,
                        HOME_NAME_X, HOME_SCORING_X, HASH_SEP, HASH_LEN, flip)

class DrawScorecard:
    def __init__(self):
        self.runner_drawer = DrawRunners()

    def draw(self, game):
        self.game = game
        self.players = game.players

        self.dwg = Drawing(debug=True, profile='full')

        self.draw_inning_stripes()
        self.draw_inning_separators()
        self.draw_inning_numbers()
        self.draw_team_boxes()
        self.draw_game()
        if self.game.innings and len(self.game.innings[0].halves) == 2:
            self.draw_pitcher_hash_marks()
            self.draw_pitcher_names()
        self.draw_score()
        self.draw_logos()

        return self.dwg

    def draw_logos(self):
        self.dwg.add(Image(get_logo(self.game.away),
                           (ORIGIN_X, ORIGIN_Y - 50), (40, 40)))
        self.dwg.add(Image(get_logo(self.game.home),
                           (ORIGIN_X + ATBAT_W + SEPARATION + ATBAT_W - 40,
                            ORIGIN_Y - 50), (40, 40)))

    def draw_score(self):
        away_score = 0
        home_score = 0
        for inning in self.game:
            for half_inning in inning:
                for atbat in half_inning:
                    away_score = atbat.away_score
                    home_score = atbat.home_score
        self.dwg.add(Text(away_score,
                          x=[ORIGIN_X + ATBAT_W], y=[ORIGIN_Y - ATBAT_HT],
                          class_='score', text_anchor='end'))
        self.dwg.add(Text(home_score,
                          x=[ORIGIN_X + ATBAT_W + SEPARATION],
                          y=[ORIGIN_Y - ATBAT_HT], class_='score',
                          text_anchor='start'))

    def get_team_box(self, id, ht):
        box = Group()
        box['id'] = id
        box['class'] = 'team-box'
        box.add(Rect((ORIGIN_X, ORIGIN_Y), (ATBAT_W, ht)))
        box.add(Line((ORIGIN_X + NAME_W, ORIGIN_Y),
                     (ORIGIN_X + NAME_W, ORIGIN_Y + ht)))
        box.add(Line((ORIGIN_X + NAME_W + SCORE_W, ORIGIN_Y),
                     (ORIGIN_X + NAME_W + SCORE_W, ORIGIN_Y + ht)))
        return box

    def draw_team_boxes(self):
        away_ht = sum(
            self.get_inning_height(inning) for inning in
            self.game.innings)
        away_team = self.get_team_box('away_team', away_ht)

        home_ht = away_ht
        if self.is_no_final_bottom():
            final_inning = self.game.innings[-1]
            home_ht = away_ht - len(final_inning.halves[0].atbats) * ATBAT_HT
        home_team = self.get_team_box('home_team', home_ht)
        flip(home_team)

        self.dwg.add(away_team)
        self.dwg.add(home_team)

    def draw_inning_stripes(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            if not i % 2:
                self.dwg.add(Rect((ORIGIN_X, y - inning_ht),
                                  (ATBAT_W, inning_ht),
                             class_='inning-fill'))

            if (not (i == len(self.game.innings) - 1 and
                     self.is_no_final_bottom())):
                if not i % 2:
                    self.dwg.add(Rect((ORIGIN_X + ATBAT_W + SEPARATION,
                                       y - inning_ht), (ATBAT_W, inning_ht),
                                 class_='inning-fill'))

    def draw_inning_separators(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings[:-1]):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            self.dwg.add(Line((ORIGIN_X, y), (ORIGIN_X + ATBAT_W, y),
                              class_='team-box'))

            if (not (i == len(self.game.innings) - 2 and
                     self.is_no_final_bottom())):
                self.dwg.add(Line((ORIGIN_X + ATBAT_W + SEPARATION, y),
                                  (ORIGIN_X + 2 * ATBAT_W + SEPARATION, y),
                                  class_='team-box'))

    def draw_inning_numbers(self):
        y = ORIGIN_Y
        for i, inning in enumerate(self.game.innings):
            inning_ht = self.get_inning_height(inning)
            y += inning_ht
            self.draw_inning_number(i + 1, y - inning_ht / 2)

    def draw_inning_number(self, num, y):
        self.dwg.add(Text(num, x=[ORIGIN_X + ATBAT_W + SEPARATION / 2], y=[y],
                          class_='inning-num', text_anchor='middle',
                          dominant_baseline='middle'))

    def draw_game(self):
        self.y = ORIGIN_Y + ATBAT_HT
        for inning in self.game.innings:
            self.draw_inning(inning)

    def draw_inning(self, inning):
        inning_start_y = self.y
        inning_end_y = 0
        for half_inning in inning.halves:
            self.y = inning_start_y
            self.draw_half_inning(half_inning)
            inning_end_y = max(inning_end_y, self.y)
        self.y = inning_end_y

    def draw_half_inning(self, half_inning):
        for atbat in half_inning.atbats:
            self.set_x_and_anchor(half_inning.num)
            is_home_team_batting = half_inning.num % 1.0
            self.draw_atbat(atbat, is_home_team_batting)
            self.y += ATBAT_HT

    def draw_atbat(self, atbat, is_home_team_batting):
        atbat_group = Group()
        atbat_group.set_desc(atbat.get_description())
        atbat_group.add(self.get_batter_name_text(atbat))
        atbat_group.add(self.get_scoring_text(atbat))
        self.runner_drawer.execute(self.y, atbat, atbat_group,
                                   is_home_team_batting)
        self.dwg.add(atbat_group)

    def set_x_and_anchor(self, inning):
        if self.is_home_team_batting(inning):
            self.name_x = HOME_NAME_X
            self.scoring_x = HOME_SCORING_X
            self.name_anchor = 'start'
        else:
            self.name_x = AWAY_NAME_X
            self.scoring_x = AWAY_SCORING_X
            self.name_anchor = 'end'

    def get_batter_name_text(self, atbat):
        return Text(self.players.get(atbat.batter),
                    x=[self.name_x], y=[self.y - TEXT_HOP],
                    class_='batter-name',
                    text_anchor=self.name_anchor)

    def get_scoring_text(self, atbat):
        text = Text(atbat.scoring.code,
                    x=[self.scoring_x], y=[self.y - TEXT_HOP],
                    class_=atbat.scoring.result,
                    text_anchor='middle')
        if atbat.scoring.code == 'Kl':
            text.text = 'K'
            text.translate(2 * self.scoring_x, 0)
            text.scale(-1, 1)
        return text

    def draw_pitcher_hash_marks(self):
        self.away_hash_ys = []
        self.home_hash_ys = []

        # Starting pitchers
        self.home_pitchers = [self.game.innings[0].halves[0].atbats[0].pitcher]
        self.away_pitchers = [self.game.innings[0].halves[1].atbats[0].pitcher]

        self.y = ORIGIN_Y
        self.draw_both_hashes()
        for inning in self.game.innings:
            inning_start = self.y
            for half_inning in inning.halves:
                for atbat in half_inning.atbats:
                    if (atbat.pitcher != self.home_pitchers[-1] and
                            atbat.pitcher != self.away_pitchers[-1]):
                        self.draw_hash(half_inning.num)
                        self.swap_pitcher(atbat.pitcher, half_inning.num)
                    self.y += ATBAT_HT
                self.y = inning_start
            self.y = (inning_start + self.get_inning_height(inning))

        self.draw_hash(1.0)
        if self.is_no_final_bottom():
            self.y = self.y - ATBAT_HT * len(inning.halves[0].atbats)
        self.draw_hash(1.5)

    def get_inning_height(self, inning):
        return ATBAT_HT * max(len(h.atbats) for h in inning.halves)

    def draw_both_hashes(self):
        self.draw_hash(1.0)
        self.draw_hash(1.5)

    def draw_hash(self, inning):
        line = Line((ORIGIN_X + ATBAT_W + HASH_SEP, self.y),
                    (ORIGIN_X + ATBAT_W + HASH_SEP + HASH_LEN, self.y))
        if self.is_home_team_batting(inning):
            flip(line)
            self.home_hash_ys.append(self.y)
            line['class'] = 'away-pitcher-hash'
        else:
            self.away_hash_ys.append(self.y)
            line['class'] = 'home-pitcher-hash'
        self.dwg.add(line)

    def swap_pitcher(self, pitcher, inning_num):
        if self.is_home_team_batting(inning_num):
            self.away_pitchers.append(pitcher)
        else:
            self.home_pitchers.append(pitcher)

    def is_no_final_bottom(self):
        return self.game.innings and not len(self.game.innings[-1].halves) == 2

    def draw_pitcher_names(self):
        self._draw_pitcher_names(self.away_hash_ys, False)
        self._draw_pitcher_names(self.home_hash_ys, True)

    def _draw_pitcher_names(self, y_array, flip):
        for i, y in enumerate(y_array[:-1]):
            y_t = (y + y_array[i + 1]) / 2
            if flip:
                x = ORIGIN_X + ATBAT_W + SEPARATION - HASH_SEP - HASH_LEN / 2
                pitcher_name = self.players[self.away_pitchers[i]]
            else:
                x = ORIGIN_X + ATBAT_W + HASH_SEP + HASH_LEN / 2
                pitcher_name = self.players[self.home_pitchers[i]]
            txt = Text(pitcher_name, x=[x], y=[y_t], text_anchor='middle',
                       dominant_baseline='middle')
            if flip:
                txt['id'] = 'away-pitcher-%02d' % i
                txt['class'] = 'away-pitcher-name'
                txt.rotate(-90, (x, y_t))
            else:
                txt['id'] = 'home-pitcher-%02d' % i
                txt['class'] = 'home-pitcher-name'
                txt.rotate(90, (x, y_t))

            self.dwg.add(txt)

    def is_home_team_batting(self, inning):
        return inning % 1.0


# pip install opencv-python --> 이미지및 영상처리를 하기 위한 라이브러리 버전 4.6

import numpy as np  #
import cv2  # 이미지 및 영상처리를 하기위한 라이브러리

face_cascade = cv2.CascadeClassifier(
    cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
)
eye_casecade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")

ff = np.fromfile(r"샘플이미지.jpg", np.uint8)
img = cv2.imdecode(ff, cv2.IMREAD_UNCHANGED)
img = cv2.resize(img, dsize=(0, 0), fx=1.0, fy=1.0, interpolation=cv2.INTER_LINEAR)

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

faces = face_cascade.detectMultiScale(gray, 1.2, 10)

for (x, y, w, h) in faces:
    face_img = img[y : y + h, x : x + w]
    # 찾아낸 얼굴 이미지 크기만큼만 가지고옴
    face_img = cv2.resize(face_img, dsize=(0, 0), fx=0.15, fy=0.15)
    # 가져온 이미지를 기존 사이즈에 0.15 배만큼 크기를 줄임
    face_img = cv2.resize(face_img, (w, h), interpolation=cv2.INTER_AREA)
    # 다시 원래 사리즈로 늘림
    img[y : y + h, x : x + w] = face_img
    # 해당 이미지 위치로 다시가서 변경된 이미지 치환
cv2.imshow("face find", img)
cv2.waitKey(0)
cv2.destroyAllWindows()

얼굴모자이크처리/얼굴찾아모자이크.py


import logging
import multiprocessing
import os
import signal
import threading
from typing import List

from unicorn.server import Server

spawn: multiprocessing.context.SpawnContext = multiprocessing.get_context("spawn")

logger = logging.getLogger('unicorn')

def run_worker(app: str, host: str, port: int):
    """ Initializes and runs a worker process with the given application server details.
    :param app: ASGI callable for the application.
    :param host: IP address on which the application server will run.
    :param port: Port on which the application server will listen for requests.
      """
    logger.info(f'Initializing worker process [{os.getpid()}]')
    Server(app=app, host=host, port=port).run()

class Manager:
    """
    This manager class is responsible for spawning worker process to handle requests of the application server.
    All the worker process can handle requests received at the defined host and port.
    Load balancing across different worker process is done by the TCP kernel.
    Request will be picked by each worker, on accept() system call, which returns a file descriptor of the connection.
    Data is only read on the  recv() system call on the connection
    """

    def __init__(self, app: str, workers: int = 1, host: str = '0.0.0.0', port: int = 8000):
        """
        Initializes the Manager class
        :param app: ASGI callable
        :param workers: Number of workers to spawn which will handle the request to application server.
        :param host: IP address on where the application server will be run
        :param port: Port on which application server will listen for request. Default to 8000.
        """
        self.app = app
        self.host = host
        self.port = port
        self.workers = workers
        self.should_wait: threading.Event = threading.Event()
        self.processes: List[spawn.Process] = []
        self.pid = os.getpid()

    def on_interrupt(self, sig, _):
        logger.info(f'Received interrupt={sig} on manager.')
        self.should_wait.set()

    def run(self):
        """Starts the Manager and spawns the required number of worker processes.

        Sets up signal listeners for interruption. After spawning the worker processes,
        it blocks the main process until an interrupt is received. Upon interruption,
        it terminates all the worker processes.
        """
        logger.info(f'Starting manager process [{os.getpid()}]')
        # Setup listeners for signal
        for sig in [signal.SIGINT, signal.SIGTERM]:
            signal.signal(sig, self.on_interrupt)

        # Spawns the required number of worker processes
        for idx in range(self.workers):
            process = spawn.Process(target=run_worker, kwargs={
                'app': self.app,
                'host': self.host,
                'port': self.port
            })
            process.start()
            self.processes.append(process)

        # Block the execution on this process till an interrupt is received
        self.should_wait.wait()

        # Terminate all the worker processes, when an interrupt is received
        for process in self.processes:
            process.terminate()
            process.join()
        logger.info(f'Closing manager process [{os.getpid()}]')


import csv
import statistics

def get_csv_rows(filepath):
    with open(filepath) as f:
        reader = csv.DictReader(f)
        return [row for row in reader], reader.fieldnames

def show_stats(filepath, column):
    rows, columns = get_csv_rows(filepath)
    print(f"Shape: {len(rows)}x{len(columns)} matrix")
    if column in columns:
        print(f"Target column: {column}")
        vals = [int(r.get(column)) for r in rows]
        print("Average:", statistics.mean(vals))
        print("Max:", max(vals))
        print("Min:", min(vals))


#!/usr/local/bin/python3.7

import statistics

def push_stack(stack_a, stack_b):
	stack_a.insert(0, stack_b.pop(0))

def swap_stack(stack_a):
	if len(stack_a) == 1:
		return
	tmp = stack_a[0]
	stack_a[0] = stack_a[1]
	stack_a[1] = tmp

def rotate_stack(stack_a):
	tmp = stack_a.pop(0)
	stack_a.append(tmp)

def reverse_rotate_stack(stack_a):
	tmp = stack_a.pop()
	stack_a.insert(0, tmp)

def run_command(command, stack_a, stack_b):
	command = command.strip()
	if command == "pa":
		push_stack(stack_a, stack_b)
	if command == "pb":
		push_stack(stack_b, stack_a)
	if command == "sa":
		swap_stack(stack_a)
	if command == "sb":
		swap_stack(stack_b)
	if command == "ss":
		swap_stack(stack_a)
		swap_stack(stack_b)
	if command == "ra":
		rotate_stack(stack_a)
	if command == "rb":
		rotate_stack(stack_b)
	if command == "rr":
		rotate_stack(stack_a)
		rotate_stack(stack_b)
	if command == "rra":
		reverse_rotate_stack(stack_a)
	if command == "rrb":
		reverse_rotate_stack(stack_b)
	if command == "rrr":
		reverse_rotate_stack(stack_a)
		reverse_rotate_stack(stack_b)

def is_sorted(stack):
	count = len(stack)
	index = 0
	while index < count - 1:
		if stack[index + 1] < stack[index]:
			return False
		index += 1
	return True

def sort_two(stack, ops_list):
	if len(stack) != 2:
		return
	if stack[0] > stack[1]:
		tmp = stack[0]
		stack[0] = stack[1]
		stack[1] = tmp
		ops_list.append("sa")

def sort_three(stack, ops_list):
	if len(stack) != 3 or is_sorted(stack):
		return
	elif stack[0] < stack[1] and stack[1] > stack[2] and stack[0] < stack[2]:
		ops_list.append("rra")
		ops_list.append("sa")
	elif stack[0] > stack[1] and stack[2] > stack[0]:
		ops_list.append("sa")
	elif stack[0] < stack[1] and stack[1] > stack[2]:
		ops_list.append("rra")
	elif stack[0] > stack[1] and stack[1] < stack[2]:
		ops_list.append("ra")
	else:
		ops_list.append("sa")
		ops_list.append("rra")

def get_min_and_pb(stack_a, stack_b, ops_list):
	info = get_info(stack_a)
	while info["index"] != 0:
		if info["index"] > len(stack_a) / 2:
			command = "rra"
		else:
			command = "ra"
		run_command(command, stack_a, stack_b)
		ops_list.append(command)
		info = get_info(stack_a)
	run_command("pb", stack_a, stack_b)
	ops_list.append("pb")

def get_info(stack):
	info = dict()
	info["min"] = min(stack)
	info["index"] = stack.index(info["min"])
	return info

def push_below_median_to_stack_b(stack_a, stack_b, ops_list, median):
	while stack_a[0] < median:
		ops_list.append("pb")
		run_command("pb", stack_a, stack_b)
	pivot = stack_a[0]
	ops_list.append("ra")
	run_command("ra", stack_a, stack_b)
	while stack_a[0] != pivot:
		if stack_a[0] < median:
			ops_list.append("pb")
			run_command("pb", stack_a, stack_b)
		else:
			ops_list.append("ra")
			run_command("ra", stack_a, stack_b)

def push_above_median_to_stack_b(stack_a, stack_b, ops_list, median):
	while stack_a[0] >= median:
		ops_list.append("pb")
		run_command("pb", stack_a, stack_b)
	pivot = stack_a[0]
	ops_list.append("ra")
	run_command("ra", stack_a, stack_b)
	while stack_a[0] != pivot:
		if stack_a[0] >= median:
			ops_list.append("pb")
			run_command("pb", stack_a, stack_b)
		else:
			ops_list.append("ra")
			run_command("ra", stack_a, stack_b)

def count_ops(index, count):
	front = index
	back = count - index
	return min(front, back)

def find_highest_and_push_to_a(stack_a, stack_b, ops_list):
	mini_index = stack_b.index(min(stack_b))
	maxi_index = stack_b.index(max(stack_b))
	mini_ops = count_ops(mini_index, len(stack_b))
	maxi_ops = count_ops(maxi_index, len(stack_b))
	if mini_ops < maxi_ops:
		is_mini = 1
		target_index = mini_index
	else:
		is_mini = 0
		target_index = maxi_index
	target = stack_b[target_index]
	if target_index < len(stack_b) / 2:
		while stack_b[0] != target:
			run_command("rb", stack_a, stack_b)
			ops_list.append("rb")
	else:
		while stack_b[0] != target:
			run_command("rrb", stack_a, stack_b)
			ops_list.append("rrb")
	ops_list.append("pa")
	run_command("pa", stack_a, stack_b)
	if is_mini == 1:
		run_command("ra", stack_a, stack_b)
		ops_list.append("ra")

def push_from_stack_b_to_a(stack_a, stack_b, ops_list):
	while len(stack_b) > 1:
		find_highest_and_push_to_a(stack_a, stack_b, ops_list)
	run_command("pa", stack_a, stack_b)
	ops_list.append("pa")

def sort_multiple(stack_a, stack_b, ops_list):
	median = statistics.median(stack_a)
	push_above_median_to_stack_b(stack_a, stack_b, ops_list, median)
	push_from_stack_b_to_a(stack_a, stack_b, ops_list)
	if is_sorted(stack_a) == True:
		return
	push_below_median_to_stack_b(stack_a, stack_b, ops_list, median)
	while is_sorted(stack_a) == False:
		ops_list.append("rra")
		run_command("rra", stack_a, stack_b)
	push_from_stack_b_to_a(stack_a, stack_b, ops_list)
	while is_sorted(stack_a) == False:
		ops_list.append("rra")
		run_command("rra", stack_a, stack_b)


# -*- coding: utf-8 -*-
"""
Created on Mon Sep 23 16:18:31 2019

@author: 13982
"""
import numpy as np
#import collections
import keras
from keras import backend as K
from keras.callbacks import EarlyStopping, ModelCheckpoint
from sklearn.model_selection import StratifiedKFold
#import csv

"""Just the cosine distance calculated from Elmo Scores"""
def getVector(line, which='left'):
    if which == 'left':
        return line[4]
    elif which == 'right':
        return line[5]

"""Get the distance of indices of aligned pairs normalized with the longer of the sentence length"""
def getIndexDistanceNormalized(line):
    aligns = line[0]
    sent_len = int(line[7])
    if '====' in aligns or '----' in aligns:
        align = ''
        if '====' in aligns:
            align = aligns.split('====')
        elif '----' in aligns:
            align = aligns.split('----')
        left = align[0][1:-1].split(', ')
        left = [int(i) for i in left]
        right = align[1][1:-1].split(', ')
        right = [int(i) for i in right]
        lsum = np.mean(left)
        rsum = np.mean(right)
        return abs(lsum - rsum) / sent_len

    else:
        align = aligns.split('-')
        left = int(align[0])
        right = int(align[1])
        return abs(left - right) / sent_len

"""Get the span of alignment candidate for each sentence"""
def getAlignmentSpan(line, which='left'):
    aligns = line[0]
    sent_len = int(line[7])
    if '====' in aligns or '----' in aligns:
        align = ''
        if '====' in aligns:
            align = aligns.split('====')
        elif '----' in aligns:
            align = aligns.split('----')

        ind = []
        if which == 'left':
            left = align[0][1:-1].split(', ')
            ind = [int(i) for i in left]
        elif which == 'right':
            right = align[1][1:-1].split(', ')
            ind = [int(i) for i in right]

        max_ind = max(ind)
        min_ind = min(ind)
        return (max_ind - min_ind) / sent_len

    else:
        return 0

"""Check if the core of the aligned pairs have the same dependency"""
def hasSameDep(line):
    pass

"""Get the gold label, changes to bool value due to implementation change"""
def getGold(line):
    if line[6] == True:
        return 1
    elif line[6] == False:
        return 0
    return None

"""????SEEMS TO WORK: Feed all vector values to the input matrix for certain example_index"""
def feedVectorToInput(input_matrix, vector, example_index, dimension, which='left'):
    for i in range(0, len(vector)):
        if which == 'left':
            input_matrix[example_index][i] = vector[i]
        elif which == 'right':
            input_matrix[example_index][dimension+i] = vector[i]

def recall_m(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

def precision_m(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision

def f1_m(y_true, y_pred):
    precision = precision_m(y_true, y_pred)
    recall = recall_m(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

class NeuralClassifier:

    def __init__(self, info_list, hidden, intermediate_dimension):
        self.model = None
        self.hidden = hidden
        self.intermediate_dimension = intermediate_dimension
        self.dimensions = []
        example_index = 0

        for line in info_list:

            left_vector = getVector(line, which='left')
            right_vector = getVector(line, which='right')
            self.dimensions.append(len(list(left_vector)))
            self.dimensions.append(len(list(right_vector)))

            example_index += 1

        """以下做法可能耗时过长，看能不能大整改"""

        """Assuming the same dimension for every token though"""
        self.dimension = max(self.dimensions)
        """Extra Feature: Any feature beyond two embeddings"""
        num_extra_features = 0
        self.input_size = (2 * self.dimension) + num_extra_features

        self.input_matrix = np.zeros((example_index, self.input_size))
        self.output_matrix = np.zeros((example_index, 1))

        example_index = 0
        for line in info_list:
            left_vector = getVector(line, which='left')
            right_vector = getVector(line, which='right')

#            sent_distance = getIndexDistanceNormalized(line)

            feedVectorToInput(self.input_matrix, left_vector, example_index, self.dimension, which='left')
            feedVectorToInput(self.input_matrix, right_vector, example_index, self.dimension, which='right')

            gold_label = getGold(line)

            self.output_matrix[example_index][0] = gold_label

            example_index += 1

    def train(self):
        """Set a random seed for reproducibility"""
        seed = 77
        np.random.seed(seed)

        """debug完就删除"""
        print("To DEBUG: ")
        try:
            print(self.input_matrix[:50])
            print(self.output_matrix[:50])
        except:
            print("DEBUGGING FAILED")

        """👇注意这里的CV用法是否正确👇"""
        kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)

        for i, (train, test) in enumerate(kfold.split(self.input_matrix, self.output_matrix)):
            print("Fold: " + str(i))

            self.model = keras.models.Sequential()
            self.model.add(keras.layers.Dense(self.intermediate_dimension, input_shape=(self.input_size,), activation="sigmoid"))
            """Adding hidden layers to create a neural network"""
            hidden = self.hidden

            evaluations = []
            for ii in range(hidden):
                self.model.add(keras.layers.Dense(self.intermediate_dimension, activation="relu", name=str(ii+1)))
                """Batch Normalization - with 'relu' activation"""
                self.model.add(keras.layers.BatchNormalization())
                self.model.add(keras.layers.Dense(1, activation="sigmoid"))
                self.model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', f1_m])

                """keras.callbacks.ModelCheckpoint will automatically save the model after each epoch"""
                callbacks = [EarlyStopping(monitor='acc', min_delta=0, patience=0, verbose=1, restore_best_weights=True),
                             ModelCheckpoint(filepath='pretrained_bert_model_V2_CV.h5', monitor='acc', save_best_only=True, verbose=1)]

                self.model.summary()
                self.model.fit(self.input_matrix[train], self.output_matrix[train], epochs=200, batch_size=128, callbacks = callbacks)
                loss, accuracy, f1 = self.model.evaluate(self.input_matrix[test], self.output_matrix[test], verbose=0)
                evaluations.append((loss, accuracy, f1))

            avg_loss = 0
            avg_acc = 0
            avg_f1 = 0
            count = 0
            for loss, acc, f1 in evaluations:
                avg_loss += loss
                avg_acc += acc
                avg_f1 += f1
                count += 1
            avg_loss /= count
            avg_acc /= count
            avg_f1 /= count

            print("Avg loss: " + str(avg_loss)[:8])
            print("Avg acc: " + str(avg_acc)[:8])
            print("Avg f1: " + str(avg_f1)[:8])

    """不太确定best model和self.model 到底是否一样"""
    def retrieve_model(self):
        return self.model
        """如果不对就换成下面的"""
#        return keras.models.load_model("pretrained_bert_model.h5")

    def retrieve_dimension(self):
        return self.dimension

    def get_input_size(self):
        return self.input_size


from django.db.models.fields import CharField, NullBooleanField
from django.core import validators
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from django.forms import fields
import vat_moss.id
from oscar_vat_moss.util import u

# The longest VAT IDs are currently 2-letter country code + 15
# characters. Make the max_length 32 to be on the safe side.
DEFAULT_MAX_LENGTH = 32

def vatin(verbose_name=_('VAT Identification Number (VATIN)'),
          name='vatin',
          verify_exists=None,
          blank=True,
          help_text=_('Required if you are associated with a business '
                      'registered for VAT in the European Union.')):
    """Convenience method to return a properly configured VATIN field."""
    return VATINField(verbose_name=verbose_name,
                      name=name,
                      verify_exists=verify_exists,
                      blank=blank,
                      help_text=help_text)

def digital_goods(verbose_name=_('Digital Goods?'),
                  name='digital_goods',
                  help_text=_("Indicates that products in this class are "
                              "Digital Goods per EU VAT regulations, "
                              "meaning the customer's location determines "
                              "the applicable VAT rate.")):
    return NullBooleanField(verbose_name=verbose_name,
                            name=name,
                            help_text=help_text)

class VATINField(CharField):
    def __init__(self, verbose_name=None, name=None,
                 verify_exists=None, **kwargs):
        kwargs['max_length'] = kwargs.get('max_length', DEFAULT_MAX_LENGTH)
        CharField.__init__(self, verbose_name, name, **kwargs)
        validator = VATINValidator(None)
        self.validators.append(validator)

    def formfield(self, **kwargs):
        # As with CharField, this will cause VATIN validation to be performed
        # twice.
        defaults = {
            'form_class': fields.CharField,
        }
        defaults.update(kwargs)
        return super(CharField, self).formfield(**defaults)

    def deconstruct(self):
        """
        deconstruct() is needed by Django's migration framework
        """
        name, path, args, kwargs = super(VATINField, self).deconstruct()
        # We have a default value for max_length; remove it in that case
        if self.max_length == DEFAULT_MAX_LENGTH:
            del kwargs['max_length']
        return name, path, args, kwargs

class VATINValidator(validators.BaseValidator):
    def __call__(self, value):
        """Verify that a VATIN is valid and exists"""
        return self.validate_vatin(value)

    def validate_vatin(self, value):
        """Validate a VATIN and check that it exists.

        :raises:
            ValidationError wrapping one of the following errors from
            vat_moss.id:
                ValueError - If the is not a string or is not in the
                format of two characters plus an identifier
                InvalidError - If the VAT ID is not valid
                WebServiceUnavailableError - If the VIES VAT ID
                service is unable to process the request - this is
                fairly common
                WebServiceError - If there was an error parsing the
                response from the server - usually this means
                something changed in the webservice
                urllib.error.URLError/urllib2.URLError - If there is
                an issue communicating with VIES or data.brreg.no
        """
        # vat_moss expects that all strings are unicode, so force it
        # here.
        try:
            vat_moss.id.validate(u(value))
        except Exception as e:
            raise ValidationError(_(str(e)))


import os
import sys
root_path = os.path.join(os.getenv("HOME"), "vqa2019")
sys.path.append(root_path)
from utils.data_provider import VQADataProvider

def load_ques2set(filename):
	ques_set = set()
	with open(filename, "r") as f:
		for row in f:
			qa = row.rstrip().split("|")
			ques_set.add(qa[1])
	return ques_set

def load_all2set(filename):
	ques_set 	= set()
	ans_set 	= set()
	img_set 	= set()
	with open(filename, "r") as f:
		for row in f:
			qa = row.rstrip().split("|")
			if(len(qa) == 3):
				img_set.add(qa[0])
				ques_set.add(qa[1])
				ans_set.add(qa[2])
			else:
				print("Abnormal Row")
	return img_set, ques_set, ans_set

def check_raw_data(filename):
	answer_set = set()
	img_id_set = set()
	ques_set = set()
	q_i_pair_set = set()
	q_a_pair_set = set()
	with open(filename, "r") as f:
		# print("total q-i pair: ", len(f))
		counter = 0
		for row in f:
			qa = row.split("|")
			img_id_set.add(qa[0])
			ques_set.add(qa[1])
			answer_set.add(qa[2].rstrip())
			counter += 1
			q_i_pair_set.add((qa[0], qa[1]))
			q_a_pair_set.add((qa[1], qa[2].rstrip()))
		print("total records: ", counter)
	return answer_set, img_id_set, ques_set, q_i_pair_set, q_a_pair_set

def check_raw_readme(filename, ans_list):
	print("readme length: ", len(ans_list))
	raw_ans, _, _, _, _ = check_raw_data(filename)
	in_data_not_readme = raw_ans.difference(set(ans_list))
	in_readme_not_data = set(ans_list).difference(raw_ans)
	return in_data_not_readme, in_readme_not_data

def check_q_i_a_pair(filename):
	_, _, _, q_i_pairs, _ = check_raw_data(filename)
	q_i_dic_for_ans = {}
	with open(filename, "r") as f:
		for row in f:
			qa = row.rstrip().split("|")
			if((qa[0], qa[1]) in q_i_dic_for_ans):
				q_i_dic_for_ans[(qa[0], qa[1])].append(qa[2])
			else:
				q_i_dic_for_ans[(qa[0], qa[1])] = [qa[2]]
	return q_i_dic_for_ans

# this function does not work for all the answers
def check_multi_answer(filename):
	# counter = 0
	sep_answer_list = []
	sep_answer_set = []
	with open(filename, "r") as f:
		for row in f:
			qa = row.rstrip().split("|")
			ans = qa[2]
			ans_ls = [a.strip() for a in ans.split(",")]
			ans_set = set([a.strip() for a in ans.split(",")])
			sep_answer_list.append(ans_ls)
			if(ans_set not in sep_answer_set):
				sep_answer_set.append(ans_set)
	return sep_answer_list, sep_answer_set

# check if there is any overlapping question in different categories
def check_ques_overlap(file1, file2):
	ques_1_set = set()
	with open(file1, "r") as f1:
		for row in f1:
			qa = row.rstrip().split("|")
			ques_1_set.add(qa[1])

	ques_2_set = set()
	with open(file2, "r") as f2:
		for row in f2:
			qa = row.rstrip().split("|")
			ques_2_set.add(qa[1])
	return ques_1_set.intersection(ques_2_set)

def check_ques_completeness():
	path = os.path.join(root_path, "data/train/All_QA_Pairs_train.txt")
	m_path = os.path.join(root_path, "data/train/QAPairsByCategory/C1_Modality_train.txt")
	p_path = os.path.join(root_path, "data/train/QAPairsByCategory/C2_Plane_train.txt")
	o_path = os.path.join(root_path, "data/train/QAPairsByCategory/C3_Organ_train.txt")
	a_path = os.path.join(root_path, "data/train/QAPairsByCategory/C4_Abnormality_train.txt")

	questions = load_ques2set(path)
	m_ques = load_ques2set(m_path)
	p_ques = load_ques2set(p_path)
	o_ques = load_ques2set(o_path)
	a_ques = load_ques2set(a_path)

	sub_ques = set()
	sub_ques.update(m_ques, p_ques, o_ques, a_ques)
	outside_ques = questions.difference(sub_ques)
	return outside_ques

def check_len(filename):
	q_len_num = {}
	a_len_num = {}
	exclude_ls = ["<break>", "<START>", "<END>", "<UNKNOWN>"]
	# exclude_ls = ["<START>", "<END>"]
	with open(filename, "r") as f:
		for row in f:
			qa = row.rstrip().split("|")
			# words_q = qa[1].split()
			words_q = VQADataProvider.text_to_list(qa[1])
			words_q = [word for word in words_q if word not in exclude_ls]
			if(len(words_q) in q_len_num):
				q_len_num[len(words_q)] += 1
			else:
				q_len_num[len(words_q)] = 1

			# words_a = qa[2].split()
			words_a = VQADataProvider.text_to_list(qa[2])
			words_a = [word for word in words_a if word not in exclude_ls]
			if(len(words_a) in a_len_num):
				a_len_num[len(words_a)] += 1
			else:
				a_len_num[len(words_a)] = 1
	return q_len_num, a_len_num

def check_img_type(folder):
	import os
	img_ls = os.listdir(folder)
	print("image number: ", len(img_ls))
	jpg_ls = [img for img in img_ls if img.endswith(".jpg")]
	print("jpg image number: ", len(jpg_ls))

def check_unique_ans_num():
	path = os.path.join(root_path, "data/train/All_QA_Pairs_train.txt")
	m_path = os.path.join(root_path, "data/train/QAPairsByCategory/C1_Modality_train.txt")
	p_path = os.path.join(root_path, "data/train/QAPairsByCategory/C2_Plane_train.txt")
	o_path = os.path.join(root_path, "data/train/QAPairsByCategory/C3_Organ_train.txt")
	a_path = os.path.join(root_path, "data/train/QAPairsByCategory/C4_Abnormality_train.txt")

	_, _, all_ans = load_all2set(path)
	_, _, m_ans = load_all2set(m_path)
	_, _, p_ans = load_all2set(p_path)
	_, _, o_ans = load_all2set(o_path)
	_, _, a_ans = load_all2set(a_path)

	sub_ans = set()
	sub_ans.update(m_ans, p_ans, o_ans, a_ans)
	outside_ans = all_ans.difference(sub_ans)

	sub_ans2 = set()
	sub_ans2.update(m_ans, p_ans, o_ans)

	return outside_ans, sub_ans2

def check_train_val_imgId_overlap():
	train_path = os.path.join(root_path, "data/train/train_ImageIDs.txt")
	val_path = os.path.join(root_path, "data/val/val_ImageIDs.txt")
	train_img_ids = []
	with open(train_path, "r") as f:
		for row in f:
			train_img_ids.append(row.strip())
	val_img_ids = []
	with open(val_path, "r") as f:
		for row in f:
			val_img_ids.append(row.strip())
	print("train+val images number: ", len(train_img_ids) + len(val_img_ids))
	comb_ids = train_img_ids + val_img_ids
	comb_ids = set(comb_ids)
	print("unique total images number: ", len(comb_ids))

if __name__ == "__main__":
	filename = os.path.join(root_path, "data/train_val/QAPairsByCategory/C4_Abnormality_train_val.txt")
	_, _, ans_set = load_all2set(filename)
	unq_words = set()
	for ans in ans_set:
		words = VQADataProvider.text_to_list(ans)
		for w in words:
			unq_words.add(w)
	print(len(unq_words))
	print(list(unq_words)[:5])
	print("<END>" in unq_words)

	# filename = "/Users/leishi/Desktop/Internship/vqa2019/ImageClef-2019-VQA-Med-Training/QAPairsByCategory/C4_Abnormality_train.txt"
	# all_ans, all_img, all_ques, all_q_i_pairs, all_a_a_pairs = check_raw_data(filename)
	# print("\nunique answer length: ", len(all_ans))
	# print("\nunique img length: ", len(all_img))
	# print("\nunique question length: ", len(all_ques))
	# # print("\nall answer")
	# # print(all_ans)
	# print("\nunique question-image pairs: ", len(all_q_i_pairs))
	# print("\nunique question-answer pairs: ", len(all_a_a_pairs))
	# # print(all_ans)

	# as_m = ["XR - Plain Film", "CT - noncontrast", "CT w/contrast (IV)", "CT - GI & IV Contrast", "CTA - CT Angiography",
	# "CT - GI Contrast", "CT - Myelogram", "Tomography", "MR - T1W w/Gadolinium", "MR - T1W - noncontrast", "MR - T2 weighted",
	# "MR - FLAIR", "MR - T1W w/Gd (fat suppressed)", "MR T2* gradient,GRE,MPGR,SWAN,SWI", "MR - DWI Diffusion Weighted",
	# "MRA - MR Angiography/Venography", "MR - Other Pulse Seq.", "MR - ADC Map (App Diff Coeff)", "MR - PDW Proton Density",
	# "MR - STIR", "MR - FIESTA", "MR - FLAIR w/Gd", "MR - T1W SPGR", "MR - T2 FLAIR w/Contrast", "MR T2* gradient GRE", "US - Ultrasound",
	# "US-D - Doppler Ultrasound", "Mammograph", "BAS - Barium Swallow", "UGI - Upper GI", "BE - Barium Enema", "SBFT - Small Bowel",
	# "AN - Angiogram", "Venogram", "NM - Nuclear Medicine", "PET - Positron Emission"]

	# as_p = ["Axial", "Sagittal", "Coronal", "AP", "Lateral", "Frontal", "PA", "Transverse", "Oblique", "Longitudinal", "Decubitus",
	# "3D Reconstruction", "Mammo - MLO", "Mammo - CC", "Mammo - Mag CC", "Mammo - XCC"]

	# as_o = ["Breast", "Skull and Contents", "Face, sinuses, and neck", "Spine and contents", "Musculoskeletal", "Heart and great vessels",
	# "Lung, mediastinum, pleura", "Gastrointestinal", "Genitourinary", "Vascular and lymphatic"]

	# # as_all = as_m + as_p + as_o

	# as_m = [a.lower() for a in as_m]
	# as_p = [a.lower() for a in as_p]
	# as_o = [a.lower() for a in as_o]
	# # as_all = [a.lower() for a in as_all]

	# filename = "/home/leishi/vqa2019/data/train/QAPairsByCategory/C3_Organ_train.txt"

	# in_data_not_readme, in_readme_not_data = check_raw_readme(filename, as_o)
	# print("\ndifference between answer of raw data and answer in readme")
	# print(in_data_not_readme)
	# print("\ndifference between answer in readme and answer of raw data")
	# print(in_readme_not_data)

	# q_i_dic_for_ans = check_q_i_a_pair(filename)
	# num_list = []
	# for key in q_i_dic_for_ans:
	# 	num_list.append(len(q_i_dic_for_ans[key]))
	# # print("\n# of answers for each question-image pair")
	# print(max(num_list), min(num_list))

	# sep_answer_list, sep_answer_set = check_multi_answer(filename)
	# # print("# of separated answers for each question-image pair")
	# # print([len(a) for a in sep_answer_list])
	# x0 = [1 for a in sep_answer_list if len(a)>1]
	# print("\n# of q-i pairs that have multiple answers")
	# print(sum(x0))
	# x1 = [1 for b in sep_answer_set if len(b)>3]
	# print("\n# of unique answer which have more than one sub answers")
	# print(sum(x1))
	# print("answers that have multiple sub answers")
	# print([b for b in sep_answer_set if len(b)>3])

	# file1 = "/Users/leishi/Desktop/Internship/vqa2019/ImageClef-2019-VQA-Med-Training/QAPairsByCategory/C4_Abnormality_train.txt"
	# file2 = "/Users/leishi/Desktop/Internship/vqa2019/ImageClef-2019-VQA-Med-Training/QAPairsByCategory/C3_Organ_train.txt"
	# overlap = check_ques_overlap(file1, file2)
	# print("\n# of overlappling questions")
	# print(overlap)

	# diff_ques = check_ques_completeness()
	# print("# of uncovered questions: ", len(diff_ques))
	# print(diff_ques)

	# filename = "/home/leishi/vqa2019/data/train/QAPairsByCategory/C4_Abnormality_train.txt"
	# filename = "/home/leishi/vqa2019/data/train/All_QA_Pairs_train.txt"
	# q_len_num, a_len_num = check_len(filename)
	# print("\nquestion length")
	# # # for k, v in sorted(q_len_num.items(), key=lambda item: item[1]):
	# # # 	print("%d: %d" % (k, v))
	# for k in sorted(q_len_num.keys()):
	# 	print("%d: %d" % (k, q_len_num[k]))
	# print("\nanswer length")
	# # # for k, v in sorted(a_len_num.items(), key=lambda item: item[1]):
	# # # 	print("%d: %d" % (k, v))
	# for k in sorted(a_len_num.keys()):
	# 	print("%d: %d" % (k, a_len_num[k]))

	# # number of classes in each category (first 3 categories)
	# filename = "/home/leishi/vqa2019/data/train/QAPairsByCategory/C3_Organ_train.txt"
	# ans_set, _, _, _, _ = check_raw_data(filename)
	# print(len(ans_set))
	# print(ans_set)

	# folder = "/home/leishi/vqa2019/data/%s/%s_images" % ("val", "val")
	# check_img_type(folder)

	# # check answer completeness
	# outside_ans, sub_ans2 = check_unique_ans_num()
	# print("\noutside answers")
	# print(outside_ans)
	# print("number of unique ansers in first 3 categories: ", len(sub_ans2))

	# # check if train and val data have overlap img ids
	# check_train_val_imgId_overlap()


import cv2
import numpy as np
import tensorflow as tf
from img2vec_pytorch import Img2Vec

def CalHistogram(img, bin=[8,8,8]):
    hist = cv2.calcHist([img],[0, 1, 2],None,[bin[0], bin[1], bin[2]],[0,256, 0, 256, 0, 256])
    hist = hist.reshape(1, -1)/ hist.sum()
    return hist

def split_image_to_tiles(img, rows=3, columns=3):
    height = img.shape[0]
    width = img.shape[1]
    h = height // rows
    w = width // columns
    tiles = []
    hindex = 0
    for i in range(0, height, h):
        windex = 0
        for j in range(0, width, w):
            addw = j + w if windex != columns - 1 else width
            addh = i + h if hindex != rows - 1 else height
            tile = img[i:addh, j:addw, :]
            tiles.append(tile)
            if windex == columns - 1:
                break
            windex += 1
        if hindex == rows - 1:
            break
        hindex += 1
    return tiles

def get_region_histogram(img, rows=3, columns=3):
    tiles = split_image_to_tiles(img, rows, columns)
    features = []
    for tile in tiles:
        histogram = CalHistogram(tile, [8,8,8])
        histogram = cv2.normalize(histogram, histogram).flatten()
        features.extend(histogram)
    features = np.array(features)
    features = features.reshape(1, -1)
    return features

def get_mobilenet(img):
    IMAGE_DIMS = (160, 160, 3)
    mnet_model = tf.keras.applications.mobilenet_v2.MobileNetV2(input_shape=IMAGE_DIMS,
                                               include_top=False, weights='imagenet')
    img=cv2.resize(img, (IMAGE_DIMS[1], IMAGE_DIMS[0]))
    im_toarray = tf.keras.preprocessing.image.img_to_array(img)
    im_toarray = np.expand_dims(img, axis=0)
    im_toarray = tf.keras.applications.mobilenet.preprocess_input(im_toarray)
    data_stack = np.vstack([im_toarray])
    feature = mnet_model.predict(data_stack)
    return feature

def CalHistogram_RGB(img):
    features = []
    for channel in range(0, 3):
        hist = cv2.calcHist([img], [channel], None, [255], [1, 256])
        features.append(hist)
    features = np.array(features)
    res = features.reshape(1, features.shape[0] * features.shape[1] * features.shape[2])
    return res

def get_dominant_color(img, number_of_colors=5):
    nrow, ncol, nchl = img.shape
    g = img.reshape(nrow*ncol,nchl)

    flags = cv2.KMEANS_RANDOM_CENTERS
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 200, .1)
    # Add one more color in case dominant color is black
    _, labels, centroids = cv2.kmeans(np.float32(g), number_of_colors + 1, None, criteria, 10, flags)
    palette = np.uint8(centroids)
    return palette

def check_color(palette, number_of_colors=5):
    result = []
    for color in palette:
        if color[0] <= 32 and color[1] <= 32 and color[2] <= 32:
            continue
        result.append(color)
        if len(result) == number_of_colors:
            break
    result = np.array(result)
    while result.shape[0] < 5:
        result = np.append(result, [[0, 0, 0]], axis=0)
    result = result.reshape(1, -1)
    return result

def get_resnet_embedding(img):
    img2vec = Img2Vec(cuda=True)
    vec = img2vec.get_vec(img, tensor=True)
    vec = vec.reshape([-1, 1])
    vec = vec.detach().numpy()
    return vec


import logging
from datetime import datetime, timedelta
from dateutil import parser
import pytz
from babel.dates import format_date, format_time
from icalendar import vDatetime
import time

import airbnb
from airbnb_automation.gcalendar import GCalendar
from airbnb_automation.notifications import Notifications
from airbnb_automation.nuki import Nuki

LOGGER = logging.getLogger('airbnb_automation')

class Airbnb():
    def __init__(self, config):
        self.config = config
        self.debug = config['debug']
        self.notifications = Notifications(config)
        self.cal = GCalendar(config)
        self.nuki = Nuki(config)

    def daily_nuki_opener_status(self):
        message = 'Nuki Opener Mode: {}\n'.format(self.nuki.opener_mode())
        self.notifications.send_telegram_private(message)

    def weekly_nuki_battery_status(self):
        message = 'Nuki Battery Critical: {}'.format(
            self.nuki.battery_critical())
        self.notifications.send_telegram_private(message)

    def weekly_messages(self):
        messages = []

        events = self.cal.get_public_calendar()
        for event in events:
            start = parser.parse(event['start']['dateTime'])
            end = parser.parse(event['end']['dateTime'])
            now = datetime.now().astimezone(pytz.utc)
            if now > start:
                if end < now + timedelta(days=7):
                    messages.append("{} odjíždí {}. ".format(
                        event['summary'],
                        format_date(end, 'EEE d MMM', locale='cs_CZ')))
            elif start < now + timedelta(days=7):
                messages.append("{} přijíždí {} a odjíždí {}.".format(
                    event['summary'],
                    format_date(start, 'EEE d MMM', locale='cs_CZ'),
                    format_date(end, 'EEE d MMM', locale='cs_CZ')))

        if len(messages) == 0:
            # message_EN = "No visitors arriving or leaving this week"
            message = "Tento týden nepřijíždí ani neodjíždí žádní hosté."
        else:
            message = "Hosté tento týden: "
            message += " ".join(messages)

        # notifications.send_sms(config['cleaner_number'], message)
        self.notifications.send_telegram_public(message)

    def cleaning_reminder(self):
        events = self.cal.get_public_calendar()
        check_out = None
        check_in = None
        difference = timedelta(days=60)

        for event in events:
            if 'dateTime' not in event['start'].keys():
                continue
            start = parser.parse(event['start']['dateTime'])
            end = parser.parse(event['end']['dateTime']).replace(hour=0)
            now = datetime.now().astimezone(pytz.utc)
            # Check out day is not today, and is less than 1 day away
            if (end > now) and (end < now + timedelta(days=1)):
                check_out = event
            else:
                if (start - now) < difference:
                    difference = start - now
                    check_in = event

        if check_out:
            message = "Připomenutí: {} odjíždí zítra v {}.".format(
                check_out['summary'],
                format_time(
                    parser.parse(check_out['end']['dateTime']),
                    'H:mm',
                    locale='cs_CZ'))
            if check_in:
                check_in_time = parser.parse(check_in['start']['dateTime'])
                message += " Příští host ({}) přijíždí {}".format(
                    check_in['summary'],
                    format_date(check_in_time, 'EEE d MMM', locale='cs_CZ'))

            # notifications.send_sms(config['cleaner_number'], message)
            self.notifications.send_telegram_public(message)

    def check_in_time(self, check_in_date):
        date = datetime.strptime(check_in_date, '%Y-%m-%d')
        check_in_time = pytz.timezone('Europe/Prague').localize(date).replace(
            hour=15).astimezone(pytz.utc)
        return check_in_time.isoformat()

    def check_out_time(self, check_out_date):
        date = datetime.strptime(check_out_date, '%Y-%m-%d')
        check_out_time = pytz.timezone('Europe/Prague').localize(date).replace(
            hour=12).astimezone(pytz.utc)
        return check_out_time.isoformat()

    def get_reservations(self):
        airbnb_api = airbnb.Api(access_token=self.config['airbnb_oauth_token'])
        calendar = airbnb_api.get_listing_calendar(
            self.config['airbnb_listing_id'], calendar_months=6)
        reservations = []

        for day in calendar['calendar']['days']:
            if not day['available'] and day['reservation']:
                reservation = day['reservation']
                event = {
                    'summary': reservation['guest']['full_name'],
                    'check_in': self.check_in_time(reservation['start_date']),
                    'check_out': self.check_out_time(reservation['end_date']),
                }
                reservations.append(event)

        reservations = list({v['summary']: v for v in reservations}.values())

        return reservations

    def calendar_sync(self):
        reservations = self.get_reservations()
        public_events = self.cal.get_public_calendar()

        for event in reservations:
            matching_public_events = [
                d for d in public_events if d['summary'] == event['summary']
            ]
            if len(matching_public_events) == 0:
                LOGGER.info(
                    "Public calendar entry for {} not found, creating".format(
                        event['summary']))
                self.cal.create_public_event(event)

                message = "Synced entry for {} to public calendar".format(
                    event['summary'])
                self.notifications.send_telegram_private(message)

                # Send message for cleaner to see bookings
                start_date = parser.parse(event['check_in'])
                end_date = parser.parse(event['check_out'])
                message_public = "Nová rezervace potvrzena: {} {} až {}".format(
                    event['summary'], format_date(start_date, locale='cs_CZ'),
                    format_date(end_date, locale='cs_CZ'))
                self.notifications.send_telegram_public(message_public)

            # else:
            #     LOGGER.info(
            #         "Public calendar entry for {} already exists".format(
            #             event['summary']))

    def nuki_sync(self):
        # Enable nuki continuous mode on check in day
        events = self.cal.get_public_calendar()
        check_in_day = False
        for event in events:
            if 'dateTime' in event['start'].keys():
                if datetime.today().date() == parser.parse(
                        event['start']['dateTime']).date():
                    check_in_day = True
                    message = "{} is checking in today, setting Nuki to continuous mode".format(
                        event['summary'])
                    LOGGER.info(message)
                    self.notifications.send_telegram_private(message)

                    count = 0
                    while count <= 5:
                        self.nuki.set_opener_mode('continuous')
                        time.sleep(
                            10
                        )
                        if self.nuki.check_opener_mode('continuous'):
                            break
                        else:
                            count += 1

                    if not self.nuki.check_opener_mode('continuous'):
                        self.notifications.send_telegram_private('PROBLEM: NUKI NOT SET TO CONTINUOUS')

        # Disable nuki continuous mode on non-check in days
        if not check_in_day:

            nuki_mode = self.nuki.opener_mode()
            count = 0
            if nuki_mode == 'continuous':
                message = "Nuki is in continuous node. Setting to disabled"
                self.notifications.send_telegram_private(message)
                while count <= 5:
                    self.nuki.set_opener_mode('disabled')
                    time.sleep(10) # allow opener mode to change before any reporting on this happens
                    if self.nuki.check_opener_mode('door'):
                        break
                    else:
                        count += 1

                if not self.nuki.check_opener_mode('door'):
                    self.notifications.send_telegram_private('PROBLEM: nuki continuous mode not disabled')

    def nuki_notifications_from_log(self):
        logs = self.nuki.log()
        for log in logs:
            log_time = parser.parse(log['date'])
            cz_log_time = log_time.astimezone(pytz.timezone('Europe/Prague'))
            if log_time + timedelta(minutes=5) > datetime.now().astimezone(
                    pytz.utc):

                # Check ring to open or continuous mode enabled
                if log['openerLog']['activeCm'] or log['openerLog']['activeRto']:
                    message = "Nuki opened door at {:d}:{:02d} for ring to open".format(cz_log_time.hour, cz_log_time.minute)
                else:
                    # Swipe
                    if log['action'] == 3:
                        message = "Nuki opened door at {:d}:{:02d} for {} due to swipe".format(cz_log_time.hour, cz_log_time.minute, log['name'])
                    # Ring but didn't open
                    else:
                        message = "Someone rung the bell at {:d}:{:02d} but Nuki did not open".format(cz_log_time.hour, cz_log_time.minute)

                self.notifications.send_telegram_private(message)
            else:
                break


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.cluster import KMeans
from sklearn import preprocessing
import random

import sys
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
dirs = os.path.join( os.path.dirname(__file__),'..')
os.sys.path.append(os.path.join( os.path.dirname(__file__), '..'))
from tools.get_data import GetData
from tools.pearson import CalcPerson

import argparse

class svm3calss(object):
    def __init__(self,  args):
        self.input_size = args.input_size
        self.num_classes =  args.num_classes
        self.learning_rate=args.learning_rate

        self.inputs = tf.placeholder(shape=[None, self.input_size], dtype=tf.float32)
        self.targets = tf.placeholder(shape=[self.num_classes, None], dtype=tf.float32)
        self.batch_size=tf.placeholder(shape=None,dtype=tf.int32,name='batch_size')

        self.bias = tf.Variable(tf.random_normal(shape=[self.num_classes,self.batch_size]))

        with tf.name_scope('rbf_kernel'):
            gamma=self.rbf_kernel()
        with tf.variable_scope('loss'):
            self.log_loss()
        with tf.variable_scope('predict_kernel'):
            self.rbf_predict_kernel(gamma)
        with tf.variable_scope('predict'):
            self.predict()
        with tf.name_scope('train'):
            self.train_op = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss)

    def rbf_kernel(self):
        gamma = tf.constant(-10.0)
        dist = tf.reduce_sum(tf.square(self.inputs), 1)
        dist = tf.reshape(dist, [-1,1])
        sq_dists = tf.multiply(2., tf.matmul(self.inputs, tf.transpose(self.inputs)))
        self.kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists)))
        return gamma
    def reshape_matmul(self,mat):
        v1 = tf.expand_dims(mat, 1)
        v2 = tf.reshape(v1, [self.num_classes, self.batch_size, 1])
        return(tf.matmul(v2, v1))

    def log_loss(self):
        first_term = tf.reduce_sum(self.bias)
        b_vec_cross = tf.matmul(tf.transpose(self.bias), self.bias)
        y_target_cross = self.reshape_matmul(self.targets)
        second_term = tf.reduce_sum(tf.multiply(self.kernel , tf.multiply(b_vec_cross, y_target_cross)),[1,2])
        self.loss = tf.reduce_sum(tf.negative(tf.subtract(first_term, second_term)))

    def rbf_predict_kernel(self,gamma):
        self.prediction_grid = tf.placeholder(shape=[None, self.input_size], dtype=tf.float32)
        rA = tf.reshape(tf.reduce_sum(tf.square(self.inputs), 1),[-1,1])
        rB = tf.reshape(tf.reduce_sum(tf.square(self.prediction_grid), 1),[-1,1])
        pred_sq_dist = tf.add(tf.subtract(rA, tf.multiply(2., tf.matmul(self.inputs, tf.transpose(self.prediction_grid)))), tf.transpose(rB))
        self.pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist)))
    def predict(self):
        prediction_output = tf.matmul(tf.multiply(self.targets,self.bias), self.pred_kernel)
        self.prediction = tf.arg_max(prediction_output-tf.expand_dims(tf.reduce_mean(prediction_output,1), 1), 0)
        self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.prediction, tf.argmax(self.targets,0)), tf.float32))

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--is-train', type=int, default=0,help='1=train, 2=test, 0=train and test')
    parser.add_argument('--num-iter', type=int, default=100,help='the number of training steps to take')
    parser.add_argument('--batch-size', type=int, default=64,help='the number of peptide')

    parser.add_argument('--learning-rate', type=float, default=1e-3,help='')
    parser.add_argument('--input-size', type=int, default=192,help='')
    parser.add_argument('--num-classes', type=int, default=3,help='')

    parser.add_argument('--intensity_num_label', type=int, default=4, help="")
    return parser.parse_args()

def get_batch_peptide(merge_list,_batch_size):
    number_of_peptide=len(merge_list[0])
    batch_peptide=[]
    seq_length=[]
    _batch_number=int(number_of_peptide/_batch_size)
    for i in range(_batch_number):
        batch_peptide.append(merge_list[0][i*_batch_size:(i+1)*_batch_size])
        seq_length.append([])
        for j in range(len(batch_peptide[i])):
            seq_length[len(batch_peptide)-1].append(len(merge_list[0][i*_batch_size+j]))
    if _batch_number*_batch_size < number_of_peptide:
        seq_length.append([])
        batch_peptide.append(merge_list[0][_batch_number*_batch_size:])
        for k in range(len(batch_peptide[-1])):
            seq_length[len(batch_peptide)-1].append(len(merge_list[0][_batch_number*_batch_size+k]))
        _batch_number+=1
    return batch_peptide,_batch_number,seq_length

def model_train(args):
    model = svm3calss(args)
    with tf.Session() as sess:
        merged = tf.summary.merge_all()
        writer = tf.summary.FileWriter("seq2seq-WGAN/svm-logs", sess.graph)
        sess.run(tf.global_variables_initializer())
         #train data
        _,_,train_X,train_y,merge_train_list=data.get_discretization_data('data/data_mm/test_test.txt',args.num_classes)
        print(str(len(merge_train_list[0]))+' train peptides ,DataShape:('+str(np.array(train_X).shape)+str(np.array(train_y).shape)+')')
        train_batch_peptide,_batch_number,seq_length=get_batch_peptide(merge_train_list,args.batch_size)
        y1 = np.array([1 if y==0 else -1 for y in train_y])
        y2 = np.array([1 if y==1 else -1 for y in train_y])
        y3 = np.array([1 if y==2 else -1 for y in train_y])
        train_y = np.array([y1, y2, y3])

        ##val data
        #_,_,test_X,test_y,merge_test_list=data.get_discretization_data('data/data_mm/test.txt',args.num_classes)
        #print(str(len(merge_test_list[0]))+' test peptides ,DataShape:('+str(np.array(test_X).shape)+str(np.array(test_y).shape)+')')

        print('..trainning')
        loss_vec = []
        batch_accuracy = []
        for Iter in range(args.num_iter):
            for i,train_piptides in enumerate(train_batch_peptide):

                train_ions=data.get_split_list2(train_piptides)

                X= train_X[train_ions]
                y= train_y[:,train_ions]
                sess.run(model.train_op, feed_dict={model.inputs: X, model.targets:  y,model.batch_size:len(X)})
                temp_loss = sess.run(model.loss, feed_dict={model.inputs:  y, model.targets:  y,model.batch_size:len(X)})
                loss_vec.append(temp_loss)

                acc_temp = sess.run(model.accuracy, feed_dict={model.inputs: X, model.targets:  y, model.prediction_grid:X,model.batch_size:len(X)})
                batch_accuracy.append(acc_temp)

                print('Step #' + str(i+1))
                print('Loss = ' + str(temp_loss))

            val_acc=0;val_loss=0
            for i, val_piptide_index in enumerate(val_batch_peptide):
                X=[];y=[]

                max_ions_number=max(val_seq_length[i])
                for j in range(len(val_piptide_index)):
                    train_ion_index=data.get_split_list(val_piptide_index[j])
                    X.append(padding_data(val_X[np.array(train_ion_index)],1,max_ions_number))
                    y.append(padding_data(val_y[np.array(train_ion_index)],0,max_ions_number))
                feed_dict_val = {
                            model.X:np.array(X),
                            model.y:np.array(y),
                            model.keep_prob:args.keep_prob,
                            model.seq_length:val_seq_length[i],
                            model.max_time:max_ions_number,
                            model.batch_size:len(X)

                    }
                loss_val, pred_val = sess.run([model.loss, model.tags],feed_dict=feed_dict_val)
                val_loss+=loss_val
                mask = (np.expand_dims(np.arange(max_ions_number), axis=0) < np.expand_dims(val_seq_length[i], axis=1))
                total_labels = np.sum(val_seq_length[i])
                correct_labels = np.sum((np.array(y) == pred_val) * mask)
                accuracy = 100.0 * correct_labels / float(total_labels)
                val_acc+=accuracy
            #val_acc,val_loss=val(args,model,val_X,val_y,val_batch_peptide,val_seq_length)
            print("Epoch: %d" % (Iter+1), "train loss: %.2f" % (train_loss/_batch_number),"train acc: %.2f%%" % (train_acc/_batch_number))
            print("Epoch: %d" % (Iter+1), "val loss: %.2f" % (val_loss/val_batch_number),"val acc: %.2f%%" % (val_acc/val_batch_number))
            result = sess.run(merged, feed_dict)
            writer.add_summary(result, Iter)

            #print('Iter[%d/%d],loss[%.4f]' % (Iter+1,args.num_iter,round(loss,4)))
        print("SaveModel:",tf.train.Saver().save(sess,'lstm/model/model.ckpt'))

def MSE(label,pred):
    return tf.reduce_mean(tf.square(pred-label))

def model_predict(args,kmodel,test_data,merge_test_list,test_label):
    print('predicting..')

    print('number of peptide:'+str(len(merge_test_list[0])))
    with tf.Session() as session:
        batch_peptide,_batch_number,_seq_length=get_batch_peptide(merge_test_list,args.batch_size)

        saver = tf.train.import_meta_graph('lstm/model/model.ckpt.meta')
        saver.restore(session, tf.train.latest_checkpoint('lstm/model/'))
        graph=tf.get_default_graph()
        inputs_X=graph.get_operation_by_name("inputs/X").outputs[0]
        batch_size=graph.get_operation_by_name("batch_size").outputs[0]
        keep_prob=graph.get_operation_by_name("keep_prob").outputs[0]
        max_time=graph.get_operation_by_name("max_time").outputs[0]
        seq_length=graph.get_operation_by_name("seq_length").outputs[0]

        pred_y=tf.get_collection("pred_network")[0]
        pred=[];aaa=[]
        mse_list=[]
        total_labels=0;correct_labels=0
        for i,(test_piptide_index) in enumerate(batch_peptide):
            X=[];y=[]
            _max_ions_number=max(_seq_length[i])
            padding_pep_num=args.batch_size-len(test_piptide_index)
            for j in range(len(test_piptide_index)):
                test_ion_index=get_split_list(test_piptide_index[j])
                X.append(padding_data(test_data[np.array(test_ion_index)],1,_max_ions_number))
                y.append(padding_data(test_label[np.array(test_ion_index)],0,_max_ions_number))

            pred_ = session.run(pred_y,feed_dict={
                inputs_X: np.array(X),
                keep_prob:args.keep_prob,
                seq_length:_seq_length[i],
                max_time:_max_ions_number,
                batch_size:len(X)
                                      })
            for k in range(len(X)):
                pred.extend(pred_[k][:_seq_length[i][k]])
                aaa.extend(y[k][:_seq_length[i][k]])

            mask = (np.expand_dims(np.arange(_max_ions_number), axis=0) < np.expand_dims(_seq_length[i], axis=1))
            total_labels += np.sum(_seq_length[i])
            correct_labels += np.sum((np.array(y) == pred_) * mask)

        accuracy = 100.0 * correct_labels / float(total_labels)
        print("test Accuracy: %.2f%%" % accuracy)
            #pred_[pred_>1]=1
            #pred_[pred_<0]=0
            #_mse=session.run(MSE(np.reshape(y,(-1,1)),pred_))
            #mse_list.append(_mse)
        cunt=0;cunt2=0
        for i in range(len(aaa)):
            if aaa[i]==0:
                cunt+=1
            if aaa[i]==pred[i]:
                cunt2+=1

        print(cunt/len(aaa))
        print(cunt2/len(aaa))
        predaaa = pd.DataFrame({"pred":pred,"label":aaa})
        predaaa.to_csv('data//SwedCAD_pred2.csv')
        min_max_scaler = preprocessing.MinMaxScaler()
        pred_minmax = min_max_scaler.fit_transform(pred)
    return pred_minmax

def get_merge_pred(merge_list,pred,data):
    print('get predict spectrum intensity list...')
    merge_list.append(data.merge_list_1label(pred))
    return merge_list
def calc_pear(test_idx,peptide,pred,merge_list,pear,data):

    pred_pd=pear.write_pred(test_idx,peptide,pred)
    merge_list=get_merge_pred(merge_list,pred_pd,data)

    person_mean=pear.get_pearson(merge_list)
    return person_mean

def test(args,data,pear):
    test_idx,peptide,pred,merge_test_list=model_predict(args,data)
    person_mean=calc_pear(test_idx,peptide,pred,merge_test_list,pear,data)
    print(person_mean)

def main(args,data,pear):
    if args.is_train==1:
        model_train(args)
    elif args.is_train==2:
        test(args,data,pear)
    else:
        model_train(args)
        test(args,data,pear)

if __name__ == '__main__':
    args=parse_args()

    data=GetData(args.intensity_num_label)
    pear=CalcPerson(args.intensity_num_label)
    main(args,data,pear)


""" IMPORT MODULES AND LIBRARIES """

from functions.utils.prompt import prompt

# Import required libraries
import os
from simple_colors import yellow, blue, green, red, cyan
import sys
from tabulate import tabulate
import pickledb
# import ipaddress
import time
import datetime

""" PICKLEDB SETUP """
# db = pickledb.load('netsploit.db', False)

""" CONFIG """
# NOTE: Change the paths to to absolute paths if needed
# UPDATE: using `functions/utils/config.py` for storing configs

# username = os.popen('whoami').read()
# username = username.replace("\n", "")

# oui_file_path = f'/home/{username}/path/to/netsploit-py/oui.txt'  # noqa
# oui_file_path = 'oui.txt'

# logs_folder_path = f'/home/{username}/path/to/netsploit-py/logs/'  # noqa
# logs_folder_path = 'logs/'

""" STARTUP """
# Print ASCII art of "NetSploit"
header = """
     __     _   __       _       _ _
  /\ \ \___| |_/ _\_ __ | | ___ (_) |_
 /  \/ / _ \ __\ \| '_ \| |/ _ \| | __|
/ /\  /  __/ |__\ \ |_) | | (_) | | |_
\_\ \/ \___|\__\__/ .__/|_|\___/|_|\__|
                  |_|
"""  # noqa
print(blue(header, 'bold'))

# Print init messages
print(yellow(' => netsploit v1.0', 'bold'))
print(yellow(' => Powered by nmap, ping and hping3', 'bold'))
print(green(' => 9 modules ready to use', 'bold'))
# print(blue(' => network-scanner - device-info - os-guesser - oui-lookup - \n      port-scanner - dos - ping - vuln-scanner - custom', 'bold'))  # noqa
print(blue(' => 5 Scanners, 3 Info, 1 Attack', 'bold'))  # noqa
# print(cyan(' => Happy Hacking! :)', 'bold'))

# Set variables for printing network configuration

gateway = os.popen("ip route show | grep -i 'default via'| awk '{print $3 }'").read()  # noqa
gateway = gateway.replace("\n", "")

up_interface = os.popen("route | awk '/Iface/{getline; print $8}'").read()
up_interface = up_interface.replace("\n", "")

# Get wireless network name (was used previously)
# n_name = os.popen('iwgetid -r').read()

# Work-around to get wireless network name on Fedora Linux:
n_name = os.popen('nmcli -t -f NAME connection show --active').read()

# Get network MAC address
n_mac = os.popen(
    "ip addr | grep 'state UP' -A1 | tail -n1 | awk '{print $2}' | cut -f1  -d'/'").read()  # noqa
n_ip = os.popen("hostname -I").read()  # Local IP address
n_host = os.popen("hostname").read()  # hostname
# Print network configuration header

print(""" \033[1;36m
╒═════════════════════════════════════════════════════════════════════════╕
│                         Your Network Configuration                      │
╘═════════════════════════════════════════════════════════════════════════╛     \033[1;m""")  # noqa

# Print network configuration , using tabulate as table.

table = [["IP Address", "MAC Address", "Gateway", "Iface", "Hostname"],
         ["", "", "", "", ""],
         [n_ip, n_mac.upper(), gateway, up_interface, n_host]]
print(tabulate(table, stralign="center",
               tablefmt="fancy_grid", headers="firstrow"))
print()
print(f'{green("[+] Please type help to view commands", "bold")}')
# print()
# print(f'{cyan("[TIP]", "bold")} You can use the {yellow("auto", "bold")} command to automate your process!')  # noqa
print()

try:
    prompt()
except KeyboardInterrupt:
    print()
    exit()


#!/usr/bin/env python3

import os
import glob
import requests

host = 'pyrite.abudhabi.nyu.edu:3005'
# host = 'localhost:5010'
uri = 'http://{}/tf_counts/1.0/api/count_worms'.format(host)

def health_check():
    uri = 'http://{}/tf_counts/1.0/api/health'.format(host)
    res = requests.get(uri, json={})
    print(res.json())

def push_counts_to_queue(dirs):
    print('Pushing {}'.format(os.path.dirname(dirs[0])))
    # dirs = [dirs[0]]
    for d in dirs:
        dd = os.path.join('/mnt/image', d, os.path.basename(d) + '-tf_counts.csv')
        data = {
            'image_path': os.path.join('/mnt/image', d),
            'counts': dd
        }

        res = requests.post(uri, json=data)
        print('Data : {}'.format(data))
        print('Res: {}'.format(res.json()))

    print('Finished {}'.format(os.path.dirname(dirs[0])))

def sanity_check():
    data = {}
    data['image_path'] = '/home/webuser/images/'
    data['counts'] = '/home/webuser/images/counts.csv'
    res = requests.post(uri, json=data)
    print('Data : {}'.format(data))
    print('Res: {}'.format(res.json()))

def sanity_check_real_world():

    dirs = [
        '2014Dec15/1155',
        '2014Dec08/966',
        '2015Nov09/3716',
        '2015Nov09/3620',
        '2015Oct26/3149',
        '2015Oct26/3053',
        '2015Oct26/3052']

    for d in dirs:
        dd = os.path.join('/mnt/image', d, os.path.basename(d) + '-tf_counts.csv')
        data = {
            'image_path': os.path.join('/mnt/image', d),
            'counts': dd
        }
        print(data)
        res = requests.post(uri, json=data)
        print(res.json())

def push_dirs_to_queue(d):
    dirs = glob.glob(d)
    for d in dirs:
        if os.path.isdir(d):
            ds = glob.glob('{}/*'.format(d))
            push_counts_to_queue(ds)

print('Health Check!')
health_check()
print('Sanity check!')
sanity_check()
# sanity_check_real_world()

print('Ok starting real world!')

push_dirs_to_queue('/mnt/image/2015*')
push_dirs_to_queue('/mnt/image/2014*')
push_dirs_to_queue('/mnt/image/2016*')
push_dirs_to_queue('/mnt/image/2017*')
push_dirs_to_queue('/mnt/image/2018*')


from rest_framework import serializers
from django.shortcuts import get_object_or_404
from rest_framework.validators import UniqueTogetherValidator
from rest_framework.serializers import ValidationError

from posts.models import Follow, Group, Comment, Post, User

class FollowSerializer(serializers.ModelSerializer):
    user = serializers.SlugRelatedField(
        slug_field='username', read_only=True,
        default=serializers.CurrentUserDefault()
    )
    following = serializers.SlugRelatedField(
        slug_field='username',
        queryset=User.objects.all()
    )

    class Meta:
        model = Follow
        fields = ('user', 'following')

        validators = [
            UniqueTogetherValidator(
                queryset=Follow.objects.all(),
                fields=('user', 'following'),
                message='The subscription must be unique'
            )
        ]

    def validate_following(self, value):
        value = get_object_or_404(User, username=value)
        if value == self.context.get('request').user:
            raise ValidationError('You can\'t subscribe to yourself')
        return value

class GroupSerializer(serializers.ModelSerializer):
    class Meta:
        model = Group
        fields = '__all__'

class PostSerializer(serializers.ModelSerializer):
    author = serializers.SlugRelatedField(
        slug_field='username', read_only=True
    )

    class Meta:
        fields = '__all__'
        model = Post

class CommentSerializer(serializers.ModelSerializer):
    author = serializers.SlugRelatedField(
        read_only=True, slug_field='username'
    )

    class Meta:
        fields = '__all__'
        model = Comment
        read_only_fields = ('post',)


import logging
logging.basicConfig(filename="dict_file.log",level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')
try:
    class dget:
        def __init__(self,a):
            if (type(a)==dict):
                self.old_dict=a
            else:
                raise Exception("this is not a dictionary")
        def dget(self,b):
            """gets value for specified key

            Args:
                b (any): enter the key
            """
            for k,v in self.old_dict.items():
                if(k==b):
                    the_value=v
            logging.info(f"the value im the dictionary is {the_value} ")
except Exception as e:
    logging.exception(e)
a=dget({"a":1,"b":2})
a.dget("b")