import os

from io import BytesIO
from PIL import Image, ImageDraw, ImageFilter, ImageFont, ImageOps
from base.classes.functions.admFunctions import fontPath

class drawLevel:
    def __init__(self):
        self.class_font_semibold = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratSemiBold.ttf'), 36)

        self.class_font_bold_name = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratBold.ttf'), 72)

        self.class_font_bold_id = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratMedium.ttf'), 42)

        self.class_font_bold_info_sans = ImageFont.truetype(
            os.path.join(fontPath.pathOpen, 'OpenSansRegular.ttf'), 36)

        self.class_font_bold_xp = ImageFont.truetype(
            os.path.join(fontPath.pathOpen, 'OpenSansBold.ttf'), 48)

        self.class_font_bold_info = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratBold.ttf'), 36)

        self.class_font_montserrat = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratRegular.ttf'), 36)

        self.class_font_montserrat_bday = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratMedium.ttf'), 36)

        self.class_font_bold_ori_bday = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratBold.ttf'), 42)

        self.class_font_role = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratBold.ttf'), 30)

        self.RadiateSansBoldCondensed = ImageFont.truetype(
            os.path.join(fontPath.pathOpen, 'OpenSansCondensedBold.ttf'), 65)

        self.class_font_montserrat_regular_nivel = ImageFont.truetype(
            os.path.join(fontPath.pathMonserrat, 'MontserratRegular.ttf'), 30)

        self.XPRadiateSansBoldCondensed = ImageFont.truetype(
            os.path.join(fontPath.pathOpen, 'RadiateSansBoldCondensed.ttf'), 85)

    @staticmethod
    def neededxp(level: int) -> int:
        return ((level * level)) + 100

    def level(
            self,
            rank: str,
            xp: str,
            xptotal: str,
            moldName: str,
            moldImg: str,
            imgxp: str,
            profile_bytes
    ) -> BytesIO:
        profile_bytes = Image.open(str(profile_bytes))
        profile_bytes_imenso = profile_bytes.copy()

        # profile_bytes = profile_bytes.resize((1270, 1270))

        bigsize = (profile_bytes.size[0] * 3, profile_bytes.size[1] * 3)
        mask = Image.new('L', bigsize, 0)
        draw = ImageDraw.Draw(mask)
        draw.ellipse((0, 0) + bigsize, fill=255)
        mask = mask.resize(profile_bytes.size, Image.Resampling.LANCZOS)
        profile_bytes.putalpha(mask)

        output = ImageOps.fit(profile_bytes, mask.size, centering=(0.5, 0.5))
        output.putalpha(mask)

        bg = Image.new('RGBA', (1270, 892), (0, 45, 62, 255))
        bg_draw = ImageDraw.Draw(bg)

        profile_bytes_imenso = profile_bytes_imenso.resize((1270, 892))
        # profile_bytes_imenso = profile_bytes_imenso.crop(
        #    (0, 420, 1280, 823+420))
        mask = Image.new("L", profile_bytes_imenso.size, 90)

        profileFundo = Image.composite(profile_bytes_imenso, bg, mask)
        profileBlur = profileFundo.filter(ImageFilter.GaussianBlur(radius=30))
        bg.paste(profileBlur, (0, 0))

        # xp_text = f'Faltam {} para chegar ao Nível {}:'.format(xp_remain, at_rank-1)

        bg_draw.rounded_rectangle(
            ((190, 768), (1089, 768 + 28)), 15, fill=(0, 45, 62))

        # xp_in = self.gradientLeft()
        # xp_im = Image.open(r"src/bg/xpFill-background.png")
        # im1 = xp_im.crop((0, 0, ((int(xp/needed_xp*100))*6), 81))
        # bg.paste(im1, (488, 558), im1)
        #
        # bg_draw.text((515, 503), xp_text, font=self.class_font_semibold, fill=(161, 177, 191))
        # bg_draw.text((870, 618), f"{xp}/{needed_xp}", font=self.class_font_bold_xp, anchor="ms", fill=(219, 239, 255))

        needed_xp = self.neededxp(int(rank))

        # Pré
        bg_draw.text((80, 797), "NÍVEL ",
                     font=self.class_font_montserrat_regular_nivel, anchor="ms", fill=(219, 239, 255))
        bg_draw.text((150, 797), f"{rank}",
                     font=self.class_font_role, anchor="ms", fill=(219, 239, 255))

        # Middle
        bg_draw.text((640, 855),
                     f"Faltam {int(needed_xp - xp):,} de XP para chegar ao Nível {rank + 1}".replace(
                         ",", "."),
                     font=self.class_font_role, anchor="ms", fill=(219, 239, 255))

        # Pós
        bg_draw.text((1089 + 70, 797), "NÍVEL ",
                     font=self.class_font_montserrat_regular_nivel, anchor="ms", fill=(219, 239, 255))
        bg_draw.text((1089 + 140, 797), f"{rank + 1}",
                     font=self.class_font_role, anchor="ms", fill=(219, 239, 255))

        xp_im = Image.open(rf"{imgxp}")
        # ((already_earned / to_reach) * 100)

        # print(xp_im.size[0])
        # print(int((xp / needed_xp) * xp_im.size[0]))
        # print(xp / needed_xp)

        im1 = xp_im.crop((0, 0, int((xp / needed_xp) * xp_im.size[0]), 81))
        bg.paste(im1, (190, 767), im1)

        color = xp_im.getpixel(((xp_im.size[0]) - 5, 7))

        if moldName is not None:
            rank_text = moldName.upper()
            bg_draw.text((635, 622), rank_text, font=self.RadiateSansBoldCondensed, fill=(
                color), anchor="ms", spacing=5)

        bg_draw.text((695, 730), f"{int(xptotal):,}".replace(",", "."),
                     font=self.XPRadiateSansBoldCondensed, anchor="rs", fill=(219, 239, 255))
        bg_draw.text((705, 730), "XP TOTAL",
                     font=self.class_font_montserrat_regular_nivel, anchor="ls", fill=(219, 239, 255))

        border_im = Image.open(r"{}".format(moldImg))
        bsize = (border_im.size[0], border_im.size[1])
        bg.paste(
            border_im, (int((1270 - bsize[0]) / 2) + 2, (int((892 - bsize[1]) / 2) - 150)), border_im)

        buffer = BytesIO()
        bg.save(buffer, 'png')
        buffer.seek(0)

        mask.close()
        bg.close()

        return buffer

drawlevel = drawLevel().level
neededxp = drawLevel().neededxp


import numpy as np
from scipy.io import loadmat
import pandas as pd
import datetime as date
from dateutil.relativedelta import relativedelta

cols = ['age', 'gender', 'path', 'face_score1', 'face_score2']

imdb_mat = 'imdb_crop/imdb.mat'
wiki_mat = 'wiki_crop/wiki.mat'

imdb_data = loadmat(imdb_mat)
wiki_data = loadmat(wiki_mat)

del imdb_mat, wiki_mat

imdb = imdb_data['imdb']
wiki = wiki_data['wiki']

imdb_photo_taken = imdb[0][0][1][0]
imdb_full_path = imdb[0][0][2][0]
imdb_gender = imdb[0][0][3][0]
imdb_face_score1 = imdb[0][0][6][0]
imdb_face_score2 = imdb[0][0][7][0]

wiki_photo_taken = wiki[0][0][1][0]
wiki_full_path = wiki[0][0][2][0]
wiki_gender = wiki[0][0][3][0]
wiki_face_score1 = wiki[0][0][6][0]
wiki_face_score2 = wiki[0][0][7][0]

imdb_path = []
wiki_path = []

for path in imdb_full_path:
    imdb_path.append('imdb_crop/' + path[0])

for path in wiki_full_path:
    wiki_path.append('wiki_crop/' + path[0])

imdb_genders = []
wiki_genders = []

for n in range(len(imdb_gender)):
    if imdb_gender[n] == 1:
        imdb_genders.append('male')
    else:
        imdb_genders.append('female')

for n in range(len(wiki_gender)):
    if wiki_gender[n] == 1:
        wiki_genders.append('male')
    else:
        wiki_genders.append('female')

imdb_dob = []
wiki_dob = []

for file in imdb_path:
    temp = file.split('_')[3]
    temp = temp.split('-')
    if len(temp[1]) == 1:
        temp[1] = '0' + temp[1]
    if len(temp[2]) == 1:
        temp[2] = '0' + temp[2]

    if temp[1] == '00':
        temp[1] = '01'
    if temp[2] == '00':
        temp[2] = '01'

    imdb_dob.append('-'.join(temp))

for file in wiki_path:
    wiki_dob.append(file.split('_')[2])

imdb_age = []
wiki_age = []

for i in range(len(imdb_dob)):
    try:
        d1 = date.datetime.strptime(imdb_dob[i][0:10], '%Y-%m-%d')
        d2 = date.datetime.strptime(str(imdb_photo_taken[i]), '%Y')
        rdelta = relativedelta(d2, d1)
        diff = rdelta.years
    except Exception as ex:
        print(ex)
        diff = -1
    imdb_age.append(diff)

for i in range(len(wiki_dob)):
    try:
        d1 = date.datetime.strptime(wiki_dob[i][0:10], '%Y-%m-%d')
        d2 = date.datetime.strptime(str(wiki_photo_taken[i]), '%Y')
        rdelta = relativedelta(d2, d1)
        diff = rdelta.years
    except Exception as ex:
        print(ex)
        diff = -1
    wiki_age.append(diff)

final_imdb = np.vstack((imdb_age, imdb_genders, imdb_path, imdb_face_score1, imdb_face_score2)).T
final_wiki = np.vstack((wiki_age, wiki_genders, wiki_path, wiki_face_score1, wiki_face_score2)).T

final_imdb_df = pd.DataFrame(final_imdb)
final_wiki_df = pd.DataFrame(final_wiki)

final_imdb_df.columns = cols
final_wiki_df.columns = cols

meta = pd.concat((final_imdb_df, final_wiki_df))

meta = meta[meta['face_score1'] != '-inf']
meta = meta[meta['face_score2'] == 'nan']

meta = meta.drop(['face_score1', 'face_score2'], axis=1)

meta = meta.sample(frac=1)

meta.to_csv('meta.csv', index=False)


# Standard library imports
from os import listdir
from os.path import join

# Third party imports
import pandas as pd

# Local application imports
import settings

class CsvWriter:
    def __init__(self, pathFinder):
        # list of countries, in the order read
        self.countries = []
        self.pathFinder = pathFinder

    def createCsvs(self):
        print('CsvWriter: ')
        print('\tCreating csv\'s...', end=" ")
        self.__createCountriesCsv()
        self.__createStatsCsv()
        self.__createIndicatorsCsv()
        print('Done')

    def __createCountriesCsv(self):
        finalDf = pd.DataFrame()
        for filename in listdir(self.pathFinder.countriesDir):
            df = pd.read_csv(join(self.pathFinder.countriesDir, filename))
            finalDf = pd.concat([finalDf, df])

        # Drop unnecessary columns
        finalDf = finalDf.drop(['Unnamed: 5', 'Unnamed: 4'], axis=1)
        # save countries code in order read
        self.countries = finalDf['Country Code'].to_list()
        # Export
        self.__exportToCsv(finalDf, settings.countriesCsv)

    def __createStatsCsv(self):
        finalDf = pd.DataFrame()

        for filename in listdir(self.pathFinder.statsDir):
            df = pd.read_csv(join(self.pathFinder.statsDir, filename), skiprows=4)
            # Get specific ID of country based on country code
            code = df['Country Code'].drop_duplicates().loc[0]
            id = self.countries.index(code) + 1

            # Invert dataframe and drop unnessecary columns
            df = df.T
            df = df.drop(['Country Name', 'Country Code', 'Indicator Name', 'Unnamed: 65'])
            # Name headers based on each 'Indicator Code'
            df = df.rename(columns=df.iloc[0]).drop(df.index[0])
            # Make Years from index to a column
            df = df.reset_index()
            df = df.rename(columns={'index': 'Year'})

            # Keep only codes starting with the given strings
            wantedColumns = ('Year', 'BM', 'EG', 'GC.TAX')
            columnFilter = [col for col in df if col.startswith(wantedColumns)]
            df = df[columnFilter]

            # insert ID column based on country code
            df.insert(0, 'Country ID', id)
            finalDf = pd.concat([finalDf, df])
        # Export
        self.__exportToCsv(finalDf, settings.statsCsv)

    def __createIndicatorsCsv(self):
        finalDf = pd.DataFrame()

        for filename in listdir(self.pathFinder.indicatorsDir):
            df = pd.read_csv(join(self.pathFinder.indicatorsDir, filename))
            df = df.drop(['Unnamed: 4'], axis = 1)
            finalDf = pd.concat([finalDf, df])

        # Keep specified non-duplicated columns
        finalDf = finalDf.drop_duplicates()
        finalDf = finalDf[finalDf['INDICATOR_CODE'].str.startswith('BM') |
                            finalDf['INDICATOR_CODE'].str.startswith('EG') |
                            finalDf['INDICATOR_CODE'].str.startswith('GC.TAX') ]
        # Export
        self.__exportToCsv(finalDf, settings.indicatorsCsv)

    def __exportToCsv(self, df, outputFileName):
        df.to_csv(join(self.pathFinder.outputDir, outputFileName), na_rep='NULL', index = False)


#coding: utf-8

# Threadpool - 여러 스레드를 특정 프로세스에 사용하도록 유지한다.
# 스레드 재사용에 목적이 있고 불필요한 스레드 생성을 방지

import sys, logging, re
import threading, queue, requests, concurrent.futures

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')

ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)

html_link_regex = \
     re.compile('<a\s(?:.*?\s)*?href=[\'"](.*?)[\'"].*?>')

urls = queue.Queue()
urls.put('http://www.google.com')
urls.put('http://br.bing.com/')
urls.put('https://duckduckgo.com/')
urls.put('https://github.com/')
urls.put('http://br.search.yahoo.com/')

result_dict = {}

# 동기화된 큐로부터 URL를 추출해 result_dict 넣는다.
def group_urls_task(urls):
    try:
        # True는 동기화된 큐에 접근을 차단
        url = urls.get(True, 0.05)
        result_dict[url] = None
        logger.info("[%s] putting url [%s] in dictionary..." % (
            threading.current_thread().name, url))
    except queue.Empty:
        logging.error('Nothing to be done, queue is empty')

# 페이지의 링크 수집
def crawl_task(url):
    links = []

    try:
        request_data = requests.get(url)
        logger.info("[%s] crawling url [%s] ..." % (
            threading.current_thread().name, url))
        links = html_link_regex.findall(request_data.text)
    except:
        logger.error(sys.exc_info()[0])
    finally:
        return (url, links)

# ThreadPoolExecutor에 정의된 worker 스레드가 비동기 방식인 병렬로 수행한다.
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as group_link_threads:
    for i in range(urls.qsize()):
        group_link_threads.submit(group_urls_task, urls)

with concurrent.futures.ThreadPoolExecutor(max_workers=3) as crawler_link_threads:
    future_tasks = {crawler_link_threads.submit(crawl_task, url): url for url in result_dict.keys()}
    for future in concurrent.futures.as_completed(future_tasks):
        result_dict[future.result()[0]] = future.result()[1]

for url, links in result_dict.items():
    logger.info("[%s] with links : [%s..." % (url, links[0]))


from django.views.generic import (
    FormView,
    ListView,
    DetailView,
    View
)
from django.shortcuts import render, HttpResponse
from users.models import User
from users.forms import UserForm

class UserListView(ListView):
    model = User
    template_name = 'user_list.html'

class UserDetailView(DetailView):

    model = User
    template_name = 'user_detail.html'

class AddUserView(FormView):

    form_class = UserForm
    template_name = 'form.html'
    success_url = "/"

    def form_valid(self, form):
        form.save()

        response = super().form_valid(form)

        return response

class UserEditView(View):

    def get(self, request, pk):

        form = UserForm()

        context = {
            'form': form
        }

        return render(
            template_name='form.html',
            context=context,
            request=request,
        )

    def post(self, request, pk):

        user = User.objects.get(pk=pk)

        username = request.POST['username']
        email = request.POST['email']

        if len(username) != 0:
            user.username = username

        if len(email) != 0:
            user.email = email

        user.save()

        context = {
            'user': user
        }

        return render(
            template_name='user_detail.html',
            context=context,
            request=request,
        )

class UserDeleteView(View):

    def get(self, request, pk):
        user = User.objects.get(pk=pk)
        user.delete()

        return HttpResponse(f'Deleted {user.username}')


import numpy as np
import cv2
def disparity_ncorr(L, R, size = 5, disparity = 30):
    """Compute disparity map D(y, x) such that: L(y, x) = R(y, x + D(y, x))

    Params:
    L: Grayscale left image
    R: Grayscale right image, same size as L

    Returns: Disparity map, same size as L, R
    """

    window_size = size  # from Fundementals of Computer Vision

    dmax = disparity * 2 + 1
    moving_range = np.arange(dmax)-disparity # -10, -9 ..0..9, 10 it has to be bigger than window_size
    height, width = L.shape
    tpl_rows = tpl_cols = window_size
    D = np.zeros(L.shape, dtype = np.float32)

    for r in range(tpl_rows//2, height - tpl_rows//2):
        for c in range(tpl_cols//2, width - tpl_cols//2):
            tpl_r_min = r - tpl_rows//2
            tpl_r_max = r + tpl_rows//2+1       ## python range max is not included
            tpl_c_min = c - tpl_cols//2
            tpl_c_max = c + tpl_cols//2+1
            tpl = L[tpl_r_min:tpl_r_max, tpl_c_min:tpl_c_max].astype(np.float32)
            mt_c_min = max(c + moving_range[0]- tpl_cols//2 ,0)
            mt_c_max = min(c + moving_range[dmax-1] + tpl_cols//2 +1 , width)
            mt = R[tpl_r_min:tpl_r_max,mt_c_min:mt_c_max].astype(np.float32)

            # slide template patch to the target matching area. Caculate and when SSD(r,c) has the minimum assign to D(r,c)
            #_, min_loc = min_ssd(tpl,mt,window_size)
            #D[r, c] = c - (mt_c_min + min_loc + tpl_rows//2)  # disparity

            #_, _, min_loc, _ = cv2.minMaxLoc(ssd(tpl,mt,window_size))
            #D[r,c] = c - (mt_c_min + min_loc[0] +  tpl_cols//2)  # disparity
            #Faster. Mine is unbearbly slow.
            _,_, _, max_loc = cv2.minMaxLoc(cv2.matchTemplate(tpl, mt, method = cv2.TM_CCORR_NORMED))
            D[r, c] = c - (mt_c_min + max_loc[0] + tpl_cols // 2)
    return D


import pika, json
from app import Post, db, app

params = pika.URLParameters("amqps://govrtnay:tMnQn4JaY1ZzT3J_ErPMtABxqMMSiW9V@dingo.rmq.cloudamqp.com/govrtnay")

connection = pika.BlockingConnection(params)

channel = connection.channel()

channel.queue_declare(queue='app')

def callback(ch, method, properties, body):
    print('Received in app')
    data = json.loads(body)
    print(data)

    with app.app_context():

        if properties.content_type == 'post_created':
            post = Post(id=data['id'], title=data['title'], image=data['image'])
            db.session.add(post)
            db.session.commit()
            print('Post Created')

        elif properties.content_type == 'post_updated':
            post = Post.query.get(data['id'])
            post.title = data['title']
            post.image = data['image']
            db.session.commit()
            print('Post Updated')

        elif properties.content_type == 'post_deleted':
            post = Post.query.get(data)
            db.session.delete(post)
            db.session.commit()
            print('Post Deleted')

    channel.basic_ack(delivery_tag=method.delivery_tag)

channel.basic_consume(queue='app',
                      on_message_callback=callback,
                      auto_ack=False)

print('Started Consuming')

channel.start_consuming()

channel.close()


import xlrd
import  xdrlib ,sys

def open_excel(file):
    try:
        data = xlrd.open_workbook(file)
        return data
    except Exception as e:
        print(str(e))

#根据省份不同，划分成数组，提取出不同省份数据起始点
def divide_by_province(table, nrows, ncols):
    province_col = ncols - 1    #province的列数索引
    province_current = table.cell(1, province_col).value
    province_index = [1, ]
    for i in range(2, nrows-1):
        if province_current != table.cell(i, province_col).value:
            province_current = table.cell(i, province_col).value
            province_index.append(i)
    return province_index

if __name__=="__main__":
    file = 'rew_data.xlsx'  #文件路径
    bound_distance = 100    #设定在多少公里内，认为是邻近城市
    by_index = 1    #使用第二个表，如果使用第一个表，by_index=0

    data = open_excel(file)
    table = data.sheets()[by_index]
    nrows = table.nrows #行数
    ncols = table.ncols #列数
    province_index = divide_by_province(table, nrows, ncols)
    print(province_index)


def slice_model(start, end, step, len):
    indices = []

    if step == ():
        step = 1

    ret = []

    if step > 0:
        # default start is 0
        if start == ():
            start = 0

        # start < 0 is from end

        elif start < 0:
            start = len + start

        # default end is len
        if end == ():
            end = len

        # end < 0 is from end
        elif end < 0:
            end = len + end

        # bounds check start
        if start < 0:
            start = 0

        val = start

        while val < end:
            if 0 <= val and val < len:
                ret += [val]
            val += step

    else:
        # default start is len - 1
        if start == ():
            start = len - 1
        elif start < 0:
            start = len + start

        # default end is -1
        if end == ():
            end = -1
        elif end < 0:
            end = len + end

        if start >= len:
            start = len - 1

        val = start
        while val > end:
            if 0 <= val and val < len:
                ret += [val]
            val += step

    return ret

def slice_string(value, start, end, step):
    indices = slice_model(start, end, step, len(value))

    ret = ""
    for i in indices:
        ret += value[i]
    return ret

def slice_list(value, start, end, step):
    indices = slice_model(start, end, step, len(value))

    ret = []
    for i in indices:
        ret += [value[i]]
    return ret

def slice_tuple(value, start, end, step):
    indices = slice_model(start, end, step, len(value))

    ret = ()
    for i in indices:
        ret += (value[i],)
    return ret

s = "hello"
l = [1, 2, 3, 4, 5]
t = (1, 2, 3, 4, 5)

def check(model, impl, mess):
    if model != impl:
        print("fail: %s (model %r impl %r)" % (mess, model, impl))
        exit(1)

# Test lots of combinations of ranges

for start in range(-5, 5):
    for end in range(-5, 5):
        for step in range(-5, 5):
            if step == 0:
                continue

            check(slice_string(s, (), (), ()), s[::], "[::]")

            check(slice_string(s, start, (), ()), s[start::], "[%d::]" % (start))

            check(slice_string(s, (), end, ()), s[:end:], "[:%d:]" % (end))

            check(
                slice_string(s, start, end, ()),
                s[start:end:],
                "[%d:%d:]" % (start, end),
            )

            check(slice_string(s, (), (), step), s[::step], "[::%d]" % (step))

            check(
                slice_string(s, start, (), step),
                s[start::step],
                "[%d::%d]" % (start, step),
            )

            check(
                slice_string(s, (), end, step), s[:end:step], "[:%d:%d]" % (end, step)
            )

            check(
                slice_string(s, start, end, step),
                s[start:end:step],
                "[%d:%d:%d]" % (start, end, step),
            )

            check(slice_list(l, (), (), ()), l[::], "[::]")

            check(slice_list(l, start, (), ()), l[start::], "[%d::]" % (start))

            check(slice_list(l, (), end, ()), l[:end:], "[:%d:]" % (end))

            check(
                slice_list(l, start, end, ()), l[start:end:], "[%d:%d:]" % (start, end)
            )

            check(slice_list(l, (), (), step), l[::step], "[::%d]" % (step))

            check(
                slice_list(l, start, (), step),
                l[start::step],
                "[%d::%d]" % (start, step),
            )

            check(slice_list(l, (), end, step), l[:end:step], "[:%d:%d]" % (end, step))

            check(
                slice_list(l, start, end, step),
                l[start:end:step],
                "[%d:%d:%d]" % (start, end, step),
            )

            check(slice_tuple(t, (), (), ()), t[::], "[::]")

            check(slice_tuple(t, start, (), ()), t[start::], "[%d::]" % (start))

            check(slice_tuple(t, (), end, ()), t[:end:], "[:%d:]" % (end))

            check(
                slice_tuple(t, start, end, ()), t[start:end:], "[%d:%d:]" % (start, end)
            )

            check(slice_tuple(t, (), (), step), t[::step], "[::%d]" % (step))

            check(
                slice_tuple(t, start, (), step),
                t[start::step],
                "[%d::%d]" % (start, step),
            )

            check(slice_tuple(t, (), end, step), t[:end:step], "[:%d:%d]" % (end, step))

            check(
                slice_tuple(t, start, end, step),
                t[start:end:step],
                "[%d:%d:%d]" % (start, end, step),
            )


from PIL import Image
import argparse

#构建ArgumentParser实例，用于处理命令行输入参数
parser = argparse.ArgumentParser()

#定义输入文件，输出文件，输出字符画的宽和高
parser.add_argument('file')
parser.add_argument('-o','--output')
parser.add_argument('--width',type = int ,default= 80)
parser.add_argument('--height',type = int, default= 80)

#解析并获取参数
args = parser.parse_args()

#输入的图片文件路径
IMG = args.file

#输出字符画的宽度
WIDTH = args.width

#输出字符画的高度
HEIGHT = args.height

#输出字符画的路径
OUTPUT = args.output

#字符画使用的字符集
ascii_char = list("$@B%8&WM#*oahkbdpqwmZO0QLCJUYXzcvunxrjft/\|()1{}[]?-_+~<>i!lI;:,\"^`'. ")

#RGB转字符串的函数
def get_char(r,g,b,alpha = 256):

    #判断 alpha 值
    if alpha == 0:
        return ' '

    #获取字符集的长度，这里为70
    length = len(ascii_char)

    #将RGB值转为灰度值gray, 灰度值范围为0~255
    gray = int(0.2126 * r + 0.7152 * g + 0.0722 *b)

    #灰度值范围为256，而字符集只有70
    unit = (256.0 + 1)/length

    #返回灰度值对应的字符
    return ascii_char[int(gray/unit)]

if __name__ == '__main__':

    #打开并调整图片的宽和高
    im = Image.open(IMG)
    im = im.resize((WIDTH,HEIGHT), Image.NEAREST)

    #初始化输出的字符串
    txt = ""

    #遍历图片中的每一行
    for i in range(HEIGHT):
        #遍历图片中的每一列
        for j in range(WIDTH):
            txt += get_char(*im.getpixel((j,i)))
        txt += '\n'
    print(txt)

    #字符画输出到文件
    if OUTPUT:
        with open(OUTPUT,'w') as f:
            f.write(txt)
    else:
        with open("output.txt,'w") as f:
            f.write(txt)


import os
from ase.io import lammpsdata
from ase.lattice.cubic import BodyCenteredCubic
import numpy as np
from matplotlib import pyplot as plt
from ase import io
from matplotlib.backends.backend_pdf import PdfPages
from ase.build.tools import sort

def distance(x0, x1, dimensions):
    delta = np.abs(x1 - x0)
    if delta[0] > (0.5 * dimensions):
        delta[0] = delta[0] - dimensions
    else:
        delta[0] = delta[0]
    if delta[1] > (0.5 * dimensions):
        delta[1] = delta[1] - dimensions
    else:
        delta[1] = delta[1]
    if delta[2] > (0.5 * dimensions):
        delta[2] = delta[2] - dimensions
    else:
        delta[2] = delta[2]
    return np.sqrt((delta ** 2).sum())

# сам кристалл
a = 1
nsize = 21
lattice_const = 3.52
atoms = BodyCenteredCubic(directions=[[a, 0, 0], [0, a, 0], [0, 0, a]],
                          size=(nsize, nsize, nsize), symbol='Li',
                          latticeconstant=lattice_const)

c = atoms.cell[0][0]

########################

pos = atoms.positions
atoms.append('Li')

del_index = nsize

temp = lattice_const / 6

# диагональ
d = []
m = []
for atm in atoms:
    if np.std(atm.position) <= lattice_const / 100:
        d.append(atm)
        m.append(atm.index)

# print(len(m))
pos_int = temp + d[del_index].position
pos_b_int = d[del_index].position - temp
# print(pos_b_int, ' ', pos_int, ' ', d[del_index - 1], d[del_index + 1])
del (d[del_index])
atoms.positions[-1] = pos_int
atoms.positions[-1] = pos_b_int

d = []
m = []
for atm in reversed(atoms):
    if np.std(atm.position) <= lattice_const / 100:
        d.append(atm)
        m.append(atm.index)

########################

DESTPATH = r"C:\Users\default.DESKTOP-J9F7KTV\OneDrive\PROJ\CODE\\"
lammpsdata.write_lammps_data(DESTPATH + "\\bulk.lammps-data", atoms)
os.chdir(DESTPATH)
os.system("run.bat")
atoms_read = io.read(DESTPATH + "\E_int_1_diag2.lammps-dump-text")
pos_a_read = atoms_read.positions

#########################

arr = []
for i in range(len(m)):
    arr.append(pos_a_read[m[i]])
arr_mod = np.array(list(np.float_(arr)))
pos_sort = sort(arr_mod, tags=arr_mod[:, 2])
p = []
for i in range(1, len(m)):
    p.append(distance(pos_sort[i], pos_sort[i - 1], c))
p.append(distance(pos_sort[len(m) - 1], pos_sort[0], c))
print(p)

# отклонение от идеального значения

dev_p = np.array(p) - lattice_const / 2 * 3**0.5
print(dev_p)

# plot
x = np.arange(0, len(m))
fig, ax = plt.subplots(figsize=(8, 6))
fig.subplots_adjust(bottom=0.15, left=0.2)
pdf = PdfPages('bulks' + str(nsize) + '.pdf')
plt.plot(x, dev_p, 'o-')

ax.set_xlabel('Atom id')
ax.set_ylabel('MD [Å]')
pdf.savefig()
pdf.close()

plt.title(str(nsize) + '×' + str(nsize) + '×' + str(nsize))

plt.show()


import numpy as np
import random
import math
import numbers

class Direct_Encoding_client():
	def __init__(self, e, d):
		# initialization of the protocol's constants
		self.e = e
		self.d = d
		# p and q are fixed, depending on the domain size and the epsilon value
		self.p = math.exp(self.e) / (math.exp(self.e) + self.d - 1)
		self.q = 1 / (math.exp(self.e) + self.d - 1)

	# encoding: simply return the value itself
	def encode(self, v):
		return v

	def perturbe(self, ret):
		x = ret
		# generate a random number in the range (0,1)
		res = random.random()

		# if it is less than p, report the real value
		if (res < self.p):
			pert = x
		else:
			# else chose one of the other values of the domain
			false_xs = [i for i in range(self.d) if i != x]

			pert = random.choice(false_xs)

		return pert

	# randomization consists of perturbing the encoded value
	def randomize(self, v):
		return self.perturbe(self.encode(v))

class Direct_Encoding_aggregator():
	def __init__(self, e, d):
		# initialization of the protocol's constants
		self.e = e
		self.d = d
		# p and q are fixed, depending on the domain size and the epsilon value
		self.p = math.exp(self.e) / (math.exp(self.e) + self.d - 1)
		self.q = 1 / (math.exp(self.e) + self.d - 1)

	def aggregate(self, config):
    		# define the needed variables from the configuration dict provided
		reported_values = config['reported_values']
		e = config['epsilon']
		d = config['d']

		# array to store the results
		results = np.zeros(d)
		n = len(reported_values)

		# compute p and q based on the espilon value and the domain size
		p = math.exp(e) / (math.exp(e) + d - 1)
		q = 1 / (math.exp(e) + d - 1)

		# compute the estimation for each value of the domain
		for i in range(d):
			sum_v = 0
			for j in reported_values:
    			# Support(i) = {i}, thus the protocol supports only the values equal to
				# the current value
				if j == i:
					sum_v += 1
			# normalize the sum by trying to extract the noise
			results[i] = ((sum_v) - n * q) / (p - q)
			# if a negative sum is generated by the normalization, convert it to zero
			if (results[i] < 0):
				results[i] = 0

		return results


#coding:utf8
"""
pip install makedown
pip install pillow
pip install requests

markdown document: https://python-markdown.github.io/
"""
import os,sys
import re
import markdown
from PIL import Image
import io
import base64
import getopt
import requests

def usage():
    print(f"using: python {sys.argv[0]} [options] [arg]")
    print(" -h, --help            : 显示使用帮助")
    print(" -t, --title           : 生成的HTML文档的title")
    print(" -o, --output          : 生成的HTML文档的保存位置")
    print("     --theme           : 生成的HTML文档额外的CSS样式")
    print("     --header          : 在正文之前（标签后边）加入html内容")
    print("     --footer          : 在正文之后（标签前边）加入html内容")
    print("     --image-base64    : 将图片转换成base64")
    print("     --max-index-depth : 将h1...h9标签编排成树状索引显示在页面的右侧，设置索引的最大深度，默认为0（不启用索引）")
    print(" -D foo=bar            : 定义变量，替换markdown文档中的“%foo%”字样")
    print(" arg                   : 要转换的markdown文件")

def download_image(url):
    resp = requests.get(url)
    return io.BytesIO(resp.content)

def image_to_base64(byteio):
    with Image.open(byteio) as image:
        image.verify()

    mime = Image.MIME[image.format]
    b64data = str(base64.b64encode(byteio.getvalue()), encoding='utf-8')

    return f"data:{mime};base64,{b64data}"

SCRIPT_DIR = os.path.split(os.path.realpath(__file__))[0]
OPTIONS = {
    "variables": {},
    "title": "",
    "output": "",
    "theme": f"{SCRIPT_DIR}/theme.css",
    "enable_image_base64": False,
    "header": "",
    "footer": "",
    "max_index_depth": 0,
}

opts, args = getopt.getopt(sys.argv[1:], "hD:t:o:", ["help", "title=", "output=", "theme=", "theme=", "header=", "footer=", "image-base64", "max-index-depth="])
if len(args) == 0:
    usage() ; quit()

for k,v in opts:
    if k == '-D':
        var = v.split("=", 1)
        OPTIONS['variables'][var[0]] = var[1]
    elif k == '--title' or k == '-t':
        OPTIONS['title'] = v
    elif k == '--output' or k == '-o':
        OPTIONS['output'] = v
    elif k == '--theme':
        OPTIONS['theme'] = v
    elif k == '--header':
        OPTIONS['header'] = v
    elif k == '--footer':
        OPTIONS['footer'] = v
    elif k == '--image-base64':
        OPTIONS['enable_image_base64'] = True
    elif k == '--max-index-depth':
        OPTIONS['max_index_depth'] = int(v);
    elif k == '--help' or k == '-h':
        usage() ; quit()

md_file = args[0]
filepath, filename = os.path.split(os.path.abspath(md_file))
name, ext = os.path.splitext(filename)

if OPTIONS['title'] == "":
    OPTIONS['title'] = name

if OPTIONS['output'] == "":
    OPTIONS['output'] = f"{name}.html"

with open(md_file, 'r') as f:
    md_text = f.read()

for varname in OPTIONS['variables']:
    md_text = md_text.replace(f"%{varname}%", OPTIONS['variables'][varname])

with open(OPTIONS['theme'], 'r') as f:
    theme_text = re.sub("\s*[\r\n]\s*", "", f.read())

index_text = None
if OPTIONS['max_index_depth'] > 0:
    with open(f"{SCRIPT_DIR}/index.js", 'r') as f:
        index_text = re.sub("\s*[\r\n]\s*", "", f.read())
        index_text = f""

header_text = ""
if OPTIONS['header']:
    with open(OPTIONS['header'], 'r') as f:
        header_text = f.read()

footer_text = ""
if OPTIONS['footer']:
    with open(OPTIONS['footer'], 'r') as f:
        footer_text = f.read()

# extensions see https://python-markdown.github.io/extensions
html = markdown.markdown(md_text, extensions=['markdown.extensions.tables'])

if OPTIONS['enable_image_base64']:
    for img in re.findall("", html):
        if img[1][:8] == 'https://' or img[1][:7] == 'http://' or img[1][:2] == '//':
            ifile = download_image(img[1])
        else:
            if os.path.isfile(img[1]):
                imgpath = img[1]
            else:
                imgpath = os.path.join(filepath, img[1])

            with open(imgpath, 'rb') as f:
                ifile = io.BytesIO(f.read())

        src = image_to_base64(ifile)

        html = re.sub(re.compile(r"()"), f"\\g<1>{src}\\g<2>", html)

output_text = f"""<!DOCTYPE html>

        <meta charset="UTF-8"/>
        <title>{OPTIONS['title']}</title>

    {index_text if not index_text is None else ""}
    {header_text}
{html}
{footer_text}

"""

with open(OPTIONS['output'], 'w') as f:
    f.write(output_text)


from stream import Stream, StreamArray, run

class feed_streams(object):
    pass

class single_item(object):
    def __init__(self, in_stream, func, **kwargs):
        self.in_stream = in_stream
        self.func = func
        self.kwargs = kwargs
        self.in_stream.subscribe(self.callback)
    def callback(self):
        for v in self.in_stream.recent[self.in_stream.start[self.callback] : self.in_stream.stop]:
            self.func(v, **self.kwargs)
        self.in_stream.start[self.callback] = self.in_stream.stop

class join_synch(object):
    def __init__(self, in_streams, func, **kwargs):
        self.in_streams = in_streams
        self.func = func
        self.kwargs = kwargs
        for in_stream in self.in_streams:
            in_stream.subscribe(self.callback)
    def callback(self):
        slices = [in_stream.recent[in_stream.start[self.callback] : in_stream.stop]
                  for in_stream in self.in_streams]
        zipped_slices = list(zip(*slices))
        for v in zipped_slices: self.func(v, **self.kwargs)
        for in_stream in self.in_streams:
            in_stream.start[self.callback] += len(zipped_slices)

class join_asynch(object):
    def __init__(self, in_streams, func, **kwargs):
        self.in_streams = in_streams
        self.func = func
        self.kwargs = kwargs
        for in_stream in self.in_streams:
            in_stream.subscribe(self.callback)
    def callback(self):
        for in_stream in self.in_streams:
            for v in in_stream.recent[in_stream.start[self.callback]: in_stream.stop]:
                self.func(v, **self.kwargs)
            in_stream.start[self.callback] = in_stream.stop

class join_timed(object):
    def __init__(self, in_streams, get_time, func, **kwargs):
        self.in_streams = in_streams
        self.get_time = get_time
        self.func = func
        self.kwargs = kwargs
        for in_stream in self.in_streams:
            in_stream.subscribe(self.callback)
    def callback(self):
        while all([in_stream.start[self.callback] < in_stream.stop
                   for in_stream in self.in_streams]):
            items = [in_stream.recent[in_stream.start[self.callback]]
                     for in_stream in self.in_streams]
            times = [self.get_time(item) for item in items]
            min_time = min(times)
            operand = [None for in_stream in self.in_streams]
            for i, in_stream in enumerate(self.in_streams):
                if times[i] == min_time:
                    operand[i] = items[i]
                    in_stream.start[self.callback] += 1
            self.func((min_time, operand), **self.kwargs)

class sliding_window(object):
    def __init__(self, in_stream, window_size, step_size, func, **kwargs):
        self.in_stream = in_stream
        self.window_size = window_size
        self.step_size = step_size
        self.func = func
        self.kwargs = kwargs
        self.in_stream.subscribe(self.callback)
    def callback(self):
        while self.in_stream.start[self.callback] + self.window_size <= self.in_stream.stop:
            start = self.in_stream.start[self.callback]
            window = self.in_stream.recent[start : start + self.window_size]
            self.func(window, **self.kwargs)
            self.in_stream.start[self.callback] += self.step_size

class subtract_mean_from_stream(object):
    def __init__(self, in_stream, window_size, func, **kwargs):
        self.in_stream = in_stream
        self.window_size = window_size
        self.func = func
        self.kwargs = kwargs
        self.in_stream.subscribe(self.callback)
    def callback(self):
        while self.in_stream.start[self.callback] + self.window_size <= self.in_stream.stop:
            start = self.in_stream.start[self.callback]
            window = self.in_stream.recent[start : start + self.window_size]
            item = window[-1] - np.mean(window)
            self.func(item, **self.kwargs)
            self.in_stream.start[self.callback] += 1

class subtract_mean_from_StreamArray(object):
    def __init__(self, in_stream, window_size, func, **kwargs):
        self.in_stream = in_stream
        self.window_size = window_size
        self.func = func
        self.kwargs = kwargs
        self.in_stream.subscribe(self.callback)
    def callback(self):
        while self.in_stream.start[self.callback] + self.window_size <= self.in_stream.stop:
            start = self.in_stream.start[self.callback]
            window = self.in_stream.recent[start : start + self.window_size]
            item = window[-1] - np.mean(window, axis=0)
            self.func(item, **self.kwargs)
            self.in_stream.start[self.callback] += 1

import numpy as np
class detect_anomaly(object):
    def __init__(self, in_stream, window_size, anomaly_size,
                     anomaly_factor, cloud_data_size,
                     cloud_func, **kwargs):
        self.in_stream = in_stream
        self.W = window_size
        self.A = anomaly_size
        self.F = anomaly_factor
        self.C = cloud_data_size
        self.cloud_func = cloud_func
        self.kwargs = kwargs
        self.in_stream.subscribe(self.callback)
        self.anomaly = False
    def callback(self):
        start = self.in_stream.start[self.callback]
        stop = self.in_stream.stop
        R = self.in_stream.recent

        while ((self.anomaly and stop - start >= self.C) or
                   (not self.anomaly and start + self.W <= stop)):
            if self.anomaly:
                data_to_cloud = R[start : start+self.C]
                self.anomaly = False
                start += self.C
                self.in_stream.set_start(self.callback, start)
                self.cloud_func(data_to_cloud, **self.kwargs)
            else:
                window = R[start : start + self.W]
                window_mean = np.mean(window[:self.A])
                window_std = np.std(window[:self.A])
                anomaly_mean = np.mean(window[-self.A: ])
                # if (np.abs(np.mean(window[-self.A: ])) >
                #         np.abs(np.mean(window)*self.F)):
                if (np.abs(anomaly_mean - window_mean) > self.F * window_std):
                    # print(window[-self.A: ].shape)
                    # print(window.shape)

                    # print(window[-self.A: ])
                    # print(window)

                    # print(window_mean, window_std, anomaly_mean, np.abs(anomaly_mean - window_mean))
                    # print()
                    self.anomaly = True
                    # print(window)
                    print('window_mean:', window_mean)
                    print('window_std: ', window_std)
                    print('anomaly_mean: ', anomaly_mean)
                    start += self.W - self.A
                    self.in_stream.set_start(self.callback, start)
                else:
                    start += 1
                    self.in_stream.set_start(self.callback, start)

def append_item_to_stream(v, out_stream):
    out_stream.append(v)

def append_item_to_StreamArray(v, out_stream):
    out_stream.append(np.stack(v, axis=0))

#------------------------------------------------------------------------
# Tests
#------------------------------------------------------------------------
def example_single_item():

    x, y, z = Stream(name='x'), Stream(name='y'), Stream(name='z')

    # Examples of functions passed to map_stream objects below
    # Function f has two parameters: an item v of the input stream
    # and a keyword parameter, out_stream, which is the output stream.
    def f(v, out_stream):
        out_stream.append(2*v)

    # This function has three parameters: the first is an item v of the
    # input stream, the second is a keyword argument (see kwargs) out_stream,
    # which is the output stream, and another keyword argument, multiplier.
    def h(v, out_stream, multiplier):
        out_stream.append(multiplier*v)

    # Set up the agent with input stream x and output stream y.
    single_item(in_stream=x, out_stream=y, func=f)
    # Set up the agent with input stream x and output stream z.
    single_item(in_stream=x, out_stream=z, func=h, multiplier=3)

    # Put values into input stream x.
    x.extend([0, 1, 2])
    # Run agents until all streams have been processed.
    run()
    # Functions f and h are called when stream x is extended.

    # Print the streams.
    x.print_recent()
    y.print_recent()
    z.print_recent()
    # y[i] = 2*x[i]), and z[i] = multiplier*x[i])

    # Put more values into input stream x.
    x.extend([3, 4])
    run()
    # Functions f and h are called when stream x is extended.

    # Print the output streams.
    x.print_recent()
    y.print_recent()
    z.print_recent()

def example_join_synch():

    def f(v, out_stream, multiplier):
        out_stream.append(sum(v) * multiplier)
        return

    x, y, z = Stream(name='x'), Stream(name='y'), Stream(name='z')
    join_synch(in_streams=[x, y], out_stream=z, func=f, multiplier=3)

    x.extend([0, 1, 2])
    y.extend([10, 11, 12, 13])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

    x.extend([3, 4, 5, 6])
    y.extend([10, 11, 12, 13, 14])
    run()
    x.print_recent()
    y.print_recent()

def example_join_asynch():
    def f(v, out_stream, multiplier):
        out_stream.append(v * multiplier)
        return

    x, y, z = Stream(name='x'), Stream(name='y'), Stream(name='z')
    join_asynch(in_streams=[x, y], out_stream=z, func=f, multiplier=3)

    x.extend([0, 1])
    y.extend([10])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

    x.extend([2])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

    y.extend([11])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

def example_join_timed():
    def get_time(v):
        return v[0]

    def f(v, out_stream):
        out_stream.append(v)
        return

    x, y, z = Stream(name='x'), Stream(name='y'), Stream(name='z')
    join_timed(in_streams=[x, y], func=f, get_time=get_time, out_stream=z)

    x.extend([(1, 0), (10, 1)])
    y.extend([(2, 'A'), (4, 'B')])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

    y.extend([(12, 'C')])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

    x.extend([(12, 2)])
    run()
    x.print_recent()
    y.print_recent()
    z.print_recent()

def example_sliding_window():
    def g(window, out_stream):
        out_stream.append(sum(window))
        return

    x, y = Stream(name='x'), Stream(name='y')
    sliding_window(in_stream=x, window_size=3, step_size=2, func=g, out_stream=y)

    x.extend(list(range(10)))
    run()
    x.print_recent()
    y.print_recent()

# EXAMPLE OF subtract_mean_from_StreamArray

def append_item_to_StreamArray(v, out_stream):
    out_stream.append(np.stack(v, axis=0))

from stream import StreamArray

def example_subtract_mean_from_StreamArray():
    xx = StreamArray(name='xx', dtype=float, dimension=2)
    yy = StreamArray(name='yy', dtype=float, dimension=2)
    subtract_mean_from_StreamArray(
        in_stream=xx, out_stream=yy, window_size=3,
        func=append_item_to_StreamArray)

    xx.extend([np.array([0., 1.]), np.array([2., 1.]),
               np.array([1., 1.]), np.array([2., 2.]),
               np.array([2., 1.])])
    run()
    xx.print_recent()
    yy.print_recent()

# EXAMPLE ILLUSTRATING example_subtract_mean_from_stream

import numpy as np

def append_item_to_stream(v, out_stream): out_stream.append(v)

def example_subtract_mean_from_stream():
    x, y = Stream(name='x'), Stream(name='y')

    subtract_mean_from_stream(
        in_stream=x, window_size=3,
        func=append_item_to_stream, out_stream=y)

    x.extend([0, 1, 2, 1, 1, 1, 2, 2, 2, 1])
    run()
    x.print_recent()
    y.print_recent()

def example_detect_anomaly():
    def cloud_func(window, ):
        print ('window ', window)
        return

    x = Stream(name='x')
    detect_anomaly(in_stream=x, window_size=4, anomaly_size=2,
                       anomaly_factor=1.1, cloud_data_size=2,
                       cloud_func=cloud_func)
    x.extend([1, 1, 2, 2, 3, 4, 7, 6, 11, 0, 3, 5, 5, 11, 11, 19, 19, 31])
    run()
    x.print_recent()

def example_detect_anomaly_with_StreamArray():
    def cloud_func(window, ):
        print ('window ', window)
        return

    x = StreamArray(name='x', dtype='float', dimension=2)

    detect_anomaly(in_stream=x, window_size=3, anomaly_size=1,
                       anomaly_factor=1.1, cloud_data_size=2,
                       cloud_func=cloud_func)

    x.extend([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [7.0, 6.0], [11.0, 6.0], [2.0, 1.0], [18.0, 16.0], [2.0, 4.0]])
    run()

    x.extend([[0.0, 1.0], [8.0, 9.0], [12.0, 15.0], [1.0, 2.0], [21.0,
    31.0], [0.0, 0.0], [0., 0.]])
    run()

if __name__ =='__main__':
    print('example_single_item')
    example_single_item()
    print('')
    print('example_join_synch')
    example_join_synch()
    print('')
    print('example_join_asynch')
    example_join_asynch()
    print('')
    print('example_join_timed')
    example_join_timed()
    print('')
    print('example_sliding_window')
    example_sliding_window()
    print('')
    print('example_subtract_mean_from_stream')
    example_subtract_mean_from_stream()
    print('')
    print('example_subtract_mean_from_StreamArray')
    example_subtract_mean_from_StreamArray()
    print('')
    print('example_detect_anomaly')
    example_detect_anomaly()
    print('')
    print('example_detect_anomaly_with_StreamArray')
    example_detect_anomaly_with_StreamArray()


"""Provides wrappers for creating and training networks."""
import logging
from typing import List

import accuracy as acc
import custom_models as cm
import joblib
from numpy import array
from tensorflow.keras import Model
from tensorflow.keras import backend as K
from tensorflow.keras.applications import EfficientNetB3, MobileNetV2
from tensorflow.keras.callbacks import (
    EarlyStopping,
    ModelCheckpoint,
    ReduceLROnPlateau,
    TensorBoard,
)
from tensorflow.keras.layers import (
    Activation,
    BatchNormalization,
    Dense,
    Dropout,
    GlobalAveragePooling2D,
    Input,
    concatenate,
)
from tensorflow.keras.losses import CategoricalCrossentropy, Huber
from tensorflow.keras.metrics import CategoricalAccuracy, MeanAbsoluteError, MeanSquaredError
from tensorflow.keras.optimizers import Adam

def create_ann(
    input_shape, dropout_rate=0.1, hidden_layers=0, hidden_layer_nodes: int = 10, bn_or_do="do"
):
    """Create the ANN part for the fusion network.

    Parameters
    ----------
    input_shape: tuple
        Shape of expected input data. To be inferred from input_data.shape

    Returns
    -------
    model: tf.keras.Model
        ANN model
    """
    if isinstance(hidden_layer_nodes, int):
        hidden_layer_nodes = [hidden_layer_nodes] * hidden_layers

    input = Input(shape=input_shape, name="input_osm")

    for i, n_nodes in enumerate(hidden_layer_nodes):
        if "x" not in locals():
            x = Dense(n_nodes, kernel_initializer="he_normal", name=f"ann_dense_{i}")(input)
        else:
            x = Dense(n_nodes, kernel_initializer="he_normal", name=f"ann_dense_{i}")(x)
        if bn_or_do == "do":
            x = Dropout(rate=dropout_rate, name=f"ann_dropout_{i}")(x)
        elif bn_or_do == "bn":
            x = BatchNormalization(name=f"ann_batchnorm_{i}")(x)
        x = Activation("relu", name=f"ann_relu_{i}")(x)

    # Create model
    model = Model(input, x, name="ANN")

    return model

def create_cnn(
    input_shape,
    cnn_model,
    skip_connections: bool = True,
    dropout_rate=0.1,
    cnn_dense_layers: List[int] = [10, 10, 10],
):
    """Create the CNN part for the fusion network.

    Parameters
    ----------
    input_shape : tuple[int]
        Shape of the expected input data. To be inferred from input_data.shape
    cnn_model : str
        Selection of network architecture.
    skip_connections : bool
        Should the model use skip connections. Default True

    Returns
    -------
    model: tf.keras.Model
    """
    # Create CNN
    cnn_input = Input(shape=input_shape, name="input_img")

    if cnn_model == "mobilenet":
        base_model = MobileNetV2(
            weights=None,
            input_shape=input_shape,
            include_top=False,
        )

    elif cnn_model == "efficientnetb3":
        base_model = EfficientNetB3(
            weights=None,
            input_shape=input_shape,
            include_top=False,
        )

    elif cnn_model == "greennet":
        base_model = cm.greennet_v2(
            input_shape=input_shape,
            skip_connections=skip_connections,
            dropout_rate=dropout_rate,
            cnn_dense_layers=cnn_dense_layers,
        )
    else:
        raise NotImplementedError

    cnn_output = base_model(cnn_input, training=True)
    return Model(cnn_input, cnn_output, name="CNN")

def create_fusion_net(
    dataset_train,
    mode="fusion",
    cnn_model="mobilenet",
    classification=False,
    dropout_rate=0.1,
    fusion_cnn_neurons=500,
    fusion_ann_neurons=500,
    cnn_dense_layers: List[int] = [10, 10, 10],
    ann_hidden_layers=0,
    ann_hidden_layer_nodes=10,
    ann_bn_or_do="do",
    fusion_hidden_layers=0,
    fusion_hidden_layer_nodes=10,
    greennet_skipconn=False,
    verbose=True,
):
    """Create the fusion network by combining a ANN and a CNN.

    Parameters
    ----------
    image_data: list(numpy.ndarray (n, m, d))
        List of image tiles. Used to determine the shape of input data for the CNN.
    osm_attributes: list(numpy.ndarray (n,))
        List of osm attribute data. Used to determine the shape of input data for the ANN.
    labels: list(numpy.ndarray (n,))
        List of osm attribute data. Used to determine the shape of input data for the output layer
        in the fusion network.

    Returns
    -------
    model: tf.keras.Model
        structure of the model
    """
    num_of_labels = dataset_train.element_spec[1]

    # ANN part for osm data
    if mode in ["fusion", "ann"]:
        num_of_attributes = dataset_train.element_spec[0]["input_osm"]
        ann = create_ann(
            num_of_attributes, dropout_rate, ann_hidden_layers, ann_hidden_layer_nodes, ann_bn_or_do
        )

    if mode in ["fusion", "cnn"]:
        img_shape = tuple(dataset_train.element_spec[0]["input_img"])
        cnn = create_cnn(
            img_shape,
            cnn_model,
            skip_connections=greennet_skipconn,
            dropout_rate=dropout_rate,
            cnn_dense_layers=cnn_dense_layers,
        )

    if mode == "fusion":
        # create the input to our final set of layers as the output of both MLP and CNN
        x_cnn = cnn.output
        if not cnn_model == "greennet":
            x_cnn = GlobalAveragePooling2D()(x_cnn)

        x_cnn = Dense(
            fusion_cnn_neurons,
            kernel_initializer="he_normal",
            activation="relu",
            name="FinalCNNLayer",
        )(x_cnn)

        x_ann = ann.output
        x_ann = Dense(
            fusion_ann_neurons,
            kernel_initializer="he_normal",
            activation="relu",
            name="FinalANNLayer",
        )(x_ann)

        x = concatenate([x_ann, x_cnn])

        if isinstance(fusion_hidden_layer_nodes, int):
            fusion_hidden_layer_nodes = [fusion_hidden_layer_nodes] * fusion_hidden_layers

        for i, n in enumerate(fusion_hidden_layer_nodes):
            x = Dense(n, name=f"fusion_dense_{i}")(x)
            if ann_bn_or_do == "do":
                x = Dropout(dropout_rate, name=f"fusion_dropout_{i}")(x)
            elif ann_bn_or_do == "bn":
                x = BatchNormalization(name=f"fusion_batchnorm_{i}")(x)
            x = Activation("relu", name=f"fusion_relu_{i}")(x)

        if classification:
            x = Dense(num_of_labels, activation="softmax", name="fusion_FusionClassificationLayer")(
                x
            )
        else:
            if ann_bn_or_do == "bn":
                # if batchnormalization is chosen, set one dropout in the right before the last
                # layer
                x = Dropout(dropout_rate, name=f"final_big_dropout_dropout")(x)
            x = Dense(num_of_labels, activation="linear", name="fusion_FusionRegressionLayer")(x)

        model = Model(inputs=[cnn.input, ann.input], outputs=x)

    elif mode == "ann":
        x = ann.output
        if ann_bn_or_do == "bn":
            # if batchnormalization is chosen, set one dropout in the right before the last
            # layer
            x = Dropout(dropout_rate, name=f"final_big_dropout_dropout")(x)
        if classification:
            x = Dense(num_of_labels, activation="softmax", name="ANNClassificationLayer")(x)
        else:
            x = Dense(num_of_labels, activation="linear", name="ANNRegressionLayer")(x)
        model = Model(inputs=ann.input, outputs=x)
    elif mode == "cnn":
        x = cnn.output
        if ann_bn_or_do == "bn":
            # if batchnormalization is chosen, set one dropout in the right before the last
            # layer
            x = Dropout(dropout_rate, name=f"final_big_dropout_dropout")(x)
        if not cnn_model == "greennet":
            x = GlobalAveragePooling2D()(x)
        if classification:
            x = Dense(num_of_labels, activation="softmax", name="CNNClassificationLayer")(x)
        else:
            x = Dense(num_of_labels, activation="linear", name="CNNRegressionLayer")(x)
        model = Model(inputs=cnn.input, outputs=x)
    else:
        raise NotImplementedError

    if verbose:
        model.summary()

    return model

def r2(y_true, y_pred):
    """Calculate R suqred from two Keras Tensors."""
    SS_res = K.sum(K.square(y_true - y_pred))
    SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
    return 1 - SS_res / (SS_tot + K.epsilon())

def compile_network(
    model, initial_learning_rate=1e-3, classification=False, n_classes=None
) -> Model:
    """Comile the network, adding loss, evaluation metrics and optimizer.

    Parameters
    ----------
    model: tf.keras.Model
        Model to be compiled

    Returns
    -------
    model: compiled tf.keras.Model
        A compiled version of the model with added loss, evaluation metrics and optimizer
    """
    # select, loss, and metrics
    if classification:
        loss_object = CategoricalCrossentropy(name="CCE")
        metrics = [
            CategoricalAccuracy(name="OA"),
        ]
    else:
        loss_object = Huber(name="huber")

        metrics = [
            MeanAbsoluteError(name="MAE", dtype=None),
            MeanSquaredError(name="MSqE", dtype=None),
            r2,
        ]

    optimizer = Adam(learning_rate=initial_learning_rate)

    logging.info(f"weights: {len(model.weights)}")
    logging.info(f"trainable_weights: {len(model.trainable_weights)}")

    model.compile(optimizer, loss_object, metrics)

    return model

def train_network(
    compiled_model,
    model_path,
    dataset_train,
    dataset_val,
    epochs,
    warmup_epochs,
    warmup_lr,
    reduce_on_plateau_patience,
    early_stopping_patience,
    log_dir,
):
    """Train the model using tf.datasets.

    Parameters
    ----------
    compiled_model: tf.Model
        Model compiled by compile_network()

    model_path: str
        file path to which the model will be saved.

    dataset_train: tf.Dataset
        A tf.Dataset containing training data and labels

    dataset_val: tf.Dataset
        A tf.Dataset containing validation data and labels

    epochs: int
        Number of epochs to train for

    Returns
    -------
    history: tf.Model.history
         Model history of the trained model
    """
    callbacks = [
        # saves new weights only when the model is better than before
        ModelCheckpoint(
            model_path,
            monitor="val_loss",
            save_weights_only=True,
            save_best_only=True,
            mode="min",
            verbose=1,
        ),
        # Start with high learning rate and reduce when val-loss not better -> yields higher gain in
        # the beginning
        ReduceLROnPlateau(
            monitor="val_loss",
            factor=0.2,
            patience=reduce_on_plateau_patience,
            min_lr=0.0000001,
            verbose=1,
        ),
        # method to avoid overfitting. when val_loss does not get better after 5 epochs, saves time
        # when training for higher numbers of epochs
        EarlyStopping(
            monitor="val_loss",
            min_delta=0,
            patience=early_stopping_patience,
            verbose=1,
            mode="auto",
            baseline=None,
            restore_best_weights=True,
        ),
    ]

    # add tensorboard callback if required
    if log_dir:
        callbacks += [
            TensorBoard(
                log_dir=log_dir,
                histogram_freq=1,
            )
        ]

    if warmup_epochs > 0:
        # change learning rate
        prior_lr = float(compiled_model.optimizer.lr)
        K.set_value(compiled_model.optimizer.lr, warmup_lr)
        wu_cb = [
            ModelCheckpoint(model_path, save_weights_only=True, save_best_only=False, verbose=1)
        ]
        if log_dir:
            wu_cb += [
                TensorBoard(
                    log_dir=log_dir,
                    histogram_freq=1,
                )
            ]

        compiled_model.fit(
            dataset_train,
            epochs=warmup_epochs,
            callbacks=wu_cb,
        )
        # revert learning rate to compiled state
        K.set_value(compiled_model.optimizer.lr, prior_lr)

    history = compiled_model.fit(
        dataset_train,
        epochs=epochs,
        callbacks=callbacks,
        validation_data=dataset_val,
        initial_epoch=warmup_epochs,
        use_multiprocessing=True,
    )

    logging.info(f"Model trained with parameters: {history.params}")
    logging.info(f"Model Trained for {len(history.history['loss'])} epochs.")

    return history

def infer_network(compiled_model, model_path, dataset, predictions_name):
    """Infer the compiled network on new data.

    Parameters
    ----------
    compiled_model: tf.Model
        model compiled by compile_network() and trained by train_network()

    dataset: tf.Dataset
        Dataset for which the inference should take place.

    predictions_name: str
        name of prediction set serving as identifier in the dataset's prediction dict

    Returns
    -------
    None
    """
    compiled_model.load_weights(model_path)
    pred = compiled_model.predict(dataset, use_multiprocessing=True)
    dataset.add_predictions(pred, predictions_name)

    return None

def train_and_evaluate_sklearn(
    learner, dataset_train, datasets_test_dict, experiment_title, experiment_path, classification
):
    """Train and evaluate a sklearn regressor.

    Parameters
    ----------
    dataset_train: tf.Dataset
        Dataset for which the inference should take place.

    datasets_test_dict: dict[tf.Dataset, tf.Dataset]
        Dictionary of different datasets the RandomForestRegressor should be evaluated against.
        Intended to supply, e.g., balanced and imbalanced datasets

    learner:
        sklearn model providing both a learner.fit and learner.predict method.

    predictions_name: str
        name of prediction set serving as identifier in the dataset's prediction dict

    Returns
    -------
    None
    """
    osm_train = array(list(x[0]["input_osm"] for x in dataset_train.as_numpy_iterator()))
    labels_train = array(list(x[1] for x in dataset_train.as_numpy_iterator()))

    learner.fit(osm_train, labels_train)

    joblib.dump(learner, f"{experiment_path}/model_{experiment_title}.pkl")

    for bal_imbal, dataset in datasets_test_dict.items():
        # extract osm information from dataset
        osm_test = array(list(x[0]["input_osm"] for x in dataset.as_numpy_iterator()))

        # extract labels from dataset
        y_labels = array(list(x[1] for x in dataset.as_numpy_iterator()))

        # extract urban_proximity from dataset
        y_urb_prox = array([i for i in dataset.get_urb_iterator()])

        # predict the model
        y_pred = learner.predict(osm_test)

        if classification:
            # overall accuracies
            acc_df = acc.df_accuracies(y_labels, y_pred)
            acc_df[0].to_csv(f"{experiment_path}/conf_mat_{experiment_title}_{bal_imbal}.csv")
            acc_df[1].to_csv(f"{experiment_path}/class_acc_{experiment_title}_{bal_imbal}.csv")
            acc_df[1].to_csv(f"{experiment_path}/class_acc_{experiment_title}_{bal_imbal}.csv")

            # assess accuracies
            acc_df_urb_gradient = acc.df_accuracies_urban_gradient(
                y_labels, y_pred, y_urb_prox, classification
            )

            acc_df_urb_gradient[0].to_csv(
                f"{experiment_path}/conf_mat_urb_gradient_{experiment_title}_{bal_imbal}.csv"
            )
            acc_df_urb_gradient[1].to_csv(
                f"{experiment_path}/class_acc_urb_gradient_{experiment_title}_{bal_imbal}.csv"
            )

            logging.info(f"\nConfusion Matrix {experiment_title} {bal_imbal}:\n{acc_df[0]}\n")
            logging.info(f"\nAccuracy metrics {experiment_title} {bal_imbal}:\n{acc_df[1]}\n")

            # plot accuracies
            acc.plot_classification(y_labels, y_pred, experiment_path, experiment_title, bal_imbal)
            acc.plot_class_based_accuracies(acc_df[1], experiment_path, experiment_title, bal_imbal)
            acc.plot_classification_urban_gradient(
                acc_df_urb_gradient[1], experiment_path, experiment_title, bal_imbal
            )

            # add predictions to dataset
            dataset.add_predictions(y_pred, experiment_title)
        else:
            acc_df = acc.df_accuracies_green_gradient(y_labels, y_pred, classification)
            # write the accuracies to file
            acc_df.to_csv(f"{experiment_path}/acc_{experiment_title}_{bal_imbal}.csv")

            # log aggregate information
            acc.log_aggregate_accuracies(acc_df, experiment_title, bal_imbal)

            # plot accuracies
            acc.plot_regression(acc_df, experiment_path, experiment_title, bal_imbal)

            # add predictions to dataset
            dataset.add_predictions(y_pred, experiment_title)


# # standard library
import warnings

# django
from django.conf.urls import url
from django.contrib import admin
from django.http import HttpResponseBadRequest
from django.http import HttpResponseForbidden
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.clickjacking import xframe_options_sameorigin

# cms
from aldryn_newsblog.admin import ArticleAdmin as BaseArticleAdmin
from aldryn_newsblog.admin import ArticleAdminForm as BaseArticleAdminForm
from aldryn_newsblog.models import Article
from aldryn_people.models import Group
from aldryn_people.models import Person
from cms import operations
from cms.admin.forms import PluginAddValidationForm
from cms.admin.placeholderadmin import _instance_overrides_method
from cms.admin.placeholderadmin import PlaceholderAdminMixin
from cms.models.pluginmodel import CMSPlugin
from cms.plugin_pool import plugin_pool

# models
from .models import ArticleCount
from .models import HeaderImage

# Register your models here.
admin.site.unregister(Article)
admin.site.unregister(Group)
admin.site.unregister(Person)

class ArticleAdminForm(BaseArticleAdminForm):
    """
    Override ArticleAdminForm to remove related field
    """
    class Meta:
        model = Article
        fields = [
            'app_config',
            'categories',
            'featured_image',
            'is_featured',
            'is_published',
            'lead_in',
            'meta_description',
            'meta_keywords',
            'meta_title',
            'owner',
            'slug',
            'tags',
            'title',
        ]

class ArticleAdmin(BaseArticleAdmin):
    """
    Override ArticleAdmin to remove related field
    Also call prefetch related on the translations
    """
    form = ArticleAdminForm
    list_display = ('title', 'publishing_date', 'is_featured', 'is_published')
    search_fields = (
        'translations__title',
    )

    fieldsets = (
        (None, {
            'fields': (
                'title',
                'author',
                'publishing_date',
                'is_published',
                'is_featured',
                'featured_image',
                'lead_in',
            )
        }),
        (_('Meta Options'), {
            'classes': ('collapse',),
            'fields': (
                'slug',
                'meta_title',
                'meta_description',
                'meta_keywords',
            )
        }),
        (_('Advanced Settings'), {
            'classes': ('collapse',),
            'fields': (
                'tags',
                'categories',
                'owner',
                'app_config',
            )
        }),
    )

    def get_queryset(self, request):
        return super().get_queryset(request).prefetch_related('translations')

    @xframe_options_sameorigin
    def add_plugin_after(self, request):
        """
        Shows the add plugin form and saves it on POST.

        Requires the following GET parameters:
            - cms_path
            - placeholder_id
            - plugin_type
            - plugin_language
            - plugin_parent (optional)
            - plugin_position (optional)
        """
        form = PluginAddValidationForm(request.GET)

        if not form.is_valid():
            # list() is necessary for python 3 compatibility.
            # errors is s dict mapping fields to a list of errors
            # for that field.
            error = list(form.errors.values())[0][0]
            return HttpResponseBadRequest(force_text(error))

        plugin_data = form.cleaned_data
        placeholder = plugin_data['placeholder_id']
        plugin_type = plugin_data['plugin_type']

        if not self.has_add_plugin_permission(
            request, placeholder, plugin_type
        ):
            message = force_text(
                _('You do not have permission to add a plugin')
            )
            return HttpResponseForbidden(message)

        parent = plugin_data.get('plugin_parent')

        if parent:
            position = parent.cmsplugin_set.count()
        else:
            position = CMSPlugin.objects.filter(
                parent__isnull=True,
                language=plugin_data['plugin_language'],
                placeholder=placeholder,
            ).count()

        plugin_data['position'] = position

        plugin_class = plugin_pool.get_plugin(plugin_type)
        plugin_instance = plugin_class(plugin_class.model, self.admin_site)

        # Setting attributes on the form class is perfectly fine.
        # The form class is created by modelform factory every time
        # this get_form() method is called.
        plugin_instance._cms_initial_attributes = {
            'language': plugin_data['plugin_language'],
            'placeholder': plugin_data['placeholder_id'],
            'parent': plugin_data.get('plugin_parent', None),
            'plugin_type': plugin_data['plugin_type'],
            'position': plugin_data['position'],
        }

        response = plugin_instance.add_view(request)

        plugin = getattr(plugin_instance, 'saved_object', None)

        uses_hook = _instance_overrides_method(
            PlaceholderAdminMixin,
            self,
            'post_add_plugin'
        )

        if plugin_instance.object_successfully_changed and uses_hook:
            warnings.warn('The post_add_plugin hook has been deprecated. '
                          'Please use placeholder operation signals instead.',
                          DeprecationWarning)
            self.post_add_plugin(request, plugin)

        if plugin_instance._operation_token:
            tree_order = placeholder.get_plugin_tree_order(plugin.parent_id)
            self._send_post_placeholder_operation(
                request,
                operation=operations.ADD_PLUGIN,
                token=plugin_instance._operation_token,
                plugin=plugin,
                placeholder=plugin.placeholder,
                tree_order=tree_order,
            )
        return response

    def get_urls(self):
        """
        Register the plugin specific urls (add/edit/copy/remove/move)
        """
        info = "%s_%s" % (
            self.model._meta.app_label,
            self.model._meta.model_name
        )

        def pat(regex, fn):
            return url(
                regex,
                self.admin_site.admin_view(fn),
                name='%s_%s' % (info, fn.__name__)
            )

        url_patterns = [
            pat(r'add-plugin/after/$', self.add_plugin_after),
        ]
        return url_patterns + super(PlaceholderAdminMixin, self).get_urls()

admin.site.register(Article, ArticleAdmin)

@admin.register(ArticleCount)
class ArticleCountAdmin(admin.ModelAdmin):
    list_display = (
        'article',
        'visits',
    )
    readonly_fields = (
        'article',
        'visits',
    )

    def has_add_permission(self, request):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

@admin.register(HeaderImage)
class HeaderImage(admin.ModelAdmin):
    list_display = (
        'name',
        'is_active',
    )
    list_filter = (
        'is_active',
    )


#!/usr/bin/env python3

"""Log Analysis Project"""

import psycopg2

"""Code"""
"""***********************************"""

"""Connects to the news database and returns a database cursor."""

def connect_to_database():
    try:
        database = psycopg2.connect("dbname=news")
        dbCursor = database.cursor()
        return dbCursor
    except DatabaseError:
        print("Unable to connect to the database")

"""Query and print: Most popular articles."""

def popularArticles(dbCursor):
    query = """
            select articles.title, count(*)from  log, articles
            where  log.path = '/article/' || articles.slug
            group by articles.title
            order by count(*) desc
            limit 3;
    """
    dbCursor.execute(query)
    x = dbCursor.fetchall()

    print()
    print()
    print('3 Most Popular Articles')
    print('**************************************************')

    for result in x:
        print('"{articleName}" - {viewCount} views'.format
              (articleName=result[0], viewCount=result[1]))

    print()

    return

"""Query and print: most popular authors."""

def popularAuthors(dbCursor):
    query = """
            select authors.name, count(*) from   log, articles, authors
            where  log.path = '/article/' || articles.slug
            and articles.author = authors.id
            group by authors.name
            order by count(*) desc;
    """
    dbCursor.execute(query)
    y = dbCursor.fetchall()

    print()
    print()
    print('Most Popular Authors')
    print('**************************************************')

    for result in y:
        print('{authorName} - {viewCount} views'.format
              (authorName=result[0], viewCount=result[1]))

    print()

    return

"""Query and print:Days error rate is greater than 1%."""

def daysGreaterThan1pError(dbCursor):
    query = """
            create view requests
            as select time::date as date, count(*) from log
            group by time::date
            order by time::date;

            create view errors
            as select time::date as date, count(*) from log
            where status != '200 OK'
            group by time::date
            order by time::date;

            create view requests_errors as
            select requests.date,
            errors.count::float / requests.count::float * 100
            as errorrate from requests, errors
            where requests.date = errors.date;

            select * from requests_errors where errorrate > 1;

    """
    dbCursor.execute(query)
    z = dbCursor.fetchall()

    print()
    print()
    print('Dates Error Rate is Greater than 1%')
    print('**************************************************')

    for result in z:
        print('{date:} - {errorRate:}% errors'.format
              (date=result[0], errorRate=result[1]))

    print()
    print()

    return

"""Call functions from code"""
"""***********************************"""

if __name__ == "__main__":
    dbCursor = connect_to_database()
    if dbCursor:
        popularArticles(dbCursor)
        popularAuthors(dbCursor)
        daysGreaterThan1pError(dbCursor)
        dbCursor.close()


# -*- coding: utf8 -*-
__author__ = "Hoseong Son <me@sookcha.com>"

import datetime
import xml.etree.ElementTree as ElementTree
from dateutil import parser
from icalendar import Calendar, Event
import requests

class Convert():
    def __init__(self, filename):
        self.filename = filename

    def get_subjects(self):
        result = []
        try:
            tree = ElementTree.parse(self.filename)
            root = tree.getroot()
        except:
            tree = ElementTree.fromstring(self.filename)
            root = tree

        for subject in root.iter('subject'):
            name = subject.find("name").get("value")
            single_subject = {}

            single_subject["name"] = name
            single_subject["professor"] = subject.find("professor").get("value")
            single_subject["info"] = list(map(
                lambda x: {
                    "day": x.get("day"),
                    "place" : x.get("place"),
                    "startAt": '{:02d}:{:02d}'.format(*divmod(int(x.get("starttime")) * 5, 60)),
                    "endAt": '{:02d}:{:02d}'.format(*divmod(int(x.get("endtime")) * 5, 60))
                }, subject.find("time").findall("data")
                )
            )
            result.append(single_subject)

        return result

    def get_calendar(self, timetable, start_date, end_date, hide_details=False):
        cal = Calendar()

        for item in timetable:
            for time in item["info"]:
                event = Event()
                event.add('summary', item["name"] if not hide_details else "수업")
                event.add('dtstart', parser.parse("%s %s" % (self.get_nearest_date(start_date, time["day"]), time["startAt"])))
                event.add('dtend', parser.parse("%s %s" % (self.get_nearest_date(start_date, time["day"]), time["endAt"])))
                event.add('rrule', {'freq': 'WEEKLY', 'until': parser.parse(end_date)})
                if time["place"] != "" and not hide_details:
                    event.add('location', time["place"])
                cal.add_component(event)

        return cal

    def export_calender_as_ics(self, cal, path):
        with open(path, 'wb') as f:
            f.write(cal.to_ical())

    def get_nearest_date(self, start_date, weekday):
        start_date = parser.parse(start_date)
        weekday = int(weekday)

        if start_date.weekday() >= weekday:
            if start_date.weekday() > weekday: start_date += datetime.timedelta(days=7)
            start_date -= datetime.timedelta(start_date.weekday() - weekday)
        else:
            start_date += datetime.timedelta(weekday - start_date.weekday())

        return start_date


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

class EncapsulatedModel:
    def __init__(self, model, ID, seed, debug=False):
        print("[+] Encapsulate Model in Swag Model")
        self.ID = ID
        self.seed = seed
        self.model = model
        self.counter = 1
        weights = np.array(list(model.parameters()))

        self.orig_shape = self.save_orig_shape(weights)
        if debug:
            print("Orig_Shape " + str(self.orig_shape))
            print("Orig_Type" + str(type(weights)))
        weights = self.local_get_weights_array(weights)
        self.new_shape = len(weights)
        if debug:
            print("New Shape of Array: " + str(self.new_shape))

        self.theta_swa = weights
        if debug:
            print("Init Shape Theta swa: " + str(self.theta_swa.shape))

        self.theta_line_squared = self.square_weights(weights)
        if debug:
            print("Init Shape Theta Line squared: " + str(self.theta_line_squared))

        self.swag_diag = None

    def get_params(self):
        return self.model.get_params()

    def save_model(self, debug=False):
            if debug:
                print("Try to save model parameters")
            torch.save(self.theta_swa, 'models/' + str(self.seed) + '/swag/theta_swa' + self.ID + '.pt')
            torch.save(self.theta_line_squared, 'models/'+ str(self.seed) + '/swag/theta_line_squared' + self.ID + '.pt')
            torch.save(self.swag_diag, 'models/' + str(self.seed) + '/swag/swag_diag' + self.ID)
            torch.save(self.model.state_dict(), 'models/'+ str(self.seed) + '/swag/swag_model' + self.ID + '.bin')
            torch.save(self.counter, 'models/' + str(self.seed) + '/swag/counter' + self.ID + '.pt')
            torch.save(self.orig_shape, 'models/' + str(self.seed) + '/swag/orig_shape' + self.ID + '.pt')
            torch.save(self.new_shape, 'models/' + str(self.seed) + '/swag/new_shape' + self.ID + '.pt')

    def load_model(self, debug=True):
            if debug:
                print("Try to load model parameters")
            self.theta_swa = torch.load('models/' + str(self.seed) + '/swag/theta_swa' + self.ID + '.pt')
            self.theta_line_squared = torch.load('models/' + str(self.seed) + '/swag/theta_line_squared' + self.ID + '.pt')
            self.swag_diag = torch.load('models/' + str(self.seed) + '/swag/swag_diag' + self.ID)
            self.model.load_state_dict(torch.load('models/' + str(self.seed) + '/swag/swag_model' + self.ID + '.bin'))
            self.counter = torch.load('models/' + str(self.seed) + '/swag/counter' + self.ID + '.pt')
            self.orig_shape = torch.load('models/' + str(self.seed) + '/swag/orig_shape' + self.ID + '.pt')
            self.new_shape = torch.load('models/' + str(self.seed) + '/swag/new_shape' + self.ID + '.pt')

    def save_orig_shape(self, weights, debug=False):
            if debug:
                print("Type of Weights Array is: " + str(type(weights)))
                print("Shape of Weights Array is: " + str(weights.shape))
            shapes = []
            for entry in weights:
                if debug:
                    print("Type of Entry" + str(type(entry)))
                shapes.append(entry.shape)
            return shapes

    def restore_orig_shape(self, weights, debug=False):
            output = []
            for entry in self.orig_shape:
                if debug:
                    print("For entry : " + str(entry))
                    print("Size of weight Array: " + str(len(weights)))
                if len(entry) == 2:
                    i = entry[0]
                    j = entry[1]
                    product = i*j
                    if debug:
                        print("Will take first X elements out of weights. X=" + str(product))
                    used_weights = weights[:product]
                    weights = weights[product:]
                    result = np.reshape(used_weights, (i, j))
                    result = torch.tensor(np.array(result))
                    if debug:
                        print("Resulting Shape " + str(result.shape))
                    result = torch.nn.Parameter(result)
                    output.append(result)
                if len(entry) == 1:
                    i = entry[0]
                    if debug:
                        print("Will take first X elements out of weights. X=" + str(i))
                    used_weights = weights[:i]
                    weights = weights[i:]
                    result = torch.tensor(np.array(used_weights))
                    if debug:
                        print("Resulting Shape " + str(result.shape))
                    result = torch.nn.Parameter(result)
                    output.append(result)
            output = np.asarray(output)
            if debug:
                print("Type of Output: " + str(type(output)))
                print("Shape of Output: " + str(output.shape))
                print("Type of Each Element: " + str(type(output[0])))
            return output

    def forward(self, x):
        y = self.model.forward(x)
        return y

    def divide(self):
        self.theta_swa = np.true_divide(self.theta_swa, self.counter)
        self.theta_line_squared = np.true_divide(self.theta_line_squared, self.counter)

    def compute_swag_diagonal(self, debug=False):
        self.divide()
        theta_swa_squared = self.square_weights(self.theta_swa)
        sub = np.subtract(self.theta_line_squared, theta_swa_squared)
        super_threshold_indices = sub < 0
        sub[super_threshold_indices] = 0
        self.swag_diag = sub

        if debug:
            print("Shape Theta swa: " + str(self.theta_swa.shape))
            print("Shape Theta swa squared: " + str(theta_swa_squared.shape))
            print("Shape Swag_Diag: " + str(sub.shape))
            min_element = np.amin(self.swag_diag)
            max_element = np.amax(self.swag_diag)
            print(str(min_element)+ " at " + str(np.argmin(self.swag_diag)))
            print(str(max_element)+ " at " +str(np.argmax(self.swag_diag)))

    def add_to_swa(self, new_weights):
        self.theta_swa = np.add(self.theta_swa, new_weights)

    def add_to_line(self, new_weights):
        new_weights = self.square_weights(new_weights)
        self.theta_line_squared = np.add(self.theta_line_squared, new_weights)

    def square_weights(self, weights):
        return np.multiply(weights, weights)

    def get_weights_array(self, weights):
        array = weights[0].data
        array = array.flatten()
        try:
            array = torch.from_numpy(array)
        except:
            x = 0
        array = array.cpu()
        for dim in range(1, len(weights)):
            tensor = weights[dim].data
            tensor = tensor.flatten().cpu()
            array = np.concatenate((array, tensor), 0)
        return array

    def local_get_weights_array(self, weights):
        weights = weights
        array = weights[0].data
        array = array.flatten()
        for dim in range(1, len(weights)):
            tensor = weights[dim].data
            tensor = tensor.flatten()
            array = np.concatenate((array, tensor), 0)
        return array

    def train_model(self):
        #Cuda
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.model = self.model.to(device)

        #Params
        MODEL_PARAMETER = self.get_params()
        lr = MODEL_PARAMETER['lr']  # learning rate
        lam = MODEL_PARAMETER['lam']  # weight decay
        epochs = MODEL_PARAMETER['epochs_per_episode'] #epochs per communication episode

        #Optimizer and Loss
        opt = optim.SGD(self.model.parameters(), lr=lr, weight_decay=lam)
        CrEnt = nn.CrossEntropyLoss()

        for epoch in range(epochs):  # loop over the dataset multiple times
            for _, data in enumerate(self.model.trainloader, 0):
                # get the inputs; data is a list of [inputs, labels]
                inputs, labels = data
                inputs, labels = inputs.to(device), labels.to(device)
                # zero the parameter gradients
                opt.zero_grad()
                # forward + backward + optimize
                outputs = self.model.forward(inputs)
                loss = CrEnt(outputs, labels)
                loss.backward()
                opt.step()
                # Here Swag Magic
                self.counter = self.counter + 1
                new_weights = np.array(list(self.model.parameters()))
                new_weights = self.get_weights_array(new_weights)
                self.add_to_swa(new_weights)
                self.add_to_line(new_weights)
            print(loss.item())
        self.compute_swag_diagonal()
        return self


from django.contrib import admin

from projects.models import (
    Dependency, Project, ProjectBuild, ProjectBuildDependency)

class ProjectDependencyInline(admin.TabularInline):
    model = Project.dependencies.through

class ProjectAdmin(admin.ModelAdmin):
    inlines = [ProjectDependencyInline]

class ProjectBuildDependencyInline(admin.TabularInline):
    model = ProjectBuild.build_dependencies.through

class ProjectBuildAdmin(admin.ModelAdmin):
    inlines = [ProjectBuildDependencyInline]

admin.site.register(Dependency)
admin.site.register(Project, ProjectAdmin)

admin.site.register(ProjectBuildDependency)
admin.site.register(ProjectBuild, ProjectBuildAdmin)


import math

from WebInfoHandle import WebInfo
from db import Session, table, SessionBase
from itertools import product

_D = 2140479  # 假装文档数是这么多
_weightIncrease = 1 / 20000  # 当收集到两万份文档时，idf = (其他文章idf+招聘网站idf)/2
session: SessionBase = Session()
_documentNum = session.query(table.Pages).count()

def getIDF(keyword: table.Keyword):
    # 计算权重
    if keyword.document_frequency == 0:
        general_idf = 12
    else:
        general_idf = math.log(_D / keyword.document_frequency)
    this_idf = math.log(_documentNum / keyword.frequency)
    weight_of_this_idf = _weightIncrease * _documentNum
    idf = (general_idf + weight_of_this_idf * this_idf) / (1 + weight_of_this_idf)
    return idf

def updateDB(webInfo: WebInfo):
    global _documentNum, _D, _weightIncrease
    print(webInfo.station + '开始')
    _documentNum += 1
    session: SessionBase = Session()
    # 创建page
    page = table.Pages()
    page.json = webInfo.getDict()
    page.pubdate = webInfo.pubdate

    # 获得station
    station: table.Station = session.query(table.Station). \
        filter(table.Station.station == webInfo.station).first()

    # 查不到则创建station
    if not station:
        station = table.Station()
        station.station = webInfo.station
        station.frequency = 0
        session.add(station)
    station.frequency += 1
    session.flush()

    # 获得专业
    specialityList = []
    for name in webInfo.speciality:
        speciality: table.Speciality = session.query(table.Speciality). \
            filter(table.Speciality.speciality == name).first()
        if not speciality:
            speciality = table.Speciality(speciality=name, frequency=0)
            session.add(speciality)
        speciality.frequency += 1
        specialityList.append(speciality)
    session.flush()

    # 获得关键字
    keywordList = []
    for name in webInfo.words:
        keyword: table.Keyword = session.query(table.Keyword). \
            filter(table.Keyword.keyword == name).first()
        if not keyword:
            keyword = table.Keyword(keyword=name, frequency=0)
            session.add(keyword)
        keyword.frequency += 1
        keywordList.append(keyword)
    session.flush()

    print(station.station + '统计wordFrequency')
    # 统计词频
    for speciality, keyword in product(specialityList, keywordList):
        wordFrequency: table.WordFrequency = session.query(table.WordFrequency). \
            filter(table.WordFrequency.r_keyword == keyword,
                   table.WordFrequency.r_speciality == speciality).first()
        if not wordFrequency:
            wordFrequency = table.WordFrequency()
            wordFrequency.r_keyword = keyword
            wordFrequency.r_speciality = speciality
            session.add(wordFrequency)
        if not wordFrequency.frequency:
            wordFrequency.frequency = 0
        wordFrequency.frequency += 1
        # 计算权重
        idf = getIDF(keyword)
        wordFrequency.weight = wordFrequency.frequency * idf

    print(station.station + '统计wordFrequencyStation')
    # 统计词频
    for keyword in keywordList:
        wordFrequencyStation: table.WordFrequencyStation = session.query(table.WordFrequencyStation). \
            filter(table.WordFrequencyStation.r_keyword == keyword,
                   table.WordFrequencyStation.r_station == station).first()
        if not wordFrequencyStation:
            wordFrequencyStation = table.WordFrequencyStation()
            wordFrequencyStation.r_keyword = keyword
            wordFrequencyStation.r_station = station
            session.add(wordFrequencyStation)
        if not wordFrequencyStation.frequency:
            wordFrequencyStation.frequency = 0
        wordFrequencyStation.frequency += 1
        # 计算权重
        idf = getIDF(keyword)
        wordFrequencyStation.weight = wordFrequencyStation.frequency * idf

    print(station.station + '统计specialityHasStation')
    # 统计specialityHasStation
    for speciality in specialityList:
        specialityHasStation: table.SpecialityHasStation = session.query(table.SpecialityHasStation). \
            filter(table.SpecialityHasStation.r_speciality == speciality,
                   table.SpecialityHasStation.r_station == station).first()
        if not specialityHasStation:
            specialityHasStation = table.SpecialityHasStation()
            specialityHasStation.r_speciality = speciality
            specialityHasStation.r_station = station
            session.add(specialityHasStation)
        if not specialityHasStation.frequency:
            specialityHasStation.frequency = 0
        specialityHasStation.frequency += 1

    page.keyword_collection = keywordList
    page.r_station = station
    page.speciality_collection = specialityList
    print(station.station+'开始commit')
    session.commit()
    print(station.station + 'commit完成')


# coding: utf-8

import time
import datetime
from functools import partial

try:
    from django.utils import timezone
except ImportError:
    raise RuntimeError('Django is required for django simple serializer.')

class TimeFormatFactory(object):
    def __init__(self):
        super(TimeFormatFactory, self).__init__()

    @staticmethod
    def datetime_to_string(datetime_time, time_format='%Y-%m-%d %H:%M:%S'):
        if isinstance(datetime_time, datetime.datetime):
            if datetime_time.tzinfo:
                datetime_time = datetime_time.astimezone(timezone.get_current_timezone())
            return datetime_time.strftime(time_format)
        elif isinstance(datetime_time, datetime.time):
            time_format = '%H:%M:%S'
        elif isinstance(datetime_time, datetime.date):
            time_format = '%Y-%m-%d'
        return datetime_time.strftime(time_format)

    @staticmethod
    def datetime_to_timestamp(datetime_time, time_format=None):
        if isinstance(datetime_time, datetime.datetime):
            if datetime_time.tzinfo:
                datetime_time = datetime_time.astimezone(timezone.get_current_timezone())
            return time.mktime(datetime_time.timetuple())
        return time.mktime(datetime_time.timetuple())

    @staticmethod
    def get_time_func(func_type='string'):
        if func_type == 'string':
            return TimeFormatFactory.datetime_to_string
        elif func_type == 'timestamp':
            return TimeFormatFactory.datetime_to_timestamp
        else:
            return TimeFormatFactory.datetime_to_string