#encoding:utf-8
from django.shortcuts import render
from django.template import RequestContext
from django.template.loader import render_to_string
from django.http import HttpResponseRedirect, HttpResponse, JsonResponse, Http404
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.core.mail import EmailMultiAlternatives
from django.contrib.auth.decorators import login_required, permission_required
from django.shortcuts import get_object_or_404
from sedes.reportes import sms_gestion
from sedes.log_usuarios import log_addition, log_change

from gestion.models import Gestion, Poblacion
from inscripcion.models import Inscripcion, Actividad_Inscripcion, Actividad_grupo_poblacion
from centrosalud.models import Centro
from actividad.models import Actividad
from grupo.models import Grupo

@permission_required('inscripcion.index_inscripcion', login_url='/login')
def index_inscripcion(request):
    sms_gestion(request)
    gestion = Gestion.objects.get(gestion = request.session['gestion'])
    inscripciones = Inscripcion.objects.filter(gestion = gestion)
    return render(request, 'inscripciones/index.html', {
        'inscripciones':inscripciones,
    })

@permission_required('inscripcion.add_inscripcion', login_url='/login')
def list_centros_inscribir(request):
    sms_gestion(request)
    gestion = Gestion.objects.get(gestion = request.session['gestion'])
    inscripciones = Inscripcion.objects.filter(gestion = gestion)
    centros = Centro.objects.exclude(id__in = inscripciones.values('centro_id'))

    return render(request, 'inscripciones/list_centros_asignar.html', {
        'centros':centros,
    })

@permission_required('inscripcion.add_inscripcion', login_url='/login')
def add_centro_inscripcion(request, centro_id):
    centro = get_object_or_404(Centro, pk = centro_id)
    gestion = Gestion.objects.get(gestion = request.session['gestion'])
    insc = Inscripcion.objects.create(
        gestion = gestion,
        centro = centro,
    )
    log_addition(request, insc, 'Inscripcion Realizada')
    sms = "Centro %s Inscrito Correctamente"% (centro.nombre)
    messages.info(request, sms)
    return HttpResponseRedirect(reverse(list_centros_inscribir))

@permission_required('inscripcion.index_activdad', login_url='/login')
def index_asignacion(request):
    sms_gestion(request)
    gestion = Gestion.objects.get(gestion = request.session['gestion'])
    inscripciones = Inscripcion.objects.filter(gestion = gestion)
    return render(request, 'asignaciones/index.html', {
        'inscripciones':inscripciones,
    })

@permission_required('inscripcion.add_actividad_inscripcion', login_url='/login')
def inscripcion_actividad_asignar(request, insc_id):
    sms_gestion(request)
    inscripcion = get_object_or_404(Inscripcion, pk = insc_id)
    asignados = Actividad_Inscripcion.objects.filter(inscripcion = inscripcion)
    actividades = Actividad.objects.exclude(id__in = asignados.values('actividad_id'))

    return render(request, 'asignaciones/inscripcion_asignacion.html', {
        'inscripcion':inscripcion,
        'asignados':asignados,
        'actividades':actividades,
    })

@permission_required('inscripcion.add_actividad_inscripcion', login_url='/login')
def add_actividad_inscripcion(request, insc_id, actividad_id):
    inscripcion = get_object_or_404(Inscripcion, pk = insc_id)
    actividad = get_object_or_404(Actividad, pk = actividad_id)
    a = Actividad_Inscripcion.objects.create(
        inscripcion = inscripcion,
        actividad = actividad,
    )
    log_addition(request, a, 'Actividad asignada')
    log_change(request, actividad, 'Asignacion realizada')
    sms = "Actividad %s Asignada Correctamente"% (actividad.nombre)
    messages.info(request, sms)
    return HttpResponseRedirect(reverse(inscripcion_actividad_asignar, args={inscripcion.id, }))

@permission_required('inscripcion.add_actividad_inscripcion', login_url='/login')
def remove_asignacion(request, asig_id):
    asignacion = get_object_or_404(Actividad_Inscripcion, pk = asig_id)
    inscripcion = Inscripcion.objects.get(id = asignacion.inscripcion_id)
    asignacion.delete()
    sms = 'Asignacion Quitada Correctamente'
    messages.info(request, sms)
    return HttpResponseRedirect(reverse(inscripcion_actividad_asignar, args={inscripcion.id, }))

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def list_actividades_add_grupo(request, insc_id):
    sms_gestion(request)
    inscripcion = get_object_or_404(Inscripcion, pk = insc_id)
    actividades_inscripcion = Actividad_Inscripcion.objects.filter(inscripcion = inscripcion)

    return render(request, 'asignaciones/list_actividades_add_grupo.html', {
        'inscripcion':inscripcion,
        'actividades_inscripcion':actividades_inscripcion,
    })

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def ajax_list_grupos_add_actividad(request, insc_acti_id):
    if request.is_ajax():
        inscripcion_actividad = get_object_or_404(Actividad_Inscripcion, pk = insc_acti_id)
        actividad_grupo = Actividad_grupo_poblacion.objects.filter(actividad_inscripcion = inscripcion_actividad)
        grupos = Grupo.objects.exclude(id__in = actividad_grupo.values('grupo_id'))
        #grupos = Grupo.objects.all()
        html = render_to_string('asignaciones/ajax_list_grupos_add_actividad.html', {
            'inscripcion_actividad':inscripcion_actividad,
            'grupos':grupos,
        }, context_instance=RequestContext(request))
        return JsonResponse(html, safe=False)
    else:
        raise Http404

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def ajax_list_poblacion_add_grupo(request):
    if request.is_ajax():
        grupo_id = request.GET['grupo']
        act_insc_id = request.GET['actividad_inscripcion']
        grupo = Grupo.objects.get(id = grupo_id)
        inscripcion_actividad = Actividad_Inscripcion.objects.get(id = act_insc_id)
        poblaciones = Poblacion.objects.all()
        html = render_to_string('asignaciones/ajax_list_poblacion_add_grupo.html', {
            'grupo':grupo,
            'poblaciones':poblaciones,
            'inscripcion_actividad':inscripcion_actividad,
        }, context_instance=RequestContext(request))
        return JsonResponse(html, safe=False)
    else:
        raise Http404

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def ajax_list_grupos_asignados(request):
    if request.is_ajax():
        insc_acti_id = request.GET['actividad_inscripcion']
        actividad_inscripcion = Actividad_Inscripcion.objects.get(id = insc_acti_id)
        grupos_asignados = Actividad_grupo_poblacion.objects.filter(actividad_inscripcion = actividad_inscripcion)
        html = render_to_string('asignaciones/ajax_list_grupos_asignados.html', {
            'grupos_asignados':grupos_asignados,
        }, context_instance=RequestContext(request))
        return JsonResponse(html, safe=False)
    else:
        raise Http404

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def asignar_grupo_actividad_inscripcion(request, grupo_id, insc_acti_id, poblacion_id):
    poblacion = get_object_or_404(Poblacion, pk = poblacion_id)
    grupo = get_object_or_404(Grupo, pk = grupo_id)
    actividad_inscripcion = get_object_or_404(Actividad_Inscripcion, pk = insc_acti_id)
    a_g_p = Actividad_grupo_poblacion.objects.create(
        actividad_inscripcion = actividad_inscripcion,
        grupo = grupo,
        poblacion = poblacion,
    )
    log_addition(request, a_g_p, 'Grupo Asignado Correctamete')
    sms = "Grupo %s asignado correctamente"% (grupo.variable)
    messages.info(request, sms)
    return HttpResponseRedirect(reverse(list_actividades_add_grupo, args={actividad_inscripcion.inscripcion_id, }))

@permission_required('inscripcion.add_actividad_grupo_poblacion', login_url='/login')
def remove_grupo_actividad_inscripcion(request, grupo_id):
    g_p_i = get_object_or_404(Actividad_grupo_poblacion, pk = grupo_id)
    inscripcion = Inscripcion.objects.get(id = g_p_i.actividad_inscripcion.inscripcion_id)
    g_p_i.delete()
    sms = "Grupo Eliminado Corrsssectamente"
    messages.info(request, sms)
    return HttpResponseRedirect(reverse(list_actividades_add_grupo, args={inscripcion.id, }))


from sys import stdin

li = []
for i in range(7):
    s = int(stdin.readline())
    if (s%2 == 1):
        li.append(s)
if li == []:
    print(-1)
else:
    print(sum(li))
    print(min(li))


from .tokens import *

from os.path import abspath, dirname, join
from parso.grammar import PythonGrammar
from typing import Optional

import parso.python.tokenize

__all__ = ['tokenize_file', 'load_grammar']

DEFAULT_GRAMMAR_PATH = abspath(join(dirname(__file__), "../python-3.4.grammar"))

def tokenize_file(filename: str, grammar: Optional[PythonGrammar] = None,
                  suppress_error_tokens: bool = False) -> TokenGenerator:
    with open(filename) as f:
        lines = f.readlines()
    start_pos = (1, 0)

    if grammar is None:
        grammar = load_grammar()

    return tokens_from_py_tokens(grammar._tokenize_lines(lines, start_pos), suppress_error_tokens)

def load_grammar(path: Optional[str] = None, version: Optional[str] = None) -> PythonGrammar:
    if path is None:
        if version is None:
            return parso.load_grammar(path=DEFAULT_GRAMMAR_PATH)
        else:
            return parso.load_grammar(version=version)
    else:
        if version is None:
            return parso.load_grammar(path=path)
        else:
            raise ValueError("Cannot specify both path and version for loading grammar.")


def seive_of_eratosthenes(n):
    composite_list = []  # Create list of composites (initially empty)
    prime_list = []  # Create list of primes (initially empty)
    # cycle through numbers from 2 to n (bound is exclusive)
    for i in range(2, n+1):
        # if i is not in the list of composites
        if i not in composite_list:
            # it is a prime, add it to the correct list
            prime_list.append(i)
            # cycle through numbers from i*i to n in steps of i
            for j in range(i*i, n+1, i):
                # add these numbers to composites
                composite_list.append(j)
    # return the list of primes.
    return prime_list

def main():
    # print out info
    print("Seive of Eratosthenes: Returns list of primes up to n.")
    # take an integer input from the user
    number = input("What is n? ")
    # run the seive and print the resulting list
    print(seive_of_eratosthenes(number))

# this runs the main() function if this file is executed
if __name__ == '__main__':
    main()


def expire(self):
        """Do the expiration of dictionary items.

        Remove items that expired by now from the dictionary.

        :Return: time, in seconds, when the next item expires or `None`
        :returntype: `float`
        """
        with self._lock:
            logger.debug("expdict.expire. timeouts: {0!r}"
                                                    .format(self._timeouts))
            next_timeout = None
            for k in self._timeouts.keys():
                ret = self._expire_item(k)
                if ret is not None:
                    if next_timeout is None:
                        next_timeout = ret
                    else:
                        next_timeout = min(next_timeout, ret)
            return next_timeout


""" Main program for Discount Code Microservice"""
""" Developer: Fawad I Sheikh
    Date: 5-Oct-2022
"""

from db_setup import Brand, Discount_Code, brand_schema, brands_schema, discount_codes_schema, discount_code_schema
from flask import Flask, request, jsonify
from flask_sqlalchemy import SQLAlchemy
from disc_codes import generate_dc, fetch_dc
import os
app = Flask(__name__)
base_dir = os.path.abspath(os.path.dirname(__file__))
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + \
    os.path.join(base_dir, 'loyalty.db')
db = SQLAlchemy(app)

#
# ------ Routes Definition ---------
#

@app.route("/")
def index():
    return "Welcome to Discount code microservice API "

@app.route("/brands")
def brands():
    brands_list = Brand.query.all()
    result = brands_schema.dump(brands_list)
    # print(result)
    return jsonify(result)

@app.route("/codes")
def codes():
    brand_id = request.args.get('brand_id')
    if brand_id:
        codes_list = Discount_Code.query.filter_by(brand_id=brand_id)
    else:
        codes_list = Discount_Code.query.all()

    result = discount_codes_schema.dump(codes_list)
    # print(result)
    if not result:
        return jsonify(message='No discount codes found'), 404

    return jsonify(result)

# ----- Generate discount_code  END POINT ----------
@app.route("/generate_disc_codes", methods=['GET', 'POST'])
def generate_disc_codes():
    """Generate discount codes, the endpoint expects two parameters
        brand_code = the brand code for which the codes will be generated
        count = how many discount codes to be generated. It is optional, default value is 10 if nothing is passed. """

    brand_id = request.args.get("brand_id")
    seed = 0

    if not brand_id:
        return jsonify(message='Please pass Brand id, it is required to generate discount codes'), 404

    count = request.args.get("count")
    if not count:
        count = 10
    else:
        """ determine count is a +ve number """
        try:
            count = int(count)
            if count <= 0:
                raise ValueError

        except ValueError:
            return {'message': 'Please send a positive numeric value for count'}, 401

    status = {'message': '', 'code': 0}
    codes = generate_dc(brand_id, count, status)

    if not codes:
        return jsonify(message=f'Codes not generated, please contact customer support. Error - {status["message"]}'), status['code']

    return jsonify(message=f'{count} Discount codes generated for Brand_id = {brand_id} ')

# ----- fetch_discount_code  END POINT ----------
@ app.route("/fetch_discount_code", methods=['GET', 'POST'])
def fetch_disc_code():
    """Fetch discount codes, the endpoint expects two parameters
        brand_id = the brand id for which the discount code will be returned
        user_id = the user_id requesting for discount code. """

    brand_id = request.args.get("brand_id")
    if not brand_id:
        return jsonify(message='Please pass Brand id, it is required to fetch discount code'), 404

    user_id = request.args.get("user_id")
    if not user_id:
        return jsonify(message='Please pass User id, it is required to fetch discount code'), 404

    status = {'message': '', 'code': 0}

    code = fetch_dc(brand_id, user_id, status)
    if not code:
        return jsonify(message=f'Code not fetched, please contact customer support. Error - {status["message"]}'), status['code']

    return jsonify(message=f'{code.discount_code} discount code is fetched for brand id = {brand_id}')

# -
# -
# ------ Main  ---------
if __name__ == "__main__":
    app.run(debug=True)


import os
from sqlite3 import connect
from hqcpq.helpers.io import read_config, join_to_project_folder

def default_path():
    config = read_config()

    project_path_fallback = join_to_project_folder(os.path.join("hqcpq", "db", "hqcpq.db"))

    database_path_value = config.get("DB", "database_path", fallback=str())
    if not os.path.exists(database_path_value):
        database_path_value = None

    return database_path_value or project_path_fallback

def initialise_db(clear: bool = False):
    database_path = default_path()
    if clear and os.path.exists(database_path):
        os.remove(database_path)

    with SQLiteConnection() as cur:
        init_script_path = join_to_project_folder(os.path.join("hqcpq", "db", "init.sql"))
        with open(init_script_path, "r") as init_file:
            init_script = init_file.read()
            cur.executescript(init_script)

class SQLiteConnection:
    def __init__(self):
        self.database_path = default_path()
        self.connection = None

    def __enter__(self):
        self.connection = connect(self.database_path)
        self.connection.execute("PRAGMA foreign_keys = ON")
        return self.connection.cursor()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connection.commit()
        self.connection.close()


import torch
import torch.nn as nn
import torch.nn.functional as F
import fvcore.nn.weight_init as weight_init

class ContrastiveHead(nn.Module):
    """MLP head for contrastive representation learning, https://arxiv.org/abs/2003.04297
    Args:
        dim_in (int): dimension of the feature intended to be contrastively learned
        feat_dim (int): dim of the feature to calculated contrastive loss

    Return:
        feat_normalized (tensor): L-2 normalized encoded feature,
            so the cross-feature dot-product is cosine similarity (https://arxiv.org/abs/2004.11362)
    """
    def __init__(self, dim_in, feat_dim):   # 1024 128
        super().__init__()
        self.head = nn.Sequential(
            nn.Linear(dim_in, dim_in),
            nn.ReLU(inplace=True),
            nn.Linear(dim_in, feat_dim),
        )
        for layer in self.head:
            if isinstance(layer, nn.Linear):
                # 用XavierFill方法初始化权重，此时bias会初始化为0
                weight_init.c2_xavier_fill(layer)

    def forward(self, x):
        feat = self.head(x)
        feat_normalized = F.normalize(feat, dim=1)
        return feat_normalized

class SupConLoss(nn.Module):
    """Supervised Contrastive LOSS as defined in https://arxiv.org/pdf/2004.11362.pdf."""

    def __init__(self, temperature=0.2, iou_threshold=0.5, reweight_func='none'):
        '''Args:
            tempearture: a constant to be divided by consine similarity to enlarge the magnitude
            iou_threshold: consider proposals with higher credibility to increase consistency.
        '''
        super().__init__()
        self.temperature = temperature
        self.iou_threshold = iou_threshold
        self.reweight_func = reweight_func

    def forward(self, features, labels, ious):
        """
        Args:
            features (tensor): shape of [M, K] where M is the number of features to be compared,
                and K is the feature_dim.   e.g., [8192, 128]
            labels (tensor): shape of [M].  e.g., [8192]
        """
        # shape[0]应该是特征的个数吧，看上面的注释，[1]是特征的维数
        # features.shape [256*bs/GPU, 128], labels.shape [256*bs/GPU], ious.shape [256*bs/GPU]
        assert features.shape[0] == labels.shape[0] == ious.shape[0]

        # 如果输入labels是个向量（一维列表），则扩展为列向量，即每个特征对应的标签
        if len(labels.shape) == 1:
            labels = labels.reshape(-1, 1)

        # 对两个张量Tensor进行逐元素的比较，若相同位置的两个元素相同，则返回True；若不同，返回False，shape[256*bs/GPU, 256*bs/GPU]
        # mask of shape [None, None], mask_{i, j}=1 if sample i and sample j have the same label
        label_mask = torch.eq(labels, labels.T).float().cuda()

        # features和 features的转置 相乘后，逐元素除以temperature，对应文中zi*zj/τ，z是已经正则化过的，即计算zi和zj的cos相似度
        # similarity的尺寸是M*M，也就是M个特征之间的相似度，一个相似度矩阵 shape[256*bs/GPU, 256*bs/GPU]
        similarity = torch.div(torch.matmul(features, features.T), self.temperature)

        # for numerical stability
        # dim=1找每行的最大值，即每个特征的最大值，因为keepdim所以返回的是[[8],[6]]这样的结构，M*1
        # 很明显，这里每行最大值应该是1，也就是自身和自身的相似度 shape [256*bs/GPU, 1]
        sim_row_max, _ = torch.max(similarity, dim=1, keepdim=True)
        # detach似乎会让requires_grad变成False，但这里意义不明
        similarity = similarity - sim_row_max.detach()

        # mask out self-contrastive
        # 和similarity相同尺寸的全1矩阵，shape[256*bs/GPU, 256*bs/GPU]
        logits_mask = torch.ones_like(similarity)
        # 对角线值填充为0
        logits_mask.fill_diagonal_(0)

        # shape[256*bs/GPU, 256*bs/GPU]
        exp_sim = torch.exp(similarity) * logits_mask

        # shape[256*bs/GPU, 256*bs/GPU]
        log_prob = similarity - torch.log(exp_sim.sum(dim=1, keepdim=True))

        # shape[256*bs/GPU]
        per_label_log_prob = (log_prob * logits_mask * label_mask).sum(1) / label_mask.sum(1)

        # keep为满足iou阈值的索引，对应论文中f(ui)里的第一项，即选出满足iou阈值的，shape [256*bs/GPU]
        keep = ious >= self.iou_threshold
        # shape 5-11 变化 决定了loss和coef的shape，其shape和torch.sum(keep)相同，也就是提取出iou大于阈值的几项
        per_label_log_prob = per_label_log_prob[keep]
        # shape 5-11 变化
        loss = -per_label_log_prob

        # 一般是none，即torch.ones_like(iou) 返回一个填充了标量值1的张量，其大小与input相同。
        # 对应论文f(ui)里的第二项，即reweighting函数g(ui)，给不同iou得分的proposal赋不同的权重
        # 根据论文消融，最好的权重似乎就是1，可以认为是没做什么
        # coef的shape和loss相同，变化的
        coef = self._get_reweight_func(self.reweight_func)(ious)
        coef = coef[keep]

        loss = loss * coef
        return loss.mean()

    @staticmethod
    def _get_reweight_func(option):
        def trivial(iou):
            return torch.ones_like(iou)
        def exp_decay(iou):
            return torch.exp(iou) - 1
        def linear(iou):
            return iou

        if option == 'none':
            return trivial
        elif option == 'linear':
            return linear
        elif option == 'exp':
            return exp_decay


from pathlib import Path
from typing import Any, List, Optional
import musicbrainzngs as mb
from ratelimit import sleep_and_retry, limits

import requests
from unitunes.services.services import (
    ServiceWrapper,
    StreamingService,
    cache,
)
from unitunes.track import AliasedString, Track
from unitunes.common_types import ServiceType
from unitunes.uri import MB_RECORDING_URI

class MusicBrainzWrapper(ServiceWrapper):
    def __init__(self, cache_root: Path) -> None:
        super().__init__("musicbrainz", cache_root)
        mb.set_useragent("unitunes", "0.1")

    @sleep_and_retry
    @limits(calls=1, period=1)
    def query_mb_api(self, query: str, params):
        headers = {"User-Agent": "unitunes"}
        r = requests.get(query, params=params, headers=headers)
        if r.status_code != 200:
            raise Exception(f"MusicBrainz API returned {r.status_code}")
        if "error" in r.json():
            raise Exception(f"MusicBrainz API returned error: {r.json()['error']}")
        return r.json()

    @cache
    def get_recording_by_id(self, id: str, use_cache=True, includes: List[str] = []):
        # Send this manually because the musicbrainzngs library doesn't return aliases for some reason
        params = {
            "inc": "+".join(includes),
            "fmt": "json",
        }
        return self.query_mb_api(
            f"http://musicbrainz.org/ws/2/recording/{id}", params=params
        )

    @cache
    def search_recordings(self, *args, use_cache=True, **kwargs):
        return mb.search_recordings(*args, **kwargs)

    @cache
    def get_release_by_id(self, *args, use_cache=True, **kwargs):
        return mb.get_release_by_id(*args, **kwargs)

class MusicBrainz(StreamingService):
    wrapper: MusicBrainzWrapper

    def __init__(self, cache_root: Path) -> None:
        super().__init__("MusicBrainz", ServiceType.MB, cache_root)
        self.wrapper = MusicBrainzWrapper(cache_root)

    def load_config(self, config) -> None:
        pass

    @staticmethod
    def parse_track(recording):
        def parse_aliases(obj) -> List[str]:
            return (
                [alias["name"] for alias in obj["aliases"]]
                if "obj" in recording
                else []
            )

        def parse_aliased_string(obj) -> Optional[AliasedString]:
            if "title" not in obj:
                return None
            return AliasedString(obj["title"], parse_aliases(obj))

        name = parse_aliased_string(recording)
        if not name:
            raise ValueError(f"Recording {recording} has no name")

        albums = []
        if "releases" in recording:
            for album in recording["releases"]:
                s = parse_aliased_string(album)
                if s:
                    albums.append(s)
        if "release-list" in recording:
            for album in recording["release-list"]:
                s = parse_aliased_string(album)
                if s:
                    albums.append(s)

        artists = []
        if "artist-credit" in recording:
            for artist in recording["artist-credit"]:
                if "artist" in artist:
                    a = artist["artist"]
                    if "name" in a:
                        aliases = (
                            [alias["alias"] for alias in a["alias-list"]]
                            if "alias-list" in a
                            else []
                        )
                        if "sort-name" in a:
                            aliases.append(a["sort-name"])
                        artists.append(AliasedString(value=a["name"], aliases=aliases))

        return Track(
            name=name,
            artists=artists,
            albums=albums,
            length=int(recording["length"]) // 1000
            if "length" in recording and recording["length"]
            else None,
            uris=[MB_RECORDING_URI.from_uri(recording["id"])],
        )

    def pull_track(self, uri: MB_RECORDING_URI) -> Track:
        results = self.wrapper.get_recording_by_id(
            id=uri.uri, includes=["releases", "artists", "aliases"]
        )
        if not results:
            raise ValueError(f"Recording {uri} not found")
        track = self.parse_track(results)
        return track

    def search_query(self, query: Any) -> List[Track]:
        # remove empty fields
        fields = {k: v for k, v in query.items() if v}

        results = self.wrapper.search_recordings(
            limit=5,
            **fields,
        )

        return list(map(self.parse_track, results["recording-list"]))

    def query_generator(self, track: Track) -> List[Any]:
        def escape_special_chars(s: str) -> str:
            # + - && || ! ( ) { } [ ] ^ " ~ * ? : \
            special_chars = [
                "\\",  # \ needs to be escaped first or else it will be escaped twice
                "+",
                "-",
                "&&",
                "||",
                "!",
                "(",
                ")",
                "{",
                "}",
                "[",
                "]",
                "^",
                '"',
                "~",
                "*",
                "?",
                ":",
            ]
            for char in special_chars:
                s = s.replace(char, f"\\{char}")
            return s

        all_fields = {
            "recording": escape_special_chars(track.name.value),
            "artist": escape_special_chars(
                " ".join([artist.value for artist in track.artists])
            ),
            "release": escape_special_chars(" ".join([a.value for a in track.albums])),
        }

        fields_to_remove = [
            [],
            ["release"],
            ["artist"],
            ["recording"],
            ["artist", "release"],
        ]

        queries = []
        for removed_fields in fields_to_remove:
            fields = {
                field: all_fields[field]
                for field in all_fields
                if field not in removed_fields
            }
            if not any(fields.values()):
                continue

            queries.append(fields)
        return queries


#pixelise an image. turning image into numbers in list

import cv2
import numpy as np

img_path = 'onechinese.png'
img = cv2.imread(img_path, 0)
img_reverted = cv2.bitwise_not(img) #255 is black and 0 is white as it is inverted
#print(img[50])

s = 1
ws = 4
p = 2

from matplotlib import pyplot as plt
plt.imshow(img_reverted, interpolation='nearest')
plt.show()

#force images of different resolutions into the same resolution

def compress(img_reverted, s, ws):
    compressedList = []

    lenImg = len(img_reverted)
    #print("////////////////")
    #print(lenImg)
    for j in range(0,lenImg -ws, s):
        listOfRow = []

        for i in range(0,lenImg-ws, s):
            sumOfPixels = 0
            for k in range(i,i + ws):
                for x in range(j,j + ws):
                    sumOfPixels = sumOfPixels + img_reverted[x][k]

            avg = sumOfPixels/(ws**2)
            listOfRow.append(avg)
            #print(len(img_reverted))

        compressedList.append(listOfRow)

    return compressedList

#print(compress(img_reverted, s, ws))
s = 4
ws = 8
compressedImg = compress(img_reverted, s, ws)

def compressMaxPooling(img_reverted, s, ws):
    compressedList = []

    lenImg = len(img_reverted)
    #print("////////////////")
    #print(lenImg)
    for j in range(0,lenImg -ws, s):
        listOfRow = []

        for i in range(0,lenImg-ws, s):
            sumOfPixels = 0
            for k in range(i,i + ws):
                for x in range(j,j + ws):
                    sumOfPixels = sumOfPixels + img_reverted[x][k]
                    value = []
                    value.append(img_reverted[x][k])
                    max1 = 0
                    lenIMG = len(value)
                    for i in range(0,lenIMG):
                        if max1 < value[i]:
                            max1 = value[i]

            listOfRow.append(max1)
            #print(len(img_reverted))

        compressedList.append(listOfRow)

    return compressedList

compressMaxPooling = compressMaxPooling(img_reverted, s, ws)
print(compressMaxPooling)

def padding(img_reverted, p):

    paddedList = []
    listOfZeros = []
    for b in range(0,len(img_reverted) + 2*p):
        listOfZeros.append(0)
    for g in range(0,len(img_reverted)):
        newList = img_reverted[g]
        for pi in range(0,p):
            newList = np.insert(newList,0,0)
            newList = np.append(newList, 0)
        newList = newList.tolist()
        paddedList.append(newList)

    for pi in range(0,p):
        paddedList.insert(0,listOfZeros)
        paddedList.append(listOfZeros)

    return paddedList

def paddingHorizontally(img_reverted, p):

    paddedList = []
    for g in range(0,len(img_reverted)):
        newList = img_reverted[g]
        for pi in range(0,p):
            newList = np.insert(newList,0,0)
            newList = np.append(newList, 0)
        newList = newList.tolist()
        paddedList.append(newList)

    return paddedList

def paddingVertically(img_reverted, p):

    paddedList = []
    listOfZeros = []
    for b in range(0,len(img_reverted)):
        listOfZeros.append(0)

    for pi in range(0,p):
        paddedList.insert(0,listOfZeros)
        paddedList.append(listOfZeros)

    return paddedList

def paddingHorizontallyLeft(img_reverted, p):

    paddedList = []
    for g in range(0,len(img_reverted)):
        newList = img_reverted[g]
        for pi in range(0,p):
            newList = np.insert(newList,0,0)

        newList = newList.tolist()
        paddedList.append(newList)

    return paddedList

def paddingHorizontallyRight(img_reverted, p):

    paddedList = []
    for g in range(0,len(img_reverted)):
        newList = img_reverted[g]
        for pi in range(0,p):

            newList = np.append(newList, 0)
        newList = newList.tolist()
        paddedList.append(newList)

    return paddedList

def paddingVerticallyTop(img_reverted, p):

    paddedList = []
    listOfZeros = []
    for b in range(0,len(img_reverted)):
        listOfZeros.append(0)

    for pi in range(0,p):
        paddedList.insert(0,listOfZeros)

    return paddedList

def paddingVerticallyBottom(img_reverted, p):

    paddedList = []
    listOfZeros = []
    for b in range(0,len(img_reverted)):
        listOfZeros.append(0)

    for pi in range(0,p):

        paddedList.append(listOfZeros)

    return paddedList

def SquaringImage(img_reverted):
    horizontalResolution = len(img_reverted[0])
    verticalResolution = len(img_reverted)

    if horizontalResolution == verticalResolution:
        return img_reverted
    else:
        if abs(horizontalResolution - verticalResolution) % 2 == 0:
            #eg. 8 by 10 img, we know how to pad

            if horizontalResolution > verticalResolution:
                difference = horizontalResolution - verticalResolution
                pad = difference / 2
                p = int(pad)
                return paddingVertically(img_reverted, p)

            elif verticalResolution > horizontalResolution:
                difference = verticalResolution - horizontalResolution
                pad = difference / 2
                p = int(pad)
                return paddingHorizontally(img_reverted, p)

        else:
            #eg. 7 by 10 img, tricky to pad

            if horizontalResolution > verticalResolution:
                difference = horizontalResolution - verticalResolution
                pad = difference / 2
                largeP = int(pad)
                smallP = difference - largeP
                img_reverted = paddingVerticallyTop(img_reverted, largeP)
                img_reverted = paddingVerticallyBottom(img_reverted, smallP)
                return img_reverted

            elif horizontalResolution < verticalResolution:
                difference = verticalResolution - horizontalResolution
                pad = difference / 2
                largeP = int(pad)
                smallP = difference - largeP
                img_reverted = paddingHorizontallyLeft(img_reverted, largeP)
                img_reverted = paddingHorizontallyRight(img_reverted, smallP)
                return img_reverted

paddedImg = padding(img_reverted, p)
lenPadded = len(paddedImg)
lenPaddedX = len(paddedImg[0])
#print(lenPadded)
#print(lenPaddedX)
for v in range(0,lenPadded):
    for w in range(0,lenPaddedX):
        paddedImg[v][w] = np.float64(paddedImg[v][w])

plt.imshow(compressedImg, interpolation='nearest')
plt.show()

#print(type(paddedImg[0][0]))

plt.imshow(paddedImg, interpolation='nearest')
plt.show()

"""Make image of any size into a square image of a particular size"""
resolutionNew = 100
def resize(img_reverted, resolutionNew = 100):

    #pad first
    img_squared = SquaringImage(img_reverted)

    widthOfImg = len(img_squared)
    print(widthOfImg)
    #width is 81
    lengthOfImg = len(img_squared[0])
    print(lengthOfImg)
    #length is 101

    if lengthOfImg != widthOfImg:
        print("Error! Lenght of image does not equal to width of image")
    else:
        #50 = 101 - ws / s  +1

        #rn = ro - ws +2p/s +1

        #ws = -(rn - 1)*s + Ro

        s = 1

        ws = -(resolutionNew - 1)*s + lengthOfImg
        ws = int(ws)
        print("/////////////////////////////////////////////")
        print(ws)
        resizedList = []

        resizedList = compress(img_squared, s, ws)

        return resizedList

resizedList = resize(img_reverted, resolutionNew)
#print(resizedList)

lenResized = len(resizedList)
lenResizedX = len(resizedList[0])
print("/////////////////////////////////////////////")
print(lenResized)
print(lenResizedX)
print("/////////////////////////////////////////////")
for v in range(0,lenResized):
    for w in range(0,lenResizedX):
        resizedList[v][w] = np.float64(resizedList[v][w])

plt.imshow(resizedList, interpolation='nearest')
plt.show()


from django.apps import apps
from django.db import migrations, models
from django.db.models import F

def populate_visiblename(apps, schema_editor):
    NoteType = apps.get_model('lsdb', 'NoteType')
    NoteType.objects.all().update(visible_name=F('name'))

class Migration(migrations.Migration):

    dependencies = [
        ('lsdb', '0030_auto_20211109_1704'),
    ]

    operations = [
        migrations.AddField(
            model_name='notetype',
            name='visible_name',
            field=models.CharField(blank=True, max_length=128, null=True),
        ),
        migrations.RunPython(populate_visiblename),
    ]


# -*- coding: utf-8 -*-
# Description: unbound netdata python.d module
# Author: Austin S. Hemmelgarn (Ferroin)
# SPDX-License-Identifier: GPL-3.0-or-later

import os
import sys

from copy import deepcopy

from bases.FrameworkServices.SocketService import SocketService
from bases.loaders import YamlOrderedLoader

PRECISION = 1000

ORDER = [
    'queries',
    'recursion',
    'reqlist',
]

CHARTS = {
    'queries': {
        'options': [None, 'Queries Processed', 'queries', 'Unbound', 'unbound.queries', 'line'],
        'lines': [
            ['ratelimit', 'ratelimited', 'absolute', 1, 1],
            ['cachemiss', 'cache_miss', 'absolute', 1, 1],
            ['cachehit', 'cache_hit', 'absolute', 1, 1],
            ['expired', 'expired', 'absolute', 1, 1],
            ['prefetch', 'prefetched', 'absolute', 1, 1],
            ['recursive', 'recursive', 'absolute', 1, 1]
        ]
    },
    'recursion': {
        'options': [None, 'Recursion Timings', 'seconds', 'Unbound', 'unbound.recursion', 'line'],
        'lines': [
            ['recursive_avg', 'average', 'absolute', 1, PRECISION],
            ['recursive_med', 'median', 'absolute', 1, PRECISION]
        ]
    },
    'reqlist': {
        'options': [None, 'Request List', 'items', 'Unbound', 'unbound.reqlist', 'line'],
        'lines': [
            ['reqlist_avg', 'average_size', 'absolute', 1, 1],
            ['reqlist_max', 'maximum_size', 'absolute', 1, 1],
            ['reqlist_overwritten', 'overwritten_requests', 'absolute', 1, 1],
            ['reqlist_exceeded', 'overruns', 'absolute', 1, 1],
            ['reqlist_current', 'current_size', 'absolute', 1, 1],
            ['reqlist_user', 'user_requests', 'absolute', 1, 1]
        ]
    }
}

# These get added too if we are told to use extended stats.
EXTENDED_ORDER = ['cache']

EXTENDED_CHARTS = {
    'cache': {
        'options': [None, 'Cache Sizes', 'items', 'Unbound', 'unbound.cache', 'stacked'],
        'lines': [
            ['cache_message', 'message_cache', 'absolute', 1, 1],
            ['cache_rrset', 'rrset_cache', 'absolute', 1, 1],
            ['cache_infra', 'infra_cache', 'absolute', 1, 1],
            ['cache_key', 'dnssec_key_cache', 'absolute', 1, 1],
            ['cache_dnscss', 'dnscrypt_Shared_Secret_cache', 'absolute', 1, 1],
            ['cache_dnscn', 'dnscrypt_Nonce_cache', 'absolute', 1, 1]
        ]
    }
}

# This is used as a templates for the per-thread charts.
PER_THREAD_CHARTS = {
    '_queries': {
        'options': [None, '{longname} Queries Processed', 'queries', 'Queries Processed',
                    'unbound.threads.queries', 'line'],
        'lines': [
            ['{shortname}_ratelimit', 'ratelimited', 'absolute', 1, 1],
            ['{shortname}_cachemiss', 'cache_miss', 'absolute', 1, 1],
            ['{shortname}_cachehit', 'cache_hit', 'absolute', 1, 1],
            ['{shortname}_expired', 'expired', 'absolute', 1, 1],
            ['{shortname}_prefetch', 'prefetched', 'absolute', 1, 1],
            ['{shortname}_recursive', 'recursive', 'absolute', 1, 1]
        ]
    },
    '_recursion': {
        'options': [None, '{longname} Recursion Timings', 'seconds', 'Recursive Timings',
                    'unbound.threads.recursion', 'line'],
        'lines': [
            ['{shortname}_recursive_avg', 'average', 'absolute', 1, PRECISION],
            ['{shortname}_recursive_med', 'median', 'absolute', 1, PRECISION]
        ]
    },
    '_reqlist': {
        'options': [None, '{longname} Request List', 'items', 'Request List', 'unbound.threads.reqlist', 'line'],
        'lines': [
            ['{shortname}_reqlist_avg', 'average_size', 'absolute', 1, 1],
            ['{shortname}_reqlist_max', 'maximum_size', 'absolute', 1, 1],
            ['{shortname}_reqlist_overwritten', 'overwritten_requests', 'absolute', 1, 1],
            ['{shortname}_reqlist_exceeded', 'overruns', 'absolute', 1, 1],
            ['{shortname}_reqlist_current', 'current_size', 'absolute', 1, 1],
            ['{shortname}_reqlist_user', 'user_requests', 'absolute', 1, 1]
        ]
    }
}

# This maps the Unbound stat names to our names and precision requiremnets.
STAT_MAP = {
    'total.num.queries_ip_ratelimited': ('ratelimit', 1),
    'total.num.cachehits': ('cachehit', 1),
    'total.num.cachemiss': ('cachemiss', 1),
    'total.num.zero_ttl': ('expired', 1),
    'total.num.prefetch': ('prefetch', 1),
    'total.num.recursivereplies': ('recursive', 1),
    'total.requestlist.avg': ('reqlist_avg', 1),
    'total.requestlist.max': ('reqlist_max', 1),
    'total.requestlist.overwritten': ('reqlist_overwritten', 1),
    'total.requestlist.exceeded': ('reqlist_exceeded', 1),
    'total.requestlist.current.all': ('reqlist_current', 1),
    'total.requestlist.current.user': ('reqlist_user', 1),
    'total.recursion.time.avg': ('recursive_avg', PRECISION),
    'total.recursion.time.median': ('recursive_med', PRECISION),
    'msg.cache.count': ('cache_message', 1),
    'rrset.cache.count': ('cache_rrset', 1),
    'infra.cache.count': ('cache_infra', 1),
    'key.cache.count': ('cache_key', 1),
    'dnscrypt_shared_secret.cache.count': ('cache_dnscss', 1),
    'dnscrypt_nonce.cache.count': ('cache_dnscn', 1)
}

# Same as above, but for per-thread stats.
PER_THREAD_STAT_MAP = {
    '{shortname}.num.queries_ip_ratelimited': ('{shortname}_ratelimit', 1),
    '{shortname}.num.cachehits': ('{shortname}_cachehit', 1),
    '{shortname}.num.cachemiss': ('{shortname}_cachemiss', 1),
    '{shortname}.num.zero_ttl': ('{shortname}_expired', 1),
    '{shortname}.num.prefetch': ('{shortname}_prefetch', 1),
    '{shortname}.num.recursivereplies': ('{shortname}_recursive', 1),
    '{shortname}.requestlist.avg': ('{shortname}_reqlist_avg', 1),
    '{shortname}.requestlist.max': ('{shortname}_reqlist_max', 1),
    '{shortname}.requestlist.overwritten': ('{shortname}_reqlist_overwritten', 1),
    '{shortname}.requestlist.exceeded': ('{shortname}_reqlist_exceeded', 1),
    '{shortname}.requestlist.current.all': ('{shortname}_reqlist_current', 1),
    '{shortname}.requestlist.current.user': ('{shortname}_reqlist_user', 1),
    '{shortname}.recursion.time.avg': ('{shortname}_recursive_avg', PRECISION),
    '{shortname}.recursion.time.median': ('{shortname}_recursive_med', PRECISION)
}

# Used to actually generate per-thread charts.
def _get_perthread_info(thread):
    sname = 'thread{0}'.format(thread)
    lname = 'Thread {0}'.format(thread)
    charts = dict()
    order = []
    statmap = dict()

    for item in PER_THREAD_CHARTS:
        cname = '{0}{1}'.format(sname, item)
        chart = deepcopy(PER_THREAD_CHARTS[item])
        chart['options'][1] = chart['options'][1].format(longname=lname)

        for index, line in enumerate(chart['lines']):
            chart['lines'][index][0] = line[0].format(shortname=sname)

        order.append(cname)
        charts[cname] = chart

    for key, value in PER_THREAD_STAT_MAP.items():
        statmap[key.format(shortname=sname)] = (value[0].format(shortname=sname), value[1])

    return (charts, order, statmap)

class Service(SocketService):
    def __init__(self, configuration=None, name=None):
        # The unbound control protocol is always TLS encapsulated
        # unless it's used over a UNIX socket, so enable TLS _before_
        # doing the normal SocketService initialization.
        configuration['tls'] = True
        self.port = 8935
        SocketService.__init__(self, configuration, name)
        self.ext = self.configuration.get('extended', None)
        self.ubconf = self.configuration.get('ubconf', None)
        self.perthread = self.configuration.get('per_thread', False)
        self.threads = None
        self.order = deepcopy(ORDER)
        self.definitions = deepcopy(CHARTS)
        self.request = 'UBCT1 stats\n'
        self.statmap = deepcopy(STAT_MAP)
        self._parse_config()
        self._auto_config()
        self.debug('Extended stats: {0}'.format(self.ext))
        self.debug('Per-thread stats: {0}'.format(self.perthread))
        if self.ext:
            self.order = self.order + EXTENDED_ORDER
            self.definitions.update(EXTENDED_CHARTS)
        if self.unix_socket:
            self.debug('Using unix socket: {0}'.format(self.unix_socket))
        else:
            self.debug('Connecting to: {0}:{1}'.format(self.host, self.port))
            self.debug('Using key: {0}'.format(self.key))
            self.debug('Using certificate: {0}'.format(self.cert))

    def _auto_config(self):
        if self.ubconf and os.access(self.ubconf, os.R_OK):
            self.debug('Unbound config: {0}'.format(self.ubconf))
            conf = YamlOrderedLoader.load_config_from_file(self.ubconf)[0]
            if self.ext is None:
                if 'extended-statistics' in conf['server']:
                    self.ext = conf['server']['extended-statistics']
            if 'remote-control' in conf:
                if conf['remote-control'].get('control-use-cert', False):
                    self.key = self.key or conf['remote-control'].get('control-key-file')
                    self.cert = self.cert or conf['remote-control'].get('control-cert-file')
                    self.port = self.port or conf['remote-control'].get('control-port')
                else:
                    self.unix_socket = self.unix_socket or conf['remote-control'].get('control-interface')
        else:
            self.debug('Unbound configuration not found.')
        if not self.key:
            self.key = '/etc/unbound/unbound_control.key'
        if not self.cert:
            self.cert = '/etc/unbound/unbound_control.pem'
        if not self.port:
            self.port = 8953

    def _generate_perthread_charts(self):
        tmporder = list()
        for thread in range(0, self.threads):
            charts, order, statmap = _get_perthread_info(thread)
            tmporder.extend(order)
            self.definitions.update(charts)
            self.statmap.update(statmap)
        self.order.extend(sorted(tmporder))

    def check(self):
        # Check if authentication is working.
        self._connect()
        result = bool(self._sock)
        self._disconnect()
        # If auth works, and we need per-thread charts, query the server
        # to see how many threads it's using.  This somewhat abuses the
        # SocketService API to get the data we need.
        if result and self.perthread:
            tmp = self.request
            if sys.version_info[0] < 3:
                self.request = 'UBCT1 status\n'
            else:
                self.request = b'UBCT1 status\n'
            raw = self._get_raw_data()
            for line in raw.splitlines():
                if line.startswith('threads'):
                    self.threads = int(line.split()[1])
                    self._generate_perthread_charts()
                    break
            if self.threads is None:
                self.info('Unable to auto-detect thread counts, disabling per-thread stats.')
                self.perthread = False
            self.request = tmp
        return result

    @staticmethod
    def _check_raw_data(data):
        # The server will close the connection when it's done sending
        # data, so just keep looping until that happens.
        return False

    def _get_data(self):
        raw = self._get_raw_data()
        data = dict()
        tmp = dict()
        for line in raw.splitlines():
            stat = line.split('=')
            tmp[stat[0]] = stat[1]
        for item in self.statmap:
            if item in tmp:
                data[self.statmap[item][0]] = float(tmp[item]) * self.statmap[item][1]
        return data


#import cPickle
#import _pickle as cPickle
from __future__ import print_function
import pickle
import os, sys
import glob, glob
from multiprocessing.dummy import Pool as ThreadPool
#import MySQLdb as mdb
import string
import re
from collections import Counter
from collections import OrderedDict
from Autocorrection import *
import nltk
from nltk.stem.porter import *
from nltk import sent_tokenize, word_tokenize
from nltk.stem.porter import *
from nltk.tag.stanford import StanfordNERTagger
from searchterms import *
import requests
import json
import argparse
import time

stemmer = PorterStemmer()

#apikey = input("input your api >")
#searchterms = search_terms(apikey)
searchterms1 = search_terms("81602583-1f6f-400a-a49f-618975025bec")
searchterms2 = search_terms("81602583-1f6f-400a-a49f-618975025bec")

st = StanfordNERTagger('/media/DataHD/r_phi_corpus/src_fred/english.all.3class.distsim.crf.ser.gz','/media/DataHD/r_phi_corpus/src_fred/stanford-ner.jar')
#st = StanfordNERTagger('F:\\Google Drive\\\Jupyter\\ICHS\\english.all.3class.distsim.crf.ser.gz','F:\\Google Drive\\\Jupyter\\ICHS\\stanford-ner.jar')

stemmer = PorterStemmer()

finpath = "/media/DataHD/baby_notes/"
#finpath = "/media/DataHD/r_phi_corpus/src_fred/input_temp/"
foutpath = "/media/DataHD/r_phi_corpus/src_fred/output_temp/"
#finpath = "input_test\\"
#foutpath = "output_test\\"

#with open("whitelist.txt",'rb') as fin:
#    whitelist = fin.read()
#whitelist = set(whitelist)

with open("whitelist.pkl", "rb") as fp:
#with open("/media/DataHD/r_phi_corpus/src_temp/whitelist.pkl", "rb") as fp:
    #whitelist = pickle.load(fp, encoding="bytes")
    whitelist = pickle.load(fp)
print('length of whitelist: {}'.format(len(whitelist)))

### initialize variables
total_records = 0
phi_containing_records = 0
screened_words = []
screens_per_file = { }
corrected_words = []
corrects_per_file = {}

### configure the regex patterns
pattern=re.compile("[^\w+]") #we're going to want to remove all special characters
pattern_sens = re.compile(r"""\b(
addr|street|lane|ave       # addres
|phone|fax|number|tel      # phone
|ip                                # ip address
|date                      # date
|md                        # MD ID
|age|year[s]?\sold         # age
|ssn|security             #SSN
)""",re.X|re.I)

'''
pattern_time = re.compile(r"""\b(
(\d{4}|\d{2})[-/:]\d{2}([-/:](\d{4}|\d{2}))+    # time-format
)""",re.X|re.I)
'''
start_time_all = time.time()

for f in glob.glob(finpath+"*.txt"):
    with open(f, encoding='utf-8', errors='ignore') as fin:
        #print f
        f_name = f.split("/")[-1]
        #f_name = f.split("\\")[-1]
        f_name = re.findall(r'\d+', f_name)[0]
        print(f_name)
        start_time_single = time.time()
        total_records += 1
        safe = True
        screens_per_file[f_name] = 0
        corrects_per_file[f_name] = 0
        #take the note name for writing
        out_name = 'whiteListed_'+ f_name
        phi_reduced = ''
        searchterms2.search("")
        note = fin.read()
        #sent_end_prev = ' '
        '''
        try:
            note = fin.read()
            # remove note id,e.g. "A30000091        2006-10-15 22:28:00.000 6789784"
            note_id = re.search(r'\s\d+\s', note).group(0)
            phi_note_id = note [:note.index(note_id)+len(note_id)]
            note = note.replace(phi_note_id, 'FILTERED ')
        except:
            continue
        '''
        # remove "-"
        note = sent_tokenize(note.replace('-', ' ').replace('/',' '))
        word_sent = [word_tokenize(sent) for sent in note]
        for sent in word_sent:
            tmp = ' '.join(sent)
            sent_tag = nltk.pos_tag_sents([sent])
            for word in sent_tag[0]:
                word_output = word[0]
                check_flag = 0
                if word[0] not in string.punctuation:
                    word_position = sent.index(word[0])

                    #print(word)
                # keep the number format, others will remove the special chars
                    if word[1] != 'CD':
                        word_output = str(pattern.sub('',word[0])) #actually remove the speical chars

                    try:
                    # nouns or Proper noun | plurals and the first letter is upperclass |
                    # the first letter is upperclass and not in the begining of the sentence
                    # numbers and in the same sentence that contains sensetive words, e.g. address, street, road
                        word_lower = word_output.lower()
                        if ((word[1] == 'NN' or word[1] == 'NNP') or
                        ((word[1] == 'NNS' or word[1] == 'NNPS') and word_output.istitle())):
                    #or ((word[1] == 'CD' or (word[1] == 'JJ') and pattern_time.findall(word)))

                            #print("1")
                            word_ori = word_output
                            autoc = autocorrect(word_output, whitelist)
                            word_modif, check_flag = autoc.autocheck()

                         #   elif word_output.lower().encode("utf8") not in whitelist and stemmer.stem(word_output.lower()).encode('utf8') not in whitelist:
                                #print("2")
                                #print(st.tag([word_output])[0])
                                #name
                            if check_flag != 1:
                                '''
                                if ((st.tag([word_ori])[0][1] != 'PERSON' and st.tag([word_ori])[0][1] != 'LOCATION') or
                                ((st.tag([word_ori])[0][1] == 'PERSON' or st.tag([word_ori])[0][1] != 'LOCATION') and not word_ori.istitle())):
                                    #print(word_output)
                                    searchterms1.search(word_ori)
                                 #   print("5")
                                    if not searchterms1.ifumls():
                                    #      print(word_output)
                                        searchterms2.search(stemmer.stem(word_ori))
                                   #     print("6")

                                        if not searchterms2.ifumls():
                                            if  check_flag == 3:
                                                word_output = word_modif + '***F***'
                                                corrected_words.append(word_ori)
                                                corrects_per_file[f_name] += 1
                                                safe = False
                                            else:
                                                screened_words.append(word_lower)
                                                word_output  = "**FILTERED**"
                                                safe = False
                                                screens_per_file[f_name] += 1
                                        else:
                                            whitelist.add(word_lower)
                                            print(word_ori, 'length of whitelist: {}'.format(len(whitelist)))

                                    else:
                                        whitelist.add(word_lower)
                                        print(word_ori, 'length of whitelist: {}'.format(len(whitelist)))

                                else:
                                    # filter names,locations
                                    #print(word_output, st.tag([word_output])[0][1])
                                    #print("7")
                                    screened_words.append(word_lower)
                                    word_output  = "**FILTERED**"
                                    safe = False
                                    screens_per_file[f_name] += 1
                                    '''
                                screened_words.append(word_lower)
                                word_output  = "**FILTERED**"
                                safe = False
                                screens_per_file[f_name] += 1

                        elif ((word[1] == 'CD' and pattern_sens.findall(tmp)) or (word[1] == 'CD' and len(word_output) > 7)):
                            #print("4")
                             #print(word_output)
                            screened_words.append(word_lower)
                            word_output  = "**FILTERED**"
                            safe = False
                            screens_per_file[f_name] += 1

                           # print(word_output)
                    except:
                        print(word[0])
                        #word_output = "+"+word_output + "+"
                        #print(sent.index(word))
                        #print(nltk.pos_tag(word))
                        pass

                phi_reduced = phi_reduced + ' ' + word_output
        if safe == False:
            phi_containing_records += 1
        fout = foutpath+"whitelisted_nlp_noNER"+f_name+".txt"

        with open(fout,"w") as phi_reduced_note:
            phi_reduced_note.write(phi_reduced)

        print(total_records, "--- %s seconds ---" % (time.time() - start_time_single))

print(total_records, "--- %s seconds ---" % (time.time() - start_time_all))
print('total records processed: {}'.format(total_records))
print('num records with phi: {}'.format(phi_containing_records))
print('length of whitelist: {}'.format(len(whitelist)))
with open("whitelist.pkl", "wb") as fout:
    pickle.dump(whitelist,fout )

screened_dict = Counter(screened_words)
screened_sorted_by_value = OrderedDict(sorted(screened_dict.items(), key=lambda x: x[1], reverse = True))

screens_per_file = OrderedDict(sorted(screens_per_file.items(), key = lambda x: x[1], reverse = True))
# print screens_per_file

# screened_to_list = {k: v for k, v in screened_sorted_by_value.items() if v > 3}
# screened_to_list = list(screened_to_list.keys()) #this is just to use for print/copy if want to examine and add to whitelist

# screened_words_out = set(screened_words) # remove redundancy
# screened_words_out = list(screened_words)
# screened_words_out.sort() # sorts normally by alphabetical order
# screened_words_out.sort(key=len) #sort by length
#print screened_words

# out_dict = open('/media/DataHD/screened_dict.pkl', 'ab+')
# pickle.dump(screened_sorted_by_value, out_dict)
# out_dict.close()

# thefile = open('/media/DataHD/screened_words.txt', 'w')
# for item in screened_words_out:
#   print>>thefile, item

#pickle.dump(screened_dict, open( "output\\words_screened_dict.pkl", "wb" ) )
#pickle.dump(screens_per_file, open( "output\\words_perFile_dict.pkl", "wb" ) )
#pickle.dump(corrects_per_file, open( "output\\corrects_per_file.pkl", "wb" ) )


from src.calculator.FakeParser import FakeParser
from src.calculator.FakeScanner import FakeScanner
from src.database.DbDao import DbDao

# We could also have a history instance variable.
# The calculator will have the responsibility of keeping the history.
# Like that, the calculator will have more responsabilities
# If we want to add an user interface, it needs to be in this class

class Calculator:

    def __init__(self) -> None:
        self.parser = FakeParser()
        self.scanner = FakeScanner()
        self.db = DbDao()
        # For the moment, we don't manage the history
        self.history = []

    def calculate(self, expression_as_string):
        scanned_input = self.scanner.scan(expression_as_string)
        calculation_result = self.parser.parse(scanned_input)
        self.db.addCalcul(expression_as_string, calculation_result)
        return calculation_result


def put(self, pid, record):
        """Handle the sort of the files through the PUT deposit files.

        Expected input in body PUT:

        .. code-block:: javascript

            [
                {
                    "id": 1
                },
                {
                    "id": 2
                },
                ...
            }

        Permission required: `update_permission_factory`.

        :param pid: Pid object (from url).
        :param record: Record object resolved from the pid.
        :returns: The files.
        """
        try:
            ids = [data['id'] for data in json.loads(
                request.data.decode('utf-8'))]
        except KeyError:
            raise WrongFile()

        record.files.sort_by(*ids)
        record.commit()
        db.session.commit()
        return self.make_response(obj=record.files, pid=pid, record=record)


import json
import os

import requests

import config

def create_images_generation(prompt: str, n: int = 1, size: str = '256x256') -> dict:
    '''Create images using Openai APIs.

    Parameters:
        - prompt (str): user query
        - n (int): number of generation
        - size (str): size of the generation, must be square, and the number must be divisible by 8

    Returns: dictionary with information from Openai API call
    '''
    try:
        url = f"{config.BASE_URL}/images/generations"
        payload = {
            "prompt": prompt,
            "n": n,
            "size": size
        }
        headers = {
            "Content-Type": "application/json",
            "Authorization": config.AUTHORIZATION
        }
        response = requests.request("POST", url, json=payload, headers=headers)
        response = json.loads(response.text)
        return response
    except:
        return {}

def create_images_edit(image_path: str, mask_path: str, prompt: str) -> dict:
    '''Create images edits from image and mask file, both files must be png.

    Parameters:
        - image_path (str): image file path
        - mask_path (str): mask file path
        - prompt (str): a prompt that will replace the mask in the original image

    Returns: dictionary with information from Openai API call
    '''
    try:
        image_file_name = os.path.basename(image_path)
        image_extention = image_path.split('.', 1)[-1]
        mask_file_name = os.path.basename(mask_path)
        mask_extention = mask_path.split('.', 1)[-1]
        url = f"{config.BASE_URL}/images/edits"
        files = [
            ('image', (image_file_name, open(image_path, 'rb'), f'image/{image_extention}')),
            ('mask', (mask_file_name, open(mask_path, 'rb'), f'image/{mask_extention}'))
        ]
        headers = {
            "Authorization": config.AUTHORIZATION
        }
        payload = {
            "prompt": prompt
        }
        response = requests.request(
            "POST", url, files=files, headers=headers, data=payload)
        response = json.loads(response.text)
        return response
    except:
        return {}

def create_images_variation(image_path: str, n: int = 1, size: str = '256x256') -> dict:
    '''Create variations of an image.

    Parameters:
        - image_path (str): image file path
        - n (int): number of generation
        - size (str): size of the generation, must be square, and the number must be divisible by 8

    Returns: dictionary with information from Openai API call
    '''
    try:
        file_name = os.path.basename(image_path)
        extention = image_path.split('.', 1)[-1]
        url = f"{config.BASE_URL}/images/variations"
        files = [
            ('image', (file_name, open(image_path, 'rb'), f'image/{extention}'))
        ]
        headers = {
            "Authorization": config.AUTHORIZATION
        }
        payload = {
            "n": n,
            "size": size
        }
        response = requests.request(
            "POST", url, files=files, headers=headers, data=payload)
        response = json.loads(response.text)
        return response
    except:
        return {}


import sys
import heapq

input = sys.stdin.readline

n, m = map(int, input().split())

start, end = map(int, input().split())
graph = [[] for _ in range(n + 1)]
distance = [0] * (n + 1)

for _ in range(m):
    a, b, cost = map(int, input().split())
    graph[a].append((cost, b))
    graph[b].append((cost, a))

def dijkstra():
    q = []
    heapq.heappush(q, (-sys.maxsize, start))

    while q:
        cost, now = heapq.heappop(q)
        cost = -cost

        if distance[now] > cost:
            continue

        for nway, nnode in graph[now]:
            temp = min(cost, nway)
            if distance[nnode] < temp:
                distance[nnode] = temp
                heapq.heappush(q, (-temp, nnode))

dijkstra()
print(distance[end])


'''
This program allows you to measure the speed of the incoming and outgoing speed of your Internet connection.

Author: Fora1ds
Version: 1.0.0
'''

from urllib.request import Request, urlopen
from urllib.parse import urlparse
from Exceptions import *
from json import loads
from time import time

SERVERS_URL = 'https://www.speedtest.net/api/js/servers'
DOWNLOAD_URL = '/download?size=25000000'
UPLOAD_URL = '/upload'

class Format:
    def __init__(self, round_byte=True, units=True, digits=3):
        self.Round = round_byte
        self.Units = units
        self.Digits = digits

    def set_type(self, byte_value, units) -> [float, str]:
        '''
        Set a new byte type:

        1. 4836849.75 -> 38.694 Mbps
        2. 4836849.75 -> 38.694
        3. 4836849.75 -> 39
        '''

        Value = byte_value

        if self.Round == True:
            Value = round(Value, self.Digits)

        if self.Units == True:
            Value = '{} {}'.format(Value, units)

        return Value

    def to_Bps(self, value) -> [float, str]:
        '''
        Convert bytes to Bps
        '''

        return self.set_type(byte_value=value / 0.125, units='Bps')

    def to_Kbps(self, value) -> [float, str]:
        '''
        Convert bytes to Kbps
        '''

        return self.set_type(byte_value=value / 125, units='Kbps')

    def to_Mbps(self, value) -> [float, str]:
        '''
        Convert bytes to Mbps
        '''

        return self.set_type(byte_value=value / 125000, units='Mbps')

    def to_Gbps(self, value) -> [float, str]:
        '''
         Convert bytes to Gbps
        '''

        return self.set_type(byte_value=value / 125000000, units='Gbps')

    def to_Tbps(self, value) -> [float, str]:
        '''
        Convert bytes to Tbps
        '''

        return self.set_type(byte_value=value / 125000000000, units='Tbps')

class SpeedTest:
    def __init__(self):
        # Standard Ookla packet size
        self.PACKET_SIZE = 25000000

        self.select_server(number=0)

    def _request(self, url, content=None, url_method='GET') -> bytes:
        '''
        Sending a request with headers, data and method
        '''

        Url = Request(url, data=content,
            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0',
                     'Accept': 'application/json, text/plain, */*',
                     'Accept-Language': 'en-US,en;q=0.5'}, method=url_method)

        try:
            URL_Request = urlopen(Url).read()
        except :
            raise NoConnectionError(message='Failed to connect to {}'.format(urlparse(url).netloc))

        return URL_Request

    def servers(self) -> list:
        '''
        Get Ookla servers
        '''

        Servers = loads(self._request(url=SERVERS_URL))

        if not Servers:
            raise SpeedTestServersError(message='No Servers Found')

        return Servers

    def select_server(self, number) -> None:
        '''
        Based on the server name, upload and download URLs are generated
        '''

        Server = self.servers()[number]
        Name = Server.get('host')

        self.DOWNLOAD = 'https://{}{}'.format(Name, DOWNLOAD_URL)
        self.UPLOAD = 'https://{}{}'.format(Name, UPLOAD_URL)

    def _get_speed(self, speed) -> float:
        '''
        Each speed is calculated according to the formula:

        Speed = PACKET_SIZE
                -----------
                T2   -   T1
        '''

        Start = time()

        try:
            if speed == 'DOWNLOAD':
                self._request(url=self.DOWNLOAD)
            else:
                self._request(url=self.UPLOAD, content=b'Q' * self.PACKET_SIZE, url_method='POST')
        except :
            raise SpeedTestError(message='Failed to get {} speed'.format(speed.lower()))

        Done = time()

        Result = self.PACKET_SIZE / (Done - Start)

        return Result

    def download(self) -> float:
        '''
        Get download speed
        '''

        return self._get_speed(speed='DOWNLOAD')

    def upload(self) -> float:
        '''
        Get upload speed
        '''

        return self._get_speed(speed='UPLOAD')


from typing import Generator
import praw
from praw import models
import requests
from os import environ

USER_AGENT = "MoringMarkArchiver (by u/hakiergrzonzo)"

session = requests.session()
session.headers.update({"User-Agent": USER_AGENT})

reddit = praw.Reddit(
    user_agent="MoringMarkArchiver (by u/hakiergrzonzo)",
    client_id="xd-riCdiH0xPRTRFIglbCQ",
    client_secret=environ["SECRET"],
)

def get_moring_mark() -> models.Redditor:
    return reddit.redditor("makmark")

def get_new_comics() -> Generator[models.Submission, None, None]:
    mark_mark = get_moring_mark()
    for submission in mark_mark.submissions.new(limit=None):
        if submission.subreddit.display_name != "TheOwlHouse":
            continue
        yield submission


import os
import sqlite3
import shutil
from lib.base_case import BaseCase
import pytest
from lib.logger import Logger

def delete_files_from_db():
    BaseCase.change_dir_to_root()
    db_name = 'FileStorage/FileStorage.db'
    conn = sqlite3.connect(db_name)
    cursor = conn.cursor()
    query = 'DROP TABLE IF EXISTS files'
    try:
        cursor.execute(query)
        conn.commit()
        return True
    finally:
        conn.close()

@pytest.fixture(scope='session', autouse=True)
def preparing_files_for_upload_and_delete_after_session():
    files = BaseCase.get_files()
    if len(files) == 0:
        raise Exception('There are no files in upload_folders')
    yield files
    # delete tests/files_for_upload/
    BaseCase.delete_upload_files(BaseCase.FILES_FOR_UPLOAD)

@pytest.fixture(scope='class', autouse=True)
def delete_files_from_server_setup(request):
    delete_files_from_db()

    def delete_uploaded_files_after_test_teardown():
        file_storage_db_path = 'FileStorage/uploaded_files'
        shutil.rmtree(file_storage_db_path)
        os.mkdir(file_storage_db_path)
        delete_files_from_db()
    request.addfinalizer(delete_uploaded_files_after_test_teardown)

@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def return_test_result_in_log(item, call):
    # execute all hooks to get report object
    outcome = yield
    rep = outcome.get_result()

    # we only look for failed test calls, not setup/teardown
    if rep.when == "call" and rep.failed:
        Logger.add_test_results('Test Failed')


import argparse

def parse():
    parser = argparse.ArgumentParser(
        prog="Bilibili downloader",
        description="Download videos from bilibili.com"
    )
    subparsers = parser.add_subparsers()
    parser_d = subparsers.add_parser("download", aliases=["d"])

    parser_d.add_argument("bvid")

    args = parser.parse_args()

    return args


# Your program asks the user for the subtotal but does not ask the user for the day of the week. Your program gets the day of the week from your computer's operating system.
# Your program correctly computes and prints the discount amount if applicable.
# Your program correctly computes and prints the sales tax amount and the total amount due.

from datetime import datetime

subtotal =0
price = -1
print('Please enter the price and quantity of each item. (enter a quantity of 0 to finish)')
while price != 0:
    price = float(input('What is your items price? $'))
    if(price == 0):
        break
    quantity = int(input('How many of this item? '))
    subtotal += (price * quantity)

print(f'Your subtotal is ${subtotal:.2f}')

day_of_the_week = datetime.now().strftime("%A")
sub_discount = 0
if ( day_of_the_week == 'Tuesday' or day_of_the_week == 'Wednesday'):

    if subtotal > 50 :
        discount = 0.1
        sub_discount = subtotal * discount
        print(f'Discount is ${sub_discount:.2f}')
    else:
        additional_amount = 50-subtotal

        print(f'To recive a 10% discount please spend an additional ${additional_amount:.2f}')

sales_tax = 0.06
tax_total = (subtotal - sub_discount) * sales_tax
print(f'Sales tax is ${tax_total:.2f}')
total_amount = subtotal - sub_discount+tax_total

print(f'Your total amount due is ${total_amount:.2f}')


queue = []
t=0

def q_push(x):
    queue.append(x)

def q_pop():
    if len(queue)==0:
        return -1
    return queue.pop(0)

# Implement stack using queue

def stack_push(x):
    q_push(x)

def stack_pop():
    n=len(queue)
    i=0
    while i<n-1:
        ele = q_pop()
        if ele == -1:
            break
        else:
            q_push(ele)
        i+=1
    q_pop()

stack_push(10)
stack_push(20)
stack_pop()
stack_push(30)
stack_push(40)
stack_pop()

print(queue)


"""汎用関数"""
from datetime import datetime
from pathlib import Path

def simple_format_time(duration):
    """
    秒数をHH時間MM分SS秒にフォーマットする
    """
    hours, remainder = divmod(duration, 3600)
    minutes, seconds = divmod(remainder, 60)
    return f"{int(hours)}時間{int(minutes)}分{int(seconds)}秒"

def create_output_directory(base_output_path: Path) -> Path:
    """
    指定したパスの配下に出力ディレクトリを作成

    出力ディレクトリはYYYYMMDDHHMMSS形式
    中間ディレクトリは自動で作成される
    """
    datetime_str = datetime.now().strftime("%Y%m%d%H%M%S")
    output_path = base_output_path / datetime_str
    output_path.mkdir(parents=True, exist_ok=True)
    return output_path


import argparse
from datetime import datetime, timedelta
from pathlib import Path

parser = argparse.ArgumentParser(
    description="Create log folders (YYYY/MM/DD) for N days into the future."
)
parser.add_argument(
    'num_days', type=int, help="Create folders for this many days"
)

num_days = parser.parse_args().num_days
dates = [datetime.now() + timedelta(days=i) for i in range(num_days)]

def fmt(n):
    return f'{n:02d}'

for date in dates:
    year, month, day = map(fmt, (date.year, date.month, date.day))
    p = Path.cwd().joinpath(year, month, day)
    p.mkdir(exist_ok=True, parents=True)
    p.parent.parent.chmod(0o2755)
    p.parent.chmod(0o2755)
    p.chmod(0o3777)


# List testing
numlist = list()
while True :
    inp = input('Enter a number: ')
    if inp == 'done' : break
    value = float(inp)
    numlist.append(value)

average = sum(numlist) / len(numlist)
print('Average:', average)
# List and define loops
friends = ['Joseph', 'Glenn', 'Sally']
for friend in friends:
    print("Happy Khmer New year: ", friend)
print("Done")


"""Calculating Least Common Multiple using lcm = ab / gcd(a,b)"""

def main():
    a, b = map(int,input().split())
    print(int(a*b / eculidean_gcd(a, b)))

def eculidean_gcd(a, b):
    """Efficient way to calculate GCD"""

    if b == 0:
        return a
    else:
        a = a % b
        return eculidean_gcd(b, a)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author: xuchaoqian
# Created Time: 2020年04月01日 星期三 16时20分42秒
from multiprocessing import Process,Pipe,Queue
from threading import Thread
import cv2
import os
import time
from recognize import recognizer_factory,deeplearning_factory

class Data_app(Process):
    '''
    load data
    '''
    def __init__(self,config,cons):
        Process.__init__(self)
        self.config=config
        self.cons=cons
        self.data_path=config['datapath']
    def run(self):
        for file_name in os.listdir(self.data_path):
            file_path=os.path.join(self.data_path,file_name)
            image_data=cv2.imread(file_path)
            [con.send(image_data) for con in self.cons]
            time.sleep(1)
    def terminate(self):
        #[con.send(None) for con in self.cons]
        [con.close for con in self.cons]
        Process.terminate(self)

class DL_app(Process):
    '''
    app for deep learning
    '''
    def __init__(self,config,con,q):
        Process.__init__(self)
        self.config=config
        self.con=con
        self.q=q
        self.factory=deeplearning_factory()
        self.recognizer=self.factory.create_recognizer(config['fruit'])
    def run(self):
        while not self.con.closed:
            data=self.con.recv()
            result=self.recognizer.recognize(data,self.config)
            self.q.put(result)
    def terminate(self):
        self.con.close()
        Process.terminate(self)

class Imgpro_app(Process):
    '''
    app for image processing
    '''
    def __init__(self,config,con,q):
        Process.__init__(self)
        self.config=config
        self.con=con
        self.q=q
        self.factory=recognizer_factory()
        self.recognizer=self.factory.create_recognizer(config['fruit'])
    def run(self):
        while not self.con.closed:
            data=self.con.recv()
            result=self.recognizer.recognize(data,self.config)
            self.q.put(result)
    def terminate(self):
        self.con.close()
        Process.terminate(self)

class Main_app(Process):
    '''
    main app:
    data load
    deep learning
    image processing
    '''
    def __init__(self,config,q):
        Process.__init__(self)
        self.config=config
        self.con1=Pipe(duplex=False)
        self.con2=Pipe(duplex=False)
        self.dl_output=Queue()
        self.imgpro_output=Queue()
        self.data_app=Data_app(self.config['data_config'],[self.con1[1],self.con2[1]])
        self.dl_app=DL_app(self.config['img_config'],self.con1[0],self.dl_output)
        self.img_app=Imgpro_app(self.config['img_config'],self.con2[0],self.imgpro_output)
        self.output=q
    def run_app(self):
        if not (self.dl_app.is_alive()):
            self.dl_app.start()
        if not (self.img_app.is_alive()):
            self.img_app.start()
        if not (self.data_app.is_alive()):
            self.data_app.start()
        if not (self.is_alive()):
            self.start()
    def run(self):
        while True:
            dl_result=self.dl_output.get()
            img_result=self.imgpro_output.get()
            #merge result
            self.output.put(img_result)
            #send control
    def terminate(self):
        if (self.data_app.is_alive()):
            self.data_app.terminate()
        if (self.dl_app.is_alive()):
            self.dl_app.terminate()
        if (self.img_app.is_alive()):
            self.img_app.terminate()
        if (self.is_alive()):
            Process.terminate(self)

class update_gui_app(Thread):
    '''
    update gui
    '''
    def __init__(self,func,q):
        Thread.__init__(self)
        self.func=func
        self.q=q
    def run(self):
        while True:
            result=self.q.get()
            self.func(result)


from django.urls import path
from . import views
from siteblog import settings
from django.conf.urls.static import static

app_name = 'blog'

urlpatterns=[
    path('', views.home, name='home'),
    path('datascience/', views.datascience, name='datascience'),
    path('webdevelopment/', views.webdevelopment, name='webdevelopment'),
    path('contact/', views.contact, name='contact'),
    path('blog/', views.PostViewList.as_view(), name='post_list'),
    path('<int:year>/<int:month>/<int:day>/<slug:post>/', views.post_detail, name='post_detail')
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


#!/usr/bin/env python
# coding: utf-8

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression

def getmacros():
    '''
    Returns a dataframe of excess returns from macro factors
    '''
    xls_file = r'C:\Users\bmcdonald\Desktop\MSCI Factors.xlsm'
    xls = pd.ExcelFile(xls_file)
    macros = xls.parse('AssetClass', index_col=0, header=None, skiprows=1,
                       names='MKT INT CRD HY CUR RF'.split())
    macros['CRD'] = 0.5*macros.CRD + 0.5*macros.HY  #TurnsC CRD returns into half IG half HY
    macros.drop('HY', axis=1, inplace=True)
    macros['RF'] = macros['RF'] / 12  #3m treasuries are annualized, so this converts to monthly
    macros.index = pd.to_datetime(macros.index).strftime('%Y-%m')
    macros = isolate_credit(macros)  #Residualizes credit returns
    macros.dropna(inplace=True)
    return macros

def isolate_credit(df, rolling=120, halflife=36):
    '''
    Takes df of macro factors as well as optional rolling periods, model (wls), and halflife arguments
    Returns df of macro factors that have isolated credit returns
    '''

    weights = [1 * (0.5 ** (1 / halflife)) ** i for i in range(rolling)][::-1]

    wls = LinearRegression()

    betas = [wls.fit(df[['MKT', 'INT']].iloc[i:i + rolling],
                     df['CRD'].iloc[i:i + rolling],
                     sample_weight=weights).coef_
             for i in np.arange(df.shape[0] - rolling)]

    mkt_crd_betas = [b[0] for b in betas]
    int_crd_betas = [b[1] for b in betas]

    # This section uses betas to adjust credit and commodity returns to isolate those factor returns
    df = df.iloc[rolling:].copy()
    df.loc[:, 'CRD'] = df['CRD'] - df['MKT'] * mkt_crd_betas - df['INT'] * int_crd_betas

    return df

def macro_prep(return_series, macros, start=None, end=None):
    returns = pd.DataFrame(return_series)

    # Opens macro factors, concats return series, subs rf, drops rf

    df = pd.concat([macros, returns], axis=1, sort=False)
    df.iloc[:, -1] = df.iloc[:, -1] - df.RF

    # Adjust Date range here if applicable
    df = df.loc[start:end]
    df.dropna(inplace=True)

    ex_macros = df.drop('RF', axis=1)

    # Creates separate df for excess returns
    ex_returns = ex_macros.iloc[:, -1]
    ex_returns = pd.DataFrame(ex_returns)
    assert ex_returns.shape[0] > 0, 'Manager returns df is empty'

    ex_macros = ex_macros.iloc[:, :4]
    assert ex_macros.shape[0] > 0, 'Macro returns df is empty'

    return ex_returns, ex_macros

# Opens Excel file with returns
xls_file = r'C:\Users\bmcdonald\Desktop\OMG Manager Returns.xlsx'
xls = pd.ExcelFile(xls_file)

# his section sets month-end date of dats
import datetime
today = datetime.date.today()
first = today.replace(day=1)
lastMonth = first - datetime.timedelta(days=1)
date = lastMonth.strftime("%Y-%m")

# Pulls macro factor data and sets ols model
macros = getmacros()
ols = LinearRegression()

# Creates an empty df for betas and errors
beta_df = pd.DataFrame(columns=['MKT INT CRD CUR'.split()])
excess_returns_df = pd.DataFrame()
predict_df = pd.DataFrame()
error_df = pd.DataFrame()

# Loops through each sheet of returns
for category in 'Index Fund Alt'.split():

    cat = xls.parse(category, index_col=0, skiprows=1)
    cat.index = pd.to_datetime(cat.index).strftime('%Y-%m')
    name = cat.columns  #Creates list of series names

    start = cat.index[120]  #Sets start date to 10 years ago

    betas = []

    #Loops through each series name
    for n in name:
        # Uses factorprep to calc excess manager and factor returns
        ex_returns, ex_macros = macro_prep(cat[n], macros, start)

        model = ols.fit(ex_macros.values,
                        ex_returns.values)

        beta = model.coef_.flatten()
        betas.append(beta)

        errors = ex_returns - model.predict(ex_macros)
        predictions = ex_returns - errors

        excess_returns_df = pd.concat([excess_returns_df, ex_returns], axis=1, sort=False)
        predict_df = pd.concat([predict_df, predictions], axis=1, sort=False)
        error_df = pd.concat([error_df, errors], axis=1, sort=False)

    df = pd.DataFrame(np.stack(betas), columns=['MKT INT CRD CUR'.split()], index=name)

    beta_df = pd.concat([beta_df, df], sort=False)

path = r'O:\Investment Research\Ben\Tools and Concepts WIP\Macro Factors'

beta_df.to_excel(path + f'\Macro Betas.xlsx')

import os
newpath = r'O:\Investment Research\Ben\Tools and Concepts WIP\Macro Factors\\' + date
if not os.path.exists(newpath):
    os.makedirs(newpath)

beta_df.to_excel(newpath + f'\ {date} - Macro Betas.xlsx')

excess_returns_df.sort_index(inplace=True)
predict_df.sort_index(inplace=True)
error_df.sort_index(inplace=True)

excess_returns_df.to_excel(newpath + f'\ {date} - Excess Returns.xlsx')
predict_df.to_excel(newpath + f'\ {date} - Model Predictions.xlsx')
error_df.to_excel(newpath + f'\ {date} - Errors.xlsx')
error_df.corr().to_excel(newpath + f'\ {date} - Macro Error Correlation.xlsx')


#
# LIBTBX_SET_DISPATCHER_NAME cxi.plotcv
#
"""Main idea:  go through the integration log files and grep out the difference
   vectors between observed and predicted spot positions.  Plot these "correction
   vectors", but do so separately for each ASIC tile, giving an independent
   check on whether a tile is positioned properly by the tile_translation
   parameters."""
from __future__ import absolute_import, division, print_function

import iotbx.phil
import sys

from xfel.merging.database.merging_database import mysql_master_phil
master_phil="""
run_numbers = None
  .type = ints
  .help = List of run numbers to be aggregated together to make the plots.
  .help = Set run_numbers to None IF the outdir_template field gives a single file containing all log results.
outdir_template = None
  .type = str
  .help = Full path directory containing the stdout logs, with %%04d tag for run number
bravais_setting_id = None
  .type = int
  .help = ID number for the Bravais setting of interest (Labelit format).  eg, 1=triclinic, 12=hexagonal
show_plots = False
  .type = bool
  .help = Show graphical plots using matplotlib
colormap = False
  .type = bool
  .help = Show colormap correction vector plots
""" + mysql_master_phil

#-----------------------------------------------------------------------
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.run_numbers is None and work_params.outdir_template is None) or
      (work_params.outdir_template is None) or
      (work_params.bravais_setting_id is None)) :
    from libtbx.utils import Usage
    raise Usage("cxi.plotcv "
                "run_numbers=16,17,18,19,20,21,22,23,24,25,26,27,71,72,73 "
                "outdir_template=/reg/data/ana11/cxi/cxi49812/scratch/april_2012/r%%04d/042/stdout "
                "bravais_setting_id=5")
  if work_params.show_plots is True:
    from matplotlib import pyplot as plt # special import

  from xfel.cxi.correction_vector_plot import run_correction_vector_plot
  run_correction_vector_plot(work_params)

  return None

if (__name__ == "__main__"):

  result = run(args=sys.argv[1:])


import os
import re
from bs4 import BeautifulSoup
from sharedutils import errlog
from parse import appender
import json

def main():
    for filename in os.listdir('source'):
        try:
            if filename.startswith('ransomhouse-zoh'):
                html_doc='source/'+filename
                file=open(html_doc, 'r')
                soup=BeautifulSoup(file,'html.parser')
                jsonpart= soup.pre.contents # type: ignore
                data = json.loads(jsonpart[0]) # type: ignore
                for element in data['data']:
                    title = element['header']
                    website = element['url']
                    description = re.sub(r'<[^>]*>', '',element['info'])
                    # stdlog(title)
                    appender(title, 'ransomhouse', description,website)
                file.close()
        except:
            errlog('ransomhouse: ' + 'parsing fail')
            pass


#import socket module
from socket import *
import sys # In order to terminate the program

serverSocket = socket(AF_INET, SOCK_STREAM)

#Prepare a server socket
HOST = '10.0.0.2'
PORT = 2010
serverSocket.bind((HOST, PORT))
# Listen to at most 1 connection at a time
serverSocket.listen(1)

while True:
    #Establish the connection
    print('Ready to serve...')
    connectionSocket, addr =  serverSocket.accept()

    try:
        message = connectionSocket.recv(1024)
        filename = message.split()[1]
        f = open(filename[1:])

        outputdata = f.read()
        #Send one HTTP header line into socket
        connectionSocket.send("HTTP/1.1 200 OK\r\n".encode())
        connectionSocket.send("Content-Type: text/html\r\n".encode())
        connectionSocket.send("\r\n".encode())

        #Send the content of the requested file to the client
        for i in range(0, len(outputdata)):
            connectionSocket.send(outputdata[i].encode())
        connectionSocket.send("\r\n".encode())

        connectionSocket.close()
    except IOError:
        #Send response message for file not found
        connectionSocket.send("HTTP/ 1.1 404 Not Found\r\n".encode())
        connectionSocket.send("Content-Type: text/html\r\n".encode())
        connectionSocket.send("\r\n".encode())
        connectionSocket.send("
404 Not Found
\r\n".encode())

serverSocket.close()
sys.exit()#Terminate the program after sending the corresponding data


from PIL import Image
import sys, os
import datetime
import random

# print(parr)
thresh = 5

def val(c, hue):
	if hue:
		return brightness(c);
	return gethue(c);

def brightness(c):
	return (c[0] + c[1] + c[2])/3;

def cont(st,curr, hue):
	return abs(val(st, hue) - val(curr, hue)) <= thresh;

def gethue(col):
	hsv = RGB_2_HSV(col);
	return hsv[0];

def sort(carr,hue, flipbool):#sort the array by brightness
	flip = 1;
	if flipbool:
		flip = posorneg();#if they want alternating, we assign random value
	if hue:
		return sorted(carr, key=lambda el: gethue(el) * flip);#sort by brightness
	# return sorted(carr, key=lambda el: gethue(el[0]));#sort by brightness
	return sorted(carr, key=lambda el: brightness(el) * flip);#sort by brightness

def posorneg():
	if random.random() > 0.5:
		return -1;
	return 1;

def RGB_2_HSV(RGB):# RGB-HSL code taken from http://stackoverflow.com/a/24153899/2009336
    ''' Converts an integer RGB tuple (value range from 0 to 255) to an HSV tuple '''

    # Unpack the tuple for readability
    R, G, B = RGB

    # Compute the H value by finding the maximum of the RGB values
    RGB_Max = max(RGB)
    RGB_Min = min(RGB)

    # Compute the value
    V = RGB_Max;
    if V == 0:
        H = S = 0
        return (H,S,V)

    # Compute the saturation value
    S = 255 * (RGB_Max - RGB_Min) // V

    if S == 0:
        H = 0
        return (H, S, V)

    # Compute the Hue
    if RGB_Max == R:
        H = 0 + 43*(G - B)//(RGB_Max - RGB_Min)
    elif RGB_Max == G:
        H = 85 + 43*(B - R)//(RGB_Max - RGB_Min)
    else: # RGB_MAX == B
        H = 171 + 43*(R - G)//(RGB_Max - RGB_Min)

    return (H, S, V)

def magic(hue, thresh=50, flip=True):#if hue is true, we sort by hue

	if len(sys.argv) > 1:
		fname = sys.argv[1];
	else:
		fname = "n.jpg"
	img = Image.open(fname);

	parr = img.load()

	for i in range(img.size[0]):	# for every pixel:
		start = 0;
		length = int(random.random() * img.size[1] / 20 + img.size[1] / 10)
		for j in range(img.size[1]):

			# parr[i,j] = (i, j, 100) # set the colour accordingly

			tp = parr[i,j];

			if (not cont(tp, parr[i,start], hue) and abs(start - j) > length) or j + 1 == img.size[1]:#broke the chain or last pixel
				shades = [];
				for e in range(start, j):
					shades.append(parr[i,e]);

				newshades = sort(shades[:], hue, flip);
				for e in range(len(newshades)):
					parr[i,e+start] = newshades[e];

				start = j+1
				length = int(random.random() * img.size[1] / 20 + img.size[1] / 10)

			# parr[i,j] = (0,100,100)

			# if (len(shades) < 100 or cont(parr[i,start], tp)) and len(shades) < 500:#if it's in threshold, add to array
			# 	shades.append( (tp, i, j) );
			# else:#we're done, apply sorted array
			# 	newshades = sort(shades);

			# 	for ind in range(len(shades)):#element is (color, i, j)
			# 		parr[newshades[ind][1],newshades[ind][2]] = newshades[ind][0];#applies new colors

			# 	shades = [];#clear old shades
			# 	start = j;
	img.show()

	detail = "brightness"
	if hue:
		detail = "hue"

	img.save(sys.argv[1] + str(datetime.datetime.now()).replace(":","") + detail + ".jpg", "JPEG");

magic(True);
magic(False);# a lot more smooth

# img = Image.merge(img.mode, parr)


from EndCompute import DDPG
from Front import Viewer
from evaluator import Evaluator
from copy import deepcopy
import util
import time

#   在gym环境中observation观测变量   3个值
#   动作是一个值
#   具体参考      https://www.jianshu.com/p/af3a7853268f

def train(agent, env, evaluate):
    #validate_episodes = evaluate.interval
    output = evaluate.save_path
    validate_steps = 200

    # agent.is_training = True  # 是不是训练状态
    step = episode_steps = 0   # episode 当前所处的回合       episode_steps 回合步数     删除了一个episode
    episode_reward = 0.  # 每一个回合的奖励总和
    num_iterations = 10000  # 一共训练多少回合
    max_episode_length = 300   # 每一个回合最大步进长度

    all_observations = None  # 环境状态，观察值
    done = False
    # train_id = None

    while step < num_iterations:    # 回合数

        step += 1     # 回合数
        while not done:
            # train_id  是指当前回合中训练的对象id
            if all_observations is None:  # 初始化环境状态  和  智能体的初始状态，一个新的回合
                all_observations = env.reset()
                train_observation = all_observations[env.train_id]
                agent.reset(train_observation)
            actions = {}
            if step <= 20:  # steop表示已经训练了多少回合    在一定的回合中采用随机动作填充刚开始的网络
                for k, v in all_observations.items():
                    actions[k] = agent.random_action()
            else:
                for k ,v in all_observations.items():
                    actions[k] = agent.select_action(v)

            next_all_observations, train_reward, done = env.step(**actions)  # 传进去每个对象对应 的动作，返回特定id的学习成果
            next_train_observation = next_all_observations[env.train_id]
            next_train_observation = deepcopy(next_train_observation)  # 深层复制，对原对象任何修改都不会同步

            if episode_steps >= max_episode_length:
                done = True

            agent.observe(train_reward, next_train_observation, done)   # 存储训练状态
            if step > 50:  #和上面选动作一致
                agent.update_policy()
            # [optional] save intermideate model    保存中间的模型数据
            if step % int(num_iterations/2) == 0:    # 如果训练过了3分之1，则保存模型参数
                agent.save_model(output)

            # update
            episode_steps += 1   #在这一回合中前进一步，一个回合的每一步
            episode_reward += train_reward     # 在一个回合中总共的奖励值，越高越好
            all_observations = next_all_observations

        #env.saveAllShips()
        #time.sleep(10)
        # [optional] evaluate          这里检测一下当前的训练结果
        if step % validate_steps == 0:
            policy = lambda x: agent.select_action(x, decay_epsilon=False)
            validate_reward = evaluate(env, policy, debug=False)    #########################这里是阶段性验证，判断是否训练成功
            util.prYellow('中间检测运行效果[Evaluate] Step_{}: mean_reward:{}'.format(step, validate_reward))

        #step += 1     # 回合数
        #if done: # end of episode    当一个回合超过了特定的步数，则认为一个回合完成
        util.prGreen('回合完结:回合中的总步数{}: episode_reward:{} steps:{}'.format(episode_steps, episode_reward, step))  # 第几个回合  回合奖励  这个回合步数
        agent.memory.append(   #  这里应该是补充存储，上面的   obverse已经存储了每一步内容
            train_observation,
            agent.select_action(train_observation),
            0, True
        )

        # reset
        done = False
        all_observations = None
        episode_steps = 0
        episode_reward = 0.
        #episode += 1  #   总体回合数
    #  补充的，最后保存模型参数
    print("保存模型参数")
    agent.save_model(output)

def test(validate_episodes, agent, env, evaluate, model_path):

    agent.load_weights(model_path)
    #agent.is_training = False
    agent.eval()
    policy = lambda x: agent.select_action(x, decay_epsilon=False)  # 不衰减  decay_epsilon

    for i in range(validate_episodes):
        validate_reward = evaluate(env, policy, debug=True, save=False)
        util.prYellow('[Evaluate] #{}: mean_reward:{}'.format(i, validate_reward))

    pass

if __name__ == "__main__":

    isTraining = True   # 训练参数/使用训练好的参数计算动作

    validate_episodes = 10    # 回合，一整个回合
    validate_steps = 20   # 每一个回合最大步数，验证需要的步数

    output = "output"   # 输出文件夹
    max_episode_length = 300   # 每一个回合最大步数

    env = Viewer()
    agent = DDPG(env.state_dim, env.action_dim)   # 环境和动作的维度
    evaluate = Evaluator(validate_episodes, validate_steps, output, max_episode_length)

    if isTraining:
        train(agent, env, evaluate)
    else:
        test(validate_episodes, agent, env, evaluate, output)
    print("train over")


# Question 1

# take input from user

num_1 = int(input( 'Please enter a number : '))
cntr = 0
total = 0
while total + cntr + 1 <= num_1 :
    cntr += 1
    total += cntr

for i in range(1,cntr+1) :
    print('#' * i)

rmn_brk = num_1 - total

print( 'Remaining bricks:', rmn_brk )

# end

#------------------------------------------------------------------------------------------------------------------------------------------

# Question 2

# take input from user

list_1 = input(' Please enter a string where multiple numbers separated by commas : ')
list_1 = [int(x) for x in list_1.split(', ')]

list_1.sort()
print( 'Sorted list:', list_1 )

list_2 = []

for i in range(len(list_1)) :
    if i == 0 :
        if list_1[i+1] - list_1[i] <= 20 :
            list_2.append(list_1[i])
        else :
            list_2.append(list_1[i+1])
    elif list_1[i] - list_1[i-1] <= 20 :
        list_2.append(list_1[i])

print( 'Criteria pass:', list_2 )

total = 0

for i in list_2 :
    total += i

avg = total / len(list_2)

print( 'Average:', avg )

# end


#Exports as csv file with how oversubscribed a class is.
# %run filename to run

from esp.program.models import Program
import csv

PROGRAM_NAME = "Splash! 2013"
DIRECTORY = "/home/shulinye/" #where you want the file saved.

cls = Program.objects.get(name=PROGRAM_NAME).classes()

csvwriter = csv.writer(open(DIRECTORY+"oversubscribed.csv", "wb"), delimiter='|')

csvwriter.writerow(["Name", "Blocks", "# Students Interested", "Max Size", "#_Students/Size"])

for c in cls:
    n = c.studentsubjectinterest_set.all().count()
    m = c.max_students()

    csvwriter.writerow([c.title.encode('ascii', 'ignore'), (', '.join(c.prettyblocks())).encode('ascii', 'ignore'), n, m, n*1.0/(m+1)])


#!/usr/bin/python3
"""Adds the State object “Louisiana” to the database.

Here we use Session as interface to the database.

Args:
    user (str): mysql username
    pass (str): mysql password
    database (str): mysql database name
"""
import sys
from model_state import Base, State
from sqlalchemy.orm import Session
from sqlalchemy import create_engine

if __name__ == "__main__":
    engine = create_engine('mysql+mysqldb://{}:{}@localhost/{}'
                           .format(sys.argv[1], sys.argv[2],
                                   sys.argv[3]), pool_pre_ping=True)

    Base.metadata.create_all(engine)
    session = Session(engine)

    louisiana = State(name='Louisiana')
    session.add(louisiana)
    session.commit()
    print(louisiana.id)
    session.close()