import os
import logging
import json
import numpy as np
from PIL import Image
from pycocotools.coco import COCO
import pycocotools.mask as maskUtils

from torch.utils.data import Dataset
logger = logging.getLogger(__name__)

class DataWrapper:
    """Simple data wrapper."""

    def __init__(self, data):
        """Initialize DataWrapper.

        Args:
            data (np.array): numpy array
        """
        self.data = data

class BoxLabelVOC(Dataset):
    """Base class for loading COCO format labels."""

    def __init__(self, ann_path, img_data_dir,
                 min_obj_size=0, max_obj_size=1e10,
                 transform=None, cfg=None,
                 **kwargs):
        """Initialize dataset.

        Args:
            ann_path (str): annotation file in json format
            img_data_dir (str): raw image directory
            min_obj_size (float): min object size
            max_obj_size (float): max object size
            transform (transform.Compose): data augmentation methods
            cfg (Hydra config): Hydra configurations
        """
        self.cfg = cfg
        self.ann_path = ann_path
        self.img_data_dir = img_data_dir
        self.min_obj_size = min_obj_size
        self.max_obj_size = max_obj_size
        self.transform = transform
        self.coco = COCO(ann_path)
        self._filter_imgs()
        self.get_category_mapping()

    def get_category_mapping(self):
        """Map category index in json to 1 based index."""
        self.cat_mapping = dict([i, i] for i in range(1, 21))

    def _filter_imgs(self):
        """Filter out bboxes based on area and H/W range."""
        anns = self.coco.dataset['annotations']
        filtered_anns = []
        for ann in anns:
            # query image info
            image_info = self.coco.loadImgs(ann['image_id'])[0]
            # check if bbox is out of bound
            is_correct_bbox = ann['bbox'][0] >= 0 and ann['bbox'][1] >= 0 and \
                (ann['bbox'][0] + ann['bbox'][2]) <= image_info['width'] and \
                (ann['bbox'][1] + ann['bbox'][3]) <= image_info['height']
            area = ann['bbox'][2] * ann['bbox'][3]
            # check if bbox area is within range
            is_correct_area = self.max_obj_size > area > self.min_obj_size
            # additionally, check bbox w/h > 2
            if is_correct_bbox and is_correct_area and ann['bbox'][2] > 2 and ann['bbox'][3] > 2:
                filtered_anns.append(ann)
        self.coco.dataset['annotations'] = filtered_anns
        num_filtered = len(self.coco.dataset['annotations']) - len(filtered_anns)
        if num_filtered > 0:
            print("***********************************")
            print(f"WARNING: {num_filtered} bboxes were filtered out.")
            print("***********************************")

    def __len__(self):
        """Total number of bboxes."""
        return len(self.coco.getAnnIds())

    def __getitem__(self, idx):
        """Per item."""
        ann = self.coco.dataset['annotations'][idx]
        img_info = self.coco.loadImgs(ann['image_id'])[0]
        h, w, file_name = img_info['height'], img_info['width'], img_info['file_name']
        img = self.get_image(file_name)

        # box mask
        mask = np.zeros((h, w))
        bbox = ann['bbox']
        x0, y0, x1, y1 = int(bbox[0]), int(bbox[1]), int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])
        mask[y0:y1 + 1, x0:x1 + 1] = 1

        data = {
            'image': img, 'mask': mask,
            'height': h, 'width': w,
            'category_id': ann['category_id'],
            'bbox': np.array([bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]], dtype=np.float32),
            'compact_category_id': self.cat_mapping[int(ann['category_id'])],
            'id': ann['id']
        }

        if self.transform is not None:
            data = self.transform(data)

        return data

    def get_image(self, file_name):
        """Load image.

        Args:
            file_name (str): relative path to an image file.
        Return:
            image (PIL image): loaded image
        """
        image = Image.open(os.path.join(self.img_data_dir, file_name)).convert('RGB')
        return image

class InstSegVOC(BoxLabelVOC):
    """Class for loading COCO format labels with instance segmentation masks."""

    def __init__(self, *args, load_mask=True, **kwargs):
        """Initialize dataset with segmentation groundtruth.

        Args:
            load_mask (bool): whether to load instance segmentation annotations
        """
        super().__init__(*args, **kwargs)
        self.load_mask = load_mask
        if load_mask:
            for ann in self.coco.dataset['annotations']:
                if not ann.get('segmentation', None):
                    raise ValueError(
                        "Please check your annotation file, "
                        "as not all annotations contain segmentation info. "
                        "Or set load_mask to False.")
        self.get_category_mapping()

    def __getitem__(self, idx):
        """Per item."""
        ann = self.coco.dataset['annotations'][idx]
        img_info = self.coco.loadImgs(ann['image_id'])[0]
        h, w, file_name = img_info['height'], img_info['width'], img_info['file_name']
        img = self.get_image(file_name)

        # box mask
        boxmask = np.zeros((h, w))
        bbox = ann['bbox']
        x0, y0, x1, y1 = int(bbox[0]), int(bbox[1]), int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])
        boxmask[y0:y1 + 1, x0:x1 + 1] = 1

        data = {'image': img, 'boxmask': boxmask,
                'height': h, 'width': w,
                'category_id': ann['category_id'],
                'bbox': np.array(
                    [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]], dtype=np.float32),
                'compact_category_id': self.cat_mapping[int(ann['category_id'])],
                'id': ann['id'],
                'image_id': ann['image_id']}

        if self.load_mask:
            # mask = np.ascontiguousarray(
            #     maskUtils.decode(maskUtils.frPyObjects(ann['segmentation'], h, w)))
            # polygons
            if isinstance(ann['segmentation'], list):
                rles = maskUtils.frPyObjects(ann['segmentation'], h, w)
                rle = maskUtils.merge(rles)
            elif 'counts' in ann['segmentation']:
                # e.g. {'counts': [6, 1, 40, 4, 5, 4, 5, 4, 21], 'size': [9, 10]}
                if isinstance(ann['segmentation']['counts'], list):
                    rle = maskUtils.frPyObjects(ann['segmentation'], h, w)
                else:
                    rle = ann['segmentation']
            else:
                raise ValueError('Please check the segmentation format.')
            mask = np.ascontiguousarray(maskUtils.decode(rle))
            if len(mask.shape) > 2:
                mask = mask.transpose((2, 0, 1)).sum(0) > 0
            mask = mask.astype(np.uint8)

            data['gtmask'] = DataWrapper(mask)
            data['mask'] = mask

        if self.transform is not None:
            data = self.transform(data)

        return data

class InstSegVOCwithBoxInput(InstSegVOC):
    """Class for loading bbox inputs with instance segmentation masks."""

    def __init__(self,
                 ann_path,
                 img_data_dir,
                 min_obj_size=0,
                 max_obj_size=1e10,
                 transform=None,
                 load_mask=True,
                 box_inputs=None):
        """Init."""
        self.load_mask = load_mask
        self.ann_path = ann_path
        self.img_data_dir = img_data_dir
        self.min_obj_size = min_obj_size
        self.max_obj_size = max_obj_size
        self.transform = transform
        self.coco = COCO(ann_path)
        self._filter_imgs()
        self.get_category_mapping()
        with open(box_inputs, "r") as f:
            self.val_coco = json.load(f)

    def __len__(self):
        """Number of samples."""
        return len(self.val_coco)

    def __getitem__(self, idx):
        """Per item."""
        ann = self.val_coco[idx]
        img_info = self.coco.loadImgs(ann['image_id'])[0]
        h, w, file_name = img_info['height'], img_info['width'], img_info['file_name']
        img = self.get_image(file_name)

        # box mask
        boxmask = np.zeros((h, w))
        bbox = np.array(ann['bbox'])
        x0, y0, x1, y1 = int(bbox[0]), int(bbox[1]), int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])
        boxmask[y0:y1 + 1, x0:x1 + 1] = 1

        if 'id' not in ann.keys():
            _id = hash(str(ann['image_id']) + ' ' + str(x0) + ' ' + str(x1) + ' ' + str(y0) + ' ' + str(y1))
        else:
            _id = ann['id']

        data = {'image': img, 'boxmask': boxmask,
                'height': h, 'width': w,
                'category_id': ann['category_id'],
                'bbox': np.array(
                    [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]], dtype=np.float32),
                'compact_category_id': self.cat_mapping[int(ann['category_id'])],
                'id': _id,
                'image_id': ann['image_id'],
                'score': ann['score']}

        if self.load_mask:
            mask = np.ascontiguousarray(maskUtils.decode(ann['segmentation']))
            if len(mask.shape) > 2:
                mask = mask.transpose((2, 0, 1)).sum(0) > 0
            mask = mask.astype(np.uint8)

            data['gtmask'] = DataWrapper(mask)
            data['mask'] = mask

        if self.transform is not None:
            data = self.transform(data)

        return data


from __future__ import absolute_import
import logging
import hashlib
import json
from datetime import datetime, timedelta
from functools import partial
from random import randint

from rq.job import Job
from rq_scheduler import Scheduler

from redash import settings, rq_redis_connection
from redash.tasks import (
    sync_user_details,
    refresh_queries,
    empty_schedules,
    refresh_schemas,
    cleanup_query_results,
    purge_failed_jobs,
    version_check,
    send_aggregated_errors,
)

logger = logging.getLogger(__name__)

rq_scheduler = Scheduler(
    connection=rq_redis_connection, queue_name="periodic", interval=5
)

def job_id(kwargs):
    metadata = kwargs.copy()
    metadata["func"] = metadata["func"].__name__

    return hashlib.sha1(json.dumps(metadata, sort_keys=True).encode()).hexdigest()

def prep(kwargs):
    interval = kwargs["interval"]
    if isinstance(interval, timedelta):
        interval = int(interval.total_seconds())

    kwargs["interval"] = interval
    kwargs["result_ttl"] = kwargs.get("result_ttl", interval * 2)

    return kwargs

def schedule(kwargs):
    rq_scheduler.schedule(scheduled_time=datetime.utcnow(), id=job_id(kwargs), **kwargs)

def periodic_job_definitions():
    jobs = [
        {"func": refresh_queries, "interval": 30, "result_ttl": 600},
        {"func": empty_schedules, "interval": timedelta(minutes=60)},
        {
            "func": refresh_schemas,
            "interval": timedelta(minutes=settings.SCHEMAS_REFRESH_SCHEDULE),
        },
        {
            "func": sync_user_details,
            "timeout": 60,
            "interval": timedelta(minutes=1),
        },
        {"func": purge_failed_jobs, "interval": timedelta(days=1)},
        {
            "func": send_aggregated_errors,
            "interval": timedelta(minutes=settings.SEND_FAILURE_EMAIL_INTERVAL),
        },
    ]

    if settings.VERSION_CHECK:
        jobs.append({"func": version_check, "interval": timedelta(days=1)})

    if settings.QUERY_RESULTS_CLEANUP_ENABLED:
        jobs.append({"func": cleanup_query_results, "interval": timedelta(minutes=5)})

    # Add your own custom periodic jobs in your dynamic_settings module.
    jobs.extend(settings.dynamic_settings.periodic_jobs() or [])

    return jobs

def schedule_periodic_jobs(jobs):
    job_definitions = [prep(job) for job in jobs]

    jobs_to_clean_up = Job.fetch_many(
        set([job.id for job in rq_scheduler.get_jobs()])
        - set([job_id(job) for job in job_definitions]),
        rq_redis_connection,
    )

    jobs_to_schedule = [
        job for job in job_definitions if job_id(job) not in rq_scheduler
    ]

    for job in jobs_to_clean_up:
        logger.info("Removing %s (%s) from schedule.", job.id, job.func_name)
        rq_scheduler.cancel(job)
        job.delete()

    for job in jobs_to_schedule:
        logger.info(
            "Scheduling %s (%s) with interval %s.",
            job_id(job),
            job["func"].__name__,
            job.get("interval"),
        )
        schedule(job)


import torch

# make vectors unit length
def normalize(vec, dim, keepLength=False, eps=0):
  length = vec.norm(dim=dim, keepdim=True)
  unit = vec / (length + eps)
  if keepLength:
    return unit, length
  return unit

# compute a gradient, possibly with AMP
def grad(inputs, outputs, gradScaler=None):
  if gradScaler is not None:
    outputs = gradScaler.scale(outputs)
  out = torch.autograd.grad(outputs=outputs,
                            inputs=inputs,
                            grad_outputs=torch.ones_like(outputs),
                            create_graph=True)[0]
  if gradScaler is not None:
    return out / gradScaler.get_scale()
  return out

# uniform random value in a range
def rand_range(rnge, shape, device):
  a, b = rnge
  return torch.rand(shape, device=device) * (b - a) + a

# iterate over count objects in batches, where count possibly not divisible by batch
def iterBatch(count, batch):
  steps = count // batch
  for i in range(steps + 1):
    if i == steps:
      if count % batch == 0:
        break
      batch = count % batch

    yield i * batch, batch


from flask import Blueprint, request, json
from flask.json import jsonify
from data.data import ProductInfoData, ProductData

product_blueprint = Blueprint('product', __name__)

@product_blueprint.route('/<category_id>', methods=["GET"])
def getProducts(category_id):
    retData = [d for d in ProductData if d['category_id'] == int(category_id)]
    return jsonify({'products': retData}), 200

@product_blueprint.route('/product/<product_id>', methods=["GET"])
def getProduct(product_id):
    retval = [
        data for data in ProductInfoData if data['id'] == int(product_id)
        ]
    return jsonify(retval[0]), 200

@product_blueprint.route('/<category_id>', methods=["POST"])
def addProduct(category_id):
    data = request.get_json()
    return data, 200

@product_blueprint.route('/', methods=["PUT"])
def updateProduct():
    data = request.get_json()
    for d in ProductInfoData:
        if d['id'] == data['id']:
            d.update(data)
    return json.dumps(
        {'success': True}), 200, {'ContentType': 'application/json'}

@product_blueprint.route('/<product_id>', methods=["DELETE"])
def deleteProduct(product_id):
    global ProductInfoData
    print(len(ProductInfoData))
    ProductInfoData = [
        d for d in ProductInfoData if d.get('id') != int(product_id)
        ]
    print(len(ProductInfoData))
    return json.dumps(
        {'success': True}), 200, {'ContentType': 'application/json'}


import pandas as pd
import numpy as np
import argparse

def run_predictions(season, game_week):
    sims_to_run = 100000
    df = pd.DataFrame(
        columns=["week", "home_team", "away_team", "home_wins", "away_wins", "draws", "home_score", "away_score"])
    # only use games before the game week we want to predict
    historical = season.loc[season["wk"] < game_week]
    # make sure we only use games that have valid scores
    historical = historical.loc[season["home_score"] > -1]

    # games to predict
    to_predict = season.loc[season["wk"] == game_week]

    # get average home and away scores for entire competition
    home_avg = historical["home_score"].mean()
    away_avg = historical["away_score"].mean()
    # loop through predicting games
    another_i = 1
    # loop through predicting games

    for i in to_predict.index:
        home_team = to_predict.loc[i, "home_team"]
        away_team = to_predict.loc[i, "away_team"]
        home_team_id = to_predict.loc[i, "home_id"]
        away_team_id = to_predict.loc[i, "away_id"]

        # average goals scored and goals conceded for home team
        home_team_average_for = historical.loc[season["home_id"] == home_team_id, "home_score"].mean()
        home_team_average_against = historical.loc[season["home_id"] == home_team_id, "away_score"].mean()

        # average goals scored and goals conceded for away team
        away_team_average_for = historical.loc[season["away_id"] == away_team_id, "away_score"].mean()
        away_team_average_against = historical.loc[season["away_id"] == away_team_id, "home_score"].mean()

        # calculate home and away offense and defense strength
        home_team_offense_strength = home_team_average_for / home_avg
        home_team_defense_strength = home_team_average_against / away_avg

        away_team_offense_strength = away_team_average_for / away_avg
        away_team_defense_strength = away_team_average_against / home_avg

        home_team_expected_goals = home_team_offense_strength * away_team_defense_strength * home_avg
        away_team_expected_goals = away_team_offense_strength * home_team_defense_strength * away_avg

        home_team_poisson = np.random.poisson(home_team_expected_goals, sims_to_run)
        away_team_poisson = np.random.poisson(away_team_expected_goals, sims_to_run)

        home_wins = np.sum(home_team_poisson > away_team_poisson) / sims_to_run * 100
        away_wins = np.sum(away_team_poisson > home_team_poisson) / sims_to_run * 100
        draws = np.sum(home_team_poisson == away_team_poisson) / sims_to_run * 100

        home_score_actual = to_predict.loc[i, "home_score"]
        away_score_actual = to_predict.loc[i, "away_score"]

        df.loc[another_i] = {
            "week": game_week,
            "home_team": home_team,
            "away_team": away_team,
            "home_wins": home_wins,
            "away_wins": away_wins,
            "draws": draws,
            "home_score": home_score_actual,
            "away_score": away_score_actual
        }
        another_i = another_i + 1
    return df

def run_tests(season, game_week):
    print('running accuracy check')
    best_score = 0
    best_threshold = 0
    games_used = 0
    for threshold in range(45, 95, 5):
        correct = 0
        total_games = 0
        possible_games = 0
        games_totally_wrong = 0
        games_wrong = 0
        for wk in range(game_week):
            if wk >= 3:
                predictions = run_predictions(season, wk)

                for i in predictions.index:
                    home_score = predictions.loc[i, "home_score"]
                    away_score = predictions.loc[i, "away_score"]
                    home_win = predictions.loc[i, "home_wins"]
                    away_win = predictions.loc[i, "away_wins"]
                    draws = predictions.loc[i, "draws"]
                    if home_score == away_score and draws > home_win and draws > away_win and draws >= threshold:
                        correct += 1
                    if home_score > away_score and home_win > draws and home_win > away_win and home_win > threshold:
                        correct += 1
                    if away_score > home_score and away_win > draws and away_win > home_win and away_win > threshold:
                        correct += 1
                    if draws > threshold or away_win > threshold or home_win > threshold:
                        total_games += 1

                    if home_score > away_score and away_win > draws and away_win > home_win and away_win > threshold:
                        games_totally_wrong += 1
                    if away_score > home_score and home_win > draws and home_win > away_win and home_win > threshold:
                        games_totally_wrong += 1

                    if away_win > home_win and away_win > draws and (home_score > away_score or home_score == away_score) and away_win > threshold:
                        games_wrong += 1

                    if home_win > away_win and home_win > draws and (away_score > home_score or away_score == home_score) and home_win > threshold:
                        games_wrong += 1

                    if draws > home_win and draws > away_win and (away_score > home_score or home_score > away_score) and draws > threshold:
                        games_wrong += 1

                    possible_games += 1

        if total_games > 0:
            score = correct / total_games * 100
        else:
            score = 0
        print('----------')
        print('threshold: ', threshold)
        print('score: ', score)
        print('games used: ', total_games)
        print('games wrong', games_wrong)
        print('games totally wrong', games_totally_wrong)
        print('----------')

        if (score > best_score or (
                score == best_score and total_games > games_used)) and total_games >= possible_games / 10:
            best_score = score
            best_threshold = threshold
            games_used = total_games
    print('')
    print('')
    print('!!!!!!=====================!!!!!!!!!!!')
    print('best threshold: ', best_threshold)
    print('score: ', best_score)
    print('games used: ', games_used)
    print('!!!!!!=====================!!!!!!!!!!!')

def print_predictions(predictions_data):
    for i in predictions_data.index:
        print('=============================')
        print('home team: ', predictions_data.loc[i, "home_team"])
        print('away team: ', predictions_data.loc[i, "away_team"])
        print('home wins: ', predictions_data.loc[i, "home_wins"])
        print('away wins: ', predictions_data.loc[i, "away_wins"])
        print('draws: ', predictions_data.loc[i, "draws"])
        print('=============================')

# initialise parser object to read from command line
parser = argparse.ArgumentParser()

# script arguments
parser.add_argument('-wk', '--week', default=100, type=int, help='put that week in, you hot bitch')
parser.add_argument('-r', '--run', action='store_true', help='run that shit')
parser.add_argument('-t', '--test', action='store_true', help='run tests to check accuracy')
parser.add_argument('-lg', '--league', default='', type=str, help='put that league in, you hot bitch')

args = parser.parse_args()

week = args.week
run = args.run
league = args.league

season_csv = pd.read_csv('full_match_stats_' + league + '.csv')

table_csv = pd.read_csv('league_table_' + league + '.csv')

if args.run and args.week:
    predictions = run_predictions(season_csv, week)
    print_predictions(predictions)
elif args.test and args.week:
    run_tests(season_csv, week)
else:
    print('need to add -r to that shit and add a week')


"""
    Time    O(N)    108 ms
    Space   O(1)    19700 KB
"""

def f():
    s = input()
    n = len(s)
    lowers = 0
    uppers = 0
    for c in s:
        if 0 <= ord(c) - ord('A') <= 25:
            uppers += 1
        elif 0 <= ord(c) - ord('a') <= 25:
            lowers += 1
    if lowers * 2 >= n:
        return s.lower()
    return s.upper()

print(f())


import cv2
import numpy as np

ScreenCut = '--------------------------------------------'
Titleup = '\n' + '【 OpenCV を使った画像処理ツール v1.2 】'
print(ScreenCut)
print(Titleup + '\n')
print('※特殊記号のない名前の画像にしましょう' + '\n')
print(ScreenCut)

def ScreenBuffer():

    ImagePath = input('\n' + '画像のディレクトリを入力：')

    DrawText = input('\n' + '描画する文字を入力：')

    #PickColor = input('\n' + '文字色のRGBを入力：')

    img = cv2.imread(ImagePath)

    print('\n' + 'Successfully ...' + '\n')
    print(ScreenCut)

    cv2.putText(img,
            DrawText,
            org=(10, 50),
            fontFace=cv2.FONT_HERSHEY_DUPLEX,
            fontScale=1.5,
            color=(130, 130, 130),
            thickness=2,
            lineType=cv2.LINE_AA)

    cv2.imshow(ImagePath, img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    ScreenBuffer()
ScreenBuffer()


import tensorflow as tf
import numpy as np
from .model import Model
from .help_functions import weight_variable, bias_variable
import pickle
from molmod.units import angstrom

def activation(tensor):
    return tf.math.softplus(tensor) - np.log(2.0)

def f_cutoff(input_tensor, cutoff, cutoff_transition_width = 0.5, float_type = tf.float32):
    r_transition = tf.where(input_tensor > cutoff, tf.zeros(tf.shape(input_tensor), dtype = float_type),
                            0.5 * (1 + tf.cos(np.pi * (input_tensor - cutoff + cutoff_transition_width) / cutoff_transition_width)))

    return tf.where(input_tensor > cutoff - cutoff_transition_width, r_transition, tf.ones(tf.shape(input_tensor), dtype = float_type))

class InteractionBlock(tf.Module):
    def __init__(self, layer_index, num_features, num_filters, filter_block, float_type = tf.float32):
        super(InteractionBlock, self).__init__()

        self.float_type = float_type
        self.num_features = num_features
        self.num_filters = num_filters

        self.int_weights = weight_variable([self.num_features, self.num_filters], 'int_weights_%d' % layer_index, stddev = 1. / np.sqrt(self.num_features))
        self.int_bias = bias_variable([1, 1, self.num_filters], 'int_bias_%d' % layer_index)

        self.int_weights2 = weight_variable([self.num_filters, self.num_filters], 'int_weights2_%d' % layer_index, stddev = 1. / np.sqrt(self.num_filters))
        self.int_bias2 = bias_variable([1, 1, self.num_filters], 'int_bias2_%d' % layer_index)

        self.int_weights3 = weight_variable([self.num_filters, self.num_features], 'int_weights3_%d' % layer_index, stddev = 1. / np.sqrt(self.num_filters))
        self.int_bias3 = bias_variable([1, 1, self.num_features], 'int_bias3_%d' % layer_index)

        self.filter_block = filter_block

    def __call__(self, atom_features, radial_features, elements_mask, neighbor_mask, smooth_cutoff_mask, gather_neighbor):
        if self.float_type == tf.float64:
            int_weights = tf.cast(self.int_weights, dtype = tf.float64)
            int_bias = tf.cast(self.int_bias, dtype = tf.float64)

            int_weights2 = tf.cast(self.int_weights2, dtype = tf.float64)
            int_bias2 = tf.cast(self.int_bias2, dtype = tf.float64)

            int_weights3 = tf.cast(self.int_weights3, dtype = tf.float64)
            int_bias3 = tf.cast(self.int_bias3, dtype = tf.float64)

        else:
            int_weights = self.int_weights
            int_bias = self.int_bias

            int_weights2 = self.int_weights2
            int_bias2 = self.int_bias2

            int_weights3 = self.int_weights3
            int_bias3 = self.int_bias3

        atom_features = (tf.tensordot(atom_features, int_weights, [[2], [0]]) + int_bias) * tf.expand_dims(elements_mask, [-1])
        neighbor_states = tf.gather_nd(atom_features, gather_neighbor) # [batches, N, J, H]

        W = self.filter_block(radial_features) * smooth_cutoff_mask

        # Normalizing this sum is important to achieve better training
        # A constant value of 70 is chosen
        conv = tf.reduce_sum(neighbor_states * W, axis = [2]) / 70.

        atom_features = activation(tf.tensordot(conv, int_weights2, [[2], [0]]) + int_bias2) * tf.expand_dims(elements_mask, [-1])
        atom_features = (tf.tensordot(atom_features, int_weights3, [[2], [0]]) + int_bias3) * tf.expand_dims(elements_mask, [-1])

        return atom_features

class FilterBlock(tf.Module):
    def __init__(self, layer_index, n_max, num_filters, float_type = tf.float32):
        super(FilterBlock, self).__init__()

        self.n_max = n_max
        self.num_filters = num_filters
        self.float_type = float_type

        self.filter_weights1 = weight_variable([self.n_max, self.num_filters], 'filter_weights_1_layer_%d' % layer_index, stddev = 1. / np.sqrt(self.n_max))
        self.filter_bias1 = bias_variable([1, 1, 1, self.num_filters], 'filter_bias_1_layer_%d' % layer_index)

        self.filter_weights2 = weight_variable([self.num_filters, self.num_filters], 'filter_weights_2_layer_%d' % layer_index, stddev = 1. / np.sqrt(self.num_filters))
        self.filter_bias2 = bias_variable([1, 1, 1, self.num_filters], 'filter_bias_2_layer_%d' % layer_index)

    def __call__(self, radial_features):
        if self.float_type == tf.float64:
            filter_weights1 = tf.cast(self.filter_weights1, dtype = tf.float64)
            filter_bias1 = tf.cast(self.filter_bias1, dtype = tf.float64)

            filter_weights2 = tf.cast(self.filter_weights2, dtype = tf.float64)
            filter_bias2 = tf.cast(self.filter_bias2, dtype = tf.float64)

        else:
            filter_weights1 = self.filter_weights1
            filter_bias1 = self.filter_bias1

            filter_weights2 = self.filter_weights2
            filter_bias2 = self.filter_bias2

        dense1 = activation(tf.tensordot(radial_features, filter_weights1, [[3], [0]]) + filter_bias1)
        return (activation(tf.tensordot(dense1, filter_weights2, [[3], [0]]) + filter_bias2))

class OutputLayer(tf.Module):
    def __init__(self, prefix_name, layer_sizes, initial_size, float_type = tf.float32):
        super(OutputLayer, self).__init__()
        self.float_type = float_type

        previous_size = initial_size
        self.layer_sizes = layer_sizes

        self.output_weights = []
        self.output_biases = []

        for i, layer_size in enumerate(self.layer_sizes):
            self.output_weights.append(weight_variable([previous_size, layer_size], prefix_name + '_weights_layer_%d' % i, stddev = 1. / np.sqrt(previous_size)))
            self.output_biases.append(bias_variable([1, 1, layer_size], prefix_name + '_bias_layer_%d' % i))

            previous_size = layer_size

    def __call__(self, atom_features):
        for i, layer_size in enumerate(self.layer_sizes):
            w = self.output_weights[i]
            b = self.output_biases[i]

            if self.float_type == tf.float64:
                w = tf.cast(w, dtype = tf.float64)
                b = tf.cast(b, dtype = tf.float64)

            atom_features = tf.tensordot(atom_features, w, [[2], [0]]) + b
            if i < len(self.layer_sizes) - 1: # No activation function for the final layer
                atom_features = activation(atom_features)

        return atom_features

class SchNet(Model):
    def __init__(self, cutoff = 5., n_max = 25, num_features = 64, start = 0.0, num_layers = 3, end = None, num_filters = -1, cutoff_transition_width = None,
                 shared_W_interactions = False, **kwargs):
        Model.__init__(self, cutoff, **kwargs)
        if end is None:
            self.end = cutoff
        else:
            self.end = end
        self.n_max = n_max
        self.start = start
        self.radial_sigma = (self.end - self.start) / self.n_max
        if cutoff_transition_width is None:
            self.cutoff_transition_width = cutoff - end
        else:
            self.cutoff_transition_width = cutoff_transition_width
        if num_filters == -1:
            self.num_filters = H
        else:
            self.num_filters = num_filters
        self.num_layers = num_layers
        self.num_features = num_features # Size of the hidden state vector
        self.shared_W_interactions = shared_W_interactions

        # Here, all variables are listed and other submodules
        self.init_features = weight_variable([100, self.num_features], 'init_features', stddev = 1.)

        if self.shared_W_interactions:
            fixed_W = FilterBlock(0, self.n_max, self.num_filters, self.float_type)

        self.interaction_blocks = []
        for layer_index in range(num_layers):
            if self.shared_W_interactions:
                filter_block = fixed_W
            else:
                filter_block = FilterBlock(layer_index, self.n_max, self.num_filters, self.float_type)

            self.interaction_blocks.append(InteractionBlock(layer_index, self.num_features, self.num_filters, filter_block, float_type = self.float_type))

        self.output_layer = OutputLayer('output_layer', [self.num_features, int(self.num_features / 2), 1], self.num_features, float_type = self.float_type)

    def save(self, output_file):
        data = {'cutoff' : self.cutoff, 'n_max' : self.n_max, 'start' : self.start, 'end' : self.end, 'shared_W_interactions' : self.shared_W_interactions, 'cutoff_transition_width' : self.cutoff_transition_width,
                'num_features' : self.num_features, 'num_layers' : self.num_layers, 'num_filters' : self.num_filters}
        pickle.dump(data, open(output_file + '.pickle', 'wb'))

    def internal_compute(self, dcarts, dists, numbers, masks, gather_neighbor):
        ''' Defining the radial features '''
        n = self.start + tf.range(self.n_max, dtype = self.float_type) / self.n_max * (self.end - self.start)
        n = tf.reshape(n, [1, 1, 1, -1])
        radial_features = tf.exp(- 0.5 * (tf.expand_dims(dists, [-1]) - n)**2 / self.radial_sigma**2) # shape [batches, N, J, n]

        # Imposing a smooth cutoff
        smooth_cutoff_mask = f_cutoff(dists, cutoff = self.cutoff, cutoff_transition_width = self.cutoff_transition_width, float_type = self.float_type) # shape [batches, N, J]
        smooth_cutoff_mask = tf.expand_dims(smooth_cutoff_mask * masks['neighbor_mask'], [-1]) # shape [batches, N, J, 1]

        if self.float_type == tf.float64:
            init_features = tf.cast(self.init_features, dtype = tf.float64)
        else:
            init_features = self.init_features

        atom_features = tf.nn.embedding_lookup(init_features, numbers * tf.cast(numbers > 0, tf.int32)) * tf.expand_dims(masks['elements_mask'], [-1])

        ''' The interaction layers '''
        for i in range(self.num_layers):
            atom_features += self.interaction_blocks[i](atom_features, radial_features, masks['elements_mask'], masks['neighbor_mask'], smooth_cutoff_mask, gather_neighbor)

        atomic_energies = tf.reshape(self.output_layer(atom_features), [self.batches, self.N])

        ''' The final energy '''
        energy = tf.reduce_sum(atomic_energies * masks['elements_mask'], [-1])

        return energy, atomic_energies


#!/usr/bin/python
# -*- coding: latin-1 -*-
import os, typing

"""
Mapping:
A = 0
C = 1
G = 2
T = 3
"""

try:
    from cdnarules import byte2QUATS
    from cdnarules import getQUAT
except ImportError:
    print("C Module failed to load, falling back to slow mode")

    def getQUAT(bit1: bool, bit2: bool):
        if not (bit1 or bit2):
            return "A"
        elif (not bit1) and bit2:
            return "C"
        elif bit1 and (not bit2):
            return "G"
        elif bit1 and bit2:
            return "T"
        else:
            print("ERROR, this might never happen.")
            return "E"

    def byte2QUATS(byte):
        res = ""
        if not isinstance(byte, int):
            byte = byte[0]
        if not isinstance(byte, str):
            byt = iter(bin(byte)[2:].rjust(8, "0"))
        else:
            byt = iter(bin(ord(byte))[2:].rjust(8, "0"))
        for x, y in zip(byt, byt):
            res += getQUAT(str2bool(x), str2bool(y))
        return res

def bin2Quaternary(filename: str):
    with open(filename, "rb") as f:
        with open(filename + ".quat", "w") as ff:
            byte = f.read(1)
            while byte != b"":
                ff.write(byte2QUATS(byte))
                byte = f.read(1)

def string2QUATS(text: typing.Union[str, bytes, typing.List[typing.AnyStr]]):
    return [byte2QUATS(x) for x in text]

def str2bool(s: str):
    return s == "1"

def quads2dna(quads: typing.Union[typing.List[int], bytes]):
    translation = {0: 'A', 1: 'C', 2: 'G', 3: 'T'}
    return "".join(translation[x] for x in quads)

def main():
    folder = "RU10_b_lq.webm"
    for filename in os.listdir(folder):
        if filename.endswith(".RU10"):
            bin2Quaternary(folder + "/" + filename)

if __name__ == "__main__":
    # main()
    filename = (
        "vergleich bzgl ACGT-verteilung/mit dna rules/RU10_logo.jpg/0.RU10"
    )  # ""RU10_b_lq.webm/1.RU10"  # "raptor.pdf"
    x = [0, 0, 0, 1, 2, 3, 1, 1, 1, ]
    print(quads2dna(x))

    def bitstring_to_bytes(s: typing.Union[str, bytes, bytearray]):
        v = int(s, 2)
        b = bytearray()
        while v:
            b.append(v & 0xff)
            v >>= 8
        return bytes(b[::-1])

    s = "001011000110100101011110001010110000110110101110011111000101000100000011111110110001011010010011101110001111101010110100"
    print("".join(string2QUATS(bitstring_to_bytes(s))))


import cv2
import os
from data_augmentation import *
import torch
import string
import editdistance
import numpy as np
from torch.utils.data import Dataset
from array_text_conversion import *

class LipsDataset(Dataset):     # inherit Dataset

    def __init__(self, alignment_path, file_list, video_padding, text_padding, split, root='./'):

        # Load attributes
        self.text_padding = text_padding
        self.alignment_path = alignment_path
        self.split = split
        self.video_padding = video_padding

        # Open the files
        try:
            file = open(file_list, 'r')
            self.videos = [path_line.strip() for path_line in file.readlines()]
        finally:
            file.close()

        # store the paths, the speakers and the name
        self.video_speaker_name = []
        for video in self.videos:
            items = video.split('/')
            # video is the path to the folder that contains the images of the video, items[-5] is the speaker (ex: s23), items[-2] is the name of the folder (ex: bram4p)
            self.video_speaker_name.append((video, items[-5], items[-2]))

    def __len__(self):
        """Return data length"""
        return len(self.video_speaker_name)

    def __getitem__(self, index):
        """
        For indexing: Dataset[i] gets to i-th sample
        """
        # Retrieve data
        data = self.video_speaker_name[index]

        # Split the data into video, speaker and name
        video = data[0]
        speaker = data[1]
        name = data[2]

        # load annotations from alignment file
        extension = '.align'
        anno_path = os.path.join(self.alignment_path, speaker, 'align', name + extension)
        annotation = self.load_annotation(anno_path)

        # load video and associated alignement file using the path
        # vid is (T, H, W, C)
        video = self.load_video(video)

        if(self.split == 'train'):
            video = self.data_augmentation(video)

        # Normalize the color (values between 0 and 1)
        video = video / 255.0

        # get the length of the video (nb of images) and the length of the alignment file (timestamps and words)
        video_length = video.shape[0]
        annotation_length = annotation.shape[0]

        # pad the videos and the alignment files to the same length (to allow stacking in the same batch)
        video = self.add_padding(video, self.video_padding)
        annotation = self.add_padding(annotation, self.text_padding)

        video = video.transpose(3, 0, 1, 2)
        video = torch.FloatTensor(video)

        annotation = torch.LongTensor(annotation)
        # convert video (T, H, W, C) --> (C, T, H, W)
        return {'video': video,
            'text': annotation,
            'text_length': annotation_length,
            'video_length': video_length}

    def is_jpeg(self, filename):
        """Return true if jpeg file"""
        idx = filename.find('.jpg') != -1
        return idx != -1

    def sort_file(self, file):
        return int(os.path.splitext(file)[0])

    def load_video(self, path):
        """
        Load a video using the path and return an array
        """
        # List directory
        loaded_files = os.listdir(path)
        # Keep only jpeg
        loaded_files = list(filter(self.is_jpeg, loaded_files))
        # Sort
        loaded_files = sorted(loaded_files, key=self.sort_file)

        # Load the image and verify it's not None
        np_array = []
        for file in loaded_files:
            file_path = os.path.join(path, file)
            image = cv2.imread(file_path)
            if image is not None:
                image = cv2.resize(image, (128, 64), interpolation=cv2.INTER_LANCZOS4)
                np_array.append(image)

        np_array = np.stack(np_array, axis=0).astype(np.float32)
        return np_array

    def data_augmentation(self, video):
        """ Apply data augmentation techniques to the input video"""
        # For this project: only horizontal flip
        #video = randomCropping(video)
        return HorizontalFlip(video)

    def load_annotation(self, path):
        """ Load the alignment file"""
        # Annotations to filer
        to_filter = ['SIL', 'SP']
        # open the file
        with open(path, 'r') as file:
            # Retrieve all words
            lines = []
            for line in file.readlines():
                lines.append(line.strip().split(' '))

            # Compute sentence
            text = []
            for line in lines:
                text.append(line[2])
            text = list(filter(lambda s: not s.upper() in to_filter, text))

        # Concatenate all words to a string
        text = ' '.join(text).upper()
        # Convert string to array
        array = convert_text_to_array(text, 1)
        return array

    def print_annotation(self, path):
        """ Print annotation file under path"""
        with open(path, 'r') as file:
            for line in file.readlines():
                print(line)

    def add_padding(self, array, length):
        """ Apply padding to the input array to equal the input length"""
        array = [array[_] for _ in range(array.shape[0])]
        size_to_add = array[0].shape
        nb_iter = length - len(array)
        for _ in range(nb_iter):
            array.append(np.zeros(size_to_add))
        return np.stack(array, axis=0)

#################################################################################################################
#################################################################################################################
#################################################################################################################

if __name__ == '__main__':
    from torch.utils.data import DataLoader
    train_dataset = LipsDataset('C:/Users/aurel/Downloads/grid_dataset_downloaded/GRID_align_txt',  'C:/Users/aurel/Downloads/grid_dataset_downloaded/paths_list_250_by_spk.txt', 75, 100, 'train')
    train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
    for (i_iter, input) in enumerate(train_loader):
        if i_iter == 0:
            print(input)


# list = ['aaa','bbb','ccc','ddd','eee']
# want = ['aaa','xxx']
#
# for i in want:
#     if i in list:
#         print(f"add {i}")
#     else:
#         print(f"we dont have {i}")

# user = ['aaa','bbb','ccc','ddd','eee','admin']
# for i in user:
#     if i == 'admin':
#         print("welcome my root")
#     else:
#         print(f'hello {i}')

# list = []
# if list:
#     list.clear()
# else:
#     print("we need users")
# print(list)

# current_users = ['aaa','Bbb','ccc','ddd','eee']
# xx = [s.lower() for s in current_users]
# print(xx)
# new_users = ['Aaa','Bbb','fff','ggg','hhh']
# for i in new_users:
#     if i.lower() in xx:
#         print(f"{i} is exist.")
#     else:
#         print(f"{i} is able to use.")

shuzi = [1,2,3,4,5,6,7,8,9]
for i in shuzi:
    if i == 1:
        print(f"{i}st")
    elif i == 2:
        print(f"{i}nd")
    elif i == 3:
        print(f"{i}rd")
    else:
        print(f"{i}th")


import requests

from settings import URL, HEADERS, USERNAME, PASSWORD
from notification import Notification

class Repo(object):

    def __init__(self, name, owner, forks, watchers, stars):
        """ Represents a repository object returned by GitHub API v3.

        Parameters
        ==========
        name: str
            Name of the repository

        owner: str
            Owner of the repository

        forks: int
            Forks count of the repository

        watchers: int
            Watchers count of the repository

        stars: int
            Stars count of the repository

        url: str
            API URL of the repository

        """
        self.name = name
        self.owner = owner
        self.forks = forks
        self.watchers = watchers
        self.stars = stars
        self.url = 'https://github.com/' + owner + '/' + name

    def __repr__(self):
        return '{}({}, {}, {}, {}, {})'.format(self.__class__.__name__,
                                               self.name, self.owner, self.forks,
                                               self.watchers, self.stars)

    @classmethod
    def from_name(cls, repo_name, owner_name):
        url = URL + '/repos/' + owner_name + "/" + repo_name
        # print("Repo = " + repo_name + ' : ' + url)
        repo = requests.get(url, headers=HEADERS, auth=(USERNAME, PASSWORD)).json()
        # handle pagination

        #owner = owner_name
        #name = repo_name
        #forks = repo['forks_count']
        #watchers = repo['watchers_count']
        #stars = repo['stargazers_count']
        #return cls(name, owner, forks, watchers, stars)
        return cls.from_Repository(repo)

    @classmethod
    def from_Repository(cls, repo):
        name = repo['name']
        owner = repo['owner']['login']
        forks = repo['forks_count']
        watchers = repo['watchers_count']
        stars = repo['stargazers_count']
        return cls(name, owner, forks, watchers, stars)

    def get_notifications(self, old_stars, old_watchers, old_forks):
        new_stars, new_watchers, new_forks = self.stars, self.watchers, self.forks

        delta_stars = new_stars - old_stars
        delta_watchers = new_watchers - old_watchers
        delta_forks = new_forks - old_forks

        notifications = []
        if delta_stars != 0:
            context = 'starr'
            if delta_stars < 0:
                context = 'un' + context # not a typo

            notifications.append(Notification.generate_message('{} users'.format(delta_stars), context, self.name))

        if delta_watchers != 0:
            context = 'watch'
            if delta_watchers < 0:
                context = 'un' + context

            notifications.append(Notification.generate_message('{} users'.format(delta_watchers), context, self.name))

        if delta_forks != 0:
            context = 'fork'
            if delta_forks < 0:
                context = 'un' + context

            notifications.append(Notification.generate_message('{} users'.format(delta_watchers), context, self.name))

        return notifications


from typing import Any, List
import cv2
import insightface
from cyuai.processors.face_analyser import get_one_face, find_similar_face
from cyuai.processors.face_reference import clear_face_reference, get_face_reference, set_face_reference
from cyuai.typing import Face, Frame
from cyuai.utilities import conditional_download, resolve_relative_path, is_image
from cyuai.processors import core

FACE_SWAPPER = None
NAME = 'ROOP.FACE-SWAPPER'

def get_face_swapper() -> Any:
    global FACE_SWAPPER
    if FACE_SWAPPER is None:
        model_path = resolve_relative_path('../models/inswapper_128.onnx')
        print(model_path)
        FACE_SWAPPER = insightface.model_zoo.get_model(model_path)
    return FACE_SWAPPER

def clear_face_swapper() -> None:
    global FACE_SWAPPER

    FACE_SWAPPER = None

def pre_check() -> bool:
    download_directory_path = resolve_relative_path('../models')
    conditional_download(download_directory_path,
                         ['https://huggingface.co/ashleykleynhans/inswapper/tree/main/inswapper_128.onnx'])
    return True

def pre_start(source_path, target_path) -> bool:
    if not is_image(source_path):
        print('输入脸对象必须是一个图片')
        return False
    elif not get_one_face(cv2.imread(source_path)):
        print('输入脸图片不存在人脸.')
        return False
    if not is_image(target_path):
        print('换脸对象必须是一个图片')
        return False
    return True

def post_process() -> None:
    clear_face_swapper()
    clear_face_reference()

def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:
    return get_face_swapper().get(temp_frame, target_face, source_face, paste_back=True)

def process_frame(source_face: Face, reference_face: Face, temp_frame: Frame) -> Frame:
    target_face = find_similar_face(temp_frame, reference_face)
    if target_face:
        temp_frame = swap_face(source_face, target_face, temp_frame)
    return temp_frame

def process_image(source_path: str, target_path: str, output_path: str) -> None:
    source_face = get_one_face(cv2.imread(source_path))
    target_frame = cv2.imread(target_path)
    reference_face = get_one_face(target_frame, 0)
    result = process_frame(source_face, reference_face, target_frame)
    cv2.imwrite(output_path, result)

def process_frames(source_path: str, temp_frame_paths: List[str]) -> None:
    source_face = get_one_face(cv2.imread(source_path))
    reference_face = get_face_reference()
    for temp_frame_path in temp_frame_paths:
        temp_frame = cv2.imread(temp_frame_path)
        print(temp_frame_path)
        result = process_frame(source_face, reference_face, temp_frame)
        cv2.imwrite(temp_frame_path, result)

def process_video(source_path: str, temp_frame_paths: List[str]) -> None:
    if not get_face_reference():
        reference_frame = cv2.imread(temp_frame_paths[0])
        reference_face = get_one_face(reference_frame, 0)
        set_face_reference(reference_face)
    core.process_video(source_path, temp_frame_paths, process_frames)


class Solution(object):
    def largestInteger(self, num):
        heap1 = []
        heap2 = []

        for i in str(num):
            if int(i)%2:
                heap2.append(-int(i))
            else:
                heap1.append(-int(i))

        heapify(heap1)
        heapify(heap2)
        res = 0

        for i in str(num):
            if int(i)%2:
                res += -heappop(heap2)
                res *= 10
            else:
                res += -heappop(heap1)
                res *= 10

        return res//10


import numpy as np
import pandas as pd
import math
import torch

def reference_to_str(df):
  df['reference'] = df.apply(lambda x: str(x['reference']), axis=1)
  return df

def remove_single_clickout_actions(df):
  print('Initial size: ' + str(df.shape[0]))
  n_action_session = df.groupby('session_id').size().reset_index(name='n_actions')

  df = (df.merge(n_action_session, left_on='session_id', right_on='session_id', how="left"))

  df = df.drop(df[(df["action_type"] == "clickout item") & (df['n_actions'] == 1)].index)
  print('Final size: ' + str(df.shape[0]))
  del df['n_actions']
  return df

def remove_nonitem_actions(df):
  df = df.drop(df[(df['action_type'] != 'interaction item image') & (df['action_type'] != 'interaction item deals') & (df['action_type'] != 'clickout item') & (df['action_type'] != 'search for item')].index)
  return df

def reduce_df(df, dim):
  df = df.head(dim)
  return pd.DataFrame(df)

def get_clickout_data(action, clickout_dict, impression_dict):
  clickout_dict[action.session_id] = action.reference
  impression_dict[action.session_id] = action.impressions.split('|')
  return action.reference

def get_list_session_interactions(group, session_dict):
  session_dict[group.session_id.values[0]] = list(group.reference.values)[-200:]

  return " ".join(list(group.reference.values))

def get_training_input(df_train):
  clickout_dict = {}
  impression_dict = {}
  session_dict = {}

  df_train['step_max'] = df_train[df_train['action_type'] == 'clickout item'].groupby(['session_id'])['step'].transform(max)
  df_train['result'] = df_train[df_train['step'] == df_train['step_max']].apply(lambda x: get_clickout_data(x, clickout_dict, impression_dict), axis = 1)
  df_train_corpus = df_train.groupby('session_id').apply(lambda x: get_list_session_interactions(x, session_dict)).reset_index(name = 'hotel_list')
  df_train = df_train.drop(df_train.index[(df_train['step'] == df_train['step_max']) & (df_train["action_type"] == "clickout item")])

  train_corpus = list(session_dict.values())

  return session_dict, clickout_dict, impression_dict, train_corpus

def get_clickout_data_test(action, clickout_dict, impression_dict):
  if math.isnan(float(action.reference)):
    clickout_dict[action.session_id] = action.step
    impression_dict[action.session_id] = action.impressions.split('|')
  return action.reference

def get_test_input(df_test):
  #Creating a NaN column for item recommendations
  df_test['item_recommendations'] = np.nan

  test_step_clickout_dict = {}
  test_impression_dict = {}
  test_sessions_dict = {}

  df_test['step_max'] = df_test[df_test['action_type'] == 'clickout item'].groupby(['session_id'])['step'].transform(max)
  df_test['result'] = df_test[df_test['step'] == df_test['step_max']].apply(lambda x: get_clickout_data_test(x, test_step_clickout_dict, test_impression_dict), axis = 1)
  df_test = df_test.drop(df_test.index[(df_test['step'] == df_test['step_max']) & (df_test["action_type"] == "clickout item")])
  df_test_corpus = df_test.groupby('session_id').apply(lambda x: get_list_session_interactions(x, test_sessions_dict)).reset_index(name = 'hotel_list')

  test_corpus = list(test_sessions_dict.values())

  return test_sessions_dict, test_step_clickout_dict, test_impression_dict, test_corpus

def get_batched_sessions(session_dict, category_dict, batchsize):
  batched_sessions = []
  temp_sessions = []
  for session_id in session_dict.keys():
    if session_id in category_dict:
      temp_sessions.append(session_id)
      if len(temp_sessions) == batchsize:
        batched_sessions.append(temp_sessions)
        temp_sessions = []

  if temp_sessions != []:
    batched_sessions.append(temp_sessions)

  return batched_sessions


# -*- coding: UTF-8 -*-
import fastText
import jieba

class FasttextTextClassifier():
    def __init__(self, train_input, test_input, output):
        '''

        :param train_input: 训练数据文件路径
        :param test_input:  验证数据文件路径
        :param output:      模型
        '''
        self.train_input = train_input
        self.test_input = test_input
        self.output = output

    def fit(self):
        '''
        模型训练
        :return:
        '''
        fastText.train_supervised(self.train_input, label="__label__").save_model(self.output)

    def load_model(self,model_path):
        '''
        文件中加载模型
        :param model_path:
        :return:
        '''
        return fastText.load_model(model_path)

    def evaluate(self):
        '''
        通过precision，recall 评估
        :return:
        '''
        fasttext_evaluate = fastText.load_model(self.output).test(self.test_input)
        rows = fasttext_evaluate[0]
        precision = fasttext_evaluate[1]
        recall = fasttext_evaluate[2]
        return precision, recall, rows

    def predict(self, text, model):
        '''
        分词后的文本内容，空格分割
        :param text:  word1 word2 word3
        :return:
        '''
        labels, probs = model.predict([text])
        label = str(labels[0][0]).replace("__label__", "")
        confidence = '%.2f' % probs[0][0]
        return label, confidence

if __name__ == "__main__":

    # 模型文件路径
    model_path = "fasttext_20190226.model"
    # 训练 && 评估
    text_classifier = FasttextTextClassifier("data_sample/fasttext_train", "data_sample/fasttext_test",model_path)
    text_classifier.fit()
    precision, recall, _ = text_classifier.evaluate()
    print("precision={0},recall={1}".format("%.2f" % precision, "%.2f" % recall))
    #precision = 0.90, recall = 0.90

    # 预测
    # text_classifier = FasttextTextClassifier("", "", model_path)
    # model = text_classifier.load_model(model_path)
    # label_to_cate = {1: 'technology', 2: 'car', 3: 'entertainment', 4: 'military', 5: 'sports'}
    # data = "空空作战主要提升了雷达还有电抗的性能，最突出的特点就是增加了对地精确打击能力。"
    # text = " ".join(jieba.lcut(data))
    # print(text)
    # label, confidence = text_classifier.predict(text, model)
    # print("input data={0}".format(data))
    # print("category={0}".format(label_to_cate[int(label)]))
    # print("confidence={0}".format(confidence))


# -*- coding: utf-8 -*-
#
# This software may be modified and distributed under the terms
# of the MIT license.  See the LICENSE file for details.

from logging import Handler

from logstash_async.constants import constants
from logstash_async.formatter import LogstashFormatter
from logstash_async.utils import import_string, safe_log_via_print
from logstash_async.worker import LogProcessingWorker
import logstash_async

class ProcessingError(Exception):
    """"""

class SynchronousLogstashHandler(Handler):
    """Sync Python logging handler for Logstash. Sends events over TCP by default.
    :param host: The host of the logstash server, required.
    :param port: The port of the logstash server, required.
    :param transport: Callable or path to a compatible transport class.
    :param ssl_enable: Should SSL be enabled for the connection? Default is False.
    :param ssl_verify: Should the server's SSL certificate be verified?
    :param keyfile: The path to client side SSL key file (default is None).
    :param certfile: The path to client side SSL certificate file (default is None).
    :param ca_certs: The path to the file containing recognized CA certificates.
    :param enable: Flag to enable log processing (default is True, disabling
                   might be handy for local testing, etc.)
    """

    # ----------------------------------------------------------------------
    # pylint: disable=too-many-arguments
    def __init__(self, host, port, transport='logstash_async.transport.TcpTransport',
                 ssl_enable=False, ssl_verify=True, keyfile=None, certfile=None, ca_certs=None,
                 enable=True, encoding='utf-8', **kwargs):
        super().__init__()
        self._host = host
        self._port = port
        self._transport_path = transport
        self._ssl_enable = ssl_enable
        self._ssl_verify = ssl_verify
        self._keyfile = keyfile
        self._certfile = certfile
        self._ca_certs = ca_certs
        self._enable = enable
        self._transport = None
        self._encoding = encoding
        self._setup_transport(**kwargs)

    # ----------------------------------------------------------------------
    def emit(self, record):
        if not self._enable:
            return  # we should not do anything, so just leave

        self._setup_transport()

        # basically same implementation as in logging.handlers.SocketHandler.emit()
        try:
            data = self._format_record(record)
            self._transport.send([data], use_logging=False)
        except Exception:
            self.handleError(record)

    # ----------------------------------------------------------------------
    def _setup_transport(self, **kwargs):
        if self._transport is not None:
            return

        transport_args = dict(
            host=self._host,
            port=self._port,
            timeout=constants.SOCKET_TIMEOUT,
            ssl_enable=self._ssl_enable,
            ssl_verify=self._ssl_verify,
            keyfile=self._keyfile,
            certfile=self._certfile,
            ca_certs=self._ca_certs,
            **kwargs)
        if isinstance(self._transport_path, str):
            transport_class = import_string(self._transport_path)
            self._transport = transport_class(**transport_args)
        elif callable(self._transport_path):
            self._transport = self._transport_path(**transport_args)
        elif hasattr(self._transport_path, 'send'):
            self._transport = self._transport_path
        else:
            raise RuntimeError(
                'Invalid transport path: must be an importable module path, '
                'a class or factory function or an instance.')

    # ----------------------------------------------------------------------
    def _format_record(self, record):
        self._create_formatter_if_necessary()
        formatted = self.formatter.format(record)
        if isinstance(formatted, str):
            formatted = formatted.encode(self._encoding)  # pylint: disable=redefined-variable-type
        return formatted + b'\n'

    # ----------------------------------------------------------------------
    def _create_formatter_if_necessary(self):
        if self.formatter is None:
            self.formatter = LogstashFormatter()

    # ----------------------------------------------------------------------
    def close(self):
        self.acquire()
        try:
            self.shutdown()
        finally:
            self.release()
        super().close()

    # ----------------------------------------------------------------------
    def shutdown(self):
        self._close_transport()

    # ----------------------------------------------------------------------
    def _close_transport(self):
        try:
            if self._transport is not None:
                self._transport.close()
        except Exception as exc:
            safe_log_via_print('error', f'Error on closing transport: {exc}')

class AsynchronousLogstashHandler(SynchronousLogstashHandler):
    """Python logging handler for Logstash. Sends events over TCP by default.
    (there are some more params coming from SynchronousLogstashHandler)
    :param database_path: The path to the file containing queued events, required.
                          Use None to use a in-memory cache.
    :param event_ttl: Amount of time in seconds to wait before expiring log messages in
                      the database. (Given in seconds. Default is None, and disables this feature)
    """

    _worker_thread = None

    # ----------------------------------------------------------------------
    # pylint: disable=too-many-arguments
    def __init__(self, host, port, database_path, transport='logstash_async.transport.TcpTransport',
                 ssl_enable=False, ssl_verify=True, keyfile=None, certfile=None, ca_certs=None,
                 enable=True, event_ttl=None, encoding='utf-8', **kwargs):

        self._database_path = database_path
        self._event_ttl = event_ttl

        super().__init__(host, port, transport,
                 ssl_enable, ssl_verify, keyfile, certfile, ca_certs,
                 enable, encoding, **kwargs)

    # ----------------------------------------------------------------------
    def emit(self, record):
        if not self._enable:
            return  # we should not do anything, so just leave

        self._setup_transport()
        self._start_worker_thread()

        # basically same implementation as in logging.handlers.SocketHandler.emit()
        try:
            data = self._format_record(record)
            AsynchronousLogstashHandler._worker_thread.enqueue_event(data)
        except Exception:
            self.handleError(record)

    # ----------------------------------------------------------------------
    def flush(self):
        if self._worker_thread_is_running():
            self._worker_thread.force_flush_queued_events()

    # ----------------------------------------------------------------------
    def _start_worker_thread(self):
        if self._worker_thread_is_running():
            return

        AsynchronousLogstashHandler._worker_thread = LogProcessingWorker(
            host=self._host,
            port=self._port,
            transport=self._transport,
            ssl_enable=self._ssl_enable,
            ssl_verify=self._ssl_verify,
            keyfile=self._keyfile,
            certfile=self._certfile,
            ca_certs=self._ca_certs,
            database_path=self._database_path,
            cache=logstash_async.EVENT_CACHE,
            event_ttl=self._event_ttl)
        AsynchronousLogstashHandler._worker_thread.start()

    # ----------------------------------------------------------------------
    @staticmethod
    def _worker_thread_is_running():
        worker_thread = AsynchronousLogstashHandler._worker_thread
        if worker_thread is not None and worker_thread.is_alive():
            return True

        return False

    # ----------------------------------------------------------------------
    def shutdown(self):
        if self._worker_thread_is_running():
            self._trigger_worker_shutdown()
            self._wait_for_worker_thread()
            self._reset_worker_thread()
            self._close_transport()
        else:
            pass

    # ----------------------------------------------------------------------
    def _trigger_worker_shutdown(self):
        AsynchronousLogstashHandler._worker_thread.shutdown()

    # ----------------------------------------------------------------------
    def _wait_for_worker_thread(self):
        AsynchronousLogstashHandler._worker_thread.join()

    # ----------------------------------------------------------------------
    def _reset_worker_thread(self):
        AsynchronousLogstashHandler._worker_thread = None


import matplotlib.pyplot as plt

def visualize(**images):
    """
    plot images in a row. include arg 'savepath=/path/to/file.ext' to save the output
    :param images: what to plot paired with desired title [title=array]
    """
    # check if the save path is there and remove it before plotting
    if 'savepath' in images.keys():
        ptf = images['savepath']
        images.pop('savepath')

    n = len(images)
    if 'mask' in images.keys():
        n += images['mask'].shape[2]-1
    plt.figure(figsize=(16, 5))
    for i, (name, image) in enumerate(images.items()):
        if name == 'mask':
            for jj in range(image.shape[2]):
                plt.subplot(1, n, i + 1 + jj)
                plt.xticks([])
                plt.yticks([])
                plt.title(f'Mask channel: {jj}'.title(), fontsize=18)
                plt.imshow(image[:, :, jj])
        elif name == 'ROI':
            plt.subplot(1, n, i + 1)
            plt.xticks([])
            plt.yticks([])
            plt.title(' '.join(name.split('_')).title(), fontsize=18)
            plt.imshow(image, cmap='gray')
        else:
            plt.imshow(image)
            if ptf:
                plt.savefig(ptf)
    plt.show()


# -*- coding: utf-8 -*-
'''
Created on Nov. 23, 2012

@author: E. LEGROS
@contact: emeline.legros@ibcp.fr
@license: GPLv3
'''
import os
from ConfigParser import Error as ConfigParserError
import logging
_log = logging.getLogger(__name__)

from mongokit import Document, ObjectId, IS
from mf.annotation import mf_decorator, MF_READ, MF_EDIT

from .connection import connection
from .config import Config
from .data import AbstractData
from .objectmanager import ObjectManager

@mf_decorator
@connection.register
class Project(Document):
    """
    Project is a class that stores all information about a project
    (owner username, inputs, outputs, job id).
    It's initialized by two parameters: the name of the owner of
    the project and the project name itself.
    """

    __collection__ = 'projects'
    __database__ = Config.config().get('app:main', 'db_name')

    structure = {'name': basestring,
                 'description': basestring,
                 'owner': ObjectId,
                 'users': [{'user': ObjectId, 'role': IS(u'manager',
                                                         u'contributor',
                                                         u'watcher')}],
                 'notebook': basestring,
                 'public': bool,
                 '_dir': basestring
                }

    required_fields = ['name', 'owner']

    default_values = {'_dir': None}

    @property
    def creation_date(self):
        """
        Get project creation date
        :returns: the attribute value
        :rtype: datetime
        """
        return self['_id'].generation_time

    def add_user(self, user, role):
        """
        add_user is a method defined to attach a new user
        and its role into the project.
        :param user: user to be associated to the project.
        :type user: :class:`User` object.
        :param role: user's role in the project.
        :type role: string.
        """
        self['users'].append({'user': user['_id'], 'role': role})

    @property
    def dir(self):
        """
        :return: the project directory
        :rtype: string
        """
        try:
            projects_store = Config.config().get("DEFAULT","projects_store")
        except ConfigParserError:
            _log.warning("cannot compute a project directory path from the config file",
                      exc_info=True)
            return None
        return os.path.join(projects_store, 'projects', str(self['_id']))

    @property
    def id(self):
        """
        :return: the unique identifier of this project
        :rtype:
        """
        return self['_id']

    @staticmethod
    def my_project_acl_filter(control, request, authenticated_userid=None):
        # find the user corresponding to the provided email
        user = connection.User.find_one({'email': authenticated_userid})
        # id is set if user exists, otherwise set to None so that the filter
        # works even for unauthenticated users.
        user_id = user['_id'] if user else None
        # admin_mode tells wether the admin user
        # is in admin mode and should access everything
        admin_mode = hasattr(request, 'session') \
                     and 'adminmode' in request.session
        if user and user.get('admin') and admin_mode:
            # admin_mode = provide everything
            project_filter = {}
        elif control == MF_READ:
            # read: elements for projects where the user is active or where he
            if user is None:
                project_filter = {'public': True}
            else:
                project_filter = {"$or": [{"users": {"$elemMatch":
                                  {'user': user_id}}}, {'public':True}]}
        elif control == MF_EDIT:
            # User must be one of project contributors or managers
            if user is None:
                project_filter = None
            else:
                project_filter = {"users": {"$elemMatch": {'user': user_id,
                                  "$or": [{'role': 'contributor'},
                                          {'role': 'manager'}]}}}
        return project_filter

    def my(self, control, request, authenticated_userid=None):
        return self.my_project_acl_filter(control,
                                          request, authenticated_userid)

class ProjectDocument(Document):
    """
    ProjectDocument is an abstract class which defines the ACLs of
    project-contained elements. The ACLs of such documents are completely
    defined by the users of the containing project
    """

    def my(self, control, request, authenticated_userid=None):
        project_filter = Project.my_project_acl_filter(control, request,
                                                       authenticated_userid)
        # return directly project filter if it allows everything or nothing
        if project_filter == {} or project_filter is None:
            return project_filter
        # otherwise select the ids of the allowed projects
        else:
            project_ids_curs = connection.Project.find(project_filter,
                                                       {'_id': 1})
            project_ids = []
            for project_id in project_ids_curs:
                project_ids.append(project_id['_id'])
            return {"project": {"$in": project_ids}}

@mf_decorator
@connection.register
class ProjectData(ProjectDocument):
    """
    ProjectData describes a data linked to a project. It is referenced by jobs
    and user data manager. A ProjectData can be persistent or not.
    Data contains an AbstractData structure defining the data available.
    """

    __collection__ = 'projects_data'
    __database__ = Config.config().get('app:main', 'db_name')

    structure = {'name': basestring,
                 'description': basestring,
                 'tags': [basestring],
                 'project': ObjectId,
                 'data': AbstractData,
                 # see ObjectManager. Only READY data can be used
                 'status': int,
                 # set to True to avoid automatic deletion
                 # by cleanup tasks
                 'persistent': bool,
                 # path to the container directory for the data
                 # if it is not stored in the datamanager pairtree
                 'path': basestring,
                 # set to True to override project visibility
                 'public': bool
                 #TODO: add data provenance information
                }

    default_values = {'persistent': False, 'public': False}

    @property
    def default_value(self):
        """
        return default_value, delegated to data property
        """
        return self['data'].default_value

    def expr_value(self):
        """
        return expr_value(), delegated to data property
        """
        return self['data'].expr_value()

    def import_to_job(self, job):
        return self['data'].import_to_job(job, self.get_file_path())

    def get_file_path(self):
        '''Get root path for files of the dataset'''
        if self['path']:
            return self['path']
        return ObjectManager.get_file_path(str(self['_id']))

    def schema(self, schema=None):
        '''Update schema if parameter is not None, return schema'''
        if schema is not None:
            self['data'] = schema
            self.save()
        return self['data']

    def status(self, status=None):
        '''Update status if parameter is not None, return status'''
        if status is not None:
            self['status'] = status
            self.save()
        return self['status']

    def save_with_history(self, files=None, msg=None):
        '''
        Save object and update git repo if available

        :param files: List of relative path of new or updated files in dataset
        :type files: list
        :param msg: optional msg for the update
        :type msg: str
        '''
        if files is not None and files and ObjectManager.use_repo:
            index = ObjectManager.get_repository_index(str(self['_id']))
            index.add(files)
            if msg is None:
                msg = "Update data"
            index.commit(msg)
        self.save()


import torch.nn as nn
import torch.nn.functional as F

class SegDecoder(nn.Module):
  '''
  Optionaly seg decoder
  '''
  def __init__(self,
                img_h,
                img_w,
                n_class,
                prior_fea_channels=64,
                refine_layers=3):
      super().__init__()
      # %10 input will be set to zero during forward propagation
      # prevents overfitting
      self.dropout = nn.Dropout2d(0.1)
      self.conv = nn.Conv2d(prior_fea_channels * refine_layers, n_class, 1)
      self.img_h = img_h
      self.img_w = img_w

  def forward(self, x):
      x = self.dropout(x)
      x = self.conv(x)
      x = F.interpolate(x,
                        size=[self.img_h, self.img_w],
                        mode='bilinear',
                        align_corners=False)
      return x