#!/usr/bin/env bash

import numpy as np
import json
import sys

def main():
  gdalinfo = json.load(sys.stdin)
  if sys.argv[1] == "1":
    zscale_for_hmm(gdalinfo)
  elif sys.argv[1] == "2":
    scale_for_postprocess(gdalinfo)
  else:
    raise Exception("arg should be 1 or 2, but its " + sys.argv[1])

def scale_for_postprocess(gdalinfo):
  geotransform = gdalinfo['geoTransform']

  xpixel_size = geotransform[1]
  ypixel_size = geotransform[5]

  assert xpixel_size > 0, 'x pixel size expected to be positive'
  assert ypixel_size < 0, 'y pixel size expected to be negative'

  # test that relative error is small
  relerr = (xpixel_size - np.abs(ypixel_size)) / xpixel_size
  assert relerr < 0.01, 'pixels not scaled properly'

  scale_factor = xpixel_size
  print(scale_factor)

def zscale_for_hmm(gdalinfo):
  geotransform = gdalinfo['geoTransform']
  xpixel_size = geotransform[1]
  ypixel_size = geotransform[5]

  assert xpixel_size > 0, 'x pixel size expected to be positive'
  assert ypixel_size < 0, 'y pixel size expected to be negative'

  # test that relative error is small
  relerr = (xpixel_size - np.abs(ypixel_size)) / xpixel_size
  assert relerr < 0.01, 'pixels not scaled properly'

  # min/max
  computed_min = gdalinfo['bands'][0]['computedMin']
  computed_max = gdalinfo['bands'][0]['computedMax']

  scale_factor = (computed_max - computed_min) / xpixel_size

  print(scale_factor)

if __name__=='__main__':
  main()


# -*- coding: utf-8 -*-

from collections.abc import Iterable

from .constants import ALLOWED_CHARACTERS
from .utils import (
    any_next_words_form_swear_word,
    get_complete_path_of_file,
    get_replacement_for_swear_word,
    read_wordlist,
)
from .varying_string import VaryingString

class Profanity:
    def __init__(self, words=None):
        """
        Args:
            words (Iterable/str): Collection of words or file path for a list of
                words to censor. `None` to use the default word list.

        Raises:
            TypeError: If `words` is not a valid type.
            FileNotFoundError: If `words` is a `str` and is not a valid file path.
        """
        if (
            words is not None
            and not isinstance(words, str)
            and not isinstance(words, Iterable)
        ):
            raise TypeError("words must be of type str, list, or None")
        self.CENSOR_WORDSET = []
        self.CHARS_MAPPING = {
            "a": ("a", "@", "*", "4"),
            "i": ("i", "*", "l", "1"),
            "o": ("o", "*", "0", "@"),
            "u": ("u", "*", "v"),
            "v": ("v", "*", "u"),
            "l": ("l", "1"),
            "e": ("e", "*", "3"),
            "s": ("s", "$", "5"),
            "t": ("t", "7"),
        }
        self.MAX_NUMBER_COMBINATIONS = 1
        self.ALLOWED_CHARACTERS = ALLOWED_CHARACTERS
        self._default_wordlist_filename = get_complete_path_of_file(
            "profanity_wordlist.txt"
        )
        if type(words) == str:
            self.load_censor_words_from_file(words)
        else:
            self.load_censor_words(custom_words=words)

    ## PUBLIC ##

    def censor(self, text, censor_char="*"):
        """Replace the swear words in the text with `censor_char`."""

        if not isinstance(text, str):
            text = str(text)
        if not isinstance(censor_char, str):
            censor_char = str(censor_char)

        if not self.CENSOR_WORDSET:
            self.load_censor_words()
        return self._hide_swear_words(text, censor_char)

    def load_censor_words_from_file(self, filename, **kwargs):
        words = read_wordlist(filename)
        self._populate_words_to_wordset(words, **kwargs)

    def load_censor_words(self, custom_words=None, **kwargs):
        """Generate a set of words that need to be censored."""
        # Replace the words from `profanity_wordlist.txt` with a custom list
        custom_words = custom_words or read_wordlist(self._default_wordlist_filename)
        self._populate_words_to_wordset(custom_words, **kwargs)

    def add_censor_words(self, custom_words):
        if not isinstance(custom_words, (list, tuple, set)):
            raise TypeError(
                "Function 'add_censor_words' only accepts list, tuple or set."
            )
        for w in custom_words:
            self.CENSOR_WORDSET.append(VaryingString(w, char_map=self.CHARS_MAPPING))

    def contains_profanity(self, text):
        """Return True if  the input text has any swear words."""
        return text != self.censor(text)

    ## PRIVATE ##

    def _populate_words_to_wordset(self, words, *, whitelist_words=None):
        if whitelist_words is not None and not isinstance(
            whitelist_words, (list, set, tuple)
        ):
            raise TypeError(
                "The 'whitelist_words' keyword argument only accepts list, tuple or set."
            )

        # Validation
        whitelist_words = whitelist_words or []
        for index, word in enumerate(whitelist_words):
            if not isinstance(word, str):
                raise ValueError(
                    "Each word in 'whitelist_words' must be 'str' type, "
                    "but '{word}' found.".format(word=type(word))
                )
            whitelist_words[index] = word.lower()

        # Populate the words into an internal wordset
        whitelist_words = set(whitelist_words)
        all_censor_words = []
        for word in set(words):
            # All words in CENSOR_WORDSET must be in lowercase
            word = word.lower()

            if word in whitelist_words:
                continue

            num_of_non_allowed_chars = self._count_non_allowed_characters(word)
            if num_of_non_allowed_chars > self.MAX_NUMBER_COMBINATIONS:
                self.MAX_NUMBER_COMBINATIONS = num_of_non_allowed_chars

            all_censor_words.append(VaryingString(word, char_map=self.CHARS_MAPPING))

        # The default wordlist takes ~5MB+ of memory
        self.CENSOR_WORDSET = all_censor_words

    def _count_non_allowed_characters(self, word):
        count = 0
        for char in iter(word):
            if char not in self.ALLOWED_CHARACTERS:
                count += 1
        return count

    def _update_next_words_indices(self, text, words_indices, start_idx):
        """Return a list of next words_indices after the input index."""
        if not words_indices:
            words_indices = self._get_next_words(
                text, start_idx, self.MAX_NUMBER_COMBINATIONS
            )
        else:
            del words_indices[:2]
            if words_indices and words_indices[-1][0] != "":
                words_indices += self._get_next_words(text, words_indices[-1][1], 1)
        return words_indices

    def _hide_swear_words(self, text, censor_char):
        """Replace the swear words with censor characters."""
        censored_text = ""
        cur_word = ""
        skip_index = -1
        next_words_indices = []
        start_idx_of_next_word = self._get_start_index_of_next_word(text, 0)

        # If there are no words in the text, return the raw text without parsing
        if start_idx_of_next_word >= len(text) - 1:
            return text

        # Left strip the text, to avoid inaccurate parsing
        if start_idx_of_next_word > 0:
            censored_text = text[:start_idx_of_next_word]
            text = text[start_idx_of_next_word:]

        # Splitting each word in the text to compare with censored words
        for index, char in iter(enumerate(text)):
            if index < skip_index:
                continue
            if char in ALLOWED_CHARACTERS:
                cur_word += char
                continue

            # Skip continuous non-allowed characters
            if cur_word.strip() == "":
                censored_text += char
                cur_word = ""
                continue

            # Iterate the next words combined with the current one
            # to check if it forms a swear word
            next_words_indices = self._update_next_words_indices(
                text, next_words_indices, index
            )
            contains_swear_word, end_index = any_next_words_form_swear_word(
                cur_word, next_words_indices, self.CENSOR_WORDSET
            )
            if contains_swear_word:
                cur_word = get_replacement_for_swear_word(censor_char)
                skip_index = end_index
                char = ""
                next_words_indices = []

            # If the current a swear word
            if cur_word.lower() in self.CENSOR_WORDSET:
                cur_word = get_replacement_for_swear_word(censor_char)

            censored_text += cur_word + char
            cur_word = ""

        # Final check
        if cur_word != "" and skip_index < len(text) - 1:
            if cur_word.lower() in self.CENSOR_WORDSET:
                cur_word = get_replacement_for_swear_word(censor_char)
            censored_text += cur_word
        return censored_text

    def _get_start_index_of_next_word(self, text, start_idx):
        """Return the index of the first character of the next word in the given text."""
        start_idx_of_next_word = len(text)
        for index in iter(range(start_idx, len(text))):
            if text[index] not in self.ALLOWED_CHARACTERS:
                continue
            start_idx_of_next_word = index
            break

        return start_idx_of_next_word

    def _get_next_word_and_end_index(self, text, start_idx):
        """Return the next word in the given text, and the index of its last character."""
        next_word = ""
        index = start_idx
        for index in iter(range(start_idx, len(text))):
            char = text[index]
            if char in self.ALLOWED_CHARACTERS:
                next_word += char
                continue
            break
        return next_word, index

    def _get_next_words(self, text, start_idx, num_of_next_words=1):
        """
        Return a list of pairs of next words and next words included with separators,
        combined with their end indices.
        For example: Word `hand_job` has next words pairs: `job`, `_job`.
        """

        # Find the starting index of the next word
        start_idx_of_next_word = self._get_start_index_of_next_word(text, start_idx)

        # Return an empty string if there are no other words
        if start_idx_of_next_word >= len(text) - 1:
            return [("", start_idx_of_next_word), ("", start_idx_of_next_word)]

        # Combine the  words into a list
        next_word, end_index = self._get_next_word_and_end_index(
            text, start_idx_of_next_word
        )

        words = [
            (next_word, end_index),
            ("%s%s" % (text[start_idx:start_idx_of_next_word], next_word), end_index),
        ]
        if num_of_next_words > 1:
            words.extend(self._get_next_words(text, end_index, num_of_next_words - 1))

        return words


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  7 00:04:33 2021

@author: sara-
"""
#Needs tp be revised

import pickle
from train_utils import bag_of_words, word_frequency
import numpy as np
from snn_model import snn_model
import keras

#Unpickle words and classes
#words = pickle.load(open('Data/words.pkl','r'))
#docs = pickle.load(open('Data/docs.pkl','r'))
print("Libreries loaded")

with open('Data/split_words.pkl','rb') as f:
   words = pickle.load(f)

with open('Data/split_docs.pkl','rb') as f:
   docs = pickle.load(f)

print("Pickle loaded")

# Training set
# bag of words per sentence

X_train = []
y_train = []

for (ids, question, answer) in docs:
    word_freq = word_frequency(words)
    # X: bag of words for each pattern_sentence
    bog = bag_of_words(question, word_freq)
    X_train.append(bog)
    # y: ids
    #label = ids.index(ids)
    num = int(ids)
    y_train.append(num)

X_train = np.array(X_train)
y_train = np.array(y_train).reshape(194,1) #Change to 200

pickle.dump(X_train,open('Data/X_train.pkl','wb'))
pickle.dump(y_train,open('Data/y_train.pkl','wb'))

print("Done")
# Instantiate model
model = snn_model(X_train, y_train, 'applemodel')
model.create_and_save(X_train, y_train, 'applemodel')

#Load model and train data
#model = keras.models.load_model("model.h5")


#!/usr/bin/env python3

import collections
import itertools
import logging
import random
import logging

import numpy as np
import torch
from tqdm import tqdm
import wandb

from domain import Domain, EquationsDomain
from policy import Policy, DecisionTransformer, RandomPolicy
from util import sample_batch

logger = logging.getLogger(__name__)

class LearningAgent:
    '''Algorithm that guides learning via interaction with the enviroment.
    Gets to decide when to start a new problem, what states to expand, when to take
    random actions, etc.

    Any learning algorithm can be combined with any Recurrent Policy.
    '''

    subtypes: dict = {}

    def learn_domain(self, d: Domain):
        "Lets the agent learn by interaction using any algorithm."
        raise NotImplementedError()

    def learn_from_experience(self):
        "Lets the agent optionally learn from its past interactions one last time before eval."

    def stats(self):
        "Returns a string with learning statistics for this agent, for debugging."
        return ""

    def get_policy(self) -> Policy:
        "Returns a Policy that encodes the current learned model."
        raise NotImplementedError()

    @staticmethod
    def new(p: Policy, config: dict):
        return LearningAgent.subtypes[config['type']](p, config)

    @staticmethod
    def register(subclass):
        LearningAgent.subtypes[subclass.__name__] = subclass
        return subclass

@LearningAgent.register
class RecurrentContrastivePolicyLearning(LearningAgent):
    "Agent that learns to discriminate positive/negative examples of actions to take."

    def __init__(self, policy, config):
        self.policy = policy
        self.config = config

        self.depth = 1
        self.training_successes = []
        self.examples = []
        self.training_problems_solved = 0
        self.optimize_every = 16

    def name(self) -> str:
        return 'ReConPoLe'

    def learn_from_environment(self, environment):
        for i in tqdm(range(10000)):
            problem = environment.sample_problem(seed=i)
            rollout = self.policy.rollout(problem, depth=self.depth)

            if rollout.success:
                print(i, problem.description, 'solved!')
                self.training_problems_solved += 1

                if self.training_problems_solved % self.optimize_every == 0:
                    self.optimize()

                # FIXME(gpoesia) add examples to training buffer

            self.training_successes.append(rollout.success)

        print(self.training_problems_solved)

    def get_policy(self):
        return self.policy

    def stats(self):
        return "{} solutions found, {:.2f} training acc".format(
            np.sum(self.training_successes),
            np.mean(self.training_successes[-100:]))

    def optimize(self):
        if not self.examples:
            return

        # FIXME(gpoesia) Run gradient steps

        raise NotImplementedError()

@LearningAgent.register
class LMPolicyLearning(LearningAgent):
    "Agent that learns to discriminate positive/negative examples of actions to take."

    def __init__(self, policy, config):
        self.policy = policy

        assert hasattr(policy, 'lm'), \
            'LMPolicyLearning only supports LM-based policies.'

        self.config = config

        self.depth = config['depth']
        self.training_successes = []
        self.examples = collections.deque(maxlen=config['max_examples'])
        self.training_problems_solved = 0
        self.optimizer = torch.optim.Adam(policy.parameters())
        self.batch_size = config['batch_size']
        self.beam_size = config['beam_size']
        self.epsilon = config['epsilon']
        self.eval_every = config['eval_every']
        self.train_temperature = config['train_rollouts_temperature']
        self.only_optimize_when_solved = config['only_optimize_when_solved']
        self.rollout_type  = config['rollout_type']
        self.search_budget = config.get('search_budget', 100)
        self.train_log = []

        self.dump_examples = config.get('dump_examples')

        self.n_evals = 0

    def name(self) -> str:
        return 'LMPolicy'

    def learn_domain(self, d: Domain):
        for i in tqdm(range(self.config['episodes'])):
            if i % self.eval_every == 0:
                self.eval(d)

            self.policy.eval()

            problem = d.generate(seed=i)
            rollout = self.run_rollout(d, problem, is_train=True)

            if rollout.success:
                logger.info('Problem #%d - %s solved!',
                            i, problem.description)
                self.training_problems_solved += 1

            self.examples.extend(self.policy.extract_examples(rollout))

            if rollout.success or not self.only_optimize_when_solved:
                self.optimize()

            self.training_successes.append(rollout.success)
            self.train_log.append(rollout)

        print(self.training_problems_solved)

    def run_rollout(self, domain, problem, is_train):
        if self.rollout_type == 'BeamSearch':
            return self.policy.rollout(domain,
                                       problem,
                                       depth=self.depth,
                                       beam_size=self.beam_size if is_train else 1,
                                       epsilon=self.epsilon if is_train else 0,
                                       temperature=self.train_temperature if is_train else 1e-2)
        elif self.rollout_type == 'BestFirstSearch':
            return self.policy.best_first_search(domain, problem, self.search_budget)
        else:
            raise NotImplementedError(f'Unknown rollout algorithm {self.rollout_type}')

    def get_policy(self):
        return self.policy

    def stats(self):
        return "{} solutions found, {:.2f} training acc".format(
            np.sum(self.training_successes),
            np.mean(self.training_successes[-100:] or [0]))

    def optimize(self):
        if not self.examples:
            logger.info('Skipping optimization since we have no examples yet.')
            return

        self.policy.train()

        logger.debug('Taking gradient steps.')

        for _ in range(self.config['gradient_steps']):
            batch = sample_batch(self.examples, self.batch_size)

            if not batch:
                continue

            logger.debug('Batch: %s', batch)

            self.optimizer.zero_grad()

            # Dump examples for debugging if that's enabled.
            if self.dump_examples:
                with open(self.dump_examples, 'a') as examples_log:
                    for e in batch:
                        examples_log.write(f'{e}\n')

            loss = self.policy.get_loss(batch)
            loss.backward()
            self.optimizer.step()

            wandb.log({'train_loss': loss})
            logger.debug('{"train_loss": %f}', loss)

    def eval(self, d: Domain):
        succ = []

        logger.info("Evaluating agent")
        self.policy.eval()

        for i in tqdm(range(self.config['eval_problems'])):
            problem = d.generate(seed=10**7 + i)
            rollout = self.run_rollout(d, problem, is_train=False)
            succ.append(rollout.success)
            logger.debug('{"eval_idx": %d, "problem": "%s", "success": %d}',
                         self.n_evals, problem.description, rollout.success)

        acc = np.mean(succ)
        logger.info('{"eval_idx": %d, "accuracy": %f}', self.n_evals, acc)

        wandb.log({"success_rate": acc})

        torch.save({
            'agent': self,
            'accuracy': acc,
            }, f'lm.{self.n_evals}.pt')

        self.n_evals += 1

if __name__ == '__main__':
    import environment
    d = EquationsDomain()
    pi = DecisionTransformer({}, list(d.action_set))

    agent = LMPolicyLearning(pi, {})
    agent.learn_domain(d)


import os
import setuptools

_setup_dir = os.path.dirname(__file__)
_readme_file = os.path.join(_setup_dir, 'README.md')
with open(_readme_file) as f:
    _long_description = f.read()

_requirements_file = os.path.join(_setup_dir, 'strictdf', 'requirements.txt')

with open(_requirements_file) as f:
    _requirements = f.read().splitlines()

setuptools.setup(
    name='strictdf',
    version='0.0.1',
    author='Damian Bourdin',
    author_email='damian.bourdin@gmail.com',
    description='',
    long_description=_long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/dbourdin/strictdf/',
    packages=setuptools.find_packages(),
    package_dir={'': '.'},
    include_package_data=True,
    install_requires=_requirements,
    classifiers=[
        'Programming Language :: Python :: 3',
        'Operating System :: OS Independent',
    ],
    python_requires='>=3.6',
)


import requests
from bs4 import BeautifulSoup
#提取章节内容
def getNovelChapter(url):
    res = requests.get(url)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text,'html.parser')
    try:
        title = soup.select('.bookname h1')[0].text
        return title + '\n    ' + '\n    '.join(a for a in soup.select('#content')[0].text.split())
    except IndexError:
        return ''

url = 'http://www.biquge.tw/0_52/'    #小说目录页的网址
headers = {'Host':'www.biquge.tw',
           'Connection':'keep-alive',
           'Cache-Control':'max-age=0',
          'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36'}
res = requests.get(url)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,'html.parser')

title=soup.select('#info')[0].h1.text    #小说名称提取
author=soup.select('#info')[0].p.text.strip()    #小说作者提取

file = open(title + '.text','w')    #保存文件
file.write(title)
file.write('\n' + author + '\n\n')

i = 0
l = int(len(soup.select('#list dd')))
while (i<l):
    ChapterUrl = url + soup.select('#list dd').a['href'].split('/')[-1]
    file.write(getNovelChapter(ChapterUrl) + '\n\n')
    i=i+1
file.close()
print('下载完成')


from scripts.helpful_scripts import get_account
from brownie import (
    UniswapV3Swap,
    UniswapV3MultiSwap,
    interface,
    network,
    config,
    UniswapV2Swap,
)

import time
from web3 import Web3
import pandas as pd
import numpy as np

def main():
    account = get_account(index=-2)
    pool = config["networks"][network.show_active()]["usdc_dai_pool"]
    poolContract = interface.IUniswapV3PoolState(pool)
    liquidity = poolContract.liquidity()
    # print(Web3.fromWei(liquidity, "ether"))
    slot0 = poolContract.slot0()
    for item in slot0:
        print(item)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

__author__ = "Christian Heider Nielsen"

import time

def benchmark_func(func, times=100000):
    """description"""
    start = time.time()
    result = None
    for _ in range(times):
        result = func()
    end = time.time()
    return end - start, result


from pathlib import Path
import pandas as pd
import re
import sys

if __name__ == "__main__":
    import argparse
    from argparse import ArgumentParser

    argparser = ArgumentParser()
    argparser.add_argument("-e", "--errortype_name", type=str, required=False)
    argparser.add_argument("-f", "--filename", type=str, default="validation-report_ndt2ud.txt", required=False)
    args = argparser.parse_args()
    fpath = args.filename
    etype = args.errortype_name

    rows = Path(fpath).read_text(encoding="utf-8").split("\n")

    error_info_regx = re.compile(r"^\[Line (\d+)(?: Sent )?(\d+)?(?: Node )?(\d+)?\]\: \[(L.*)\] (.*)(\[[0-9]*, [0-9]*\])?(.*)?$", flags=re.DOTALL)
    errors = []
    print("Report summary:")
    for row in rows:
        m = error_info_regx.fullmatch(row)
        if m is None:
            print(row)
            continue
        errors.append(m.groups())

    df = pd.DataFrame(errors, columns=["line", "sent","node", "errortype", "message", "relevant_nodes", "message2"])

    print("Different types of errors:")
    type_counts = df.errortype.value_counts()
    print(type_counts)

    if etype is not None:
        df[df.errortype.str.contains(etype)].to_csv(f"error_{etype}.csv", index=False)


from itertools import combinations

import pandas as pd
from gensim.models.doc2vec import Doc2Vec

from config import DIR_PATH, DIR_LABELED_DATA
from modelling_config import PATH_TO_D2V_MODEL_ABSTRACTS, PATH_TO_D2V_MODEL_TITLES
from utils.helpers import convert_to_set, compute_diff, cosine_sim
from utils.pipeline import *
from utils.text_proc import extract_name_parts, extract_acronyms, extract_middle_name_initial
from utils.tf_idf import create_corpus_from_docs, create_corpus_from_docs_keywords, \
    extract_top_keywords, extract_top_ngrams

class Disambiguator():
    DOC2VEC_ABSTRACTS = Doc2Vec.load(PATH_TO_D2V_MODEL_ABSTRACTS)
    DOC2VEC_TITLES = Doc2Vec.load(PATH_TO_D2V_MODEL_TITLES)
    DOC2VEC_ABSTRACTS_EMPTY_VEC = DOC2VEC_TITLES.infer_vector([''])

    attr_to_attr_creation_method = {
        'signature': '_prepare_signature',
        'first_name': '_prepare_name_parts',
        'middle_name': '_prepare_name_parts',
        'middle_name_init': '_prepare_name_parts',
        'last_name': '_prepare_name_parts',
        'journal': '_prepare_journal',
        'aff_org': '_prepare_aff_ner',
        'aff_loc': '_prepare_aff_ner',
        'aff': '_prepare_aff',
        'country': '_prepare_country',
        'city': '_prepare_city',
        'coauthors': '_prepare_coauthors',
        'top_words': '_prepare_top_keywords',
        'top_ngrams': '_prepare_top_ngrams',
        'top_bigrams': '_prepare_top_bigrams',
        'abstract_doc2vec': '_prepare_doc2vec_abstract',
        'title_doc2vec': '_prepare_doc2vec_title',
        'acronyms': '_prepare_acronyms',
        'block_size': '_prepare_signature_based_attributes',
        'block_signature_diversity': '_prepare_signature_based_attributes',
        'perc_matching_signatures': '_prepare_signature_based_attributes',
        'signature_frequency': '_prepare_signature_based_attributes',
        'arxiv_class': '_prepare_arxiv_class',
        'keywords': '_prepare_keywords',
        'age': '_prepare_age',
        'abstract_bool': '_prepare_abstract_bool',
        'arxiv_class_bool': '_prepare_arxiv_class_bool',
        'keywords_bool': '_prepare_keywords_bool',
        'aff_bool': '_prepare_aff_bool',
        'acronyms_bool': '_prepare_acronyms_bool',
        'all_sig_compatible': '_prepare_signature_based_attributes',
        'matching_signatures': '_prepare_signature_based_attributes'
    }

    feature_to_pw_comp_method = {
        'signature_match': ('signature', is_signature_compatible),
        'first_name_equal': ('first_name', is_equal_or_nickname),
        'first_name_textual_dist': ('first_name', lambda x, y: textual_distance(x, y, "jaro_winkler")),
        'first_name_phonetic_dist': ('first_name', lambda x, y: phonetic_distance(x, y, "dmetaphone")),
        # 'first_name_variant': ('first_name', ), # TODO: implement usage of synonym table
        'middle_name_equal': ('middle_name', are_middle_names_equal),
        'middle_name_init_equal': ('middle_name_init', are_name_parts_equal),
        'journal_equal': ('journal', is_equal),
        'journal_astro': ('journal', lambda x, y: contains_word(x, y, "astro")),
        'journal_astron': ('journal', lambda x, y: contains_word(x, y, "astron")),
        'journal_astrop': ('journal', lambda x, y: contains_word(x, y, "astrop")),
        'journal_geo': ('journal', lambda x, y: contains_word(x, y, "geo") or contains_word(x, y, "earth")),
        'journal_top': ('journal', is_in_specific_journal_subset),
        # 'orcid_id_equal': ('orcid_id', is_equal),
        'aff_sim': ('aff', jaro_winkler_dist),
        'country_equal': ('country', is_intersection_nonempty),
        'city_equal': ('city', is_intersection_nonempty),
        'coauthors_match': ('coauthors', is_contained_set),
        'top_words_match': ('top_words', is_contained_set),
        'top_ngrams_match': ('top_ngrams', is_contained_set),
        'top_bigrams_match': ('top_bigrams', is_contained_set),
        'aid_equal': ('aid', is_equal_num),
        'abstract_sim': (
            'abstract_doc2vec', lambda x, y: cosine_sim_doc2vec_abs(x, y, Disambiguator.DOC2VEC_ABSTRACTS_EMPTY_VEC)),
        'title_sim': ('title_doc2vec', cosine_sim),
        'acronyms_match': ('acronyms', compute_idf_sum_of_acronyms),
        'block_size': ('block_size', keep_value),
        'block_signature_diversity': ('block_signature_diversity', keep_value),
        'perc_matching_signatures': ('perc_matching_signatures', compute_diff),
        'signature_frequency': ('signature_frequency', compute_diff),
        'arxiv_class_match': ('arxiv_class', compute_idf_sum_of_arxiv_class),
        'keywords_match': ('keywords', compute_idf_sum_of_keywords),
        'age_max': ('age', lambda x, y: max(x, y)),
        'age_diff': ('age', compute_diff),
        'abstract_bool_match': ('abstract_bool', compare_existence),
        'arxiv_class_bool_match': ('arxiv_class_bool', compare_existence),
        'keywords_bool_match': ('keywords_bool', compare_existence),
        'aff_bool_match': ('aff_bool', compare_existence),
        'acronyms_bool_match': ('acronyms_bool', compare_existence),
        'all_sig_compatible': ('all_sig_compatible', keep_value),
        'sig_comp_w_incomp_sigs': ('matching_signatures', are_signatures_compatible),
        'last_name_chinese': ('last_name', lambda x, y: 1 if x in Disambiguator.chinese_surnames else 0)
    }

    target = 'aid_equal'
    all_features = tuple([a for a in feature_to_pw_comp_method.keys()])
    numerical_features = ['age_diff', 'block_size', 'abstract_sim', 'title_sim', 'block_signature_diversity',
                          'aff_sim', 'perc_matching_signatures', 'signature_frequency', 'acronyms_match',
                          'arxiv_class_match', 'keywords_match', 'age_max', 'first_name_textual_dist',
                          'first_name_phonetic_dist', 'sig_comp_w_incomp_sigs']  # TODO: last attribute in fact 0 or 1
    features_requiring_missing_value_imputation = ['acronyms_match', 'arxiv_class_match', 'abstract_sim', 'aff_sim',
                                                   'first_name_textual_dist', 'first_name_phonetic_dist']

    def __init__(self, ai):
        # e.g. ai = 'miller.j
        self.ai = ai
        # Stores document data from file
        self.block = None
        # Stores features (preprocessed data)
        self.attributes = pd.DataFrame()
        # Stores pairwise comparisons of features
        self.feature_matrix = None
        self.pw_combs = None
        self.use_for_modelling = []

    def populate_block(self, verbose=False):
        if verbose:
            print('Populating block from file')
        datafile = join(DIR_PATH, DIR_LABELED_DATA, f'{self.ai}.xlsx')
        self.block = pd.read_excel(datafile)

        # Reset index, sort by year, author
        self.block = self.block.sort_values(by=['year', 'author']).reset_index(drop=True)
        if verbose:
            print(f'Populated block with {len(self.block)} authorships')

    def ignore_ambiguous_authorships(self):
        self.block = self.block[pd.notnull(self.block['aid'])]

    def prepare_attributes_for_modelling(self, create_attrs, select_attrs=('document_id', 'year', 'aid')):
        self._select_attributes(list(select_attrs))
        # These are the block related attributes
        signature_attrs = ['signature', 'block_signature_diversity', 'matching_signatures',
                           'perc_matching_signatures', 'signature_frequency',
                           'block_size', 'all_sig_compatible']
        # These attributes can be created first independently of others
        first_attrs = [a for a in create_attrs if a not in signature_attrs]
        print("first attrs", first_attrs)
        # Make first the first attrs
        create_attrs_methods = set([Disambiguator.attr_to_attr_creation_method[a] for a in first_attrs])
        for m in create_attrs_methods:
            getattr(self, m)()

        # Create signature and block attributes afterwards (they need first_name, etc to exist!)
        # print(self.attributes.columns)

        if 'signature' in create_attrs:
            print("signature in create_attrss")
            self._prepare_signature()
            print(self.attributes.columns)

            if set(signature_attrs).issubset(set(create_attrs)):
                print("yes all sig attrs are there")
                self._prepare_signature_based_attributes()

        self._filter_attributes()

    def build_all_pw_comparisons(self, features, store_pw_combs=False, squared=False):

        use_attrs = [v[0] for k, v in Disambiguator.feature_to_pw_comp_method.items()]
        use_attrs = [attr for attr in self.attributes.columns if attr in use_attrs]

        # Store document_id to keep track of data origin later
        combs = Disambiguator.build_pw_comparisons(self.attributes[['document_id'] + use_attrs], squared=squared)
        filtered_attr_to_method = {f: v for f, v in Disambiguator.feature_to_pw_comp_method.items() if
                                   f in features and v[0] in use_attrs}

        self.feature_matrix = pd.DataFrame()
        for f, a_m in sorted(filtered_attr_to_method.items()):
            attr, method = a_m[0], a_m[1]
            self.feature_matrix[f] = combs.apply(lambda x: method(x[attr + '_x'], x[attr + '_y']), axis=1)

        if store_pw_combs:
            self.pw_combs = combs

    def _select_attributes(self, attr_names):
        for a in attr_names:
            self.attributes[a] = self.block[a]
        self.use_for_modelling.extend(attr_names)

    def _prepare_aff(self):
        self.attributes['aff'] = self.block['affiliation'].fillna("")
        self.use_for_modelling.append('aff')

    def _prepare_name_parts(self):
        self.attributes['last_name'] = self.block['author'].map(lambda x: x.split(",")[0]).unique()[0]
        if 'first_name' in self.attributes.columns and 'middle_name' in self.attributes.columns:
            self.attributes['middle_name_init'] = self.block['author'].map(extract_middle_name_initial)
            self.use_for_modelling.extend(['middle_name_init'])
        else:
            self.attributes['first_name'], self.attributes['middle_name'], self.attributes['middle_name_init'] = zip(
                *self.block['author'].map(extract_name_parts))

            self.use_for_modelling.extend(['last_name', 'first_name', 'middle_name', 'middle_name_init'])

    def _prepare_country(self):
        self.attributes['country'] = self.block['country'].map(convert_to_set)
        self.use_for_modelling.append('country')

    def _prepare_city(self):
        self.attributes['city'] = self.block['city'].map(convert_to_set)
        self.use_for_modelling.append('city')

    def _prepare_journal(self):
        self.attributes['journal'] = self.block['journal'].apply(str.lower)
        self.use_for_modelling.append('journal')

    def _prepare_top_keywords(self):
        corpus = create_corpus_from_docs_keywords(self.block)
        self.attributes['top_words'] = self.block.apply(lambda x: set([l[0] for l in extract_top_keywords(x, corpus)]),
                                                        axis=1)

        self.use_for_modelling.append('top_words')

    def _prepare_top_ngrams(self, ngram_range=(1, 2), lim=10):
        corpus = create_corpus_from_docs(self.block)
        self.attributes['top_ngrams'] = self.block.apply(lambda x: set(extract_top_ngrams(x, corpus,
                                                                                ngram_range=ngram_range,
                                                                                lim=lim)), axis=1)
        self.use_for_modelling.append('top_ngrams')

    def _prepare_top_bigrams(self, ngram_range=(2, 2), lim=10):
        corpus = create_corpus_from_docs(self.block)
        self.attributes['top_bigrams'] = self.block.apply(lambda x: set(extract_top_ngrams(x, corpus,
                                                                                ngram_range=ngram_range,
                                                                                lim=lim)), axis=1)
        self.use_for_modelling.append('top_bigrams')

    def _prepare_coauthors(self):
        self.attributes['coauthors'] = self.block.apply(extract_coauthors, axis=1)

        self.use_for_modelling.append('coauthors')

    def _prepare_signature(self):
        assert {'first_name', 'middle_name', 'middle_name_init'}.issubset(
            set(self.attributes.columns)), "Required columns not in attributes"
        self.attributes['signature'] = self.attributes.apply(lambda r: author_signature(r, self.ai), axis=1)

        """Some names occur written in one word or with a '-' connecting first and middle name.
        If such a word occurs, then split it."""

        unique_signatures = self.attributes['signature'].unique()
        signatures_double_name = [s for s in unique_signatures if s[0] != '' and s[1] != '']
        if signatures_double_name != []:
            signatures_double_name = {s[0] + s[1]: s for s in
                                      signatures_double_name}  # not necessarily unique but should be extremely rare
            first_name_only = [s[0] for s in unique_signatures if s[0] != '' and s[1] == '']

            for s in first_name_only:
                if s in signatures_double_name.keys():
                    cond_name = self.attributes.signature == (s, '')
                    # print("Signature that could also be split into two parts: ", s)
                    first, middle = signatures_double_name[s][0], signatures_double_name[s][1]
                    print("first and middle: ", first, middle)
                    self.attributes.loc[cond_name, 'first_name'] = first
                    self.attributes.loc[cond_name, 'middle_name'] = middle
                    self.attributes.loc[cond_name, 'middle_name_init'] = middle[0]
                    same_cond = (self.attributes.first_name == first) & (self.attributes.middle_name == middle)

                    self.attributes.loc[same_cond, 'signature'] = self.attributes.loc[same_cond, :].apply(
                        lambda x: (x['first_name'], x['middle_name']), axis=1)

        self.use_for_modelling.append('signature')

    def _prepare_signature_based_attributes(self):
        # Requires to know all signatures present in the block
        assert ('signature') in self.attributes.columns, 'Required column signature not in attributes'

        block_size = len(self.block)
        unique_signatures = list(self.attributes['signature'].unique())

        # How many distinct (first, middle) signatures in block
        self.attributes['block_signature_diversity'] = np.round(len(unique_signatures) / block_size, 3)

        # list of all compatible signatures with s in first position
        self.attributes['matching_signatures'] = self.attributes.signature.map(
            lambda s: [s] + [s1 for s1 in [i for i in unique_signatures if i != s]
                             if is_signature_compatible(s, s1) == "true"])
        # With how many of those signatures each particular record matches
        # I.e. in block with (john s) and (jake simon) a pub with (j s) matches 2
        self.attributes['perc_matching_signatures'] = self.attributes.signature.apply(
            lambda s: sum([int(x == 'true') for x in [is_signature_compatible(s, s1) for s1 in unique_signatures]]))
        self.attributes['perc_matching_signatures'] = np.round(self.attributes['perc_matching_signatures'] / len(
            unique_signatures), 3)

        # How frequent the signature is
        # I.e. in block with (john s) = 3 and (jake simon) = 1, freques are 0.75 and 0.25
        count_sign = lambda s: self.attributes.signature.value_counts().loc[[s]].values[0]
        self.attributes['signature_frequency'] = self.attributes.signature.apply(
            count_sign) / block_size
        self.attributes['signature_frequency'] = self.attributes['signature_frequency'].map(lambda x: np.round(x, 3))

        # smallest quantile for which the block size value is larger than length of current block
        block_size_quantiles = fetch_block_sizes()
        self.attributes['block_size'] = min([q for q, v in block_size_quantiles.items() if v >= len(self.block)])

        unique_signatures = self.attributes["signature"].unique()
        unique_signature_pairs = set(combinations(unique_signatures, 2))
        incompatible_sig_pairs = [sig_pair for sig_pair in unique_signature_pairs if
                                  is_signature_compatible(*sig_pair) != 'true']
        print(incompatible_sig_pairs)

        if len(incompatible_sig_pairs) > 0:
            self.attributes['all_sig_compatible'] = 'false'
        else:
            self.attributes['all_sig_compatible'] = 'true'

        # self.use_for_modelling.extend(
        #     ['block_size', 'block_signature_diversity', 'perc_matching_signatures', 'signature_frequency', 'all_sig_compatible'])
        self.use_for_modelling.extend(
            ['block_size', 'block_signature_diversity', 'all_sig_compatible', 'matching_signatures'])

    def _prepare_doc2vec_abstract(self):
        self.attributes['abstract_doc2vec'] = self.block['abstract'].map(tokenize)
        self.attributes['abstract_doc2vec'] = self.attributes['abstract_doc2vec'].map(
            lambda x: Disambiguator.DOC2VEC_ABSTRACTS.infer_vector(x))
        self.use_for_modelling.append('abstract_doc2vec')

    def _prepare_doc2vec_title(self):
        self.attributes['title_doc2vec'] = self.block['title'].map(tokenize)
        self.attributes['title_doc2vec'] = self.attributes['title_doc2vec'].map(
            lambda x: Disambiguator.DOC2VEC_TITLES.infer_vector(x))
        self.use_for_modelling.append('title_doc2vec')

    def _prepare_acronyms(self):
        self.attributes['acronyms'] = self.block['abstract'].map(str) + ' ' + self.block['title']
        self.attributes['acronyms'] = self.attributes['acronyms'].map(extract_acronyms)

        self.use_for_modelling.append('acronyms')

    def _prepare_arxiv_class(self):

        self.attributes['arxiv_class'] = self.block['arxiv_class'].fillna('')
        self.attributes['arxiv_class'] = self.attributes['arxiv_class'].map(extract_arxiv_classes)
        self.attributes['arxiv_class'] = self.attributes['arxiv_class'].map(lambda x: set() if x == {''} else x)

        self.use_for_modelling.append('arxiv_class')

    def _prepare_keywords(self):
        self.attributes['keywords'] = self.block['keywords'].fillna('')
        self.attributes['keywords'] = self.attributes['keywords'].map(extract_keywords)
        self.attributes['keywords'] = self.attributes['keywords'].map(lambda x: set() if x == {''} else x)

        self.use_for_modelling.append('keywords')

    def _prepare_age(self):
        self.attributes['age'] = 2018 - self.block[
            'year']  # TODO: replace 2018 by this year in production or more recent data
        self.use_for_modelling.append('age')

    def _prepare_abstract_bool(self):
        self.attributes['abstract_bool'] = self.block['abstract'].map(lambda x: not is_missing(x))
        self.use_for_modelling.append('abstract_bool')

    def _prepare_arxiv_class_bool(self):
        self.attributes['arxiv_class_bool'] = self.block['arxiv_class'].map(lambda x: not is_missing(x))
        self.use_for_modelling.append('arxiv_class_bool')

    def _prepare_keywords_bool(self):
        self.attributes['keywords_bool'] = self.block['keywords'].map(lambda x: not is_missing(x))
        self.use_for_modelling.append('keywords_bool')

    def _prepare_aff_bool(self):
        self.attributes['aff_bool'] = self.block['affiliation'].map(lambda x: not is_missing(x))
        self.use_for_modelling.append('aff_bool')

    def _prepare_acronyms_bool(self):
        # TODO: improve code later. first 2 lines are repeated from _prepare_acronyms
        self.attributes['acronyms_bool'] = self.block['abstract'].map(str) + ' ' + self.block['title']
        self.attributes['acronyms_bool'] = self.attributes['acronyms_bool'].map(extract_acronyms)
        self.attributes['acronyms_bool'] = self.attributes['acronyms_bool'].map(lambda x: len(x) > 0)
        self.use_for_modelling.append('acronyms_bool')

    def _filter_attributes(self):
        self.attributes = self.attributes[sorted(self.use_for_modelling)]

    @staticmethod
    def build_pw_comparisons(df, squared):
        if squared is True:
            combs = list(product(df.values.tolist(), repeat=2))
        else:
            combs = list(combinations(df.values.tolist(), 2))
        combs = [item[0] + item[1] for item in combs]  # flatten combs which is a list of tuples of len 2
        new_cols = [c + '_x' for c in df.columns] + [c + '_y' for c in df.columns]
        combs = pd.DataFrame(combs, columns=new_cols)
        return combs

    # TODO: experimental; should be extended by other ethnicities
    chinese_surnames = ['wang', 'li', 'zhang', 'liu', 'chen', 'yang', 'huang', 'zhao', 'wu', 'zhou', 'xu', 'sun', 'ma',
                        'zhu', 'hu', 'guo', 'he', 'lin', 'gao', 'luo', 'zheng', 'liang', 'xie', 'song', 'tang', 'xu',
                        'deng', 'han', 'feng', 'cao', 'peng', 'zeng', 'xiao', 'tian', 'dong', 'pan', 'yuan', 'cai',
                        'jiang', 'yu', 'yu', 'du', 'ye', 'cheng', 'wei', 'su', 'lu', 'ding', 'ren', 'lu', 'yao', 'shen',
                        'zhong', 'jiang', 'cui', 'tan', 'lu', 'fan', 'wang', 'liao', 'shi', 'jin', 'wei', 'jia', 'xia',
                        'fu', 'fang', 'zou', 'xiong', 'bai', 'meng', 'qin', 'qiu', 'hou', 'jiang', 'yin', 'xue', 'yan',
                        'duan', 'lei', 'long', 'li', 'shi', 'tao', 'he', 'mao', 'hao', 'gu', 'gong', 'shao', 'wan',
                        'qin', 'wu', 'qian', 'dai', 'yan', 'ou', 'mo', 'kong', 'xiang', 'chang']

TARGET = Disambiguator.target
ALL_FEATURES = Disambiguator.all_features
NUMERICAL_FEATURES = Disambiguator.numerical_features
CATEGORICAL_FEATURES = [a for a in Disambiguator.all_features if a not in NUMERICAL_FEATURES]

def fetch_block_sizes(quantiles=(0.5, 0.7, 0.9, 1.0)):
    """Compute block size quantiles and return them as a dictionary"""
    df = pd.read_csv(join(PATH_DATA, 'ai_block_sizes_entire_db.csv'))
    block_size_quantiles = df[['count']].quantile(q=quantiles)

    return block_size_quantiles.to_dict()['count']


import json
import re
from typing import Dict, AnyStr

import requests

CONFUSABLES_URL = "http://www.unicode.org/Public/security/latest/confusables.txt"

# 064F ;	0313 ;	MA	# ( ُ → ̓ ) ARABIC DAMMA → COMBINING COMMA ABOVE	#
CONFUSABLES_RE = re.compile(r"(?P<source>.*)\s;\s(?P<target>.*)\s;\sMA\s.*")

def get_confusables(url: AnyStr = CONFUSABLES_URL):
    """Download the url to parse"""
    data = requests.get(url).text
    return data

def parse_confusables(text: str) -> Dict[AnyStr, AnyStr]:
    """convert the consumables format from the site format to a dict"""
    out = {}
    for line in text.splitlines():
        if not line or line.startswith("#"):
            continue
        match = CONFUSABLES_RE.fullmatch(line)
        if not match:
            print(f"Unmatched line: {line!r}")
            continue

        match_dict = match.groupdict()
        assert match_dict["source"] not in out, f"Duplicate entry in confusables found: {match_dict['source']} -> " \
                                                f"{match_dict['target']} ({line!r})"

        source_codepoint = match_dict["source"]
        target_codepoint = match_dict["target"]
        out[source_codepoint] = target_codepoint

    return out

def convert_codepoint(codepoint: str):
    """convert codepoints (including multu byte ones) to the bytes they represent"""
    split = codepoint.split(" ")
    out = ""
    for point in split:
        out += chr(int(point, 16))

    return out

def convert_to_chars(codepoint_dict: Dict[AnyStr, AnyStr]):
    """Create a dict of real characters from a dict of codepoints"""
    out = {}
    for source, target in codepoint_dict.items():
        source_chr = convert_codepoint(source)
        target_chr = convert_codepoint(target)
        out[source_chr] = target_chr

    return out

def create_json():
    """Create json files of confusables for use in other programs"""
    confusables = get_confusables()
    codepoint_dict = parse_confusables(confusables)
    chr_dict = convert_to_chars(codepoint_dict)

    with open("codepoints.json", "w") as f:
        json.dump(codepoint_dict, f, indent=2)

    with open("chrs.json", "w") as f:
        json.dump(chr_dict, f, indent=2)

if __name__ == '__main__':
    create_json()


class rectangle:
    def __init__(self,length,width):
        self.__length=length
        self.__width=width
    def __lt__(self,a2):
        area1=self.__length*self.__width
        area2=a2.__length*a2.__width
        if(area1<area2):
            return(True)
        else:
            return(False)
print("Enter the Details of Rectangle:1")
l1=int(input("Lenght:"))
w1=int(input("width:"))
r1=rectangle(l1,w1)
print("Enter the Details of Rectangle:2")
l2=int(input("Lenght:"))
w2=int(input("width:"))
r2=rectangle(l2,w2)
if(r1<r2):
    print("Rectangle 2 is larger!!")
else:
    print("Rectangle 1 is larger!!")


from __future__ import annotations

import uuid
from typing import TYPE_CHECKING, Iterator

import pandas as pd
import pytest

from great_expectations.datasource.fluent.pandas_datasource import DataFrameAsset
from great_expectations.expectations.expectation_configuration import (
    ExpectationConfiguration,
)

if TYPE_CHECKING:
    from great_expectations.checkpoint import Checkpoint
    from great_expectations.checkpoint.checkpoint import CheckpointResult
    from great_expectations.core import ExpectationSuite, ExpectationValidationResult
    from great_expectations.data_context import CloudDataContext
    from great_expectations.datasource.fluent import (
        BatchRequest,
        DataAsset,
        PandasDatasource,
    )
    from great_expectations.validator.validator import Validator

@pytest.fixture(scope="module")
def pandas_test_df() -> pd.DataFrame:
    d = {
        "string": ["a", "b", "c"],
        "datetime": [
            pd.to_datetime("2020-01-01"),
            pd.to_datetime("2020-01-02"),
            pd.to_datetime("2020-01-03"),
        ],
    }
    df = pd.DataFrame(data=d)
    return df

@pytest.fixture(scope="module")
def datasource(
    context: CloudDataContext,
    datasource_name: str,
) -> PandasDatasource:
    """Test Adding and Updating the Datasource associated with this module.
    Note: There is no need to test Get or Delete Datasource.
    Those assertions can be found in the datasource_name fixture."""
    datasource = context.sources.add_pandas(
        name=datasource_name,
    )
    assert datasource.name == datasource_name
    new_datasource_name = f"ds{uuid.uuid4().hex}"
    datasource.name = new_datasource_name
    datasource = context.sources.add_or_update_pandas(
        datasource=datasource,
    )
    assert (
        datasource.name == new_datasource_name
    ), "The datasource was not updated in the previous method call."
    datasource.name = datasource_name
    datasource = context.add_or_update_datasource(  # type: ignore[assignment]
        datasource=datasource,
    )
    assert (
        datasource.name == datasource_name
    ), "The datasource was not updated in the previous method call."
    return datasource

def dataframe_asset(
    datasource: PandasDatasource,
    asset_name: str,
) -> DataFrameAsset:
    return datasource.add_dataframe_asset(
        name=asset_name,
    )

@pytest.fixture(scope="module", params=[dataframe_asset])
def data_asset(
    datasource: PandasDatasource,
    get_missing_data_asset_error_type: type[Exception],
    request,
) -> Iterator[DataAsset]:
    """Test the entire Data Asset CRUD lifecycle here and in Data Asset-specific fixtures."""
    asset_name = f"da_{uuid.uuid4().hex}"
    yield request.param(
        datasource=datasource,
        asset_name=asset_name,
    )
    datasource.delete_asset(asset_name=asset_name)
    with pytest.raises(get_missing_data_asset_error_type):
        datasource.get_asset(asset_name=asset_name)

@pytest.fixture(scope="module")
def batch_request(
    data_asset: DataAsset,
    pandas_test_df: pd.DataFrame,
    in_memory_batch_request_missing_dataframe_error_type: type[Exception],
) -> BatchRequest:
    """Build a BatchRequest depending on the types of Data Assets tested in the module."""
    if isinstance(data_asset, DataFrameAsset):
        with pytest.raises(in_memory_batch_request_missing_dataframe_error_type):
            data_asset.build_batch_request()
        batch_request = data_asset.build_batch_request(dataframe=pandas_test_df)
    else:
        batch_request = data_asset.build_batch_request()
    return batch_request

@pytest.fixture(scope="module")
def expectation_suite(
    context: CloudDataContext,
    expectation_suite: ExpectationSuite,
) -> ExpectationSuite:
    """Add Expectations for the Data Assets defined in this module.
    Note: There is no need to test Expectation Suite CRUD.
    Those assertions can be found in the expectation_suite fixture.
    """
    expectation_suite.add_expectation(
        expectation_configuration=ExpectationConfiguration(
            expectation_type="expect_column_values_to_not_be_null",
            kwargs={
                "column": "string",
                "mostly": 1,
            },
        )
    )
    return expectation_suite

@pytest.mark.cloud
def test_interactive_validator(
    context: CloudDataContext,
    validator: Validator,
):
    """Test interactive evaluation of the Data Assets in this module using an existing Validator.
    Note: There is no need to test getting a Validator or using Validator.head(). That is already
    tested in the validator fixture.
    """
    expectation_validation_result: ExpectationValidationResult = (
        validator.expect_column_values_to_not_be_null(
            column="datetime",
            mostly=1,
        )
    )
    assert expectation_validation_result.success

@pytest.mark.xfail(
    reason="1.0 API requires a backend change. Test should pass once #2623 is merged"
)
@pytest.mark.cloud
def test_checkpoint_run(checkpoint: Checkpoint):
    """Test running a Checkpoint that was created using the entities defined in this module."""
    checkpoint_result: CheckpointResult = checkpoint.run()
    assert checkpoint_result.success


def unikati(s):
    seznam = []
    for x in s:
        if x not in seznam:
            seznam.append(x)
    return seznam

def avtor(tvit):

    b = tvit.split(" ")
    return b[0][:-1]

def vsi_avtorji(tviti):
    seznam = []
    for x in tviti:
        a = avtor(x)
        seznam.append(a)

    seznam1= unikati(seznam)
    return seznam1

def izloci_besedo(beseda):
    for x in beseda:              #gres cez vse znake v besedi
        if x.isalnum() == False:   #ce 1. znak ni alfanumeričen ga odstraniš
            beseda = beseda[1:]
        else:
            break           #prenehaš ko prideš do prve črke ali številke

    a = beseda[::-1]            #obrnes besedo okoli in ponoviš
    for y in a:
        if y.isalnum()==False:
            a = a[1:]
        else:
            break
    return a[::-1]          #ko returnaš, ponovno obrneš besedo

def se_zacne_z(tvit, c):
    seznam = []
    b = tvit.split(" ")   #b= seznam splitanih besed

    for beseda in b:    #gres cez vse besede v b
        if beseda[0] == c:   #ce se beseda zacne z c, jo dodas na seznam
            seznam.append(beseda)

    koncniseznam = []
    for beseda1 in seznam:    #gres seenkrat cez seznam in vedno klices funkcijo izloci_besedo
        koncniseznam.append(izloci_besedo(beseda1))

    return koncniseznam

def zberi_se_zacne_z(tviti, c):
    seznam = []

    for tvit in tviti:
        a = se_zacne_z(tvit,c)
        seznam.extend(a)

    seznam = unikati(seznam)
    return seznam

def vse_afne(tviti):

    seznam = zberi_se_zacne_z(tviti, "@")
    return seznam

def vsi_hashtagi(tviti):
    seznam = zberi_se_zacne_z(tviti, "#")
    return seznam

def vse_osebe(tviti):
    seznam_avtorjev = vsi_avtorji(tviti)
    seznam_avtorjev.extend(vse_afne(tviti))

    seznam_avtorjev = sorted(unikati(seznam_avtorjev))

    return seznam_avtorjev

def custva(tviti, hashtagi):
    seznam = []
    for tvit in tviti:
        besede = tvit.split(" ")
        for beseda in besede:
            for beseda1 in hashtagi:
                if beseda == "#"+beseda1:
                    seznam.append(avtor(tvit))

    seznam = unikati(seznam)

    return sorted(seznam)

def se_poznata(tviti, oseba1, oseba2):
    se_poznata= False
    for tvit in tviti: #gres skozi vsak tvit
        avtor1 = avtor(tvit)   #zapises avtorja tvita
        seznam = se_zacne_z(tvit,"@")  #naredis seznam vseh oseb, ki jih je avtor omenil

        if oseba1 == avtor1 and oseba2 in seznam:  #ce je oseba 1 avtor in je oseba 2 med
            se_poznata = True          # seznamom omenjenih oseb nastavi se_poznata na True

    return se_poznata

import unittest

class TestTviti(unittest.TestCase):
    tviti = [
        "sandra: Spet ta dež. #dougcajt",
        "berta: @sandra Delaj domačo za #programiranje1",
        "sandra: @berta Ne maram #programiranje1 #krneki",
        "ana: kdo so te @berta, @cilka, @dani? #krneki",
        "cilka: jst sm pa #luft",
        "benjamin: pogrešam ano #zalosten",
        "ema: @benjamin @ana #split? po dvopičju, za začetek?",
    ]

    def test_unikat(self):
        self.assertEqual(unikati([1, 2, 1, 1, 3, 2]), [1, 2, 3])
        self.assertEqual(unikati([1, 3, 2, 1, 1, 3, 2]), [1, 3, 2])
        self.assertEqual(unikati([1, 5, 4, 3, 2]), [1, 5, 4, 3, 2])
        self.assertEqual(unikati([1, 1, 1, 1, 1]), [1])
        self.assertEqual(unikati([1]), [1])
        self.assertEqual(unikati([]), [])
        self.assertEqual(unikati(["Ana", "Berta", "Cilka", "Berta"]), ["Ana", "Berta", "Cilka"])

    def test_avtor(self):
        self.assertEqual(avtor("janez: pred dvopičjem avtor, potem besedilo"), "janez")
        self.assertEqual(avtor("ana: malo krajse ime"), "ana")
        self.assertEqual(avtor("benjamin: pomembne so tri stvari: prva, druga in tretja"), "benjamin")

    def test_vsi_avtorji(self):
        self.assertEqual(vsi_avtorji(self.tviti), ["sandra", "berta", "ana", "cilka", "benjamin", "ema"])
        self.assertEqual(vsi_avtorji(self.tviti[:3]), ["sandra", "berta"])

    def test_izloci_besedo(self):
        self.assertEqual(izloci_besedo("@ana"), "ana")
        self.assertEqual(izloci_besedo("@@ana!!!"), "ana")
        self.assertEqual(izloci_besedo("ana"), "ana")
        self.assertEqual(izloci_besedo("!#$%\"=%/%()/Ben-jamin'"), "Ben-jamin")

    def test_vse_na_crko(self):
        self.assertEqual(se_zacne_z("Benjamin $je $skocil! Visoko!", "$"), ["je", "skocil"])
        self.assertEqual(se_zacne_z("Benjamin $je $skocil! #Visoko!", "$"), ["je", "skocil"])
        self.assertEqual(se_zacne_z("ana: kdo so te @berta, @cilka, @dani? #krneki", "@"), ["berta", "cilka", "dani"])

    def test_zberi_na_crko(self):
        self.assertEqual(zberi_se_zacne_z(self.tviti, "@"), ['sandra', 'berta', 'cilka', 'dani', 'benjamin', 'ana'])
        self.assertEqual(zberi_se_zacne_z(self.tviti, "#"), ['dougcajt', 'programiranje1', 'krneki', 'luft', 'zalosten', 'split'])

    def test_vse_afne(self):
        self.assertEqual(vse_afne(self.tviti), ['sandra', 'berta', 'cilka', 'dani', 'benjamin', 'ana'])

    def test_vsi_hashtagi(self):
        self.assertEqual(vsi_hashtagi(self.tviti), ['dougcajt', 'programiranje1', 'krneki', 'luft', 'zalosten', 'split'])

    def test_vse_osebe(self):
        self.assertEqual(vse_osebe(self.tviti), ['ana', 'benjamin', 'berta', 'cilka', 'dani', 'ema', 'sandra'])

class TestDodatna(unittest.TestCase):
    tviti = [
        "sandra: Spet ta dež. #dougcajt",
        "berta: @sandra Delaj domačo za #programiranje1",
        "sandra: @berta Ne maram #programiranje1 #krneki",
        "ana: kdo so te @berta, @cilka, @dani? #krneki",
        "cilka: jst sm pa #luft",
        "benjamin: pogrešam ano #zalosten",
        "ema: @benjamin @ana #split? po dvopičju, za začetek?",
    ]

    def test_custva(self):
        self.assertEqual(custva(self.tviti, ["dougcajt", "krneki"]), ["ana", "sandra"])
        self.assertEqual(custva(self.tviti, ["luft"]), ["cilka"])
        self.assertEqual(custva(self.tviti, ["meh"]), [])

    def test_se_poznata(self):
        self.assertTrue(se_poznata(self.tviti, "ana", "berta"))
        self.assertTrue(se_poznata(self.tviti, "ema", "ana"))
        self.assertFalse(se_poznata(self.tviti, "sandra", "ana"))
        self.assertFalse(se_poznata(self.tviti, "cilka", "luft"))
        self.assertFalse(se_poznata(self.tviti, "cilka", "balon"))

if __name__ == "__main__":
    unittest.main()


from constants import NUM_ROWS, NUM_COLS
from helpers import player_turn, create_board, play_again

def main():
    lets_play_again = True
    max_score = NUM_ROWS * NUM_COLS // 2

    while lets_play_again:
        # initialise values
        board = create_board()
        player_turns = 0
        player_score = 0

        # main logic
        game_over = False
        while not game_over:
            success = player_turn(board, player_score, player_turns)
            if success:
                player_score += 1
            player_turns += 1
            if player_score >= max_score:
                game_over = True
        print()
        print('Congartulations!')
        print('You completed the game in', player_turns, 'turn')
        print('Your score is', player_score, '.', sep='')

        if not play_again():
            lets_play_again = False
    print('Goodbye!')

if __name__ == '__main__':
    main()


import numpy as np
np.random.seed(123)
from matplotlib import pyplot as plt
import time

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Convolution2D, MaxPooling2D
from keras.utils import np_utils

from keras.datasets import mnist

(X_train, y_train), (X_test, y_test) = mnist.load_data()
print(X_train.shape)

X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)

X_train = X_train.astype('uint8')
X_test = X_test.astype('uint8')

Y_train = np_utils.to_categorical(y_train, 10)
Y_test = np_utils.to_categorical(y_test, 10)

model = Sequential()
model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(28,28,1), data_format="channels_last"))
model.add(Convolution2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))

model.compile(loss='categorical_crossentropy',
	      optimizer='adam',
	      metrics=['accuracy'])

model.fit(X_train, Y_train,
	epochs=10, verbose=1)

score = model.evaluate(X_test, Y_test, verbose=1)
print(score)

model.save("my_model.h5")

data = X_test[4,:,:,0]
print(Y_test[4,:])
data =data.reshape(1,28,28,1)
start_time = time.time()
out = model.predict(data)
print("--- %s seconds ---" % (time.time() - start_time))
print(out)

data = X_test[0,:,:,0]
print(Y_test[4,:])
data =data.reshape(1,28,28,1)
start_time = time.time()
out = model.predict(data)
print("--- %s seconds ---" % (time.time() - start_time))
print(out)

data = X_test[1,:,:,0]
print(Y_test[4,:])
data =data.reshape(1,28,28,1)
start_time = time.time()
out = model.predict(data)
print("--- %s seconds ---" % (time.time() - start_time))
print(out)

data = X_test[2,:,:,0]
print(Y_test[4,:])
data =data.reshape(1,28,28,1)
start_time = time.time()
out = model.predict(data)
print("--- %s seconds ---" % (time.time() - start_time))
print(out)


__author__ = ["Markus Löning"]
__all__ = ["ESTIMATOR_TEST_PARAMS", "EXCLUDED_ESTIMATORS", "EXCLUDED_TESTS"]

from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

from sktime.classification.compose import ColumnEnsembleClassifier
from sktime.classification.compose import TimeSeriesForestClassifier
from sktime.classification.frequency_based import RandomIntervalSpectralForest
from sktime.classification.interval_based import TimeSeriesForest
from sktime.classification.shapelet_based import ShapeletTransformClassifier
from sktime.forecasting.arima import AutoARIMA
from sktime.forecasting.compose import DirectRegressionForecaster
from sktime.forecasting.compose import DirectTimeSeriesRegressionForecaster
from sktime.forecasting.compose import EnsembleForecaster
from sktime.forecasting.compose import RecursiveRegressionForecaster
from sktime.forecasting.compose import RecursiveTimeSeriesRegressionForecaster
from sktime.forecasting.compose import StackingForecaster
from sktime.forecasting.compose import TransformedTargetForecaster
from sktime.forecasting.exp_smoothing import ExponentialSmoothing
from sktime.forecasting.model_selection import ForecastingGridSearchCV
from sktime.forecasting.model_selection import SingleWindowSplitter
from sktime.forecasting.naive import NaiveForecaster
from sktime.forecasting.theta import ThetaForecaster
from sktime.performance_metrics.forecasting import sMAPE
from sktime.regression.compose import TimeSeriesForestRegressor
from sktime.transformers.series_as_features.compose import ColumnTransformer
from sktime.transformers.series_as_features.compose import RowTransformer
from sktime.transformers.series_as_features.dictionary_based import SFA
from sktime.transformers.series_as_features.interpolate import TSInterpolator
from sktime.transformers.series_as_features.reduce import Tabularizer
from sktime.transformers.series_as_features.shapelets import ContractedShapeletTransform
from sktime.transformers.series_as_features.shapelets import ShapeletTransform
from sktime.transformers.series_as_features.summarize import FittedParamExtractor
from sktime.transformers.series_as_features.summarize import TSFreshFeatureExtractor
from sktime.transformers.series_as_features.summarize import (
    TSFreshRelevantFeatureExtractor,
)
from sktime.transformers.single_series.adapt import SingleSeriesTransformAdaptor
from sktime.transformers.single_series.detrend import Detrender

# TODO fix estimators to pass all tests
EXCLUDED_ESTIMATORS = [
    "ElasticEnsemble",
    "KNeighborsTimeSeriesClassifier",
    "ProximityForest",
    "ProximityStump",
    "ProximityTree",
]

EXCLUDED_TESTS = {
    "ShapeletTransformClassifier": ["check_fit_idempotent"],
    "ContractedShapeletTransform": ["check_fit_idempotent"],
}

TRANSFORMER = StandardScaler()
TRANSFORMERS = [
    ("transformer1", RowTransformer(TRANSFORMER)),
    ("transformer2", RowTransformer(TRANSFORMER)),
]
REGRESSOR = LinearRegression()
TIME_SERIES_CLASSIFIER = TimeSeriesForest(n_estimators=5, random_state=1)
TIME_SERIES_CLASSIFIERS = [
    ("tsf1", TIME_SERIES_CLASSIFIER),
    ("tsf2", TIME_SERIES_CLASSIFIER),
]
FORECASTER = ExponentialSmoothing()
FORECASTERS = [("ses1", FORECASTER), ("ses2", FORECASTER)]
STEPS = [
    ("transformer", Detrender(ThetaForecaster())),
    ("forecaster", NaiveForecaster()),
]
ESTIMATOR_TEST_PARAMS = {
    DirectRegressionForecaster: {"regressor": REGRESSOR},
    RecursiveRegressionForecaster: {"regressor": REGRESSOR},
    DirectTimeSeriesRegressionForecaster: {
        "regressor": make_pipeline(Tabularizer(), REGRESSOR)
    },
    RecursiveTimeSeriesRegressionForecaster: {
        "regressor": make_pipeline(Tabularizer(), REGRESSOR)
    },
    TransformedTargetForecaster: {"steps": STEPS},
    EnsembleForecaster: {"forecasters": FORECASTERS},
    StackingForecaster: {"forecasters": FORECASTERS, "final_regressor": REGRESSOR},
    Detrender: {"forecaster": FORECASTER},
    ForecastingGridSearchCV: {
        "forecaster": NaiveForecaster(strategy="mean"),
        "cv": SingleWindowSplitter(fh=1),
        "param_grid": {"window_length": [2, 5]},
        "scoring": sMAPE(),
    },
    SingleSeriesTransformAdaptor: {"transformer": StandardScaler()},
    ColumnEnsembleClassifier: {
        "estimators": [
            (name, estimator, 0) for (name, estimator) in TIME_SERIES_CLASSIFIERS
        ]
    },
    FittedParamExtractor: {
        "forecaster": FORECASTER,
        "param_names": ["smoothing_level"],
    },
    RowTransformer: {"transformer": TRANSFORMER},
    ColumnTransformer: {
        "transformers": [(name, estimator, [0]) for name, estimator in TRANSFORMERS]
    },
    # ARIMA requires d > start where start = 0 for full in-sample predictions
    AutoARIMA: {
        "d": 0,
        "suppress_warnings": True,
        "max_p": 2,
        "max_q": 2,
        "seasonal": False,
    },
    ShapeletTransformClassifier: {"n_estimators": 5, "time_contract_in_mins": 0.125},
    ContractedShapeletTransform: {"time_contract_in_mins": 0.125},
    ShapeletTransform: {
        "max_shapelets_to_store_per_class": 1,
        "min_shapelet_length": 3,
        "max_shapelet_length": 4,
    },
    TSFreshFeatureExtractor: {"disable_progressbar": True, "show_warnings": False},
    TSFreshRelevantFeatureExtractor: {
        "disable_progressbar": True,
        "show_warnings": False,
        "ml_task": "classification",
        "fdr_level": 0.01,
    },
    TSInterpolator: {"length": 10},
    RandomIntervalSpectralForest: {"n_estimators": 5, "acf_lag": 10},
    SFA: {"return_pandas_data_series": True},
    TimeSeriesForest: {"n_estimators": 5},
    TimeSeriesForestClassifier: {"n_estimators": 5},
    TimeSeriesForestRegressor: {"n_estimators": 5},
}

# these methods should not change the state of the estimator, that is, they should
# not change fitted parameters or hyper-parameters
NON_STATE_CHANGING_METHODS = [
    "predict",
    "predict_proba",
    "decision_function",
    "transform",
    "inverse_transform",
]


import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np

from utils import print_debug

class Classifier(nn.Module):
    def __init__(self, args=None) -> None:
        super().__init__()
        self.debug = args.debug
        self.latent_size = args.latent_size
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=5, stride=1, padding=2)
        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=5, stride=1, padding=2)
        self.conv3 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.max_pool = nn.MaxPool2d(2, 2)
        self.fc = nn.Linear(in_features=256*4*4, out_features=self.latent_size)
        self.fc_out = nn.Linear(in_features=self.latent_size, out_features=10)
        # self.fc1 = nn.Linear(in_features=128*8*8, out_features=512)
        # self.fc2 = nn.Linear(in_features=512, out_features=10)
        self.bn1 = nn.BatchNorm2d(num_features=64)
        self.bn2 = nn.BatchNorm2d(num_features=128)
        self.bn3 = nn.BatchNorm2d(num_features=256)
        self.bn4 = nn.BatchNorm1d(num_features=self.latent_size)
        # self.bn5 = nn.BatchNorm1d(num_features=512)
        self.dropout = nn.Dropout(p=args.dropout, inplace=False)
        self.act = args.act_fn

    def forward(self, x):
        print_debug(x, self.debug, name='\nClassifier: input')
        x = self.act(self.bn1(self.conv1(x)))
        print_debug(x, self.debug, name='Classifier: after conv1')
        x = self.max_pool(x)
        print_debug(x, self.debug, name='Classifier: after maxpool')
        x = self.act(self.bn2(self.conv2(x)))
        print_debug(x, self.debug, name='Classifier: after conv2')
        x = self.max_pool(x)
        print_debug(x, self.debug, name='Classifier: after maxpool')

        x = self.act(self.bn3(self.conv3(x)))
        print_debug(x, self.debug, name='Classifier: after conv3')
        x = self.max_pool(x)
        print_debug(x, self.debug, name='Classifier: after maxpool')
        x = x.view(-1, 4*4*256)
        print_debug(x, self.debug, name='Classifier: after reshape')
        x = self.act(self.bn4(self.fc(x)))
        print_debug(x, self.debug, name='Classifier: after fc')
        x = self.dropout(x)
        self.latent_vector = x
        x = self.fc_out(x)
        print_debug(x, self.debug, name='Classifier: after fc_out')

        # x = x.view(-1, 8*8*128)
        # print_debug(x, self.debug, name='Classifier: after reshape')
        # x = self.act(self.bn5(self.fc1(x)))
        # print_debug(x, self.debug, name='Classifier: after fc1')
        # x = self.dropout(x)
        # if not self.train:
        #     self.features = x
        # x = self.fc2(x)
        # print_debug(x, self.debug, name='Classifier: after fc2')

        return x

def compute_accuracy(outputs, labels):
    with torch.no_grad():
        batch_size = labels.size(0)
        pred = outputs.max(1)[1]
        acc = pred.eq(labels).sum().item() * 100.0 / batch_size
        return acc

def train_classifier(args, classifier=None, train_dataloader=None, test_dataloader=None, device=None):
    # python autoencoder.py --lr 0.001 --wd 0.1 --epochs 100 --dropout 0 --act relu
    print(f'\n\nTraining CIFAR-10 Classifier\n\n')

    if classifier is None:
        classifier = Classifier(args=args).to(device)
    optim = torch.optim.AdamW(classifier.parameters(), lr=args.lr, weight_decay=args.wd)
    loss_fn = nn.CrossEntropyLoss()
    lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer=optim, T_max=args.epochs * len(train_dataloader))

    classifier.train()
    for epoch in range(args.epochs):
        classifier.train()
        tr_accs = []
        for images, labels in train_dataloader:
            images, labels = images.to(device), labels.to(device)
            outputs = classifier(images)
            loss = loss_fn(outputs, labels)

            optim.zero_grad()
            loss.backward()
            optim.step()
            lr_scheduler.step()

            tr_acc = compute_accuracy(outputs, labels)
            tr_accs.append(tr_acc)

        train_acc = np.mean(tr_accs)
        test_acc = evaluate_classifier(classifier, test_dataloader=test_dataloader, device=device)

        print(f'Epoch {epoch}  train {train_acc:.2f}  test {test_acc:.2f}  LR {lr_scheduler.get_last_lr()[0]:.4f}')

    checkpoint_path = f'checkpoints/cifar_classifier_{args.latent_size}.pth'
    print(f'\n\nSaving classifier model checkpoint to {checkpoint_path}')
    torch.save(classifier, checkpoint_path)
    # torch.save(classifier, f'checkpoints/{args.tag}cifar_classifier_{args.latent_size}.pth')
    return classifier

def evaluate_classifier(classifier, test_dataloader=None, device=None):
    te_accs = []
    classifier.eval()
    with torch.no_grad():
        for images, labels in test_dataloader:
            images, labels = images.to(device), labels.to(device)
            outputs = classifier(images)
            te_acc = compute_accuracy(outputs, labels)
            te_accs.append(te_acc)
    return np.mean(te_accs)


import requests
try:
    # Make an API request to the Dad Jokes API
    response = requests.get('https://icanhazdadjoke.com/', headers={'Accept': 'application/json'})

    # Check if the response status code indicates success (200 OK)
    if response.status_code == 200:
        # Parse the JSON response to retrieve the joke
        data = response.json()
        joke = data['joke']
        print(f"Dad Joke: {joke}")
    else:
        print(f"API request failed with status code: {response.status_code}")
except requests.exceptions.RequestException as e:
    # Handle network-related exceptions (e.g., no internet connection)
    print(f"An error occurred: {e}")
except Exception as e:
    # Handle other exceptions
    print(f"An unexpected error occurred: {e}")


from __future__ import with_statement
from divisi2.sparse import SparseMatrix

def movielens_titles(filebase):
    d = {}
    with open(filebase+'.item') as f:
        for line in f:
            if line.strip():
                id, title = line.split('|')[:2]
                id = int(id)
                d[id] = title
    return d

def movielens_ratings(filebase):
    titles = movielens_titles(filebase)
    seen = set()
    with open(filebase+'.data') as f:
        for line in f:
            if line.strip():
                user_id, movie_id, rating = line.split('\t')[:3]
                user_id = int(user_id)
                movie_id = int(movie_id)
                rating = float(rating)
                assert 1 <= rating <= 5
                movie_title = titles[movie_id]
                if (movie_title, user_id) not in seen:
                    seen.add((movie_title, user_id))
                    yield (rating, movie_title, user_id)


from setuptools import setup

def readme():
    with open('README.md') as f:
        return f.read()

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(name='faced',
      version='0.1',
      description='Face detection using deep learning',
      long_description=readme(),
      classifiers=[
        'Development Status :: 3 - Alpha',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3',
        'Topic :: Text Processing :: Linguistic',
      ],
      keywords='face detection deep learning cnn neural network',
      url='http://github.com/iitzco/faced',
      author='Ivan Itzcovich',
      author_email='i.itzcovich@gmail.com',
      license='MIT',
      packages=['faced'],
      scripts=["bin/faced"],
      install_requires=required,
      python_requires='>3.4',
      include_package_data=True,
      zip_safe=False)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
import os
import sys
import commands
import jx_common

xjdf=0
totalitemnum=5
checkconfigbz=-1
notconfignum=0
configsuccessnum=0
jxitemsjcount=0

def Header_fwkzrzsq():
    printjg='''
    *******************************************************************
    *                                                                 *
    *            Linux基线规范检查和配置：3、访问控制及认证授权       *
    *                                                                 *
    *******************************************************************
            '''
    jx_common.write_report(3,jx_common.JXGL_Report,'a',printjg)

#No3.1、【配置要求】：用户缺省访问权限
def Userdefault_access(op):
    global xjdf,totalitemnum,jxitemsjcount,notconfignum,configsuccessnum,checkconfigbz
    jxitemsjcount+=1
    filegroups=['/etc/profile','/etc/csh.login','/etc/csh.cshrc','/etc/bashrc']
    No3_1="★★★★★★★★★★No3.1、【配置要求】：用户缺省访问权限★★★★★★★★★★"
    jxzt=0
    linenum=1
    filelinenum=0
    umask=""
    exppzlist=""
    sbbzcount=0
    ycbzcount=0
    for f in filegroups:
        command="cat -n %s|grep -v '#'|grep 'umask'|sed -n '$p'" % f
        umaskstr=commands.getoutput(command).strip()
        if umaskstr!="":
           try:
              umask=umaskstr.split()[-1]
           except Exception:
              umask="-1"
           try:
              filelinenum=int(umaskstr.split()[0])
           except Exception:
              filelinenum=-1
           if umask!="-1" and filelinenum!=-1:
              if f=='/etc/profile':
                 if umask!='027':
                    exppzlist+="%d [/etc/profile]:umask %s.\n" % (filelinenum,umask)
              else:
                 if umask!='077':
                    exppzlist+="%d [%s]:umask %s.\n" % (filelinenum,f,umask)
           else:
               ycbzcount+=1
        else:
           exppzlist+="  [%s] 未配置umask" % f
    if op=='--check':
       if exppzlist!="":
             jxzt=2
       else:
           jxzt=0
    elif op=='--config':
         exppzlistgroups=exppzlist.strip('\n').split('\n')
         if exppzlist!="":
            for e in exppzlistgroups:
                try:
                   tmp_linenum=int(e.split()[0])
                except Exception:
                   tmp_linenum=-1
                tmp_umaskvalue=e.split()[-1].strip('.')
                tmp_filename=e.split('/')[-1].split(']')[0]
                cmdstr="cat /etc/%s|grep -E '^umask'" % tmp_filename
                cmdvalue=commands.getoutput(cmdstr).strip()
                if tmp_filename=='profile':
                   if cmdvalue!="":
                      cmdstr="sed -i '%ds/%s/027/g'  /etc/profile" % (tmp_linenum,tmp_umaskvalue)
                   else:
                      cmdstr="echo 'umask 027' >> /etc/profile"
                else:
                   if cmdvalue!="":
                      cmdstr="sed -i '%ds/%s/077/g'  /etc/%s" % (tmp_linenum,tmp_umaskvalue,tmp_filename)
                   else:
                      cmdstr="echo 'umask 077' >> /etc/%s" % tmp_filename
                cmdstatus=os.system(cmdstr)
                if cmdstatus!=0:
                   sbbzcount+=1
            if sbbzcount>0:
               jxzt=2
            else:
               jxzt=1
         else:
            jxzt=0
    if op=='--check':
       if jxzt==2:
          printjg='''
%s
【符合度得分】： 0
【扫描结果】：[异常]按照用户缺省访问权限检查,不符合基线规范。
不规范列表如下：
%s
                  ''' % (No3_1,exppzlist)
          checkconfigbz=1
       elif jxzt==3:
            printjg='''
%s
【符合度得分】：0
【扫描结果】：[异常]按照用户缺省访问权限检查，检查%s文件中umask出现异常，不符合基线规范。
                  ''' % (No3_1,filegroups)
            checkconfigbz=1
       else:
            printjg='''
%s
【符合度得分】：1
【扫描结果】：[正常]按照用户缺省访问权限检查，umask值都符合基线配置规范。
                  ''' %  No3_1
            xjdf+=1
            jx_common.fhd_score+=1
            checkconfigbz=0
    elif op=='--config':
            if jxzt==2 :
               printjg='''
%s
配置结果：[失败]按照用户缺省访问权限检查，存在更新umask配置失败。
失败列表如下：
%s
               ''' % (No3_1,exppzlist)
               checkconfigbz=2
            elif jxzt==1:
                 printjg='''
%s
配置结果：[成功]按照用户缺省访问权限检查，配置需更新umask都全部成功！！
需要更新列表如下：
%s
               ''' % (No3_1,exppzlist)
                 xjdf+=1
                 checkconfigbz=0
            else:
                printjg='''
%s
配置结果：[正常]按照用户缺省访问权限检查,%s中umask值都符合基线配置规范。
               ''' % (No3_1,filegroups)
                notconfignum+=1
                checkconfigbz=0
    else:
        printjg="无"
    jx_common.write_report(checkconfigbz,jx_common.JXGL_Report,'a',printjg)

#No3.2、【配置要求】：授权账户SSH访问控制
def Sshaccess_control(op):
    global xjdf,totalitemnum,jxitemsjcount,notconfignum,configsuccessnum,checkconfigbz
    jxitemsjcount+=1
    sshdconfiggroups=['Protocol 2','LogLevel INFO','MaxAuthTries 5','IgnoreRhosts yes','RhostsAuthentication no','RhostsRSAAuthentication no','HostbasedAuthentication no','PermitEmptyPasswords no','PermitUserEnvironment no','ClientAliveInterval 60','ClientAliveCountMax 3','LoginGraceTime 60','X11Forwarding yes']
    sshconfigfile=jx_common.SSHD_Config_FILE
    No3_2="★★★★★★★★★★No3.2、【配置要求】：授权账户SSH访问控制★★★★★★★★★★"
    jxzt=0
    linenum=1
    filelinenum=0
    umask=""
    exppzlist=""
    successlist=""
    gxcgcount=0
    sbbzcount=0
    for s in sshdconfiggroups:
        command="cat -n %s|grep -v '#'|grep '%s'" % (sshconfigfile,s)
        pzgroupsstr=commands.getoutput(command).strip()
        if pzgroupsstr!="":
           pzgroups=pzgroupsstr.strip('\n').split('\n')
           pznum=len(pzgroups)
           n=1
           if pznum>1:
              if op=='--check':
                 sbbzcount+=1
                 exppzlist+="%d [%s] 配置重复.\n" % (linenum,s)
                 linenum+=1
              elif op=='--config':
                   for p in pzgroups:
                       try:
                          filelinenum=int(p.split()[0])
                       except Exception:
                          filelinenum=-1
                       if n!=pznum:
                          cmdstr="sed -i '%ds/^/#/g' %s" % (filelinenum,sshconfigfile)
                          status=os.system(cmdstr)
                          if status!=0:
                             exppzlist+="%d <%d> [%s] 删除重复配置失败\n" % (linenum,filelinenum,s)
                             linenum+=1
                             sbbzcount+=1
                          else:
                             successlist+="%d <%d> [%s] 重复配置更新成功\n" % (gxcgcount,filelinenum,s)
                             gxcgcount+=1
                       n+=1
        else:
           if op=='--check':
              exppzlist+="%d [%s] 未配置.\n" % (linenum,s)
              linenum+=1
              sbbzcount+=1
           elif op=='--config':
                cmdstr="echo '%s' >> %s" % (s,sshconfigfile)
                cmdstatus=os.system(cmdstr)
                if cmdstatus!=0:
                   exppzlist+="%d [%s] 添加配置失败\n" % (linenum,s)
                   linenum+=1
                   sbbzcount+=1
                else:
                   successlist+="%d [%s] 未配置已添加成功.\n" % (gxcgcount,s)
                   gxcgcount+=1
    if exppzlist!="":
       jxzt=2
    if op=='--config':
       if sbbzcount==0 and gxcgcount>0:
          jxzt=1

    if op=='--check':
       if jxzt==2:
          printjg='''
%s
【符合度得分】： 0
【扫描结果】：[异常]按照授权账户SSH访问控制检查,%d项配置均不符合基线规范。
不规范列表如下：
%s
                  ''' % (No3_2,sbbzcount,exppzlist)
          checkconfigbz=1
       else:
            printjg='''
%s
【符合度得分】：1
【扫描结果】：[正常]按照授权账户SSH访问控制检查，ssh相关配置都符合基线配置规范。
                  ''' %  No3_2
            xjdf+=1
            jx_common.fhd_score+=1
            checkconfigbz=0
    elif op=='--config':
            if jxzt==2 :
               printjg='''
%s
配置结果：[失败]按照授权账户SSH访问控制检查，存在%d项更新SSH配置失败。
配置失败列表如下：
%s
               ''' % (No3_2,sbbzcount,exppzlist)
               checkconfigbz=2
            elif jxzt==1:
                 printjg='''
%s
配置结果：[成功]按照授权账户SSH访问控制检查，%d项配置更新ssh都全部成功！！
配置列表如下：
%s
               ''' % (No3_2,gxcgcount,successlist)
                 xjdf+=1
                 checkconfigbz=0
            else:
                printjg='''
%s
配置结果：[正常]按照授权账户SSH访问控制检查，ssh配置全部符合基线配置规范。
               ''' % No3_2
                notconfignum+=1
                checkconfigbz=0
    else:
        printjg="无"
    jx_common.write_report(checkconfigbz,jx_common.JXGL_Report,'a',printjg)

#No3.5、【配置要求】:检查是否使用PAM认证模块禁止wheel组之外的用户su为root
def Restrictroot_Pam_directlogin(op):
    global xjdf,totalitemnum,jxitemsjcount,notconfignum,configsuccessnum,checkconfigbz
    jxitemsjcount+=1
    sshdconfiggroups=['Protocol 2','LogLevel INFO','MaxAuthTries 5','IgnoreRhosts yes','RhostsAuthentication no','RhostsRSAAuthentication no','HostbasedAuthentication no','PermitEmptyPasswords no','PermitUserEnvironment no','ClientAliveInterval 60','ClientAliveCountMax 3','LoginGraceTime 60','X11Forwarding yes']
    osversion=int(jx_common.get_osversion())
    No3_5="★★★★★★★★★★No3.5、【配置要求】：检查是否使用PAM认证模块禁止wheel组之外的用户su为root★★★★★★★★★★"
    jxzt=0
    exppzlist=""
    successlist=""
    gxcgcount=0
    sbbzcount=0
    linenum=1
    xquserbz=0
    SU_COMMONUser=jx_common.SU_COMMONUser
    cmdstr="cat -n /etc/pam.d/su|grep -v '#'|grep 'pam_wheel.so use_uid'"
    cmd_use_uidvalue=commands.getoutput(cmdstr).strip()
    cmdstr1="cat /etc/group|grep '^wheel'|awk -F ':' '{ print $NF }'"
    wheelusers=commands.getoutput(cmdstr1).strip()
    if op=='--check':
       if osversion==6:
          cmdstr="cat -n /etc/pam.d/su|grep -v '#'|grep 'pam_wheel.so'|grep 'group=wheel'"
          pamwheelvalue=commands.getoutput(cmdstr).strip()
          if pamwheelvalue=="":
             exppzlist+="[%d][/etc/pam.d/su]中auth required pam_wheel.so group=wheel未配置.\n" % linenum
             sbbzcount+=1
       elif osversion==7:
            cmdstr="cat /etc/login.defs|grep -v '#'|grep 'SU_WHEEL_ONLY'|grep 'yes'"
            suwheelvalue=commands.getoutput(cmdstr).strip()
            if suwheelvalue=="":
               exppzlist+="[%d][/etc/login.defs]中SU_WHEEL_ONLY yes未配置.\n" % linenum
               sbbzcount+=1
       if cmd_use_uidvalue=="":
          exppzlist+="[%d][/etc/pam.d/su]中auth		required	pam_wheel.so use_uid未配置.\n" % linenum
          sbbzcount+=1
       for u in SU_COMMONUser:
           cmdstr="id %s >/dev/null 2>&1" % u
           cmdstatus=os.system(cmdstr)
           cmdstr1="passwd -S %s|awk '{ print $2 }'" % u
           pwdstatus=''.join(commands.getoutput(cmdstr1)).replace('\n','')
           if cmdstatus!=0 or 'L' in pwdstatus:
              continue
           else:
              if u not in wheelusers.split(','):
                 exppzlist+="[%d]%s用户未添加到wheel组.\n" % (linenum,u)
                 sbbzcount+=1
                 linenum+=1
    elif op=='--config':
         if osversion==6:
            cmdstr="cat -n /etc/pam.d/su|grep -v '#'|grep 'pam_wheel.so'|grep 'group=wheel'"
            pamwheelvalue=commands.getoutput(cmdstr).strip()
            if pamwheelvalue=="":
               cmdstr="echo 'auth required pam_wheel.so group=wheel' >> /etc/pam.d/su"
               cmdstatus=os.system(cmdstr)
               if cmdstatus!=0:
                  sbbzcount+=1
                  exppzlist+="[/etc/pam.d/su]中添加auth required pam_wheel.so group=wheel配置失败.\n"
               else:
                  gxcgcount+=1
                  successlist+="[/etc/pam.d/su]中添加auth required pam_wheel.so group=wheel]配置成功.\n"
         elif osversion==7:
              cmdstr1="cat /etc/login.defs|grep -v '#'|grep 'SU_WHEEL_ONLY'|grep 'yes'"
              suwheelvalue=commands.getoutput(cmdstr1).strip()
              if suwheelvalue=="":
                 cmdstr="echo 'SU_WHEEL_ONLY yes' >> /etc/login.defs"
                 cmdstatus=os.system(cmdstr)
                 if cmdstatus!=0:
                    sbbzcount+=1
                    exppzlist+="[/etc/login.defs]中添加SU_WHEEL_ONLY yes配置失败.\n"
                 else:
                    gxcgcount+=1
                    successlist+="[/etc/login.defs]中添加SU_WHEEL_ONLY yes配置成功.\n"
         if cmd_use_uidvalue=="":
            cmdstr="echo 'auth		required	pam_wheel.so use_uid' >> /etc/pam.d/su"
            cmdstatus=os.system(cmdstr)
            if cmdstatus!=0:
               sbbzcount+=1
               exppzlist+="[/etc/pam.d/su]中添加auth		required	pam_wheel.so use_uid配置失败.\n"
            else:
               gxcgcount+=1
               successlist+="[/etc/pam.d/su]中添加auth		required	pam_wheel.so use_uid配置成功.\n"
         for u in SU_COMMONUser:
             cmdstr="cat /etc/passwd|grep '%s'" % u
             cmdvalue=commands.getoutput(cmdstr).strip()
             if cmdvalue!="":
                cmd="passwd -S %s|awk '{ print $2 }'" % u
                pwdstatus=''.join(commands.getoutput(cmd)).replace('\n','')
                if 'L' not in pwdstatus:
                    cmdstr="usermod -G wheel %s" % u
                    cmdstatus=os.system(cmdstr)
                    if cmdstatus!=0:
                       exppzlist+="选取未锁定普通用户[%s],加入wheel组失败.\n" % u
                       sbbzcount+=1
                    else:
                       gxcgcount+=1
                       successlist+="选取未锁定普通业务用户[%s],加入wheel组成功.\n" % u
    if exppzlist!="":
       jxzt=2
    if op=='--config':
       if sbbzcount==0 and gxcgcount>0:
          jxzt=1
    if op=='--check':
       if jxzt==2:
          printjg='''
%s
【符合度得分】： 0
【扫描结果】：[异常]检查是否使用PAM认证模块禁止wheel组之外的用户su为root,%d项配置均不符合基线规范。
不规范列表如下：
%s
                  ''' % (No3_5,sbbzcount,exppzlist)
          checkconfigbz=1
       else:
            printjg='''
%s
【符合度得分】：1
【扫描结果】：[正常]检查是否使用PAM认证模块禁止wheel组之外的用户su为root，相关配置都符合基线配置规范。
                  ''' %  No3_5
            xjdf+=1
            jx_common.fhd_score+=1
            checkconfigbz=0
    elif op=='--config':
            if jxzt==2 :
               printjg='''
%s
配置结果：[失败]检查是否使用PAM认证模块禁止wheel组之外的用户su为root，存在%d项更新配置失败。
配置失败列表如下：
%s
               ''' % (No3_5,sbbzcount,exppzlist)
               checkconfigbz=2
            elif jxzt==1:
                 printjg='''
%s
配置结果：[成功]检查是否使用PAM认证模块禁止wheel组之外的用户su为root，%d项配置更新都全部成功！！
配置列表如下：
%s
               ''' % (No3_5,gxcgcount,successlist)
                 xjdf+=1
                 checkconfigbz=0
            else:
                printjg='''
%s
配置结果：[正常]检查是否使用PAM认证模块禁止wheel组之外的用户su为root，配置全部符合基线配置规范。
               ''' % No3_5
                notconfignum+=1
                checkconfigbz=0
    else:
        printjg="无"
    jx_common.write_report(checkconfigbz,jx_common.JXGL_Report,'a',printjg)


from aiohttp.web_urldispatcher import View

from server.models import Bike
from server.permissions.permission import RoutePermissionError, Permission
from server.service.access.issues import get_issues

class BikeNotInUse(Permission):
    """Asserts that the given bike is not being used."""

    async def __call__(self, view: View, bike: Bike, **kwargs):
        if view.rental_manager.is_in_use(bike):
            raise RoutePermissionError("The requested bike is in use.")

class BikeNotBroken(Permission):
    """Asserts that the given bike has no active issues."""

    def __init__(self, *, max_issues=0):
        """The maximum number of issues before denying access."""
        self.max_issues = max_issues

    async def __call__(self, view: View, bike: Bike, **kwargs):
        if len(await get_issues(bike=bike, is_active=True)) > self.max_issues:
            raise RoutePermissionError("The requested bike is broken.")

class BikeIsConnected(Permission):

    async def __call__(self, view: View, bike: Bike, **kwargs):
        if not view.bike_connection_manager.is_connected(bike):
            raise RoutePermissionError("The requested bike is not connected.")


# -*- coding: utf-8 -*-
from django.contrib import admin
from myadmin.admin import admin_site
from django.contrib.admin import DateFieldListFilter
from .models import *
from rangefilter.filter import DateRangeFilter

class ConsultantAdmin(admin.ModelAdmin):
    list_per_page = 10
    list_display = ('last_name', 'first_name', 'consultant_num', 'phone_number', 'email', 'status', 'refferal_url', 'url_to_personal_room',)
    # date_hierarchy = 'last_name'
    list_filter = ('status', 'last_name', 'first_name', 'middle_name', 'citizenship', 'city', 'region', ('registered_date', DateRangeFilter),)
    filter_horizontal = ('user_lead', 'user_lead_1', 'user_lead_2',)
    search_fields = (
        'last_name',
        'first_name',
        'middle_name',
        'city',
        'region',
        'consultant_num',
        'passport_data',
        'birthday',
        'street',
        'num_home',
        'num_apartment',
        'email',
        'phone_number',
        'user_led',
        'user_led_1',
        'user_led_2',
    )
    fieldsets = (
        ('Персональные данные', {
            'fields': (
                ('email',),
                ('last_name',),
                ('first_name',),
                ('middle_name',),
                # ('passport_data',),
                ('birthday',),
                ('phone_number',),
                ('citizenship',),
            ),
        },),
        ('Адрес', {
            'fields': (
                ('region',),
                ('city',),
                ('street',),
                ('num_home',),
                ('num_apartment',),
            ),
        },),
        ('Технические данные', {
            'fields': (
                ('consultant_num', 'status',),
            ),
        },),
        ('Рферальные данные', {
            'fields': (
                ('refferal_url','url_to_personal_room',),
                ('user_led', 'user_led_1', 'user_led_2',),
            ),
        },),
        ('Списки рефералов консультанта', {
            'fields': (
                ('user_lead',), ('user_lead_1',), ('user_lead_2',),
            ),
        },),
    )

class RelatedConsultantAdmin(admin.ModelAdmin):
    list_per_page = 10
    list_display = ('last_name', 'first_name',  'middle_name', 'consultant_num', 'refferal_url', 'url_to_personal_room', 'email',)
    list_filter = ('last_name', 'first_name', 'middle_name', 'consultant_num', ('registered_date', DateRangeFilter))
    filter_horizontal = ('user_lead', 'user_lead_1', 'user_lead_2',)
    search_fields = (
        'last_name',
        'first_name',
        'middle_name',
        'email',
        'user_led',
        'user_led_1',
        'user_led_2',
    )
    fieldsets = (
        ('Персональная данные', {
            'fields': (
                ('email',),
                ('last_name',),
                ('first_name',),
            ),
        },),
        ('Технические данные', {
            'fields': (
                ('consultant_num', 'status',),
            ),
        },),
        ('Рферальные данные', {
            'fields': (
                ('refferal_url', 'url_to_personal_room',),
                ('user_led', 'user_led_1', 'user_led_2',),
            ),
        },),
        ('Списки рефералов консультанта', {
            'fields': (
                ('user_lead',), ('user_lead_1',), ('user_lead_2',),
            ),
        },),
    )
# Register your models here.

admin_site.register(User, ConsultantAdmin)
admin_site.register(RefferalConsultant, ConsultantAdmin)
admin_site.register(RelatedConsultant, RelatedConsultantAdmin)


from sys import stdin
from math import comb
input = stdin.readline

def inv(a, n):
    if(a == 1):
        return 1
    return int((1 - inv(n % a, a) * n) / a + n)

for case in range(int(input())):
    n = int(input())
    a = list(map(int, input().split()))
    num_ones = sum(a)
    incorrect_inds = num_ones - sum(a[-num_ones:])
    mod = 998244353
    ans = 0
    total_pairs = comb(n, 2)
    for i in range(incorrect_inds, 0, -1):
        ans += (total_pairs * inv(i ** 2, mod))
        ans %= mod

    if ans == 0:
        print(0)
        continue

    p, q = ans.as_integer_ratio()

    print((p * inv(q, mod)) % mod)

CodeForces/PyPy 3-64/1754E | Wish I Knew How to Sort/180248463.py