from datasets import load_dataset
import json
import requests
from collections import defaultdict

from langchain.llms import OpenAI
from langchain import PromptTemplate, FewShotPromptTemplate
from langchain.prompts.example_selector import LengthBasedExampleSelector
from inference_api import YiyanInferenceApi, HfInferenceApi
import regex
import sys

from mpi4py import MPI

dataset_mapping_url = f"https://raw.githubusercontent.com/SJTU-LIT/ceval/main/subject_mapping.json"

llm = YiyanInferenceApi("yiyan-007", debug=False)
# llm = OpenAI(model_name="text-davinci-003")

def get_data_mapping(mapping_url):
    headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/json'
        }
    response = requests.request("GET", mapping_url, headers=headers)
    return json.loads(response.content.decode('utf-8'))

def escape_format(text : str) -> str :
    return text.replace('{', '{{').replace('}', '}}')

def gen_by_fewshot_prompt(task_name, fewshot_set, test_set, output) :

    prompt_template = """问题：{question} \n选项\nA: {A}\nB: {B}\nC: {C}\nD: {D}\n思考：{explanation}\n答案：{answer}\n\n"""

    example_prompt = PromptTemplate(
        input_variables = ['question', 'A', 'B', 'C', 'D', 'explanation', 'answer'],
        template = prompt_template
    )

    fewshot_examples = []
    for id in range(len(fewshot_set)) :
        fewshot_examples.append({'question': escape_format(fewshot_set[id]['question']),
                                 'A': escape_format(fewshot_set[id]['A']),
                                 'B': escape_format(fewshot_set[id]['B']),
                                 'C': escape_format(fewshot_set[id]['C']),
                                 'D': escape_format(fewshot_set[id]['D']),
                                 'explanation': escape_format(fewshot_set[id]['explanation']),
                                 'answer': fewshot_set[id]['answer']})

    example_selector = LengthBasedExampleSelector(
        examples = fewshot_examples,
        example_prompt = example_prompt,
        max_length = 1024,
        get_text_length = lambda x: len(x) # tokens per Chinese character
    )

    few_shot_prompt = FewShotPromptTemplate(
        example_selector = example_selector,
        example_prompt = example_prompt,
        prefix = f"请作为一个{task_name}科目的考生，参考示例回答单项选择题，示例如下：\n",
        suffix = """下面是要回答的问题，可以先写出思考过程，然后给出ABCD的一个选项作为答案，确保用<答案：>作为答案的前缀\n
                    问题：{question}\n选项\nA: {A}\nB: {B}\nC: {C}\nD: {D}\n思考：""",

        input_variables = ['question', 'A', 'B', 'C', 'D'],
        example_separator = "\n\n"
    )

    correct, total = 0, 0
    for id in range(len(test_set)) :
        try :
            grounded_prompt = few_shot_prompt.format(question = escape_format(test_set[id]['question']),
                                                            A = escape_format(test_set[id]['A']),
                                                            B = escape_format(test_set[id]['B']),
                                                            C = escape_format(test_set[id]['C']),
                                                            D = escape_format(test_set[id]['D']))
            completion = llm(grounded_prompt)
            llm_answer = extract_answer(completion)
            answer = test_set[id]['answer']
            print(f"question: {task_name}-{id}, llm_answer={llm_answer}, answer={answer}, completion={repr(completion)}",
                  file = output)
            total += 1
            if llm_answer == answer :
                correct += 1
        except Exception as e :
            print(f"Exception {e} in answer generation for question <{id}> in task <{task_name}>")

    return (correct, total)

def extract_answer(completion) :

    # string find part
    answer_prefixes = ['答案：', '答案:', '答案为', '答案是', '答案选',
                       '故选', '选项中最接近的是', '选项为', '答案应该是',
                       '因此选择', '因此答案为：', '即选项', '故选择',
                       '正确的选项是', '答案选', '因此答案为 ', '故选 ']
    for answer_prefix in answer_prefixes :
        if answer_prefix in completion :
            pos = completion.rfind(answer_prefix) + len(answer_prefix)
            if pos < len(completion) :
                answer = completion[pos]
                if answer == '{' and pos + 1 < len(completion) :
                    answer = completion[pos + 1]
                if answer in ['A', 'B', 'C', 'D'] :
                    return answer
    # regex part
    answer_patterns = [r'选项\((.*?)\)为正确', r'选项(.*?)为正确',
                       r'选项\((.*?)\)正确', r'选项(.*?)正确',
                       r'选项\((.*?)\)是正确', r'选项(.*?)是正确',
                       r'选择\((.*?)\)为正确', r'选择(.*?)为正确',
                       r'因此\((.*?)\)选项正确', r'因此(.*?)选项正确',
                       r'答案：<\((.*?)\)>', r'答案：<(.*?)>']

    for pattern in answer_patterns :
        match = regex.search(pattern, completion)
        if match :
            answer = match.group(1)
            if answer in ['A', 'B', 'C', 'D'] :
                return answer

    return 'Err'

if __name__ == "__main__" :

    data_mapping = get_data_mapping(dataset_mapping_url)
    g_correct, g_total = 0.0, 0.0

    tested_subject = 0
    topK = len(data_mapping.keys())

    # distribute with MPI
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()

    output = f'yiyan-ceval-result-rank{rank}.log'
    with open(output, 'w') as f :
        for id, name in enumerate(data_mapping.keys()) :
            if id % comm.size == rank : # process a sub-task
                dataset=load_dataset(r"ceval/ceval-exam",name=name)
                # print(dataset['dev'][0])
                (correct, total) = gen_by_fewshot_prompt(data_mapping[name][1], dataset['dev'], dataset['val'], f)
                g_correct += correct
                g_total += total
                print(f"Task <{name}>: correct={correct}, total={total}", file=f)
                print(f"Global accuracy so far: {g_correct/g_total:.4f}(g_correct={g_correct}, g_total={g_total})", file=f)

                tested_subject += 1
                if tested_subject >= topK :
                    break


# Enter your code here. Read input from STDIN. Print output to STDOUT
from collections import deque

d = deque()

for x in range(int(input())):
    op, _, num = input().partition(" ")
    eval(f'd.{op}({num})')

print(*d)


import pandas as pd
from konlpy.tag import Okt
import re
from collections import Counter
from wordcloud import WordCloud
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np

okt = Okt()

def insta_morph(data):
    data_list = []
    data_list.append(data)
    df_set = pd.concat(data_list)
    df_set.duplicated(["게시글"])
    df = df_set.drop_duplicates(["게시글"], keep="first")

    new_list = []
    for row in df["게시글"]:
        if (row != ''):
            new = re.sub('[^#A-Za-z0-9ㄱ-ㅣ가-힣]+', ' ', str(row))
        if (new == ''):
            del new
        else:
            morph = okt.pos(new)
            new_list.append(morph)

    return new_list

def googlemap_morph(data):
    data_list = []
    data_list.append(data)
    df_set = pd.concat(data_list)
    df_set.duplicated(["review_text"])
    df = df_set.drop_duplicates(["review_text"], keep="first")

    new_list=[]
    for row in df["review_text"]:
        if(row !=''):
            new = re.sub('[^#A-Za-z0-9ㄱ-ㅣ가-힣]+', ' ',str(row))
            new = re.sub('원문|번역|google|제공', '', str(row)) #번역 리뷰에 뜨는 문구 제거
        if (new==''):
            del new
        else:

            morph = okt.pos(new)
            new_list.append(morph)

    return new_list

def insta_list(new_list):
    count_list1 = []
    count_list2 = []
    for sentence in new_list:
        for word, tag in sentence:
            if (tag in ['Noun', 'Adjective'] and len(word) > 1):  # 동사가 아니고 두 글자 이상일 때
                count_list1.append((word))
            if tag in 'Hashtag' and len(word) > 1:
                count_list2.append((word))

    counts1 = Counter(count_list1)
    counts2 = Counter(count_list2)
    words = counts1.most_common(100)
    hashtags = counts2.most_common(100)

    common_list=[]

    for i in range(100):
        m = okt.pos(words[i][0])
        common_list.append((words[i][0], m[0][1], words[i][1], hashtags[i][0], hashtags[i][1]))
    return words, hashtags, common_list

def googlemap_list(new_list): #구글맵 리뷰
    count_list1 = []
    for sentence in new_list:
        for word, tag in sentence:
            if (tag in ['Noun', 'Adjective'] and len(word) > 1):  # 동사가 아니고 두 글자 이상일 때
                count_list1.append((word))

    counts1 = Counter(count_list1)
    words = counts1.most_common(100)
    common_list = []

    for i in range(100):
        m = okt.pos(words[i][0])
        common_list.append((words[i][0], words[i][1], m[0][1]))
    return words, common_list

def wordcloud(words):
    wc = WordCloud(
        font_path='C:\Windows\Fonts\MALGUNBD.TTF',
        width=800,
        colormap='Accent_r',
        height=800,
        background_color="White")

    wc.generate_from_frequencies(dict(words))
    return wc

def save_insta_file(key,year, wc, common_list):
    wc.to_file('wordcloud_insta_' + key + '_' + year + '.png')
    word_count = pd.DataFrame(common_list, columns=['단어', '품사', '빈도수', '해시태그', '개수'])
    word_count.to_csv('instagram_' + key + '_' + year + '_most_common.csv', mode='wt', encoding='utf-8-sig',
                      header=True, index=False)
    xpos = np.arange(len(word_count['해시태그']))
    xtext = list(word_count["해시태그"])
    return word_count, xpos, xtext

def save_googlemap_file(key,year, wc, common_list):
    wc.to_file('wordcloud_googlemap_'+key+'_'+year+'.png')
    word_count=pd.DataFrame(common_list, columns=['단어', '빈도수', '품사'])
    word_count.to_csv('googlemap_'+key+'_'+year+'_most_common.csv', mode='wt', encoding='utf-8-sig', header=True, index=False)
    xpos = np.arange(len(word_count['단어']))
    xtext = list(word_count["단어"])
    return word_count,xpos,xtext

def draw_googlemap_graph(word_count, xpos, xtext, key, year):

    plt.title(year+ '년 ' + key+ "park most common words(googlemap)")
    plt.xlabel('단어')
    plt.ylabel('빈도수')
    plt.yticks(xpos, xtext)
    plt.ylim(0, 10)
    plt.barh(word_count['단어'], word_count['빈도수'], color='green', height=0.7)
    plt.tight_layout()
    plt.savefig('bar_googlemap_' + key + '_' + year + '.png')
    return
def draw_insta_graph(word_count, xpos, xtext, key, year):
    plt.title(year + '년 ' + key + "park most common words(instagram)")
    plt.xlabel('해시태그')
    plt.ylabel('개수')
    plt.yticks(xpos, xtext)
    plt.ylim(0, 10)
    plt.barh(word_count['해시태그'], word_count['개수'], color='green', height=0.7)
    plt.tight_layout()
    plt.savefig('bar_instagram_' + key + '_' + year + '.png')

key=['daejeo','samrak']
year=['2018','2019']

googlemap_data=[pd.read_csv('googlemap_daejeo_2018.csv', index_col=False),
        pd.read_csv('googlemap_daejeo_2019.csv', index_col=False),
         pd.read_csv('googlemap_samrak_2018.csv', index_col=False),
          pd.read_csv('googlemap_samrak_2019.csv', index_col=False)]

insta_data=[pd.read_csv('instagram_daejeo_2018.csv', index_col=False),
            pd.read_csv('instagram_daejeo_2019.csv', index_col=False),
            pd.read_csv('instagram_samrak_2018.csv', index_col=False),
            pd.read_csv('instagram_samrak_2019.csv', index_col=False)]

font_name = mpl.font_manager.FontProperties(fname='C:\Windows\Fonts\MALGUNBD.TTF').get_name()
mpl.rc('font', family=font_name)

k=0
for i in range (len(key)):
    for j in range(len(year)):
        googlemap_review = googlemap_morph(googlemap_data[k])
        common_words, common_list = googlemap_list(googlemap_review)
        wc=wordcloud(common_words)
        word_count, xpos, xtext = save_googlemap_file(key[i], year[j], wc, common_list)
        draw_googlemap_graph(word_count, xpos, xtext, key[i], year[j])
        plt.cla()
        k=k+1

p=0
for i in range (len(key)):
    for j in range(len(year)):
        insta_review = insta_morph(insta_data[p])
        common_words, common_tag, common_list = insta_list(insta_review)
        wc=wordcloud(common_words)
        word_count, xpos, xtext = save_insta_file(key[i], year[j], wc, common_list)
        draw_insta_graph(word_count, xpos, xtext, key[i], year[j])
        plt.cla()
        p=p+1


import base64
import glob
from typing import Dict

import dash_bootstrap_components as dbc
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
from pathlib import Path
from .utils import get_dataset_info, generate_kpi_card, get_reuses, filter_reuses, generate_badge, slugify, \
    MLJAR_INFO_DICT
import json
with open("conf.json", "r") as f:
    URL_MAIN = json.load(f)["URL_MAIN"]
# Path

DATA_PATH = Path("./assets/datasets")
encoded_image_validated = base64.b64encode(open(DATA_PATH.parent.joinpath("quality.png"), 'rb').read()).decode()

def generate_etalab_cards(experiment_path: Path):
    notebook_html = [Path(p) for p in glob.glob(experiment_path.as_posix() + "/*.html")]

    if not notebook_html:
        return html.P("Aucune expÃ©rience disponible pour ce jeu de donnÃ©es ðŸ˜ž")
    list_cards = []
    for notebook in notebook_html:
        reuse_card = dbc.Card(
            [
                dbc.CardImg(src="./assets/jupyter_logo.png", top=True),
                dbc.CardBody(
                    [
                        html.H4(notebook.stem, className="card-title"),
                        html.P("A complete ML pipeline."
                               ),
                        dbc.Button("Voir expÃ©rience", color="primary", href=notebook.as_posix(), target="_blank",
                                   external_link=True),
                    ]
                ),
            ],

        )
        list_cards.append(reuse_card)
    return dbc.CardDeck(list_cards, className="w-25")

def generate_reuses_cards(resuses_dict: Dict):
    ml_reuses_dict = filter_reuses(resuses_dict)
    if not ml_reuses_dict:
        return html.P(
            "Aucune rÃ©utilisation disponible pour ce jeu de donnÃ©es ðŸ˜ž Ajoutez votre rÃ©utilisation sur data.gouv.fr !")
    list_cards = []
    for reuse in ml_reuses_dict:
        reuse_card = dbc.Card(
            [
                dbc.CardImg(src=reuse["image_thumbnail"], top=True),
                dbc.CardBody(
                    [
                        html.H4(reuse["title"], className="card-title"),
                        html.P(f"{reuse['description'][:100]}[...]", className="card-text",
                               ),
                        dbc.Button("Voir rÃ©utilisation", color="primary", href=reuse["url"], target="_blank"),
                    ]
                ),
            ],

        )
        list_cards.append(reuse_card)
    return dbc.CardDeck(list_cards, className="w-25")

def generate_table(dataset_id: str, table_type: str = "dict_data.csv"):
    display_table_path = DATA_PATH.joinpath(f"resources/{dataset_id}/{table_type}")
    if not display_table_path.exists():
        html_table = html.H5("AperÃ§u du dictionnaire des variables non disponible")
    else:
        table_df = pd.read_csv(display_table_path)
        if "leaderboard" in table_type:
            # add links to mljar table
            table_df["name"] = table_df[["name", "url"]].apply(lambda row: html.A(html.P(row["name"]), href=row["url"],
                                                                                target="_blank"), axis=1)
            table_df = table_df.drop(["url"], axis=1)
        html_table = html.Div(dbc.Table.from_dataframe(table_df, striped=True, size="sm", borderless=True),
                              style={"height": "300px", "overflow": 'auto'})
    return html_table

def generate_mljar_table(dataset_id: str, target_variable: str, base_url: str):
    display_table_path = DATA_PATH.joinpath(f"resources/{dataset_id}/automl_{slugify(target_variable)}/leaderboard.csv")
    experiment_path = display_table_path.parent
    if not display_table_path.exists():
        html_table = html.H5("MLJAR profile preview not available")
    else:
        table_df = pd.read_csv(display_table_path)
        table_df["metric_value"] = table_df["metric_value"].round(decimals=3)
        algorithm_urls = [html.A(html.P(n), href=experiment_path.joinpath(f"{n}/README.html").as_posix(),
                                 target="_blank")
                          for n in table_df.name]
        table_df["name"] = algorithm_urls
        html_table = html.Div(dbc.Table.from_dataframe(table_df, striped=True, size="sm", borderless=True),
                              style={"height": "300px", "overflow": 'auto'})
    return html_table, display_table_path.parent.joinpath("README.html")

def generate_stats_df(dataset_id: str, table_type: str = "statistics_summary.csv"):
    display_stats_path = DATA_PATH.joinpath(f"resources/{dataset_id}/{table_type}")
    if not display_stats_path.exists():
        statistics_df = html.H5("RÃ©sumÃ© statistique non disponible")
    else:
        statistics_df = pd.read_csv(display_stats_path)
    return statistics_df

def generate_dataset_page(dataset_url: str, datasets_df: pd.DataFrame):
    dataset_id = dataset_url.split("/")[-1]
    dataset_row = datasets_df[datasets_df["dgf_resource_id"] == dataset_id].iloc[0]
    if dataset_row.empty:
        return dbc.Container(html.H2("This dataset was not found in our catalog."))
    dataset_dict = get_dataset_info(dataset_row)
    dictionary_table = generate_table(dataset_dict["dgf_resource_id"], table_type="dict_data.csv")
    target_vars = list(MLJAR_INFO_DICT[dataset_id].keys())
    statistics_df = generate_stats_df(dataset_dict["dgf_resource_id"], table_type="statistics_summary.csv")
    pandas_profile_url = DATA_PATH.joinpath(f"resources/{dataset_id}/{dataset_id}_pandas_profile.html")
    experiments_url = DATA_PATH.joinpath(f"resources/{dataset_id}/our_experiments/")
    container = dbc.Container([
        # html.H4(generate_badge("Go back", url="/openml/", background_color="red")),
        html.H5(generate_badge("Accueil", url=URL_MAIN, background_color="#cadae6", new_tab=False)),
        html.Title("DGML: Data Gouv pour le Machine Learning"),
        html.H2([dataset_dict["title"],
                 html.Img(id="validated-img2",
                          src="data:image/png;base64,{}".format(encoded_image_validated),
                          style={'height': '3%', 'width': '3%', "float": "right"},
                          hidden=not dataset_dict["is_validated"]),
                 dbc.Tooltip("Ce jeu de donnÃ©es a Ã©tÃ© sÃ©lectionnÃ© et analysÃ© manuellement.",
                             target="validated-img2",
                             style={'font-size': 13}
                             )
                 ]),
        html.P(dataset_dict["description"]),
        # html.H4(generate_badge("Dataset in data.gouv.fr", url=dataset_dict['dgf_dataset_url'], background_color="#5783B7")),
        html.H4(
            generate_badge("Jeu de donnÃ©es sur data.gouv.fr", url=dataset_dict['dgf_dataset_url'],
                           background_color="#6d92ad")),
        html.Hr(style={"marginBottom": "20px"}),
        html.H3("Dictionnaire des variables"),
        dictionary_table,
        html.H4(generate_badge("Dictionnaire des variables complet", url=dataset_dict['dict_url'],
                               background_color="#6d92ad")),
        html.Hr(style={"marginBottom": "20px"}),
        html.H3("Statistiques Descriptives"),

        dbc.CardDeck(
            [
                generate_kpi_card("Nombre de colonnes", statistics_df['Number of variables']),
                generate_kpi_card("Nombre de lignes", statistics_df['Number of lines']),
                generate_kpi_card("Valeurs manquantes",
                                  f"{round(float(statistics_df['Percentage of missing cells']), 2)} %"),
                generate_kpi_card("Variables numÃ©riques", statistics_df['Numeric']),
                generate_kpi_card("Variables catÃ©gorielles", statistics_df['Categorical']),
                generate_kpi_card("Haute CardinalitÃ©", statistics_df['High cardinality variables']),
                generate_kpi_card("Haute Correlation", statistics_df['High correlation variables']),
                # generate_kpi_card("Skewed", 10),
            ]
        ),
        html.H4(
            generate_badge("Pandas Profile complet", url=pandas_profile_url.as_posix(), background_color="#6d92ad")),
        html.Hr(style={"marginBottom": "20px"}),
        html.H3("Auto Machine Learning"),
        html.Div([html.P(children=f"ModÃ¨les entraÃ®nÃ©s en utilisant comme target variable : "),
                  # dcc.Dropdown(
                  #     id="task-model-select",
                  #     options=[{"label": i, "value": i} for i in ["Classification", "Regression"]],
                  #     multi=False,
                  #     value="Classification",
                  #     clearable=False,
                  #     style={"min-width": "120px", "marginLeft": "3px"}
                  # ),
                  # html.P("using ", style={"marginLeft": "6px"}),
                  dcc.Dropdown(
                      id="target-var-model-select",
                      options=[{"label": i, "value": i} for i in target_vars],
                      multi=False,
                      value=target_vars[0],
                      clearable=False,
                      style={"min-width": "250px", "width": "auto", "marginLeft": "3px"}

                  ),
                  # html.P("as target variable.", style={"marginLeft": "6px"})

                  ], style={"display": 'flex'}),
        html.Div(id="mljar-div", style={"height": "300px", "overflow": 'auto'}),
        html.H4(id="mljar-link-badge", children=None),
        html.Hr(style={"marginBottom": "20px"}),
        html.H3("RÃ©utilisations Machine Learning (data.gouv.fr)"),
        #generate_reuses_cards(get_reuses(dataset_dict["dgf_dataset_id"])),
        html.Hr(style={"marginBottom": "20px"}),
        html.H3("Nos ExpÃ©riences"),
        # html.P("Check out our experiments on this dataset : "),
        generate_etalab_cards(experiments_url),
        # html.H4(generate_badge("See notebook", url=dataset_dict['etalab_xp_url'], background_color="#cadae6")),
        html.Hr(style={"marginBottom": "20px"}),
        # html.H3("Load Data"),
        # html.Hr(style={"marginBottom": "20px"}),
    ])

    return container


import pathlib
import unittest

import yaml

project_root = pathlib.Path(__file__).resolve().parent.parent.parent

class TestActionYml(unittest.TestCase):

    def test_cicd_workflow(self):
        with open(project_root / 'action.yml', encoding='utf-8') as r:
            action = yaml.safe_load(r)

        with open(project_root / '.github/workflows/publish.yml', encoding='utf-8') as r:
            cicd = yaml.safe_load(r)

        docker_image_steps = cicd.get('jobs', []).get('publish-docker-image', {}).get('steps', [])
        docker_image_step = [step
                             for step in docker_image_steps
                             if step.get('name') == 'Publish Test Results']
        self.assertEqual(1, len(docker_image_step))
        docker_image_run = docker_image_step[0].get('run')
        self.assertTrue(docker_image_run)
        vars = [var[7:-1].lower() if var.startswith('"') and var.endswith('"') else var[6:].lower()
                for line in docker_image_run.split('\n')
                for part in line.split(' ')
                for var in [part.strip()]
                if var.startswith('INPUT_') or var.startswith('"INPUT_')]

        self.assertEqual(sorted(list(action.get('inputs', {}).keys()) + ['log_level', 'root_log_level']), sorted(vars))


import csv
import itertools
from abc import ABC, abstractmethod
from datetime import timedelta

from django.http import HttpResponse, StreamingHttpResponse
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page

from rest_framework.views import APIView

from eric.appointments.config import CALENDAR_EXPORT_CACHE_SECONDS, DISPLAY_EXPORT_CACHE_SECONDS
from eric.appointments.utils import Echo
from eric.shared_elements.models import Meeting, Resource
from eric.site_preferences.models import options as site_preferences

class PublicCSVExportView(APIView, ABC):
    authentication_classes = ()
    permission_classes = ()
    download_file_name = "export.csv"

    def get(self, request, *args, **kwargs):
        response = self.build_http_response()
        response["Content-Disposition"] = f'attachment; filename="{self.download_file_name}"'
        return response

    def build_http_response(self):
        """
        Returns the CSV data as standard HTTP response.
        """

        response = HttpResponse(content_type="text/csv")
        writer = csv.writer(response, delimiter=";")
        for row in self.headers_and_rows:
            writer.writerow(row)

        return response

    def build_stream_response(self):
        """
        Returns the CSV data as streaming response.
        Might become necessary, if there is a huge amount of data.
        """

        pseudo_buffer = Echo()
        writer = csv.writer(pseudo_buffer, delimiter=";")
        streaming_content = (writer.writerow(row) for row in self.headers_and_rows)
        response = StreamingHttpResponse(streaming_content, content_type="text/csv")
        return response

    @property
    def headers_and_rows(self):
        """Iterator for all CSV rows (headers + content)"""
        return itertools.chain((self.csv_headers,), self.csv_rows)

    @property
    @abstractmethod
    def csv_rows(self):
        """Returns an iterator for data rows"""
        pass

    @property
    @abstractmethod
    def csv_headers(self):
        """Returns the CSV headers as list"""
        pass

@method_decorator(cache_page(CALENDAR_EXPORT_CACHE_SECONDS), "dispatch")
class CalendarCsvView(PublicCSVExportView):
    """
    Outputs anonymous CSV data of study room bookings within the next two weeks for simple calendars.
    """

    download_file_name = "calendar-export.csv"

    @property
    def study_room_bookings(self):
        """
        Gets all bookings within the next two weeks.
        """

        start_of_today = timezone.now().date()
        in_two_weeks = start_of_today + timedelta(days=14)
        return (
            Meeting.objects.study_room_bookings()
            .filter(deleted=False)
            .intersecting_interval(start_of_today, in_two_weeks)
            .order_by("date_time_start")
        )

    @property
    def csv_headers(self):
        return [
            "appointment_id",
            "resource_name",
            "start",
            "end",
        ]

    @property
    def csv_rows(self):
        return (
            [
                appointment.pk,
                appointment.resource.name,
                appointment.date_time_start.isoformat(),
                appointment.date_time_end.isoformat(),
            ]
            for appointment in self.study_room_bookings
        )

@method_decorator(cache_page(DISPLAY_EXPORT_CACHE_SECONDS), "dispatch")
class DisplayCsvView(PublicCSVExportView):
    """
    Outputs anonymous CSV data of study room bookings from today for simple displays.
    """

    download_file_name = "display-export.csv"

    @property
    def study_room_bookings(self):
        """
        Returns upcoming bookings of today.
        """

        now = timezone.now()
        end_of_today = now.replace(hour=23, minute=59, second=59, microsecond=999_999)
        return (
            Meeting.objects.study_room_bookings()
            .filter(deleted=False)
            .intersecting_interval(now, end_of_today)
            .order_by("date_time_start")
        )

    @property
    def bookings_per_room(self):
        number = site_preferences.bookings_per_room_in_study_room_booking_display_export
        assert number > 0, "Export is configured to have no columns"
        return number

    @property
    def csv_headers(self):
        headers = [
            "RaumId",
            "Raumbezeichnung",
        ]

        # append the per-appointment headers with a two-digit index (e.g. "TerminStart01")
        for index in range(1, self.bookings_per_room + 1):
            for key in ["TerminStart", "TerminEnde", "Laufend", "TerminBezeichnung", "Organisator", "TerminDatum"]:
                headers.append(f"{key}{index:02d}")

        headers.extend(
            [
                "Datum",
                "Template",
            ]
        )

        return headers

    def format_date(self, date):
        return timezone.localtime(date)

    @property
    def csv_rows(self):
        study_rooms = Resource.objects.study_rooms().order_by("study_room_info__room_id")
        return (self.row_for_room(resource) for resource in study_rooms)

    def row_for_room(self, resource: Resource):
        study_room = resource.study_room_info
        now = timezone.now()

        cells = [study_room.room_id, study_room.name]

        # add available booking information
        bookings = self.study_room_bookings.filter(resource=resource)[: self.bookings_per_room]
        for booking in bookings:
            start_time = self.format_date(booking.date_time_start).strftime("%H:%M")
            end_time = self.format_date(booking.date_time_end).strftime("%H:%M")
            running_flag = booking.date_time_start <= now < booking.date_time_end
            appointment_name = booking.title
            organiser_name = booking.created_by.userprofile.first_name_and_last_name
            booking_date = self.format_date(booking.date_time_start).strftime("%d.%m.%Y")
            cells.extend([start_time, end_time, running_flag, appointment_name, organiser_name, booking_date])

        # add empty cells if there are not enough bookings to show
        empty_booking_count = self.bookings_per_room - len(bookings)
        empty_cells = [""] * 6 * empty_booking_count
        cells.extend(empty_cells)

        current_date = now.strftime("%d.%m.%Y")
        template = study_room.display_design.key
        cells.extend([current_date, template])

        return cells


import sys, os, re, pprint
# UDPipe tokenize.py
from .helpers import *
from .config import *

from .utils import Batch

def file2fast_text(in_file_1, in_file_2, out_file, empty_dict=None):
    """
    Converts two tokenized text files into fast_text format
    Empty lines as well original number of lines will be saved to empty_dict dictionary
    args:
        in_file_1 (string): valid path to file 1
        in_file_2 (string): valid path to file 1
        out_file  (string): path where output should be saved to
        empty_dict  (dict): a dictionary where empty line indeces and number of lines in the input files will be saved to
    """
    print('Writing:', out_file)
    with open(in_file_1, 'r') as srcf, \
         open(in_file_2, 'r') as trgf, \
         open(out_file, 'w') as outf:
        empty_lines = []
        for i, (src, trg) in enumerate(zip(srcf, trgf)):
            src = src.strip()
            trg = trg.strip()
            if src and trg:
                outf.write(src + ' ||| ' + trg + '\n')
            else:
                empty_lines.append(i)

    if empty_dict != None:
        filename = os.path.basename(out_file)
        if filename.endswith('.fast_text'):
            filename = filename.rsplit('.', 1)[0]
        empty_dict[filename] = empty_lines
        empty_dict[filename+'.no_lines'] = i+1

def dir2fast_text(input_dir):
    """
    Finds language pairs based on notation in form de-en.de, de-en.en and converts them into fast text format using file2fast_text()
    args:
        input_dir (string): path to input directory
    """
    input_dir, output_dir = create_same_level_output_dir(input_dir, 'merged')
    pairs_dict = get_pairs(input_dir, verbose=False)
    for lang_pair in pairs_dict:
        print(lang_pair)
        for lang in pairs_dict[lang_pair]:
            assert not pairs_dict[lang_pair][lang].endswith('.conll'), 'Fast Text format cannot be applied to .conll-files!'
    print('Pairs found:')
    pprint.pprint(pairs_dict)

    print('Merging..')
    empty_dict = {}
    for pair in pairs_dict:
        lang_1, lang_2 = pair.split('-')
        root = root = pairs_dict[pair][lang_1].split('.')[0]

        filename_d1 = f'{root}.{lang_1}-{lang_2}.fast_text'
        outputpath_d1 = os.path.join(output_dir, filename_d1)

        inputpath_l1 = os.path.join(input_dir, pairs_dict[pair][lang_1])
        inputpath_l2 = os.path.join(input_dir, pairs_dict[pair][lang_2])
        file2fast_text(inputpath_l1, inputpath_l2, outputpath_d1, empty_dict)

        filename_d2 = f'{root}.{lang_2}-{lang_1}.fast_text'

        outputpath_d2 = os.path.join(output_dir, filename_d2)
        file2fast_text(inputpath_l2, inputpath_l1, outputpath_d2, empty_dict)

    empty_dict_path = os.path.join(output_dir, 'empty.dict')
    with open(empty_dict_path, 'w') as f:
        f.write(repr(empty_dict))
    print('.. done.')
    return output_dir

def align(input_dir, use_shell=False, args=None):
    """
    Expects 'input_dir' to be the path to the directory with tokens. Tokens will be converted to fast text and saved
    into a directory 'merged'. Batchfiles for alignment jobs will be created and submitted to slurm job system
    args:
        input_dir (string): path to token directory
        use_shell (bool)  : trys to perform alignment in shell instead of slurm sbatch (experimental)
        args (None or argparser.args): additional arguments to modify parameters such es memory, timelimit or partition to run on

    """

    input_dir = dir2fast_text(input_dir=input_dir)
    # changing input dir to merged_dir for alignment input
    input_dir, output_dir = create_same_level_output_dir(input_dir, 'alignment')

    merged_files = list(filter(lambda x: re.match(r'.*\.[a-z]{2}-[a-z]{2}', x), os.listdir(input_dir)))
    print(f'Valid input files  ("{input_dir}"):')
    pprint.pprint(merged_files)
    print()
    print('Create and submit batchfiles..')
    batch = Batch(name=f'alignment', n_cpus='2', log_dir='alignment', timelimit='04:00:00', args=args)

    for filename in merged_files:
        outputname = filename
        if filename.endswith('.fast_text'):
            outputname = filename.rsplit('.', 1)[0]
        pair = 'al_'+outputname.split('.')[-1]
        batch.name = pair

        batch.align(input_dir=input_dir, output_dir=output_dir, filename=filename, outputname=outputname)
        batch.shell() if use_shell else batch.submit()


'''
compute_collision() will compute the collision of all the entities with
a Point while compute_first_collision() will always return its first entry.
Especially if a point is on an element edge this can be tricky.
You may also want to compare with the Cell.contains(Point) tool.
'''
# Script by Rudy at https://fenicsproject.discourse.group/t/
#           any-function-to-determine-if-the-point-is-in-the-mesh/275/3
import dolfin
from vedo.dolfin import plot
from vedo import printc, pointcloud

n  = 4
Px = 0.5
Py = 0.5
mesh = dolfin.UnitSquareMesh(n, n)
bbt = mesh.bounding_box_tree()
collisions = bbt.compute_collisions(dolfin.Point(Px, Py))
collisions1st = bbt.compute_first_entity_collision(dolfin.Point(Px, Py))
printc("collisions    : ", collisions)
printc("collisions 1st: ", collisions1st)

for cell in dolfin.cells(mesh):
    contains = cell.contains(dolfin.Point(Px, Py))
    printc("Cell", cell.index(), "contains P:", contains, c=contains)

###########################################
pt = pointcloud.Point([Px, Py], c='blue')

plot(mesh, pt, text=__doc__)


class Solution:
    def minimumDifference(self, nums: List[int]) -> int:
        n = len(nums) // 3

        second = nums[-n:]
        heapify(second)
        total = sum(second)
        list1 = [0 for _ in range(len(nums))]
        list1[2 * n] = total
        for i in range(2 * n - 1, n - 1, -1):
            heappush(second, nums[i])
            total -= second[0]
            heappop(second)
            total += nums[i]
            list1[i] = total

        first = [-nums[i] for i in range(n)]
        heapify(first)
        total = sum(first) * -1
        res = total - list1[n]

        for i in range(n, 2 * n):
            total += nums[i]
            heappush(first, -nums[i])
            total -= -first[0]
            heappop(first)
            res = min(res, total - list1[i + 1])
        return res

2163. Minimum Difference in Sums After Removal of Elements.py


# pylint: disable=too-many-lines

"""
Contains all of the Fides resources modeled as Pydantic models.
"""
from __future__ import annotations

from enum import Enum
from typing import Any, Dict, List, Optional, Union
from warnings import warn

from pydantic import (
    AnyUrl,
    BaseModel,
    ConstrainedStr,
    Field,
    HttpUrl,
    PositiveInt,
    root_validator,
    validator,
)

from fideslang.validation import (
    FidesKey,
    FidesVersion,
    check_valid_country_code,
    deprecated_version_later_than_added,
    has_versioning_if_default,
    is_deprecated_if_replaced,
    matching_parent_key,
    no_self_reference,
    parse_data_type_string,
    sort_list_objects_by_name,
    unique_items_in_list,
    valid_data_type,
)

# Reusable Validators
country_code_validator = validator("third_country_transfers", allow_reuse=True)(
    check_valid_country_code
)
matching_parent_key_validator = validator("parent_key", allow_reuse=True, always=True)(
    matching_parent_key
)
no_self_reference_validator = validator("parent_key", allow_reuse=True)(
    no_self_reference
)
has_versioning_if_default_validator = validator(
    "is_default", allow_reuse=True, always=True
)(has_versioning_if_default)
deprecated_version_later_than_added_validator = validator(
    "version_deprecated", allow_reuse=True
)(deprecated_version_later_than_added)
is_deprecated_if_replaced_validator = validator("replaced_by", allow_reuse=True)(
    is_deprecated_if_replaced
)

# Reusable Fields
name_field = Field(description="Human-Readable name for this resource.")
description_field = Field(
    description="A detailed description of what this resource is."
)
meta_field = Field(
    default=None,
    description="An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects.",
)

class FidesModel(BaseModel):
    """The base model for most top-level Fides objects."""

    fides_key: FidesKey = Field(
        description="A unique key used to identify this resource."
    )
    organization_fides_key: FidesKey = Field(
        default="default_organization",
        description="Defines the Organization that this resource belongs to.",
    )
    tags: Optional[List[str]] = None
    name: Optional[str] = name_field
    description: Optional[str] = description_field

    class Config:
        "Config for the FidesModel"
        extra = "ignore"
        orm_mode = True

class DefaultModel(BaseModel):
    """
    A model meant to be inherited by versioned parts of the Default Taxonomy.
    """

    version_added: Optional[str] = Field(
        default=None,
        description="The version of Fideslang in which this label was added.",
    )
    version_deprecated: Optional[str] = Field(
        default=None,
        description="The version of Fideslang in which this label was deprecated.",
    )
    replaced_by: Optional[FidesKey] = Field(
        default=None,
        description="The new name, if applicable, for this label after deprecation.",
    )
    is_default: bool = Field(
        default=False,
        description="Denotes whether the resource is part of the default taxonomy or not.",
    )

    _has_versioning_if_default: classmethod = has_versioning_if_default_validator
    _deprecated_version_later_than_added: classmethod = (
        deprecated_version_later_than_added_validator
    )
    _is_deprecated_if_replaced: classmethod = is_deprecated_if_replaced_validator

    @validator("version_added")
    @classmethod
    def validate_verion_added(
        cls, version_added: Optional[str], values: Dict
    ) -> Optional[str]:
        """
        Validate that the `version_added` field is a proper FidesVersion
        """
        if not version_added:
            return None

        FidesVersion.validate(version_added)
        return version_added

    @validator("version_deprecated")
    @classmethod
    def validate_version_deprecated(
        cls, version_deprecated: Optional[str], values: Dict
    ) -> Optional[str]:
        """
        Validate that the `version_deprecated` is a proper FidesVersion
        """
        if not version_deprecated:
            return None

        FidesVersion.validate(version_deprecated)
        return version_deprecated

class DataResponsibilityTitle(str, Enum):
    """
    The model defining the responsibility or role over
    the system that processes personal data.

    Used to identify whether the organization is a
    Controller, Processor, or Sub-Processor of the data
    """

    CONTROLLER = "Controller"
    PROCESSOR = "Processor"
    SUB_PROCESSOR = "Sub-Processor"

class IncludeExcludeEnum(str, Enum):
    """
    Determine whether or not defined rights are
    being included or excluded.
    """

    ALL = "ALL"
    EXCLUDE = "EXCLUDE"
    INCLUDE = "INCLUDE"
    NONE = "NONE"

class DataSubjectRightsEnum(str, Enum):
    """
    The model for data subject rights over
    personal data.

    Based upon chapter 3 of the GDPR
    """

    INFORMED = "Informed"
    ACCESS = "Access"
    RECTIFICATION = "Rectification"
    ERASURE = "Erasure"
    PORTABILITY = "Portability"
    RESTRICT_PROCESSING = "Restrict Processing"
    WITHDRAW_CONSENT = "Withdraw Consent"
    OBJECT = "Object"
    OBJECT_TO_AUTOMATED_PROCESSING = "Object to Automated Processing"

class LegalBasisEnum(str, Enum):
    """
    Deprecated. The model for allowable legal basis categories on data uses.
    """

    CONSENT = "Consent"
    CONTRACT = "Contract"
    LEGAL_OBLIGATION = "Legal Obligation"
    VITAL_INTEREST = "Vital Interest"
    PUBLIC_INTEREST = "Public Interest"
    LEGITIMATE_INTEREST = "Legitimate Interests"

class LegalBasisForProcessingEnum(str, Enum):
    """
    The model for allowable legal basis categories on privacy declarations.

    Based upon article 6 of the GDPR
    """

    CONSENT = "Consent"
    CONTRACT = "Contract"
    LEGAL_OBLIGATION = "Legal obligations"
    VITAL_INTEREST = "Vital interests"
    PUBLIC_INTEREST = "Public interest"
    LEGITIMATE_INTEREST = "Legitimate interests"

class LegalBasisForProfilingEnum(str, Enum):
    """The model for describing the legal basis under which profiling is performed"""

    EXPLICIT_CONSENT = "Explicit consent"
    CONTRACT = "Contract"
    AUTHORISED_BY_LAW = "Authorised by law"

class LegalBasisForTransfersEnum(str, Enum):
    """
    The model for describing the legal basis under which data is transferred

    We currently do _not_ enforce this enum on the `legal_basis_for_transfers`
    field, because the set of allowable values seems to be changing frequently
    and without clear notice in upstream, public data sources.
    """

    ADEQUACY_DECISION = "Adequacy Decision"
    SCCS = "SCCs"
    BCRS = "BCRs"
    SUPPLEMENTARY_MEASURES = "Supplementary Measures"
    OTHER = "Other"

class SpecialCategoriesEnum(str, Enum):
    """
    Deprecated. Special Categories Enum that was used on Data Uses.
    """

    CONSENT = "Consent"
    EMPLOYMENT = "Employment"
    VITAL_INTEREST = "Vital Interests"
    NON_PROFIT_BODIES = "Non-profit Bodies"
    PUBLIC_BY_DATA_SUBJECT = "Public by Data Subject"
    LEGAL_CLAIMS = "Legal Claims"
    PUBLIC_INTEREST = "Substantial Public Interest"
    MEDICAL = "Medical"
    PUBLIC_HEALTH_INTEREST = "Public Health Interest"

class SpecialCategoryLegalBasisEnum(str, Enum):
    """
    The model for the legal basis for processing special categories of personal data
    on privacy declarations

    Based upon article 9 of the GDPR
    """

    CONSENT = "Explicit consent"
    EMPLOYMENT = "Employment, social security and social protection"
    VITAL_INTEREST = "Vital interests"
    NON_PROFIT_BODIES = "Not-for-profit bodies"
    PUBLIC_BY_DATA_SUBJECT = "Made public by the data subject"
    LEGAL_CLAIMS = "Legal claims or judicial acts"
    PUBLIC_INTEREST = "Reasons of substantial public interest (with a basis in law)"
    MEDICAL = "Health or social care (with a basis in law)"
    PUBLIC_HEALTH_INTEREST = "Public health (with a basis in law)"
    RESEARCH = "Archiving, research and statistics (with a basis in law)"

# Privacy Data Types
class DataCategory(FidesModel, DefaultModel):
    """The DataCategory resource model."""

    parent_key: Optional[FidesKey]

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

class DataQualifier(FidesModel, DefaultModel):
    """The DataQualifier resource model."""

    parent_key: Optional[FidesKey]

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

class Cookies(BaseModel):
    """The Cookies resource model"""

    name: str
    path: Optional[str]
    domain: Optional[str]

    class Config:
        """Config for the cookies"""

        orm_mode = True

class DataSubjectRights(BaseModel):
    """
    The DataSubjectRights resource model.

    Includes a strategy and optionally a
    list of data subject rights to apply
    via the set strategy.
    """

    strategy: IncludeExcludeEnum = Field(
        description="Defines the strategy used when mapping data rights to a data subject.",
    )
    values: Optional[List[DataSubjectRightsEnum]] = Field(
        description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.",
    )

    @root_validator()
    @classmethod
    def include_exclude_has_values(cls, values: Dict) -> Dict:
        """
        Validate the if include or exclude is chosen, that at least one
        value is present.
        """
        strategy, rights = values.get("strategy"), values.get("values")
        if strategy in ("INCLUDE", "EXCLUDE"):
            assert (
                rights is not None
            ), f"If {strategy} is chosen, rights must also be listed."
        return values

class DataSubject(FidesModel, DefaultModel):
    """The DataSubject resource model."""

    rights: Optional[DataSubjectRights] = Field(description=DataSubjectRights.__doc__)
    automated_decisions_or_profiling: Optional[bool] = Field(
        description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.",
    )

class DataUse(FidesModel, DefaultModel):
    """The DataUse resource model."""

    parent_key: Optional[FidesKey] = None
    legal_basis: Optional[LegalBasisEnum] = Field(
        description="Deprecated. The legal basis category of which the data use falls under. This field is used as part of the creation of an exportable data map.",
    )
    special_category: Optional[SpecialCategoriesEnum] = Field(
        description="Deprecated. The special category for processing of which the data use falls under. This field is used as part of the creation of an exportable data map.",
    )
    recipients: Optional[List[str]] = Field(
        description="Deprecated. An array of recipients when sharing personal data outside of your organization.",
    )
    legitimate_interest: Optional[bool] = Field(
        description="Deprecated. A boolean representation of if the legal basis used is `Legitimate Interest`. Validated at run time and looks for a `legitimate_interest_impact_assessment` to exist if true.",
    )
    legitimate_interest_impact_assessment: Optional[AnyUrl] = Field(
        description="Deprecated. A url pointing to the legitimate interest impact assessment. Required if the legal bases used is legitimate interest.",
    )

    _matching_parent_key: classmethod = matching_parent_key_validator
    _no_self_reference: classmethod = no_self_reference_validator

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of Data Use fields pending deprecation.
        """
        deprecated_fields = [
            "legal_basis",
            "recipients",
            "special_category",
            "legitimate_interest",
            "legitimate_interest_impact_assessment",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

    @validator("legitimate_interest", always=True)
    @classmethod
    def set_legitimate_interest(cls, value: bool, values: Dict) -> bool:
        """Sets if a legitimate interest is used."""
        if values["legal_basis"] == "Legitimate Interests":
            value = True
        return value

    @validator("legitimate_interest_impact_assessment", always=True)
    @classmethod
    def ensure_impact_assessment(cls, value: AnyUrl, values: Dict) -> AnyUrl:
        """
        Validates an impact assessment is applied if a
        legitimate interest has been defined.
        """
        if values["legitimate_interest"]:
            assert (
                value is not None
            ), "Impact assessment cannot be null for a legitimate interest, please provide a valid url"
        return value

# Dataset
class DatasetFieldBase(BaseModel):
    """Base DatasetField Resource model.

    This model is available for cases where the DatasetField information needs to be
    customized. In general this will not be the case and you will instead want to use
    the DatasetField model.

    When this model is used you will need to implement your own recursive field in
    to adding any new needed fields.

    Example:

    ```py
    from typing import List, Optional
    from fideslang import DatasetFieldBase

    class MyDatasetField(DatasetFieldBase):
        custom: str
        fields: Optional[List[MyDatasetField]] = []
    ```
    """

    name: str = name_field
    description: Optional[str] = description_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.",
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="A Data Qualifier that applies to this field. Note that this field holds a single value, therefore, the property name is singular.",
    )
    retention: Optional[str] = Field(
        description="An optional string to describe the retention policy for a dataset. This field can also be applied more granularly at either the Collection or field level of a Dataset.",
    )

class EdgeDirection(str, Enum):
    """Direction of a FidesDataSetReference"""

    FROM = "from"
    TO = "to"

class FidesDatasetReference(BaseModel):
    """Reference to a field from another Collection"""

    dataset: FidesKey
    field: str
    direction: Optional[EdgeDirection]

class FidesMeta(BaseModel):
    """Supplementary metadata used by the Fides application for additional features."""

    references: Optional[List[FidesDatasetReference]] = Field(
        description="Fields that current field references or is referenced by. Used for drawing the edges of a DSR graph.",
        default=None,
    )
    identity: Optional[str] = Field(
        description="The type of the identity data that should be used to query this collection for a DSR."
    )
    primary_key: Optional[bool] = Field(
        description="Whether the current field can be considered a primary key of the current collection"
    )
    data_type: Optional[str] = Field(
        description="Optionally specify the data type. Fides will attempt to cast values to this type when querying."
    )
    length: Optional[PositiveInt] = Field(
        description="Optionally specify the allowable field length. Fides will not generate values that exceed this size."
    )
    return_all_elements: Optional[bool] = Field(
        description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False."
    )
    read_only: Optional[bool] = Field(
        description="Optionally specify if a field is read-only, meaning it can't be updated or deleted."
    )

    @validator("data_type")
    @classmethod
    def valid_data_type(cls, value: Optional[str]) -> Optional[str]:
        """Validate that all annotated data types exist in the taxonomy"""
        return valid_data_type(value)

class FidesopsMetaBackwardsCompat(BaseModel):
    """Mixin to convert fidesops_meta to fides_meta for backwards compatibility
    as we add DSR concepts to fideslang"""

    def __init__(self, **data: Union[Dataset, DatasetCollection, DatasetField]) -> None:
        """For Datasets, DatasetCollections, and DatasetFields, if old fidesops_meta field is specified,
        convert this to a fides_meta field instead."""
        fidesops_meta = data.pop("fidesops_meta", None)
        fides_meta = data.pop("fides_meta", None)
        super().__init__(
            fides_meta=fides_meta or fidesops_meta,
            **data,
        )

class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat):
    """
    The DatasetField resource model.

    This resource is nested within a DatasetCollection.
    """

    fides_meta: Optional[FidesMeta] = None

    fields: Optional[List[DatasetField]] = Field(
        description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).",
    )

    @validator("fides_meta")
    @classmethod
    def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]:
        """Validate upfront that the return_all_elements flag can only be specified on array fields"""
        if not meta_values:
            return meta_values

        is_array: bool = bool(
            meta_values.data_type and meta_values.data_type.endswith("[]")
        )
        if not is_array and meta_values.return_all_elements is not None:
            raise ValueError(
                "The 'return_all_elements' attribute can only be specified on array fields."
            )
        return meta_values

    @validator("fields")
    @classmethod
    def validate_object_fields(  # type: ignore
        cls,
        fields: Optional[List["DatasetField"]],
        values: Dict[str, Any],
    ) -> Optional[List["DatasetField"]]:
        """Two validation checks for object fields:
        - If there are sub-fields specified, type should be either empty or 'object'
        - Additionally object fields cannot have data_categories.
        """
        declared_data_type = None
        field_name: str = values.get("name")  # type: ignore

        if values.get("fides_meta"):
            declared_data_type = values["fides_meta"].data_type

        if fields and declared_data_type:
            data_type, _ = parse_data_type_string(declared_data_type)
            if data_type != "object":
                raise ValueError(
                    f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field."
                )

        if (fields or declared_data_type == "object") and values.get("data_categories"):
            raise ValueError(
                f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead"
            )

        return fields

# this is required for the recursive reference in the pydantic model:
DatasetField.update_forward_refs()

class FidesCollectionKey(ConstrainedStr):
    """
    Dataset.Collection name where both dataset and collection names are valid FidesKeys
    """

    @classmethod
    def validate(cls, value: str) -> str:
        """
        Overrides validation to check FidesCollectionKey format, and that both the dataset
        and collection names have the FidesKey format.
        """
        values = value.split(".")
        if len(values) == 2:
            FidesKey.validate(values[0])
            FidesKey.validate(values[1])
            return value
        raise ValueError(
            "FidesCollection must be specified in the form 'FidesKey.FidesKey'"
        )

class CollectionMeta(BaseModel):
    """Collection-level specific annotations used for query traversal"""

    after: Optional[List[FidesCollectionKey]]
    skip_processing: Optional[bool] = False

class DatasetCollection(FidesopsMetaBackwardsCompat):
    """
    The DatasetCollection resource model.

    This resource is nested within a Dataset.
    """

    name: str = name_field
    description: Optional[str] = description_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.",
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="Array of Data Qualifier resources identified by `fides_key`, that apply to all fields in the collection.",
    )
    retention: Optional[str] = Field(
        description="An optional string to describe the retention policy for a Dataset collection. This field can also be applied more granularly at the field level of a Dataset.",
    )
    fields: List[DatasetField] = Field(
        description="An array of objects that describe the collection's fields.",
    )

    fides_meta: Optional[CollectionMeta] = None

    _sort_fields: classmethod = validator("fields", allow_reuse=True)(
        sort_list_objects_by_name
    )
    _unique_items_in_list: classmethod = validator("fields", allow_reuse=True)(
        unique_items_in_list
    )

class ContactDetails(BaseModel):
    """
    The contact details information model.

    Used to capture contact information for controllers, used
    as part of exporting a data map / ROPA.

    This model is nested under an Organization and
    potentially under a system/dataset.
    """

    name: str = Field(
        default="",
        description="An individual name used as part of publishing contact information. Encrypted at rest on the server.",
    )
    address: str = Field(
        default="",
        description="An individual address used as part of publishing contact information. Encrypted at rest on the server.",
    )
    email: str = Field(
        default="",
        description="An individual email used as part of publishing contact information. Encrypted at rest on the server.",
    )
    phone: str = Field(
        default="",
        description="An individual phone number used as part of publishing contact information. Encrypted at rest on the server.",
    )

class DatasetMetadata(BaseModel):
    """
    The DatasetMetadata resource model.

    Object used to hold application specific metadata for a dataset
    """

    resource_id: Optional[str]
    after: Optional[List[FidesKey]]

class Dataset(FidesModel, FidesopsMetaBackwardsCompat):
    """The Dataset resource model."""

    meta: Optional[Dict] = meta_field
    data_categories: Optional[List[FidesKey]] = Field(
        description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.",
    )
    data_qualifier: Optional[FidesKey] = Field(
        description="Deprecated. Array of Data Qualifier resources identified by `fides_key`, that apply to all collections in the Dataset.",
    )
    fides_meta: Optional[DatasetMetadata] = Field(
        description=DatasetMetadata.__doc__, default=None
    )
    joint_controller: Optional[ContactDetails] = Field(
        description="Deprecated. " + ContactDetails.__doc__,
    )
    retention: Optional[str] = Field(
        description="Deprecated. An optional string to describe the retention policy for a dataset. This field can also be applied more granularly at either the Collection or field level of a Dataset.",
    )
    third_country_transfers: Optional[List[str]] = Field(
        description="Deprecated. An optional array to identify any third countries where data is transited to. For consistency purposes, these fields are required to follow the Alpha-3 code set in [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3).",
    )
    collections: List[DatasetCollection] = Field(
        description="An array of objects that describe the Dataset's collections.",
    )

    _sort_collections: classmethod = validator("collections", allow_reuse=True)(
        sort_list_objects_by_name
    )
    _check_valid_country_code: classmethod = country_code_validator
    _unique_items_in_list: classmethod = validator("collections", allow_reuse=True)(
        unique_items_in_list
    )

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of Dataset fields pending deprecation.
        """
        deprecated_fields = [
            "joint_controller",
            "data_qualifier",
            "retention",
            "third_country_transfers",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

# Evaluation
class ViolationAttributes(BaseModel):
    "The model for attributes which led to an evaluation violation"

    data_categories: List[str] = Field(
        description="A list of data categories which led to an evaluation violation.",
    )
    data_subjects: List[str] = Field(
        description="A list of data subjects which led to an evaluation violation.",
    )
    data_uses: List[str] = Field(
        description="A list of data uses which led to an evaluation violation.",
    )
    data_qualifier: str = Field(
        description="The data qualifier which led to an evaluation violation.",
    )

class Violation(BaseModel):
    "The model for violations within an evaluation."

    violating_attributes: ViolationAttributes = Field(
        description=ViolationAttributes.__doc__
    )
    detail: str = Field(
        description="A human-readable string detailing the evaluation violation.",
    )

class StatusEnum(str, Enum):
    "The model for possible evaluation results."

    FAIL = "FAIL"
    PASS = "PASS"

class Evaluation(BaseModel):
    """
    The Evaluation resource model.

    This resource is created after an evaluation is executed.
    """

    fides_key: FidesKey = Field(
        description="A uuid generated for each unique evaluation.",
    )
    status: StatusEnum = Field(description=StatusEnum.__doc__)
    violations: List[Violation] = Field(
        default=[],
        description=Violation.__doc__,
    )
    message: str = Field(
        default="",
        description="A human-readable string response for the evaluation.",
    )

    class Config:
        "Config for the Evaluation"
        extra = "ignore"
        orm_mode = True

# Organization
class ResourceFilter(BaseModel):
    """
    The ResourceFilter resource model.
    """

    type: str = Field(
        description="The type of filter to be used (i.e. ignore_resource_arn)",
    )
    value: str = Field(
        description="A string representation of resources to be filtered. Can include wildcards.",
    )

class OrganizationMetadata(BaseModel):
    """
    The OrganizationMetadata resource model.

    Object used to hold application specific metadata for an organization
    """

    resource_filters: Optional[List[ResourceFilter]] = Field(
        description="A list of filters that can be used when generating or scanning systems."
    )

class Organization(FidesModel):
    """
    The Organization resource model.

    This resource is used as a way to organize all other resources.
    """

    # It inherits this from FidesModel but Organizations don't have this field
    organization_parent_key: None = Field(
        default=None,
        description="An inherited field from the FidesModel that is unused with an Organization.",
    )
    controller: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    data_protection_officer: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    fidesctl_meta: Optional[OrganizationMetadata] = Field(
        description=OrganizationMetadata.__doc__,
    )
    representative: Optional[ContactDetails] = Field(
        description=ContactDetails.__doc__,
    )
    security_policy: Optional[HttpUrl] = Field(
        description="Am optional URL to the organization security policy."
    )

# Policy
class MatchesEnum(str, Enum):
    """
    The MatchesEnum resource model.

    Determines how the listed resources are matched in the evaluation logic.
    """

    ANY = "ANY"
    ALL = "ALL"
    NONE = "NONE"
    OTHER = "OTHER"

class PrivacyRule(BaseModel):
    """
    The PrivacyRule resource model.

    A list of privacy data types and what match method to use.
    """

    matches: MatchesEnum = Field(
        description=MatchesEnum.__doc__,
    )
    values: List[FidesKey] = Field(
        description="A list of fides keys to be used with the matching type in a privacy rule.",
    )

class PolicyRule(BaseModel):
    """
    The PolicyRule resource model.

    Describes the allowed combination of the various privacy data types.
    """

    name: str
    data_categories: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_uses: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_subjects: PrivacyRule = Field(
        description=PrivacyRule.__doc__,
    )
    data_qualifier: FidesKey = Field(
        default="aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified",
        description="The fides key of the data qualifier to be used in a privacy rule.",
    )

class Policy(FidesModel):
    """
    The Policy resource model.

    An object used to organize a list of PolicyRules.
    """

    rules: List[PolicyRule] = Field(
        description=PolicyRule.__doc__,
    )

    _sort_rules: classmethod = validator("rules", allow_reuse=True)(
        sort_list_objects_by_name
    )

# Registry
class Registry(FidesModel):
    """
    The Registry resource model.

    Systems can be assigned to this resource, but it doesn't inherently
    point to any other resources.
    """

# System
class DataProtectionImpactAssessment(BaseModel):
    """
    The DataProtectionImpactAssessment (DPIA) resource model.

    Contains information in regard to the data protection
    impact assessment exported on a data map or Record of
    Processing Activities (RoPA).

    A legal requirement under GDPR for any project that
    introduces a high risk to personal information.
    """

    is_required: bool = Field(
        default=False,
        description="A boolean value determining if a data protection impact assessment is required. Defaults to False.",
    )
    progress: Optional[str] = Field(
        description="The optional status of a Data Protection Impact Assessment. Returned on an exported data map or RoPA.",
    )
    link: Optional[AnyUrl] = Field(
        description="The optional link to the Data Protection Impact Assessment. Returned on an exported data map or RoPA.",
    )

class PrivacyDeclaration(BaseModel):
    """
    The PrivacyDeclaration resource model.

    States a function of a system, and describes how it relates
    to the privacy data types.
    """

    name: Optional[str] = Field(
        description="The name of the privacy declaration on the system.",
    )
    data_categories: List[FidesKey] = Field(
        description="An array of data categories describing a system in a privacy declaration.",
    )
    data_use: FidesKey = Field(
        description="The Data Use describing a system in a privacy declaration.",
    )
    data_qualifier: Optional[FidesKey] = Field(
        description="Deprecated. The fides key of the data qualifier describing a system in a privacy declaration.",
    )
    data_subjects: List[FidesKey] = Field(
        default_factory=list,
        description="An array of data subjects describing a system in a privacy declaration.",
    )
    dataset_references: Optional[List[FidesKey]] = Field(
        description="Referenced Dataset fides keys used by the system.",
    )
    egress: Optional[List[FidesKey]] = Field(
        description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied."
    )
    ingress: Optional[List[FidesKey]] = Field(
        description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied."
    )
    features: List[str] = Field(
        default_factory=list, description="The features of processing personal data."
    )
    flexible_legal_basis_for_processing: Optional[bool] = Field(
        description="Whether the legal basis for processing is 'flexible' (i.e. can be overridden in a privacy notice) for this declaration.",
    )
    legal_basis_for_processing: Optional[LegalBasisForProcessingEnum] = Field(
        description="The legal basis under which personal data is processed for this purpose."
    )
    impact_assessment_location: Optional[str] = Field(
        description="Where the legitimate interest impact assessment is stored"
    )
    retention_period: Optional[str] = Field(
        description="An optional string to describe the time period for which data is retained for this purpose."
    )
    processes_special_category_data: bool = Field(
        default=False,
        description="This system processes special category data",
    )
    special_category_legal_basis: Optional[SpecialCategoryLegalBasisEnum] = Field(
        description="The legal basis under which the special category data is processed.",
    )
    data_shared_with_third_parties: bool = Field(
        default=False,
        description="This system shares data with third parties for this purpose.",
    )
    third_parties: Optional[str] = Field(
        description="The types of third parties the data is shared with.",
    )
    shared_categories: List[str] = Field(
        default_factory=list,
        description="The categories of personal data that this system shares with third parties.",
    )
    cookies: Optional[List[Cookies]] = Field(
        description="Cookies associated with this data use to deliver services and functionality",
    )

    @validator("data_qualifier")
    @classmethod
    def deprecate_data_qualifier(cls, value: FidesKey) -> FidesKey:
        """
        Warn that the `data_qualifier` field is deprecated, if set.
        """
        if value is not None:
            warn(
                "The data_qualifier field is deprecated, and will be removed in a future version of fideslang.",
                DeprecationWarning,
            )

        return value

    class Config:
        """Config for the Privacy Declaration"""

        orm_mode = True

class SystemMetadata(BaseModel):
    """
    The SystemMetadata resource model.

    Object used to hold application specific metadata for a system
    """

    resource_id: Optional[str] = Field(
        description="The external resource id for the system being modeled."
    )
    endpoint_address: Optional[str] = Field(
        description="The host of the external resource for the system being modeled."
    )
    endpoint_port: Optional[str] = Field(
        description="The port of the external resource for the system being modeled."
    )

class FlowableResources(str, Enum):
    """
    The resource types with which DataFlows can be created.
    """

    DATASET = "dataset"
    SYSTEM = "system"
    USER = "user"

class DataFlow(BaseModel):
    """
    The DataFlow resource model.

    Describes a resource model with which a given System resource communicates.
    """

    fides_key: FidesKey = Field(
        ...,
        description="Identifies the System or Dataset resource with which the communication occurs. May also be 'user', to represent communication with the user(s) of a System.",
    )
    type: str = Field(
        ...,
        description=f"Specifies the resource model class for which the `fides_key` applies. May be any of {', '.join([member.value for member in FlowableResources])}.",
    )
    data_categories: Optional[List[FidesKey]] = Field(
        description="An array of data categories describing the data in transit.",
    )

    @root_validator(skip_on_failure=True)
    @classmethod
    def user_special_case(cls, values: Dict) -> Dict:
        """
        If either the `fides_key` or the `type` are set to "user",
        then the other must also be set to "user".
        """

        if values["fides_key"] == "user" or values["type"] == "user":
            assert (
                values["fides_key"] == "user" and values["type"] == "user"
            ), "The 'user' fides_key is required for, and requires, the type 'user'"

        return values

    @validator("type")
    @classmethod
    def verify_type_is_flowable(cls, value: str) -> str:
        """
        Assert that the value of the `type` field is a member
        of the `FlowableResources` Enum.
        """

        flowables = [member.value for member in FlowableResources]
        assert value in flowables, f"'type' must be one of {', '.join(flowables)}"
        return value

class System(FidesModel):
    """
    The System resource model.

    Describes an application and includes a list of PrivacyDeclaration resources.
    """

    registry_id: Optional[int] = Field(
        description="The id of the system registry, if used.",
    )
    meta: Optional[Dict] = meta_field
    fidesctl_meta: Optional[SystemMetadata] = Field(
        description=SystemMetadata.__doc__,
    )
    system_type: str = Field(
        description="A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc.",
    )
    data_responsibility_title: Optional[DataResponsibilityTitle] = Field(
        description="Deprecated. The responsibility or role over the system that processes personal data",
    )
    egress: Optional[List[DataFlow]] = Field(
        description="The resources to which the system sends data."
    )
    ingress: Optional[List[DataFlow]] = Field(
        description="The resources from which the system receives data."
    )
    privacy_declarations: List[PrivacyDeclaration] = Field(
        description=PrivacyDeclaration.__doc__,
    )
    joint_controller: Optional[ContactDetails] = Field(
        description="Deprecated. " + ContactDetails.__doc__,
    )
    third_country_transfers: Optional[List[str]] = Field(
        description="Deprecated. An optional array to identify any third countries where data is transited to. For consistency purposes, these fields are required to follow the Alpha-3 code set in ISO 3166-1.",
    )
    administrating_department: Optional[str] = Field(
        default="Not defined",
        description="An optional value to identify the owning department or group of the system within your organization",
    )
    data_protection_impact_assessment: Optional[DataProtectionImpactAssessment] = Field(
        description="Deprecated. " + DataProtectionImpactAssessment.__doc__,
    )
    vendor_id: Optional[str] = Field(
        description="The unique identifier for the vendor that's associated with this system."
    )
    dataset_references: List[FidesKey] = Field(
        default_factory=list,
        description="Referenced Dataset fides keys used by the system.",
    )
    processes_personal_data: bool = Field(
        default=True,
        description="This toggle indicates whether the system stores or processes personal data.",
    )
    exempt_from_privacy_regulations: bool = Field(
        default=False,
        description="This toggle indicates whether the system is exempt from privacy regulation if they do process personal data.",
    )
    reason_for_exemption: Optional[str] = Field(
        description="The reason that the system is exempt from privacy regulation."
    )
    uses_profiling: bool = Field(
        default=False,
        description="Whether the vendor uses data to profile a consumer in a way that has a legal effect.",
    )
    legal_basis_for_profiling: List[LegalBasisForProfilingEnum] = Field(
        default_factory=list,
        description="The legal basis (or bases) for performing profiling that has a legal effect.",
    )
    does_international_transfers: bool = Field(
        default=False,
        description="Whether this system transfers data to other countries or international organizations.",
    )
    legal_basis_for_transfers: List[str] = Field(
        default_factory=list,
        description="The legal basis (or bases) under which the data is transferred.",
    )
    requires_data_protection_assessments: bool = Field(
        default=False,
        description="Whether this system requires data protection impact assessments.",
    )
    dpa_location: Optional[str] = Field(
        description="Location where the DPAs or DIPAs can be found."
    )
    dpa_progress: Optional[str] = Field(
        description="The optional status of a Data Protection Impact Assessment"
    )
    privacy_policy: Optional[AnyUrl] = Field(
        description="A URL that points to the system's publicly accessible privacy policy."
    )
    legal_name: Optional[str] = Field(
        description="The legal name for the business represented by the system."
    )
    legal_address: Optional[str] = Field(
        description="The legal address for the business represented by the system."
    )
    responsibility: List[DataResponsibilityTitle] = Field(
        default_factory=list,
        description=DataResponsibilityTitle.__doc__,
    )
    dpo: Optional[str] = Field(
        description="The official privacy contact address or DPO."
    )
    joint_controller_info: Optional[str] = Field(
        description="The party or parties that share the responsibility for processing personal data."
    )  # Use joint_controller_info in favor of joint_controller
    data_security_practices: Optional[str] = Field(
        description="The data security practices employed by this system."
    )
    cookie_max_age_seconds: Optional[int] = Field(
        description="The maximum storage duration, in seconds, for cookies used by this system."
    )
    uses_cookies: bool = Field(
        default=False, description="Whether this system uses cookie storage."
    )
    cookie_refresh: bool = Field(
        default=False,
        description="Whether the system's cookies are refreshed after being initially set.",
    )
    uses_non_cookie_access: bool = Field(
        default=False,
        description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.",
    )
    legitimate_interest_disclosure_url: Optional[AnyUrl] = Field(
        description="A URL that points to the system's publicly accessible legitimate interest disclosure."
    )

    _sort_privacy_declarations: classmethod = validator(
        "privacy_declarations", allow_reuse=True
    )(sort_list_objects_by_name)

    _check_valid_country_code: classmethod = country_code_validator

    @root_validator
    @classmethod
    def deprecate_fields(cls, values: Dict) -> Dict:
        """
        Warn of System fields pending deprecation.
        """
        deprecated_fields = [
            "joint_controller",
            "third_country_transfers",
            "data_responsibility_title",
            "data_protection_impact_assessment",
        ]
        for field in deprecated_fields:
            if values.get(field) is not None:
                warn(
                    f"The {field} field is deprecated, and will be removed in a future version of fideslang.",
                    DeprecationWarning,
                )
        return values

    @validator("privacy_declarations", each_item=True)
    @classmethod
    def privacy_declarations_reference_data_flows(
        cls,
        value: PrivacyDeclaration,
        values: Dict,
    ) -> PrivacyDeclaration:
        """
        Any `PrivacyDeclaration`s which include `egress` and/or `ingress` fields must
        only reference the `fides_key`s of defined `DataFlow`s in said field(s).
        """

        for direction in ["egress", "ingress"]:
            fides_keys = getattr(value, direction, None)
            if fides_keys is not None:
                data_flows = values[direction]
                system = values["fides_key"]
                assert (
                    data_flows is not None and len(data_flows) > 0
                ), f"PrivacyDeclaration '{value.name}' defines {direction} with one or more resources and is applied to the System '{system}', which does not itself define any {direction}."

                for fides_key in fides_keys:
                    assert fides_key in [
                        data_flow.fides_key for data_flow in data_flows
                    ], f"PrivacyDeclaration '{value.name}' defines {direction} with '{fides_key}' and is applied to the System '{system}', which does not itself define {direction} with that resource."

        return value

    class Config:
        """Class for the System config"""

        use_enum_values = True

# Taxonomy
class Taxonomy(BaseModel):
    """
    Represents an entire taxonomy of Fides Resources.

    The choice to not use pluralized forms of each resource name
    was deliberate, as this would have caused huge amounts of complexity
    elsewhere across the codebase.
    """

    data_category: List[DataCategory] = Field(default_factory=list)
    data_subject: Optional[List[DataSubject]] = Field(default_factory=list)
    data_use: Optional[List[DataUse]] = Field(default_factory=list)
    data_qualifier: Optional[List[DataQualifier]] = Field(default_factory=list)

    dataset: Optional[List[Dataset]] = Field(default_factory=list)
    system: Optional[List[System]] = Field(default_factory=list)
    policy: Optional[List[Policy]] = Field(default_factory=list)

    registry: Optional[List[Registry]] = Field(default_factory=list)
    organization: List[Organization] = Field(default_factory=list)


import torch
import torchvision as tv
import torchvision.transforms as transforms
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torch.utils.data import DataLoader
from torch.autograd import Variable

import load_data
import wide_resnet as wrn

def random_choose(X_train, y_train, batch_size = 256):
    print(len(X_train))
    if (batch_size >= len(X_train)):
        return X_train, y_train
    indices = torch.from_numpy(np.random.choice(len(X_train), batch_size))

    return X_train.index_select(0, indices), y_train.index_select(0, indices)

def test_wrn(model):
    total_loss = 0
    correct = 0
    size = len(X_test)
    total = 0 # num of samples
    transformer = transforms.Compose([
        transforms.ToTensor()
    ])

    testset = tv.datasets.CIFAR10(root='./data', train=False, download=True, transform=transformer)
    testloader = DataLoader(testset, batch_size=100, shuffle=False, num_workers=2)

    calculate_loss = nn.CrossEntropyLoss()

    test_string = "# of batches: {}, loss: {}\n"
    print("Test 100 samples each time")

    with torch.no_grad():
        for step, (X, y) in enumerate(testloader):
            y_bar = model(X)
            loss = calculate_loss(y_bar, y)

            total_loss += loss.item()
            _, predicted = torch.max(y_bar.data, 1)

            correct += predicted.eq(y.data).cpu().sum()
            total += len(X)

            # save output
            with open("output.txt", "a") as output_file:
                output_file.write(test_string.format(step, loss))

            print(test_string.format(step, loss))

    with open("output.txt", "a") as output_file:
        output_file.write("Total loss: {}, correct: {}, total: {}, succ ratio: {}".format(total_loss, correct, total, 1.0*correct/total))

    print("Total loss: {}, correct: {}, total: {}, succ ratio: {}".format(total_loss, correct, total, 1.0*correct/total))

# training with all examples as torch.Tensor that are not modified
def naive_training(X_train, y_train, epoch, batch_size = 256, learning_rate = 0.1):
    # depth, widening factor, dropout rate from WRN paper, number of classes
    model = wrn.Wide_ResNet(16, 10, 0.3, 10)
    loss = 0
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)

    calculate_loss = nn.CrossEntropyLoss()

    output_string = "Numpy Array - Step {}, loss: {}\n"
    for i in range(epoch):
        optimizer.zero_grad()

        # SGD
        X_batch, y_batch = random_choose(X_train, y_train, batch_size = batch_size)
        y_bar = model(X_batch)

        loss = calculate_loss(y_bar, y_batch)
        loss.backward()

        optimizer.step()

        # save output
        with open("output.txt", "a") as output_file:
            output_file.write(output_string.format(i, loss))

        # print to thescreen also
        print(output_string.format(i, loss))

    return model

# training with Dataloader
def nt():
    model = wrn.Wide_ResNet(16, 10, 0.3, 10)

    optimizer = optim.SGD(model.parameters(), lr=0.1)
    calculate_loss = nn.CrossEntropyLoss()

    loss = 0
    transformer = transforms.Compose([
        transforms.ToTensor()
    ])
    trainset = tv.datasets.CIFAR10(root='./data', train=True, download=True, transform=transformer)
    trainloader = DataLoader(trainset, batch_size=256, shuffle=False, num_workers=2)

    output_string = "Dataloader - Step {}, loss: {}\n"
    for step, (X, y) in enumerate(trainloader):
        optimizer.zero_grad()

        y_bar = model(X)

        loss = calculate_loss(y_bar, y)
        loss.backward()

        optimizer.step()

        # save output
        with open("output.txt", "a") as output_file:
            output_file.write(output_string.format(step, loss))

    return model

if __name__ == '__main__':

    X_train, y_train, X_test, y_test, label_names = load_data.load()

    X_train = torch.Tensor(X_train)
    y_train = torch.from_numpy(y_train)

    #model = naive_training(X_train, y_train, 200)
    model = nt()
    test_wrn(model)

    '''
    x, y = random_choose(X_train, y_train, 2)
    '''


import pygame as pg

def Speak(name):

    volume=0.8
    # set up the mixer
    freq = 29999     # audio CD quality
    bitsize = -16    # unsigned 16 bit
    channels = 2     # 1 is mono, 2 is stereo
    buffer = 2048    # number of samples (experiment to get best sound)
    pg.mixer.init(freq, bitsize, channels, buffer)
    # volume value 0.0 to 1.0
    pg.mixer.music.set_volume(volume)
    clock = pg.time.Clock()
    try:
        pg.mixer.music.load(name)
    except pg.error:
        return
    pg.mixer.music.play()
    while pg.mixer.music.get_busy():
        clock.tick(30)


from classes import Neat, Gene, Network
from math import fabs
from time import sleep

neat = Neat(2, 1)

for i in range(0, 200):

	if i % 5 == 0: sleep(0.5)

	for organism in neat.getPopulation():
		network = organism.createNetwork()

		'''
		result = network.evaluate([1])
		error = fabs(1.0 - result[0])

		if error < 0.00001:
			#raise Exception("GENERATION %s" % i)
			organism.fitness = 100000
			continue

		organism.fitness = 1.0 / error

		'''
		error = 0.0

		result = network.evaluate([0, 0])
		error += fabs(0.0 - result[0])

		result = network.evaluate([0, 1])
		error += fabs(1.0 - result[0])

		result = network.evaluate([1, 0])
		error += fabs(1.0 - result[0])

		result = network.evaluate([1, 1])
		error += fabs(0.0 - result[0])

		if error == 0:
			organism.fitness = 100000
			continue

		organism.setFitness((4 - error)**2)

		#print len(organism.node_genes) + len(organism.connection_genes)
	neat.epoch()


## Tight-binding module

'''Tight-binding module: Setup of tight-binding models'''

from math import *
import numpy as np

import sys
sys.path.append('../lattice')
import lattice as ltc

'''Matrix setup'''

def stid(r,fl,Nall):
    '''
    Matrix indices for the fermion with fl at site r
    r: Lattice site
    flavor: Flavor index
    Nall=[Nltc,Nfl]: Lattice dimension, flavor number
    '''
    return Nall[1]*ltc.rid(r,Nall[0])+fl

def stnum(Nall):
    '''
    State number
    '''
    return Nall[0][0][0]*Nall[0][0][1]*Nall[0][0][2]*Nall[0][1]*Nall[1]

def termmat(Mt,mt,r0,fl0,r1,fl1,Nall):
    '''
    Assign matrix elements: Assign the coupling mt between states (r0,fl0) and (r1,fl1) to the matrix Mt under Hermitian condition
    r=[nr,sl]: Lattice site at Bravais lattice site nr and sublattice sl
    fl: Flavor index
    Nall=[Nbl,Nsl,Nfl]: Bravais lattice dimension, sublattice number, flavor number
    '''
    Mt[stid(r0,fl0,Nall),stid(r1,fl1,Nall)]+=mt
    Mt[stid(r1,fl1,Nall),stid(r0,fl0,Nall)]+=np.conj(mt)

'''Set Hamiltonian'''

def tbham(htb,rs,Nall,bc,ltype):
    '''
    Tight-binding Hamiltonian: Assign the couplings htb=[v0,-t1,-t2] to the Hamiltonian H.
    v0: Onsite potential
    t1 and t2: Nearest and second neighbor hoppings
    The factor 1/2 is to cancel double counting from the Hermitian assignment in termmat
    '''
    H=np.zeros((stnum(Nall),stnum(Nall)),dtype=complex)
    for r in rs:
        # Pairs at Bravais lattice site bls
        pairst=ltc.pairs(r,Nall[0][0],bc,ltype)
        # Add matrix elements for the pairs
        [termmat(H,(1./2.)*htb[nd],pairt[0],fl,pairt[1],fl,Nall) for nd in range(len(pairst)) for pairt in pairst[nd] for fl in range(Nall[1])]
    return H

def paulimat(n):
    '''
    Pauli matrices
    '''
    if(n==0):
        return np.array([[1.,0.],[0.,1.]])
    elif(n==1):
        return np.array([[0.,1.],[1.,0.]])
    elif(n==2):
        return np.array([[0.,-1.j],[1.j,0.]])
    elif(n==3):
        return np.array([[1.,0.],[0.,-1.]])

def hamsite(H,vs,rs,Nall):
    '''
    Onsite potentials: Assign the couplings vs=[v[r0],v[r1],....] to the Hamiltonian H.
    v[r]=[v0[r],v1[r],v2[r],v3[r]]: Set the potential v0[r]*sigma_0+v1[r]*sigma_1+v2[r]*sigma_2+v3[r]*sigma_3 to the site r
    The factor 1/2 is to cancel double counting from the Hermitian assignment in termmat
    '''
    for r in rs:
        # Onsite potential
        vr=vs[ltc.rid(r,Nall[0])]
        vrm=(1./2.)*(vr[0]*paulimat(0)+vr[1]*paulimat(1)+vr[2]*paulimat(2)+vr[3]*paulimat(3))
        # Add matrix elements for the pairs
        [termmat(H,(1./2.)*vrm[fl0,fl1],r,fl0,r,fl1,Nall) for fl0 in range(Nall[1]) for fl1 in range(Nall[1])]

'''Functions of density matrix'''

def projdenmat(U,n0,n1,Nst):
    '''
    Generate the density matrix by projecting on the n0-th to n1-th states of the unitary operator U=[u0,u1,u2,....].
    '''
    UT=U.conj().T
    # Project to only the Noc occupied states
    D=np.diag(np.array(n0*[0.]+(n1-n0)*[1.]+(Nst-n1)*[0.]))
    return np.linalg.multi_dot([U,D,UT])

def pairdenmat(P,r0,r1,Nall):
    '''
    Generate the 2x2 density matrix of a pair of lattice sites.
    '''
    return np.array([[P[stid(r0,fl0,Nall),stid(r1,fl1,Nall)] for fl1 in range(Nall[1])] for fl0 in range(Nall[1])])

def paircharge(Pt,r0,r1,Nall):
    '''
    Compute the charge of a pair of lattice sites. The onsite charge is real, while the offsite charge can be complex.
    '''
    return np.trace(pairdenmat(Pt,r0,r1,Nall))

def pairspin(Pt,r0,r1,Nall):
    '''
    Compute the spin of a pair of lattice sites. The onsite spin is real, while the offsite spin can be complex.
    '''
    return np.array([np.trace(np.dot(pairdenmat(Pt,r0,r1,Nall),(1./2.)*paulimat(n))) for n in [1,2,3]])

def chargeorder(Pt,rs,Nall,ltype):
    '''
    Compute the charge order of the whole lattice. Return the lists of the site and bond orders and their maximal values.
    '''
    # Site order
    schs=[paircharge(Pt,r,r,Nall).real for r in rs]
    # Extract the order as the deviation from the average
    schsavg=sum(schs)/len(schs)
    schs=[schs[nr]-schsavg for nr in range(len(schs))]
    schsa=[abs(schs[nr]) for nr in range(len(schs))]
    schsmax=max(schsa)
    # Bond order
    bc=1
    bchs=[paircharge(Pt,pairt[0],pairt[1],Nall) for r in rs for pairt in ltc.pairs(r,Nall[0][0],bc,ltype)[1]]
    # Extract the order as the deviation from the average
    bchsavg=sum(bchs)/len(bchs)
    bchs=[bchs[nr]-bchsavg for nr in range(len(bchs))]
    # Distinguish the real and imaginary bonds
    bchsr=[bchs[nb].real for nb in range(len(bchs))]
    bchsra=[abs(bchsr[nb]) for nb in range(len(bchsr))]
    bchsi=[bchs[nb].imag for nb in range(len(bchs))]
    bchsia=[abs(bchsi[nb]) for nb in range(len(bchsi))]
    bchsrmax,bchsimax=max(bchsra),max(bchsia)
    return [[schs,bchsr,bchsi],[schsmax,bchsrmax,bchsimax]]

def spinorder(Pt,rs,Nall,ltype):
    '''
    Compute the spin order of the whole lattice. Return the lists of the site and bond orders and their maximal values.
    '''
    # Site order
    ssps=[pairspin(Pt,r,r,Nall).real for r in rs]
    sspsn=[np.linalg.norm(ssps[nr]) for nr in range(len(ssps))]
    sspsmax=max(sspsn)
    # Bond order
    bc=1
    bsps=[pairspin(Pt,pairt[0],pairt[1],Nall) for r in rs for pairt in ltc.pairs(r,Nall[0][0],bc,ltype)[1]]
    # Distinguish the real and imaginary bonds
    bspsr=[bsps[nb].real for nb in range(len(bsps))]
    bspsi=[bsps[nb].imag for nb in range(len(bsps))]
    bspsrn=[np.linalg.norm(bspsr[nb]) for nb in range(len(bspsr))]
    bspsin=[np.linalg.norm(bspsi[nb]) for nb in range(len(bspsi))]
    bspsrmax,bspsimax=max(bspsrn),max(bspsin)
    return [[ssps,bspsr,bspsi],[sspsmax,bspsrmax,bspsimax]]

def setpairspin(Pt,Pr,r0,r1,Nall):
    '''
    Set the spin of a pair of lattice sites.
    '''
    [termmat(Pt,(1./2.)*Pr[fl0,fl1],r0,fl0,r1,fl1,Nall) for fl0 in range(Nall[1]) for fl1 in range(Nall[1])]


import tkinter
import time

# handler alram event
def alarm():
    print("Calling the pizza company.")

# handler for rining doorbell
def doorbell():
    print("Ding Dong!")
    time.sleep(5) # I was watching TV, so a bit slow to response to doorbell
    print("Opening the door.")

# handler for phone rings
def phonecall():
    print("Answering the phone.")

# create buttons and assign callbacks
root = tkinter.Tk()
tkinter.Button(root, text="Ring Doorbell", command = doorbell).pack()
tkinter.Button(root, text="Call Phone", command = phonecall).pack()

# set timer for 4 second
root.after(4000, alarm)

# start the event loop
root.mainloop()

11.Learning Python - JM/Event Driven Programming/alarms_doorbells.py


from cola import Cola

def verificar_elemento(cola, elemento):
    #Dada una cola y un elemento, verifica si el elemento esta en la cola.
    lista_datos = []
    resultado = False
    while not cola.esta_vacia():
        lista_datos.append(cola.desencolar())

    for dato in lista_datos:
        if dato == elemento:
            resultado = True
        cola.encolar(dato)
    return resultado


import dash
from dash import html, dcc, Input, Output, State, callback
import dash_bootstrap_components as dbc

import sys
# adding charts to the system path

from charts.colors import graphics_colors as gc

def make_dropdown(id, value, options = [], background = gc['dropdown_background']):
    dropdown = dcc.Dropdown(
    options,
    id = id,
    value = value,
    style = {
        "cursor": "pointer",
        "font-size":"12px"},
    className = "p-2",
    clearable=False)
    return dropdown

def make_card (header:str, id:str, className = "card text-white bg-primary mb-3"):
    card = dbc.Card(
        [
            dbc.CardHeader(header),
            dbc.CardBody(
                [
                    dcc.Markdown(id = id)

                ]
            )
        ],
        className = className
    )

    return card


import sqlite3

#define connection and cursor

connection = sqlite3.connect('store_movies.db')

cursor = connection.cursor()

#create movies table

command = "CREATE TABLE IF NOT EXISTS movies(movie_name TEXT, lead_actor TEXT, actress TEXT, YearOfRelease INTEGER, Director TEXT)"

cursor.execute(command)