# Lab 2, AE7785
# Richard Agbeyibor & Bhushan Pawaskar
# rotate robot to follow object

import rclpy
from rclpy.node import Node
from geometry_msgs.msg import Point
from geometry_msgs.msg import Twist

class RotateRobot(Node):

    def __init__(self):
        # Calls the superclass constructor from Node
        super().__init__('rotate_robot')
        # subscribes to /object/position topic
        self._position_subscriber = self.create_subscription(Point,
                                                         '/object/position',
                                                         self._position_callback,
                                                         1)
        self._position_subscriber
        # publishes to cmd/vel topic
        self._vel_publisher = self.create_publisher(Twist,
                                                      '/cmd_vel',
                                                      5)
        self._vel_publisher # supposedly prevents unused variable warning

        # Declares an object_position variable
        self.object_position = Point()
        # Declares a velocity variable
        self.velocity = Twist()

    def _position_callback(self, Point):
        self.object_position = Point
        self.get_logger().info('Received an object at: x= %3.0f,y= %3.0f' %(self.object_position.x, self.object_position.y))
        self._cmd_vel()

    def _cmd_vel(self):
        if (self.object_position.x < 120.0):
            self.velocity.angular.z = 1.0
        elif (self.object_position.x > 200.0):
            self.velocity.angular.z = -1.0
        else:
            self.velocity.angular.z = 0
        self._vel_publisher.publish(self.velocity)
        self.get_logger().info('Published a velocity')

def main():
    # init routine needed for ROS2
    rclpy.init()
    # Create a class object to be used
    robot_controller = RotateRobot()

    # Trigger callback processing
    rclpy.spin(robot_controller)

    # command robot to stop before exiting
    robot_controller.velocity.angular.z = 0
    robot_controller._vel_publisher.publish(robot_controller.velocity)

    #clean up and shutdown
    robot_controller.destroy_node()
    rclpy.shutdown()

if __name__ == '__main__':
    main()


def is_prime_number(x):
    for i in range(2,x):
        if x % i == 0:
            return False
    return True

def solution(num):
    res = 0
    for i in range(1000000):
        if num == 0:
            break
        else:
           if is_prime_number(i) == True:
                res = i
                num-=1
    return res

N = int(input())
print(solution(N))


def read(cls, proto):
    """
    :param proto: capnp TwoGramModelProto message reader
    """
    instance = object.__new__(cls)
    super(TwoGramModel, instance).__init__(proto=proto.modelBase)

    instance._logger = opf_utils.initLogger(instance)

    instance._reset = proto.reset
    instance._hashToValueDict = {x.hash: x.value
                                 for x in proto.hashToValueDict}
    instance._learningEnabled = proto.learningEnabled
    instance._encoder = encoders.MultiEncoder.read(proto.encoder)
    instance._fieldNames = instance._encoder.getScalarNames()
    instance._prevValues = list(proto.prevValues)
    instance._twoGramDicts = [dict() for _ in xrange(len(proto.twoGramDicts))]
    for idx, field in enumerate(proto.twoGramDicts):
      for entry in field:
        prev = None if entry.value == -1 else entry.value
        instance._twoGramDicts[idx][prev] = collections.defaultdict(int)
        for bucket in entry.buckets:
          instance._twoGramDicts[idx][prev][bucket.index] = bucket.count

    return instance


#! usr/bin/python3.5
import numpy as np
import PCApackage as pac
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from sklearn.decomposition import PCA
import os

os.system('wget ftp://ftp.ncbi.nlm.nih.gov/geo/datasets/GDS6nnn/GDS6248/soft/GDS6248.soft.gz')
os.system('gunzip GDS6248.soft.gz')
os.system('tail -n +141 GDS6248.soft > GDS6248.softer') #getting rid of the redundant lines
os.system('rm GDS6248.soft')
os.system('head -n -1 GDS6248.softer > GDS6248.soft') #one last redundant line
os.system('rm GDS6248.softer')

#In the following loop I'm keeping the float values while skipping the strings by setting the ValueError exception
temp = []
with open('GDS6248.soft') as f:
	for l in f:
		temp2=[]
		for x in l.split()[2:]:
			try:
				temp2.append(float(x))
			except ValueError:
				pass
		temp.append(temp2)

X=np.array(temp)
Color = ['m' for x in range(3)] + ['c' for x in range(24)] + ['r' for x in range(24)] #Color scheme for the samples

plt.figure()
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.5, hspace=0.5)

green = mpatches.Patch(color='m',label='baseline')
cyan = mpatches.Patch(color='c',label='normal diet')
red = mpatches.Patch(color='r',label='high fat diet')

Y=pac.PCA(X,k=2, F=False)[0]
plt.subplot(2,2,1)
plt.title('After Linear PCA')
plt.scatter(Y[0], Y[1], c=Color)
plt.legend(bbox_to_anchor=(1, -.1), loc=2, borderaxespad=0.,handles=[green, cyan, red], prop={'size':10})
Y=pac.PPCA(X, 2)
plt.subplot(2,2,2)
plt.title('After PPCA')
plt.scatter(Y[0], Y[1], c=Color)

Y=pac.PPCA(X, M=2, variance=False)
plt.subplot(2,2,3)
plt.title('After PPCA, zero sigma')
plt.scatter(Y[0], Y[1], c=Color)

my_pca=PCA(n_components=2)
Y = my_pca.fit_transform(X.T).T
plt.subplot(2,2,4)
plt.title('After Sklearn PCA')
plt.scatter(Y[0], Y[1], c=Color)
plt.show()


#!/usr/bin/python3

import argparse
import csv
import json
import logging
import os
import sqlite3 as sl
import sys
import time
import urllib.parse
from datetime import datetime
from pathlib import Path

import requests
from __init__ import __prog__, __version__
from dotenv import load_dotenv
from rich.console import Console
from rich.progress import track

# Get the current timestamp
now = datetime.now()
dt_formatted = now.strftime("%d/%m/%Y %H:%M:%S")

# Create filename to save messages if provider is down
# Format YYYY-MM-DD
dt_formatted_filename = now.strftime("%Y-%m-%d")

# Get Process ID
pid = os.getpid()

# Construct filename to save message state
filename = f"ratatoskr_{dt_formatted_filename}_{pid}.json"

# Create a custom logger
logger = logging.getLogger(__name__)

# Create handlers
c_handler = logging.StreamHandler()
f_handler = logging.FileHandler(f"ratatoskr_{dt_formatted_filename}_{pid}.log")
c_handler.setLevel(logging.INFO)
f_handler.setLevel(logging.INFO)

# Create formatters and add it to handlers
c_format = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
f_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format)

# Add handlers to the logger
logger.addHandler(c_handler)
logger.addHandler(f_handler)

# Define header values
USERAGENT = f"ratatoskr-{__version__}"

# Init rich console
console = Console()

# Load .env file
load_dotenv()

def verify_environment(environment_variable):
    """Verify if a provided environment variable has a value and return that value if true"""

    value = os.getenv(environment_variable)
    if not value:
        logger.error(f"No environment variable defined for {environment_variable}")
        return None
    else:
        logger.info(
            f"Identified environment variable is present {environment_variable}"
        )
        return value

def verify_gitlab_token(session):
    """Check API if access token is valid

    session - Requests session object with headers applied for GitLab token auth
    """

    query_url = "https://gitlab.com/api/v4/personal_access_tokens"
    response = session.get(query_url, timeout=5)
    response_json = response.json()

    # Check if active
    if response.ok:
        logger.info(f"Verified GitLab Token is active")
        return True

    # If we don't have 2XX status code
    if not response.ok:
        # Check if we have expired GitLab Token
        if response_json["message"] == "401 Unauthorized":
            console.print(
                f"[!] Error - Unauthorized, verify GitLab token!",
                style="bold red",
            )
            logger.error(f"Invalid token for GitLab!")
            return None

def verify_github_token(session):
    """Check API if access token is valid

    session - Requests session object with headers applied for GitHub token auth
    """
    query_url = "https://api.github.com/user"
    response = session.get(query_url, timeout=5)
    response_json = response.json()

    # Check if we have expired GitHub Token
    if response.status_code == 401:
        console.print(
            f"[!] Error - Unauthorized, verify GitHub token is accurate and not expired! {response_json['message']}",
            style="bold red",
        )
        logger.error(f"Invalid token for GitHub!")
        return None

    # Check if active
    if response.ok:
        logger.info(f"Verified GitHub Token is active")
        return True

def get_ratelimit_status(session):
    """Get the rate limit status from GitHub API"""

    query_url = "https://api.github.com/rate_limit"
    response = session.get(query_url, timeout=5)

    if response.status_code == 200:
        requests_remaining = int(response.headers["X-RateLimit-Remaining"])
        requests_reset_time = int(response.headers["X-RateLimit-Reset"])
        result = (requests_remaining, requests_reset_time)
        return result
    if response.status_code != 200:
        return None

def get_urls(filename):
    """Read in list of GitHub repositories to monitor releases"""

    # We use a list of tuples instead of a dictionary since the owners (or keys)
    # may be duplicates for different repos by same owner in URL list
    repositories = []

    with open(filename) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=",")
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                line_count += 1
                pass
            else:
                components = row[0].split("/")
                owner = components[3]
                repo = components[4]
                if "gitlab" in row[0]:
                    combo = (owner, repo, "gitlab")
                    repositories.append(combo)
                    line_count += 1
                elif "github" in row[0]:
                    combo = (owner, repo, "github")
                    repositories.append(combo)
                    line_count += 1
                else:
                    line_count += 1
        console.print(f"[*] INFO - Processed {line_count} lines.", style="bold green")
        return repositories

def get_gitlab_latest_release(session, repository):
    """Get latest release for given GitLab public project

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'gitlab')
    """

    # GitLab API will take URL-encoded namespace/project URI component
    project = urllib.parse.quote_plus(f"{repository[0]}/{repository[1]}")
    query_url = f"https://gitlab.com/api/v4/projects/{project}/releases"
    response = session.get(query_url, timeout=5)

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()

        if response_json == [] and response.status_code == 200:
            console.print(
                f"\n[!] INFO - No release found for project {repository[0]}/{repository[1]}",
                style="bold yellow",
            )
            return None

    if response.status_code == 404:
        console.print(
            f"[!] WARN - Project {repository[0]}/{repository[1]} was not found at {query_url} be sure to confirm the URL",
            style="bold red",
        )

    try:
        latest_release = response_json[0]["_links"].get("self")
    except KeyError:
        return None

    if latest_release:
        return latest_release

def get_gitlab_latest_commit(session, repository):
    """Get latest commit for given GitLab public project

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'gitlab')
    """

    # GitLab API will take URL-encoded namespace/project URI component
    project = urllib.parse.quote_plus(f"{repository[0]}/{repository[1]}")
    query_url = f"https://gitlab.com/api/v4/projects/{project}/repository/commits"
    response = session.get(query_url, timeout=5)

    # Check if JSON
    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()

        # Check if project does not exist
        if response.status_code == 404:
            console.print(
                f"[!] WARN - Project {repository[0]}/{repository[1]} was not found at {query_url} be sure to confirm the URL",
                style="bold red",
            )

        # Check if we have expired GitLab Token
        if response.status_code == 401:
            console.print(
                f"[!] Error - Your GitLab token is expired! {response_json}",
                style="bold red",
            )
            return None

        latest_commit = response_json[0]

        # Return GitLab latest commit URL
        if latest_commit.get("web_url"):
            return latest_commit["web_url"]
    else:
        return None

def get_latest_release(session, repository):
    """Get the latest GitHub release for given repository

    session - A requests Session object
    repository - A tuple like ('Owner', 'Repo', 'github')
    """

    # Sample input ('outflanknl', 'RedELK', 'github')

    query_url = (
        f"https://api.github.com/repos/{repository[0]}/{repository[1]}/releases/latest"
    )
    response = session.get(query_url, timeout=5)

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()
        if response_json.get("html_url"):
            return response_json["html_url"]
    else:
        return None

def get_latest_commit(session, repository):
    """Get the latest commit for a given list of repos in ('Owner', 'Repo') format"""

    query_url = f"https://api.github.com/repos/{repository[0]}/{repository[1]}/commits"
    response = session.get(query_url, timeout=5)

    if not response:
        return None

    if "json" in response.headers.get("Content-Type"):
        response_json = response.json()
        latest_commit = response_json[0]
        if latest_commit.get("html_url"):
            return latest_commit["html_url"]
    else:
        return None

def update_tracker(connection, update):
    """Update tracker DB with the latest release and commit

    connection - SQLite DB Connection Object
    update - List of commit, release, datetime, GitHub username, repo name, and whether GitLab or GitHub
    """

    # update_object = [commit, release, dt_formatted, repo[0], repo[1], repo[2]]
    # sample SQL entry = its-a-feature|Mythic|0|https://github.com/its-a-feature/Mythic/commit/75a46ef1c06e58ffaed2b036c3e4adf67b72bbc4|12/05/2021 12:37:32|github

    sql = "UPDATE repo SET latest_commit = ?, latest_release = ?, last_updated = ? WHERE  owner = ? AND repo = ? AND website = ?"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, update)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to update repo {update[4]} in tracker DB",
            style="bold red",
        )
        console.print(f"{e}")
        sys.exit(1)

def insert_repo(connection, newrepo):
    """Insert newly identified repository to track

    connection - SQLite DB Connection Object
    newrepo - List of user, repo URL, last updated timestamp, GitHub or GitLab
    """

    # newrepo = [repo[0], repo[1], dt_formatted, repo[2]]

    sql = "insert into repo (owner, repo, last_updated, website) values(?, ?, ?, ?)"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, newrepo)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to insert new repo into tracker DB", style="bold red"
        )
        console.print(f"{e}")
        sys.exit(1)

def confirm_table(connection):
    """Verify if repo table has been created

    connection - SQLite DB Connection Object
    """

    cursor = connection.cursor()
    with connection:
        cursor.execute("select * FROM sqlite_master WHERE type='table' and name='repo'")
        data = cursor.fetchall()
        if len(data) == 0:
            return None
        else:
            console.print(f"[+] INFO - Table already exists", style="bold green")
            return True

def delete_repo(connection, repo):
    """Delete repository from tracker db

    connection - SQLite DB Connection Object
    repo - List of owner and repo
    """

    sql = "DELETE FROM repo WHERE owner = ? AND repo = ?"

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(sql, repo)
    except sl.IntegrityError as e:
        console.print(
            f"[!] ERROR - Unable to delete repo from tracker DB", style="bold red"
        )
        console.print(f"{e}")
        sys.exit(1)

def confirm_repo(connection, repo):
    """Verify if the owner and repository name is already setup in the tracker database

    connection - SQLite DB Connection Object
    repo - List of owner and repo
    """

    cursor = connection.cursor()
    with connection:
        cursor.execute(
            "select * from repo WHERE owner = ? AND repo = ?",
            [repo[0], repo[1]],
        )
        data = cursor.fetchall()
        if len(data) == 0:
            return None
        else:
            return True

def bootstrap_db(connection):
    """Bootstrap sqlite3 db with REPO table

    connection - SQLite DB Connection Object
    """

    try:
        cursor = connection.cursor()
        with connection:
            cursor.execute(
                "create table repo (owner, repo, latest_release, latest_commit, last_updated, website)"
            )
    except sl.IntegrityError as e:
        console.print(f"[!] ERROR - Unable to create repo table", style="bold red")
        console.print(f"{e}")
        sys.exit(1)

def dump_table(connection):
    """Print the tracker database

    connection - SQLite DB Connection Object
    """

    cursor = connection.cursor()
    with connection:
        data = cursor.execute("select * from repo")
        for row in data:
            print(row)

def read_repositories(connection):
    """Return all repositories in the tracker database

    connection - SQLite DB Connection Object
    """

    repositories = []

    cursor = connection.cursor()
    with connection:
        data = cursor.execute("select * from repo")
        for row in data:
            repositories.append(row)
    return repositories

def save_messages(data, filename):
    """Write messages as JSON to disk in the event webhook is unsuccessful

    data - JSON object
    filename - Filname to write to disk
    """

    try:
        with open(filename, "rt") as fh:
            existing_data = json.load(fh)
    except FileNotFoundError as notfound:
        console.print(
            f"[!] WARN - Filename [blue]{filename}[/blue] does not exist [i]will create[/i]: {notfound}",
            style="bold yellow",
        )
        existing_data = []
    except IOError as e:
        console.print(
            f"[!] ERROR - Unable to read file [blue]{filename}[/blue]: {e}",
            style="bold yellow",
        )
        existing_data = []

    # Update the dict object with new data passed to function
    existing_data.append(data)

    # Write our merged JSON object to disk in the event we have to resend the messages
    with open(filename, "wt") as fh:
        json.dump(existing_data, fh)

    console.print(
        f"[!] WARN - Wrote messages to file [blue]{filename}[/blue]",
        style="bold yellow",
    )

def send_webhook(message, webhook_url, provider, filename):
    """Send web request to webhook URL

    message - Message text for chat notification
    webhook_url - Webhook URL to send request to
    provider - Webhook provider to format message
    filename - Filename to write to disk in event of failure
    """

    # https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook
    if provider == "msteams":
        data = {"Text": message}

    # https://api.slack.com/messaging/webhooks
    if provider == "slack":
        data = {"text": message}

    # https://discord.com/developers/docs/resources/webhook
    if provider == "discord":
        data = {"content": message}

    # https://docs.rocket.chat/guides/administration/admin-panel/integrations
    if provider == "rocketchat":
        data = {
            "emoji": ":chipmunk:",
            "attachments": [
                {"title": "ratatoskr notify", "text": message, "color": "#764FA5"}
            ],
        }

    # HTTP POST to our Webhook URL
    r = requests.post(webhook_url, json=data)

    # Verify successful status code via .ok method on response object
    # Most APIs return 200, Discord returns 204
    if not r.ok:
        console.print(
            f"\n[!] ERROR - Webhook was unsuccessful status code is {r.status_code}: {r.text}",
            style="bold red",
        )
        # Save messages to disk in event we don't succesfully POST
        save_messages(message, filename)

        # Return condition and response object
        return (False, r)

    if r.ok:
        return (True, r)

def parse_arguments():
    """Parse command-line arguments"""

    # Define parser
    parser = argparse.ArgumentParser()

    # Define main group
    arg_group_1 = parser.add_argument_group()
    arg_group_1.add_argument(
        "-p",
        "--provider",
        type=str,
        choices=["rocketchat", "discord", "msteams", "slack"],
        help="provider to use; required with --check",
    )
    arg_group_1.add_argument(
        "-c",
        "--check",
        action="store_true",
        help="check for new repository releases and commits",
    )

    # Create mutually exclusive group
    arg_group_2 = arg_group_1.add_mutually_exclusive_group()
    arg_group_2.add_argument(
        "-l",
        "--load",
        action="store_true",
        help="load the repositories to watch into the database",
    )
    arg_group_2.add_argument(
        "-v", "--version", action="version", version=f"{__prog__} {__version__}"
    )
    arg_group_2.add_argument(
        "-e",
        "--examples",
        action="store_true",
        help="display usage examples and exit",
    )

    # Parse our arguments into internal variables
    args = parser.parse_args()

    # Cleaner variable names
    load = args.load
    check = args.check
    provider = args.provider
    examples = args.examples

    # Display examples if user requested
    if examples:
        console.print(f"Check latest commits and releases and notify Microsoft Teams")
        console.print(
            f"    {__prog__} [cyan]--check[/cyan] [cyan]--provider[/cyan] msteams",
            style="bold blue",
        )
        console.print(f"Check latest commits and releases and notify Discord")
        console.print(
            f"    {__prog__} [cyan]--check[/cyan] [cyan]--provider[/cyan] discord",
            style="bold blue",
        )
        console.print(f"Load the latest reference CSV into tracker database")
        console.print(f"    {__prog__} [cyan]--load[/cyan]", style="bold blue")
        sys.exit(0)

    # Check and webhook provider are required to format the payload
    if check and provider is None:
        console.print(
            f"[!] ERROR - Chat provider was not provided by [green]--provider[/green] argument!",
            style="bold red",
        )
        sys.exit(1)

    return {"Load": load, "Check": check, "Provider": provider}

def prepare_database(filename):
    """Prepare the database

    filename - SQLite database filename to track repos
    """

    # Check if tracker.db file exists or not
    p = Path(filename)

    if p.exists():
        console.print(f"[+] INFO - [blue]{filename}[/blue] exists", style="bold green")
        # DB Connection
        con = sl.connect(filename, timeout=5)

        # Confirm the table exists, use the connection object
        confirm_result = confirm_table(con)

        # If the table already exists in the tracker.db file
        if confirm_result:
            console.print(
                f"[+] INFO - Tracker database is already prepared", style="bold green"
            )
            return (True, con)

        # If the table does not exist, create it!
        if not confirm_result:
            console.print(
                f"[+] INFO - Preparing database tables in [blue]{filename}[/blue] file..",
                style="bold green",
            )

            # We want to try and handle the sqlite OperationalError condition at least
            try:
                bootstrap_db(con)
                return (True, con)
            except sl.OperationalError as e:
                console.print(
                    f"[!] ERROR - Database has already been initialized!",
                    style="bold red",
                )
                console.print(f"{e}")
                return (False, con)

    # If the file does not exist, create it and prepare it
    if not p.exists():
        console.print(
            f"[!] WARN - [blue]{filename}[/blue] does not exist, creating..",
            style="bold yellow",
        )
        # DB Connection
        con = sl.connect(filename, timeout=5)

        # We want to try and handle the sqlite OperationalError condition at least
        try:
            bootstrap_db(con)
            return (True, con)
        except sl.OperationalError as e:
            console.print(
                f"[!] ERROR - Database has already been initialized!",
                style="bold red",
            )
            console.print(f"{e}")
            return (False, con)

def main():
    """Main function"""

    # Print a pretty header to console
    console.print(f" :chipmunk:  ~ Ratatoskr the Norse Squirrel God ~ :chipmunk:")

    # High-level function to parse arguments
    arguments = parse_arguments()

    # Verify tokens and webhook
    github_token = verify_environment("GITHUB_TOKEN")
    gitlab_token = verify_environment("GITLAB_TOKEN")

    # Exit if we don't have GitHub API token
    if not github_token:
        console.print(
            f"[!] ERROR - No GitHub Personal Access Token in environment variables",
            style="bold red",
        )
        sys.exit(1)

    # Exit if we don't have GitLab API Token
    if not gitlab_token:
        console.print(
            f"[!] ERROR - No GitLab Personal Access Token in environment variables",
            style="bold red",
        )
        sys.exit(1)

    # Parse provider and format for env check
    if arguments["Provider"]:
        prefix = arguments["Provider"].upper()

        # Verify that our provider webhook is in the environment
        webhook_url = verify_environment(f"{prefix}_WEBHOOK")

        # Exit if we don't have webhook URL
        if not webhook_url:
            console.print(
                f"[!] ERROR - No webhook URL found in environment variables",
                style="bold red",
            )
            sys.exit(1)

    # Define our headers
    github_custom_headers = {
        "Accept": "application/vnd.github.v3+json",
        "User-Agent": USERAGENT,
        "Authorization": "token {}".format(github_token),
    }
    gitlab_custom_headers = {
        "Accept": "application/vnd.github.v3+json",
        "User-Agent": USERAGENT,
        "PRIVATE-TOKEN": "{}".format(gitlab_token),
    }

    # Create reusable HTTP session object
    s_github = requests.Session()
    s_gitlab = requests.Session()

    # Update Session headers
    s_github.headers.update(github_custom_headers)
    s_gitlab.headers.update(gitlab_custom_headers)

    # Prepare the tracker.db file before loading data
    db_prep_result = prepare_database("tracker.db")

    # Ensure we got True from previous function call
    if not db_prep_result[0]:
        console.print(f"[!] ERROR - Preparing database!", style="bold red")
        sys.exit(1)

    # Use a friendly name for our connection object
    db_connection_handler = db_prep_result[1]

    # Check rate limits
    github_ratelimit_response = get_ratelimit_status(s_github)

    # Check GitHub token validity
    result = verify_github_token(s_github)

    # Check GitLab token validity
    result = verify_gitlab_token(s_gitlab)

    if github_ratelimit_response is None:
        console.print(f"[!] ERROR Unable to confirm rate limits", style="bold red")
        sys.exit(1)

    # If user provided --load argument, read CSV and load into tracker
    if arguments["Load"]:
        # Extract all the URLs from the first column in the CSV
        repositories = get_urls("GitHub_Tools_List.csv")

        console.print(
            f"[+] Loading repositories to monitor into tracker..", style="bold green"
        )

        # We enumerate over all repository URLs
        for repo in track(
            sequence=repositories,
            description="Loading...",
            update_period=1.0,
            auto_refresh=False,
        ):
            # Check if already tracking in database
            confirmation = confirm_repo(db_connection_handler, repo)

            # Verify that repository has been inserted into tracker
            if confirmation:
                pass
            # If it has not been inserted, load it in
            elif not confirmation:
                # Prepare our SQL insert
                newrepo = [repo[0], repo[1], dt_formatted, repo[2]]
                # Perform insert by passing our DB handler and new repository
                insert_repo(db_connection_handler, newrepo)
        sys.exit(0)

    # If user provided --check argument, read all repos from DB
    if arguments["Check"]:
        # Read tracker.db and populate all our repositories in memory
        repositories = read_repositories(db_connection_handler)

        for count, repo in enumerate(
            track(
                sequence=repositories,
                description="Checking repositories...",
                update_period=1.0,
            )
        ):
            if repo[5] == "github":
                # Get latest release URL
                release = get_latest_release(s_github, repo)
                # Get latest commit URL
                commit = get_latest_commit(s_github, repo)
            if repo[5] == "gitlab":
                # Get latest release URL
                release = get_gitlab_latest_release(s_gitlab, repo)
                # Get latest commit URL
                commit = get_gitlab_latest_commit(s_gitlab, repo)

            # Check if latest release matches DB
            if repo[2] != release and release is not None:
                console.print(
                    f"\n[+] New release for repository {repo[1]}: {release}",
                    style="bold green",
                )

                # Update the database
                update = [commit, release, dt_formatted, repo[0], repo[1], repo[5]]
                update_tracker(db_connection_handler, update)

                # Send notification to webhook
                message = f"New release for repository {repo[1]}: {release}"
                response = send_webhook(
                    message, webhook_url, arguments["Provider"], filename
                )

                # If response code is 429, backoff
                if response[1].status_code == 429:
                    delay_time = 60
                    console.print(
                        f"[!] WARN - Too many requests, backing off for [blue]{delay_time}[/blue] seconds"
                    )
                    time.sleep(delay_time)
                    response = send_webhook(
                        message, webhook_url, arguments["Provider"], filename
                    )

            if repo[3] != commit and commit is not None:
                console.print(
                    f"\n[+] New commit for repository {repo[1]}: {commit}",
                    style="bold green",
                )

                # Update the database
                update = [commit, release, dt_formatted, repo[0], repo[1], repo[5]]
                update_tracker(db_connection_handler, update)

                # Send notification to webhook
                message = f"New commit for repository {repo[1]}: {commit}"
                response = send_webhook(
                    message, webhook_url, arguments["Provider"], filename
                )

                # If response code is 429, backoff
                if response[1].status_code == 429:
                    delay_time = 60
                    console.print(
                        f"[!] WARN - Too many requests, backing off for [blue]{delay_time}[/blue] seconds"
                    )
                    time.sleep(delay_time)
                    response = send_webhook(
                        message, webhook_url, arguments["Provider"], filename
                    )

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy import Field

class BaseParent(scrapy.Item):
    base = Field() #上面的基本信息
    threat = Field() # 威胁情报检测
    IP = Field()  #IP反查，IP分析
    child_domains = Field() #子域名
    whois = Field() # Whois
    certificates = Field() # 数字证书
    uuid = Field()
    keyword = Field()

# 基本信息
class Base(scrapy.Item):
    ip = Field() # ip地址
    address = Field() # 地理位置
    asn = Field() # ASN
    tags = Field() # 标签
    domain_provider = Field() # 域名服务商
    domain_server = Field() # 域名服务器
    alex_rank = Field() # Alex 排名

# 威胁情报
class Threat(scrapy.Item):
    threat_test = Field()  # 威胁情报检测
    sample = Field() # 威胁情报样本

# 威胁情报检测
class Test(scrapy.Item):
    source = Field() # 情报源
    find_time = Field() # 发现时间
    type = Field() # 情报类型

# 相关样本
class Sample(scrapy.Item):
    SHA256 = Field()
    family = Field() # 病毒家族
    type = Field() # 病毒类型
    rate = Field() # 病毒检出率
    analysis_time = Field() # 分析时间
# IP反查，IP分析
class Ip(scrapy.Item):
    his_domains = Field() # 历史域名记录 History
    domains = Field() # 指向同一IP的域名列表
    his_ana = Field() #历史解析

# 历史解析记录
class Hisdomain(scrapy.Item):
    time = Field()
    domain = Field()

# 历史解析记录
class Hisana(scrapy.Item):
    time = Field()
    ips = Field()
    country = Field()
    province = Field()
#  Whois
class Whois(scrapy.Item):
    cu_registy = Field() # 当前注册信息
    his_registy = Field() # 历史注册信息

# 当前注册信息
class CRegisty(scrapy.Item):
    registrar = Field()
    institution = Field()
    email = Field()
    address = Field()
    phone = Field()
    re_time = Field()
    over_time = Field()
    update_time = Field()
    provider = Field()
    server = Field()

# 历史注册信息
class HRegisty(scrapy.Item):
    date = Field()
    message = Field()


from time import sleep
from PodSixNet.Server import Server
from PodSixNet.Channel import Channel

class ClientChannel(Channel):
    def Network(self, data):
        self.gameid = data["gameid"]
        print(data)

    def Network_placeBox(self, data):
        # deconsolidate all of the data from the dictionary
        playerTurn = data["playerTurn"]
        pijlx = data["pijlx"]
        playerNR = data["playerNR"]
        # id of game given by server at start of game
        self.gameid = data["gameid"]
        # tells server to place box
        self._server.placeBox(playerTurn, pijlx, data, self.gameid, playerNR)

    def Network_movePijl(self,data):
        pijlx = data["pijlx"]
        self.gameid = data["gameid"]
        self._server.movePijl(pijlx, self.gameid, data)

    def Close(self):
        self._server.close(self.gameid)

class vieropeenrijServer(Server):
    channelClass = ClientChannel

    def __init__(self, *args, **kwargs):
        Server.__init__(self, *args, **kwargs)
        self.games = []
        self.queue = None
        self.currentIndex = 0
        self.numPlayers = 0

    def Connected(self, channel, addr):
        self.numPlayers += 1
        print('new connection:', channel)
        print(self.queue)
        if self.queue == None:
            self.currentIndex += 1
            channel.gameid = self.currentIndex
            self.queue = Game(channel, self.currentIndex)
        else:
            channel.gameid = self.currentIndex
            self.queue.player[(self.numPlayers-1)%self.queue.playerAantal] = channel

        if self.numPlayers > 1 and self.numPlayers%self.queue.playerAantal == 0:
            for i in range(self.queue.playerAantal):
                self.queue.player[i].Send({"action": "startgame", "player": i, "gameid": self.queue.gameid,"playerAantal": self.queue.playerAantal})
            self.games.append(self.queue)
            for a in self.games:
                print(a)
            self.queue = None

    def movePijl(self,pijlx,gameid, data):
        game = [a for a in self.games if a.gameid == gameid]
        if len(game) == 1:
            game[0].movePijl(pijlx, data)

    def placeBox(self, playerTurn, pijlx, data, gameid, playerNR):
        game = [a for a in self.games if a.gameid == gameid]
        if len(game) == 1:
            game[0].placeBox(playerTurn, pijlx, data, playerNR)
    def close(self,gameid):
        try:
            game = [a for a in self.games if a.gameid == gameid][0]
            for i in range(game.playerAantal):
                game.player[i].Send({"action": "close", "gameid": gameid})
        except:
            pass

class Game:  # controleren
    def __init__(self, player0, currentIndex):
        # whose turn
        self.Turn = 1
        self.playerAantal = 2
        # dimensions tiles game board
        self.boardBoxH = 7
        self.boardBoxW = 14
        # define game board dimensions
        self.board = [[0 for x in range(self.boardBoxW)] for y in range(self.boardBoxH)]
        # initialize the players including the one who started the game
        self.player=[player0,None,None,None]
        # gameid of game
        self.gameid = currentIndex

    def movePijl(self, pijlx, data):
        for i in range(self.playerAantal):
            self.player[i].Send(data)

    def placeBox(self, playerTurn, pijlx, data, playerNR):
        # make sure it's their turn
        if playerNR == self.Turn:
            #  and self.board[0][pijlx]==0:
            # self.board[0][pijlx]=self.playerTurn
            if self.playerAantal > self.Turn:
                self.Turn += 1
            else:
                self.Turn = 1
            data["playerTurn"] = self.Turn
        # send data and turn data to each player
        for i in range(self.playerAantal):
            self.player[i].Send(data)

print("STARTING SERVER ON LOCALHOST")
server = vieropeenrijServer(localaddr=("LOCALHOST", 31425))

while 1:
    server.Pump()
    sleep(0.01)

 # def updateServer():
 #    print("Clock is ticking")
 #    vieropenrijServer.Pump()
 #    sleep(0.0001)


print("Как тебя зовут?")
a = input()
b = (".")
a = a + b
print("Привет,", a,"У тебя был хороший день? ")
c = input()
if (c == "Da"):
    print("Это не может не радовать!")
if (c == "Net"):
    print("Что случилось?")


from django.urls import path, include
from . import views
from .api import PostResource, UserResource

app_name = 'blog'

urlpatterns = [
    path('', views.index, name='index'),
    path('posts/<id>/', views.detail, name='detail'),
    path('create_post/', views.create_post, name='create_post'),
    path('posts/<id>/edit', views.edit_post, name='edit'),
    path('posts/<id>/delete', views.delete_post, name='delete'),
    path('posts/<id>/like', views.like_post, name='like'),
    path('posts/<id>/like_index', views.like_post_index, name='like_index'),
    path('api/posts/', include(PostResource.urls())),
    path('api/users/', include(UserResource.urls())),

]


from django.shortcuts import render,get_object_or_404,redirect
from django.utils import timezone
from .models import Plat,Good,Post,love
from .forms import goods_form,ReplyForm
from plat import models
from accounts.models import UserInfo
from django.contrib.auth.decorators import login_required
from utils.pagination import Pagination
from django.db.models import Q
from django.http import JsonResponse
from django.forms.models import model_to_dict

# Create your views here.
def all_good(request):
    count = models.Good.objects.filter().count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=9)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter().all()[req.start:req.end]

    p1 = models.Good.objects.filter(plat='书籍教材').count()
    p2 = models.Good.objects.filter(plat='电子设备').count()
    p3 = models.Good.objects.filter(plat='生活用品').count()
    p_counts = {
        'p1': p1,
        'p2': p2,
        'p3': p3,
    }

    c1 = models.UserInfo.objects.filter(Institute='信息工程学院').count()
    c2 = models.UserInfo.objects.filter(Institute='自动化学院').count()
    c3 = models.UserInfo.objects.filter(Institute='计算机学院').count()
    c4 = models.UserInfo.objects.filter(Institute='外国语学院').count()
    c5 = models.UserInfo.objects.filter(Institute='轻化学院').count()
    c6 = models.UserInfo.objects.filter(Institute='物理学院').count()
    counts = {
        'c1': c1,
        'c2': c2,
        'c3': c3,
        'c4': c4,
        'c5': c5,
        'c6': c6,
    }

    name = request.GET.get('name')
    if name:
        count = models.Good.objects.filter(good_name=name).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(good_name=name).all()[req.start:req.end]

    plat = request.GET.get('plat')
    if plat:
        count = models.Good.objects.filter(plat=plat).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(plat=plat).all()[req.start:req.end]

    username = request.GET.get('user')
    user = UserInfo.objects.filter(username=username).first()
    if user:
        count = models.Good.objects.filter(starter=user).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]

    online = request.GET.get('online')
    if online:
        count = models.Good.objects.filter(online=1).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(online=1).all()[req.start:req.end]

    argue = request.GET.get('argue')
    if argue:
        count = models.Good.objects.filter(lower=1).count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter(lower=1).all()[req.start:req.end]

    time_order = request.GET.get('time_order')
    if time_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('-last_updated').all()[req.start:req.end]

    count_order = request.GET.get('count_order')
    if count_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('views').all()[req.start:req.end]

    price_order = request.GET.get('price_order')
    if price_order:
        count = models.Good.objects.filter().count()
        p = request.GET.get('p')
        path_info = request.META.get('PATH_INFO')
        req = Pagination(p, count, per_page_count=9)
        page_str = req.page_str(path_info)
        goods = models.Good.objects.filter().order_by('price').all()[req.start:req.end]

    return render(request, 'allgoods.html', {'goods': goods,"page_str": page_str,'counts':counts,'p_counts':p_counts})

@login_required
def plat_goods(request):
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    count = models.Good.objects.filter(starter=user).count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=3)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]
    return render(request, 'goods.html',{'goods':goods,"page_str": page_str})

@login_required
def add_goods(request):
    form = goods_form(request.POST,request.FILES)
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    if request.method == "POST":
        form = goods_form(request.POST,request.FILES)
        if form.is_valid():
            data = {
            'starter': user,
        }
            data.update(form.cleaned_data)
            img = request.FILES.get('good_photo')
            v = Good.objects.create(**data)
            return redirect("/plat/my_good")
        else:
            print(form.errors)
    return render(request, 'add_bak.html', {'form':form})

@login_required
def edit_goods(request,id):
    obj = models.Good.objects.filter(id=id).first()
    form = goods_form(instance=obj)
    if request.method == "POST":
        form = goods_form(request.POST,request.FILES,instance=obj)
        if form.is_valid():
            img = request.FILES.get('good_photo')
            v = models.Good.objects.filter(id=id).update(**form.cleaned_data)
            return redirect("/plat/my_good")
        else:
            print(form.errors)
    return render(request, 'edit_goods.html', {'form': form,'id':id})

@login_required
def del_goods(request,id):
    goods = models.Good.objects.filter(id=id)
    goods.delete()
    return redirect("/plat/my_good")

@login_required
def good_detail(request,id):
    goods = models.Good.objects.filter(id=id)
    good = models.Good.objects.filter(id=id).first()
    post_list = models.Post.objects.filter(topic=good).all()
    username = good.starter
    user = UserInfo.objects.filter(username=username).first()
    user_goods = models.Good.objects.filter(starter=user)
    userdetail = UserInfo.objects.filter(username=username).first()
    love_user = request.user
    love_user = UserInfo.objects.filter(username=love_user).first()
    fav = models.love.objects.filter(user_id=love_user, good_id=good).first()
    good.views +=1
    good.save()

    if request.method =='POST':
        message = request.POST['message']
        good = models.Good.objects.filter(id=id).first()
        username = request.user
        userreal = UserInfo.objects.filter(username=username).first()
        post = Post()
        post.message = message
        post.updated_at = timezone.now()
        post.created_by = userreal
        post.topic = good
        post.save()
        return redirect("/plat/good_%s" % id)
    return render(request, 'good_details.html', {'goods':goods,'userdetail':userdetail,'post_list':post_list,'user_goods':user_goods,'thisgood':good,'fav':fav})

def home(request):
    plats = Plat.objects.all()
    good = models.Good.objects.filter()
    return render(request, 'index.html', {'plats': plats,'good':good})

@login_required
def loved(request,id):
    good = models.Good.objects.filter(id=id).first()
    loved= love()
    username = request.user
    user = UserInfo.objects.filter(username=username).first()
    loved.user_id = user
    loved.good_id = good
    loved.save()
    return redirect("/plat/good_%s" % id)

@login_required
def unloved(request,id):
    good = models.Good.objects.filter(id=id).first()
    username = request.user
    user = UserInfo.objects.filter(username=username).first()
    fav = models.love.objects.filter(user_id=user,good_id=good)
    fav.delete()
    return redirect("/plat/good_%s" % id)

@login_required
def love_goods(request):
    username = request.user
    userdetail = UserInfo.objects.filter(username=username).first()
    goods = models.Good.objects.filter(starter=userdetail)
    good = []
    fav = models.love.objects.filter(user_id=userdetail)
    fav_self = models.love.objects.filter(user_id=userdetail).count()
    good_self = models.Good.objects.filter(starter=userdetail).count()
    if fav:
       good = fav
    return render(request, 'love_good.html', {'userdetail':userdetail,'goods': goods,'good':good,'fav_self':fav_self,'good_self':good_self})

@login_required
def my_good(request):
    user = request.user
    user = UserInfo.objects.filter(username=user).first()
    username = request.user
    userdetail = UserInfo.objects.filter(username=username).first()
    fav_self = models.love.objects.filter(user_id=userdetail).count()
    count = models.Good.objects.filter(starter=user).count()
    p = request.GET.get('p')
    path_info = request.META.get('PATH_INFO')
    req = Pagination(p, count, per_page_count=3)
    page_str = req.page_str(path_info)
    goods = models.Good.objects.filter(starter=user).all()[req.start:req.end]
    good_self = models.Good.objects.filter(starter=userdetail).count()
    return render(request, 'my_good.html', {'userdetail':userdetail,'goods': goods,"page_str": page_str,'fav_self':fav_self,'good_self':good_self})

@login_required
def delete_post(request,id):
    post = models.Post.objects.filter(id=id).first()
    id = post.topic.id
    post.delete()
    return redirect("/plat/good_%s" % id)

def reply_post(request):
    # 二级回复
    comment_id = request.POST.get('comment_id')
    author_to_id = request.POST.get('author_to_id')
    if comment_id and author_to_id:
        try:
            comment = Post.objects.get(id=comment_id)
            author_to = UserInfo.objects.get(id=author_to_id)
            form = ReplyForm(data=request.POST)
            if form.is_valid():
                new_reply = form.save(commit=False)
                new_reply.comment = comment
                new_reply.author_from = request.user
                new_reply.author_to = author_to
                new_reply.save()
                return JsonResponse({'status': 'ok'})
        except Post.DoesNotExist or UserInfo.DoesNotExist:
            return JsonResponse({'status': 'ko'})
    return JsonResponse({'status': 'ko'})


from _config import MAX_LENGTH
from torch.utils.data import Dataset
from collections import namedtuple, Counter
import random
import json
import os

Pair = namedtuple('Pair', ('utterance', 'response', 'rating', 'conversation_id', 'opening_line'))
numeric_ratings = {'Poor':0, 'Not Good':1, 'Passable':2, 'Good':3, 'Excellent':4}

def standardise_sentence(sentence):
    sentence = sentence.replace(',',' , ').replace('.',' . ').replace('?',' ? ').replace('!',' ! ').replace('(',' ( ').replace(')',' ) ').replace('%',' % ')
    while '  ' in sentence:
        sentence = sentence.replace('  ',' ')
    return sentence.lower()

def load_alexa_pairs(fname='train.json', dir='./data/amazon'):
    pairs = []
    data = json.loads(open(os.path.join(dir, fname), 'r').read())
    for c_id, conversation in data.items():
        first = True
        for utterance, response in zip(conversation['content'][:-1],conversation['content'][1:]):
            u_sentence, r_sentence = standardise_sentence(utterance['message']), standardise_sentence(response['message'])
            if response['turn_rating'] not in [''] and len(u_sentence.split(' ')) < MAX_LENGTH and len(r_sentence.split(' ')) < MAX_LENGTH:
                pairs.append(Pair(utterance=u_sentence,
                                  response=r_sentence,
                                  rating=numeric_ratings[response['turn_rating']],
                                  conversation_id=c_id,
                                  opening_line=first))
                first = False
    return pairs

class AlexaDataset(Dataset):
    """Amazon Alexa Conversations dataset."""

    def __init__(self, json=None, dir='./data/amazon', rare_word_threshold=0):
        """
        Args:
            json (string): Name of json file to load
            dir (string): Directory where json is stored
            rare_word_threshold: Remove pairs which contain words that appear less than or equal to threshold
        """
        self.data = []
        self.rare_word_threshold = rare_word_threshold
        if json is not None:
            self.add_pairs_from_json(json, dir)
        else:
            for f in [f for f in os.listdir(dir) if f.endswith('.json')]:
                print("Added {} to dataset".format(f))
                self.add_pairs_from_json(f, dir)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        return self.data[index]

    @property
    def conversation_ids(self):
        return set([p.conversation_id for p in self.data])

    @property
    def opening_lines(self):
        return [self.get_conversation(c)[0].utterance for c in self.conversation_ids]

    def balance_data(self):
        min_obs = 10000000
        for r in numeric_ratings.values():
            obs = len([p for p in self.data if p.rating==r])
            min_obs = obs if obs < min_obs else min_obs
        new_data = []
        for r in numeric_ratings.values():
            r_data = [p for p in self.data if p.rating==r]
            if len(r_data) >= min_obs:
                r_data = r_data[:min_obs]
                new_data += r_data
        self.data = new_data
        self.ids = list(set([p.conversation_id for p in self.data]))

    def add_pairs_from_json(self, json, dir='./data/amazon'):
        self.data += load_alexa_pairs(json, dir)
        self._trim_rare_words(self.rare_word_threshold)
        self.ids = list(set([p.conversation_id for p in self.data]))

    def random_opening_line(self):
        return random.choice([p.utterance for p in self.data if p.opening_line])

    def get_conversation(self, id):
        return [p for p in self.data if p.conversation_id == id]

    def random_conversation(self):
        id = 0
        while id == 0:
            id = random.choice(self.ids)
        return [p for p in self.data if p.conversation_id == id]

    def _rare_words(self, threshold=3):
        count = Counter()
        for pair in self.data:
            for word in set(pair.utterance.split(' ')):
                count[word] += 1
        rare_words = set([word for word, cnt in count.items() if cnt <= threshold])
        return rare_words

    def _trim_rare_words(self, threshold=3):
        rare_words = self._rare_words(threshold)
        new_data = []
        for pair in self.data:
            words = pair.utterance.split(' ') + pair.response.split(' ')
            if set(words).isdisjoint(rare_words):
                new_data.append(pair)
        print('{} pairs trimmed, {} remain'.format(len(self.data) - len(new_data), len(new_data)))
        self.data = new_data

    def trimPairsToVocab(self, voc):
        keepPairs = []
        for pair in self.data:
            keep = True
            for word in pair.utterance.split(' ') + pair.response.split(' '):
                if word not in voc.word2index:
                    keep = False
            if keep:
                keepPairs.append(pair)
        print('=============={}=============='.format(len(self.data)-len(keepPairs)))
        self.data = keepPairs

    def add_scrambled_training_data(self, addition_rate=0.2, rating='Poor'):
        pairs = []
        for i in range(int(len(self.data) * addition_rate)):
            utterance = random.choice(self.data).utterance
            response = random.choice(self.data).response
            pairs.append(Pair(utterance=utterance,
                              response=response,
                              rating=numeric_ratings[rating],
                              conversation_id='0',
                              opening_line=False))
        self.data += pairs

if __name__ == '__main__':
    data = AlexaDataset(rare_word_threshold=2)
    print(len(data))
    print(data[0])
    c = data.get_conversation('t_bde29ce2-4153-4056-9eb7-f4ad710505fe')
    s = data.random_conversation()
    data.add_scrambled_training_data()


import sqlite3
from sqlite3 import Error

def createConnection(dbFile):
    """
        create a database connection to the SQLite database
        specified by db_file
    """
    conn = None
    try:
        conn = sqlite3.connect(dbFile)
    except Error as e:
        print(e)
    return conn


import shutil
import datetime
import string
import subprocess

import cmn
import glob
import os
import time
from zipfile import ZipFile

log_file = 'zipper.log'

def remove_log():
    global log_file
    with open(log_file, "a+") as f:
        f.write('\n')
        f.write('--------------------------------------------------------------------------------\n')
        f.write('\n')

def log(message):
    global log_file
    strmsg = time.strftime('%Y.%m.%d %T') + ' ' + message + '\n'
    with open(log_file, "a+") as f:
        f.write(strmsg)

def remover(item):
    iterator = 0
    pick = time.time()
    with ZipFile('/u02/app/oracle/oradata/datastore/logs/odadb1_aud/log.zip', 'w') as zip:
        for file in glob.glob(item["file_mask"]):
            if os.stat(file).st_mtime < pick - item["delay_in_days"] * 86400:
                zip.write(file)
                os.remove(file)
                iterator += 1
                if iterator % 1000 == 0:
                    log('temporary deleted files by mask {mask} is {iterator}'
                        .format(mask=item["file_mask"], iterator=iterator))

    log('deleted files by mask {mask} is {iterator}'.format(mask=item["file_mask"], iterator=iterator))

def checkfile(file):
    cmn.log.debug('found %s',file)

def copy_file(source, dest):
    # copy file to destination
    cmn.log.debug('file %s copy to %s' , file, dest)
    if dest[-4]=='.':
        make_dir(generate_mask(dest)[0:string.rfind(generate_mask(dest),'/')])
        shutil.copyfile(source, generate_mask(dest))
    else:
        make_dir(generate_mask(dest))
        shutil.copy(source, generate_mask(dest))

def remove_file(file):
    os.remove(file)

def move_file(source, dest):
    # move file to destination
    cmn.log.debug('file %s move to %s' , source, dest)
    make_dir(generate_mask(dest))
    shutil.move(source, generate_mask(dest))

def find_files_by_date( source, delta):
    cmn.log.debug('find_files_by_date action %s', delta)
    pick = time.time()
    res = []
    if delta is None: delta = 0
    for iter_file in glob.glob(source):
        if os.stat(iter_file).st_mtime < pick - delta * 86400:
            res.append(iter_file)
    return res

            # if action=='check':
            #     checkfile(iter_file)
            # if action == 'cp':
            #     copy_file(iter_file, dest)

def findfiles(action, source, destination,delta):
    cmn.log.debug('findfiles action %s', delta)
    pick = time.time()
    if delta is None: delta = 0
    for iter_file in glob.glob(source):
        if os.stat(iter_file).st_mtime < pick - delta * 86400:
            if action=='check':
                checkfile(iter_file)
            if action == 'cp':
                copy_file(iter_file, destination)

def conf_iterator():
    cmn.log.debug('conf_iterator')
    cmn.log.debug(cmn.config)
    for iterKey, iterValue in cmn.config.items():
       if iterValue['action'] in ('check','cp'):
           findfiles(iterValue['action'],iterValue['source'],iterValue.get('dest') ,iterValue.get('delta'))

def generate_mask(mask):
    cmn.log.debug('generate_mask')
    return generate_date(mask)

def make_dir(dir):
    # create directory if not exists
    if not os.path.exists(dir):
        os.makedirs(dir)

def generate_date(input_string):
    today = datetime.date.today()
    first = today.replace(day=1)
    last_month = first - datetime.timedelta(days=1)
    var_char = last_month.strftime("%Y.%m")
    return input_string.replace("%YM%", var_char)

def copy_to_archive(files, archive):
    make_dir(generate_mask(archive)[0:string.rfind(generate_mask(archive), '/')])
    for file in files:
        with ZipFile(generate_mask(archive), 'a') as zip:
            zip.write(file)
            os.remove(file)

def copy_to_server(file, user, server, path ):
    cmd = 'scp {file} {user}@{server}:{path}'.format(file=file, user=user, server=server, path=path)
    subprocess.call(cmd, shell=True)

def move_to_server(file, user, server, path ):
    copy_to_server(file, user, server, path)
    remove_file(file)

if __name__ == "__main__":
    # cli.read_cli_flags(sys.argv[1:])
    # cli.configlogger(filename=cmn.args['logfile'], level='debug')
    # utl_config.read_config()
    # conf_iterator()
    # make_dir(generate_mask('/u02/app/oracle/oradata/datastore/database_logs/test/%YM%/'))
    # copy_file('/u01/app/oracle/diag/rdbms/eisgs00/eisgs001/trace/alert_eisgs001.log', '/u02/app/oracle/oradata/datastore/database_logs/test/odadb1/%YM%/')
    # move_file('/u02/app/oracle/oradata/datastore/database_logs/test/odadb1/2022.05/alert_eisgs001.log', '/u02/app/oracle/oradata/datastore/database_logs/test/odadb2/%YM%/')
    # copy_to_server('/home/oracle/holdi.sql', 'oracle', 'odadb2','/home/oracle/')
    # remove_file('/home/oracle/holdi.sql')
    # copy_file('/home/oracle/holdi.sql','/home/oracle/holdi1.sql')
    # move_to_server('/home/oracle/holdi1.sql', 'oracle', 'odadb2','/home/oracle/')
    filelist  = find_files_by_date('/u01/app/oracle/admin/eisgs03/adump/*.aud', 1)
    copy_to_archive(filelist, '/u02/app/oracle/oradata/datastore/database_logs/odadb1/%YM%/aud.zip')

    # utl_config.write_config()


from ctfdist.paths import *
from ctfdist.descent import *
from ctfdist.helper_functions import *
from sklearn.linear_model import LogisticRegression
from scripts.toy_helper import *
from scripts.plot_helper import *

# setup
random_seed = 2338
random_state = np.random.RandomState(random_seed)
outcome_column_name = 'y'
group_column_name = 's'
minority_group = 0
majority_group = 1

train_size = 0.30
test_size = 0.20
audit_size = 1.0 - train_size - test_size

# generate full dataset
ps = 0.5
coefs = np.array([5.0, -2.5, -2.5])

param_min = {
    'coefs': coefs,
    'group_label': minority_group,
    'px': [0.9, 0.2, 0.2],
    'n_samples': 50000,
    }

param_maj = {
    'coefs': coefs,
    'group_label': majority_group,
    'px': [0.1, 0.5, 0.5],
    'n_samples': 50000,
    }

df_min, h_min = generate_toy_dataset(**param_min)
df_maj, h_maj = generate_toy_dataset(**param_maj)
df = df_min.append(df_maj)

#### Data Splitting ####

df_train, df_audit, df_test = train_test_audit_split(df = df,
                                                     train_size = train_size,
                                                     test_size = test_size,
                                                     audit_size = audit_size,
                                                     group_column_name = group_column_name,
                                                     outcome_column_name = outcome_column_name,
                                                     random_state = random_state)

#### Models ####

#handle for p(S = 1 | X = x) -- model to get true outcomes for minority group
hs_min = h_min['get_px']
hs_maj = h_maj['get_px']
predict_group = lambda x: np.multiply(ps, hs_maj(x)) / (np.multiply(ps, hs_maj(x)) + np.multiply(1.0 - ps, hs_min(x)))

# handle for p(Y = 1 | X = x) -- model to get true outcomes for minority group:
get_py_min = h_min['get_py']
get_py_maj = h_maj['get_py']
predict_ytrue = lambda x: get_py_min(x).flatten()
predict_ytrue_maj = lambda x: get_py_maj(x).flatten()

# train predictive model
X, Y, S = df_to_XYS(df_train, minority_group = minority_group)
f_hat = LogisticRegression(penalty = 'l2',  C = 100, fit_intercept = False)
f_hat.fit(X = X, y = Y)
pos_class_idx = np.flatnonzero(f_hat.classes_ > 0)
predict_prob = lambda x: f_hat.predict_proba(x)[:, pos_class_idx].flatten()
predict_yhat = lambda x: 2.0 * np.greater(predict_prob(x), 0.5) - 1.0

#### Distribution Descent Experiment ####

script_metric_weights = {
    'cal': {'cal': 1.0},
    'fpr': {'fpr': 1.0},
    'fnr': {'fnr': 1.0},
    'fdr': {'fdr': 1.0},
    'da_out': {'da_out': 1.0},
    'da_in': {'da_in': 1.0},
    'da_mix': {'da_out': 0.8, 'da_in': 0.2},
    }

data_name = 'toy'
metric_name = 'fpr'
metric_weights = script_metric_weights[metric_name]
figure_name = '%s/%s_descent_%s.pdf' % (results_dir, data_name, metric_name)

### SANITY CHECK PERFORMANCE
np.mean(S == np.greater(predict_group(X), 0.5))

train_stats_df = get_performance_metrics(df = df_train,
                                         data_type = 'train',
                                         predict_yhat = predict_yhat,
                                         predict_prob = predict_prob)

audit_stats_df = get_performance_metrics(df = df_audit,
                                         data_type = 'audit',
                                         predict_yhat = predict_yhat,
                                         predict_prob = predict_prob)

test_stats_df = get_performance_metrics(df = df_test,
                                        data_type = 'test',
                                        predict_yhat = predict_yhat,
                                        predict_prob = predict_prob)

train_stats_df[train_stats_df['s'] >= 0][['s', metric_name]]

# Fit Counterfactual Distribution
ctf_args = {
    'df': df_audit,
    'metric_weights': metric_weights,
    #
    'predict_yhat': predict_yhat,
    'predict_group':predict_group,
    'predict_ytrue': predict_ytrue,
    'predict_ytrue_maj': predict_ytrue_maj,
    #
    'outcome_column_name': outcome_column_name,
    'group_column_name':group_column_name,
    'minority_group':minority_group,
    #
    'random_state':random_state,
    }

ctf_dist = CounterfactualDistribution(**ctf_args)
df_audit_min, df_audit_maj = split_by_group(df_audit)
results = ctf_dist.fit(df_min_test = df_audit_min, df_maj_test = df_audit_maj, max_iterations = 1e2, step_size = 0.1)
f, _ = plot_descent_profile(results)
f.savefig(figure_name)

# Build Preprocessor
pp = ctf_dist.build_preprocessor(df_audit)
stats_df = get_performance_metrics(df = df_test,
                                   data_type = 'test',
                                   pp = pp,
                                   predict_yhat = predict_yhat,
                                   predict_prob = predict_prob,
                                   minority_group = minority_group)

stats_df[stats_df['s'] >= 0][['s', metric_name, 'dist_type']]


import sys

var = input("Please enter an integer:")

if not var.isdecimal():
    sys.exit("ERROR: Input is not an integer")
var = int(var)
while var >= 0:
    print(var)
    var -= 2


"""
@brief      test log(time=4s)
@author     Xavier Dupre
"""
import os
import unittest
import warnings
import logging
from io import StringIO
from docutils.parsers.rst import directives
from pyquickhelper.pycode import get_temp_folder, ExtTestCase
from pyquickhelper.helpgen import rst2html
from pyquickhelper.helpgen.sphinxm_custom_app import CustomSphinxApp
from pyquickhelper.sphinxext.sphinximages.sphinxtrib.images import ImageDirective

class TestImageExtension(ExtTestCase):

    def setUp(self):
        logger = logging.getLogger('gdot')
        logger.disabled = True

    def test_post_parse_sn(self):
        directives.register_directive("image", ImageDirective)

    def test_thumbnail(self):
        from docutils import nodes as skip_

        content = """
                    test a directive
                    ================

                    before

                    .. thumbnail:: http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/_static/project_ico.png
                        :width: 10
                        :height: 20
                        :download: 1

                    after

                    this code shoud appear
                    """.replace("                    ", "")
        content = content.replace('u"', '"')

        logger2 = logging.getLogger("video")

        log_capture_string = StringIO()
        ch = logging.StreamHandler(log_capture_string)
        ch.setLevel(logging.DEBUG)
        logger2.addHandler(ch)
        with warnings.catch_warnings(record=True):
            html = rst2html(content,  # fLOG=fLOG,
                            writer="custom", keep_warnings=True,
                            directives=None)

        warns = log_capture_string.getvalue()
        if warns:
            raise Exception(warns)

        t1 = "this code shoud not appear"
        if t1 in html:
            raise Exception(html)

        t1 = "this code shoud appear"
        if t1 not in html:
            raise Exception(html)

        t1 = "_images"
        if t1 not in html:
            raise Exception(html)

        t1 = "linkedin"
        if t1 in html:
            raise Exception(html)

        temp = get_temp_folder(__file__, "temp_sphinx_thumbnail")
        with open(os.path.join(temp, "out_image.html"), "w", encoding="utf8") as f:
            f.write(html)

    def test_sphinx_ext_thumbnail_html(self):
        temp = get_temp_folder(__file__, "temp_sphinx_ext_thumbnail_html")
        src_ = os.path.join(temp, "..", "data", "image")

        app = CustomSphinxApp(src_, temp)
        app.build()

        index = os.path.join(temp, "index.html")
        self.assertExists(index)
        img = os.path.join(temp, '_images', 'im.png')
        self.assertExists(img)

if __name__ == "__main__":
    unittest.main()


from aws_cdk import (
    core,
    aws_ec2 as ec2,
    aws_cognito as cg,
    aws_lambda as lambda_,
    aws_iam as iam,
    aws_ssm as ssm,
)

class CognitoStack(core.Stack):

    def __init__(self, scope: core.Construct, construct_id: str,
                 region: str, account: str, es_domain_name: str,
                 vpc_id: str, security_group_id: str,
                 **kwargs) -> None:
        super().__init__(scope, construct_id, **kwargs)

        vpc = ec2.Vpc.from_lookup(
            self,
            "VPC",
            vpc_id=vpc_id
        )

        sg = ec2.SecurityGroup.from_lookup(
            self,
            "SecurityGroup",
            security_group_id=security_group_id
        )

        # lambda_role = iam.Role(
        #     self,
        #     "OctankBaseCognito_PostConfirmationLambdaRole",
        #     role_name="OctankBaseCognito_PostConfirmationLambdaRole",
        #     assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
        #     managed_policies=[
        #         iam.ManagedPolicy.from_aws_managed_policy_name("AWSLambdaFullAccess"),
        #         iam.ManagedPolicy.from_aws_managed_policy_name("AmazonDynamoDBFullAccess"),
        #         iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEC2FullAccess"),
        #     ]
        # )

        # self._lambda_code = lambda_.Code.from_cfn_parameters()

        # Post Confirmation Lambda
        # post_confirmation_lambda = lambda_.Function(
        #     self,
        #     "OctankCognitoPostConfirmation",
        #     function_name="OctankCognitoPostConfirmation",
        #     handler="lambda_handler.handler",
        #     runtime=lambda_.Runtime.PYTHON_3_8,
        #     code=self._lambda_code,
        #     timeout=core.Duration.seconds(7),
        #     role=lambda_role,
        #     vpc=vpc,
        #     vpc_subnets=ec2.SubnetSelection(
        #         subnet_type=ec2.SubnetType.PRIVATE
        #     ),
        #     security_groups=[sg]
        # )

        self._user_pool = cg.UserPool(
            self,
            "KeehyunUserPool",
            user_pool_name="KeehyunUserPool",
            self_sign_up_enabled=True,
            sign_in_aliases=cg.SignInAliases(
                email=True
            ),
            user_verification=cg.UserVerificationConfig(
                email_subject="Verify your email",
                email_body="Hello, Thanks for signing up! Your verification code is {####}",
                email_style=cg.VerificationEmailStyle.CODE,
            ),
            auto_verify=cg.AutoVerifiedAttrs(email=True),
            standard_attributes=cg.StandardAttributes(
                email=cg.StandardAttribute(
                    required=True,
                    mutable=False
                ),
            ),
            password_policy=cg.PasswordPolicy(
                min_length=8,
                require_lowercase=True,
                require_digits=True,
                temp_password_validity=core.Duration.days(3)
            ),
            account_recovery=cg.AccountRecovery.EMAIL_ONLY,
            # lambda_triggers=cg.UserPoolTriggers(
            #     post_confirmation=post_confirmation_lambda
            # )
        )

        client = self._user_pool.add_client(
            "KeehyunUserPoolAppClient",
            user_pool_client_name="KeehyunUserPoolAppClient",
            generate_secret=False
        )

        self._user_pool.add_domain(
            "KeehyunCognitoDomain",
            cognito_domain=cg.CognitoDomainOptions(domain_prefix="keehyun")
        )

        core.CfnOutput(
            self,
            "UserPoolID",
            value=self._user_pool.user_pool_id
        )

        core.CfnOutput(
            self,
            "AppClientID",
            value=client.user_pool_client_id
        )

        # Identity Pool
        self._identity_pool = cg.CfnIdentityPool(
            self,
            "KeehyunIdentityPool",
            identity_pool_name="KeehyunIdentityPool",
            allow_unauthenticated_identities=False,
            cognito_identity_providers=[
                cg.CfnIdentityPool.CognitoIdentityProviderProperty(
                    client_id=client.user_pool_client_id,
                    provider_name=self._user_pool.user_pool_provider_name
                )
            ]
        )

        unauthenticated_role = iam.Role(
            self,
            "KeehyunCognitoDefaultUnauthenticatedRole",
            role_name="KeehyunCognitoDefaultUnauthenticatedRole",
            assumed_by=iam.FederatedPrincipal(
                federated='cognito-identity.amazonaws.com',
                conditions={
                    "StringEquals": {
                       "cognito-identity.amazonaws.com:aud": self._identity_pool.ref,
                    },
                    "ForAnyValue:StringLike": {
                       "cognito-identity.amazonaws.com:amr": "unauthenticated",
                    },
                },
                assume_role_action="sts:AssumeRoleWithWebIdentity"
            )
        )

        unauthenticated_role.add_to_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                actions=[
                    "mobileanalytics:PutEvents",
                    "cognito-sync:*",
                ],
                resources=["*"]
            )
        )

        authenticated_role = iam.Role(
            self,
            "KeehyunCognitoDefaultAuthenticatedRole",
            role_name="KeehyunCognitoDefaultAuthenticatedRole",
            assumed_by=iam.FederatedPrincipal(
                federated='cognito-identity.amazonaws.com',
                conditions={
                    "StringEquals": {
                        "cognito-identity.amazonaws.com:aud": self._identity_pool.ref,
                    },
                    "ForAnyValue:StringLike": {
                        "cognito-identity.amazonaws.com:amr": "authenticated",
                    },
                },
                assume_role_action="sts:AssumeRoleWithWebIdentity"
            )
        )

        authenticated_role.add_to_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                actions=[
                    "mobileanalytics:PutEvents",
                    "cognito-sync:*",
                    "cognito-identity:*"
                ],
                resources=["*"]
            )
        )

        cg.CfnIdentityPoolRoleAttachment(
            self,
            "DefaultValid",
            identity_pool_id=self._identity_pool.ref,
            roles={
                'unauthenticated': unauthenticated_role.role_arn,
                'authenticated': authenticated_role.role_arn
            }
        )

        # ElasticSearch Admin Group
        es_admin_role = iam.Role(
            self,
            "KeehyunCognitoESAdminRole",
            role_name="KeehyunCognitoESAdminRole",
            assumed_by=iam.FederatedPrincipal(
                federated='cognito-identity.amazonaws.com',
                conditions={
                    "StringEquals": {
                        "cognito-identity.amazonaws.com:aud": self._identity_pool.ref,
                    },
                    "ForAnyValue:StringLike": {
                        "cognito-identity.amazonaws.com:amr": "authenticated",
                    },
                },
                assume_role_action="sts:AssumeRoleWithWebIdentity"
            )
        )

        es_admin_role.add_to_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                resources=[f"arn:aws:es:{region}:{account}:domain/{es_domain_name}/*"],
                actions=["es:ESHttp*"]
            )
        )

        cg.CfnUserPoolGroup(
            self,
            "KeehyunUserPoolGroup",
            user_pool_id=self._user_pool.user_pool_id,
            group_name="ESAdmin",
            description="ElasticSearch Admin User",
            role_arn=es_admin_role.role_arn,
            precedence=0
        )

        ssm.StringParameter(
            self,
            "UserPoolIDStringParameter",
            parameter_name="user-pool-id",
            string_value=self._user_pool.user_pool_id
        )

        ssm.StringParameter(
            self,
            "UserPoolARNStringParameter",
            parameter_name="user-pool-arn",
            string_value=self._user_pool.user_pool_arn
        )

        ssm.StringParameter(
            self,
            "IdentityPoolIDStringParameter",
            parameter_name="identity-pool-id",
            string_value=self._identity_pool.ref
        )

    # @property
    # def lambda_code(self):
    #     return self._lambda_code


#This python script works for any csv file with two column that the
#first column as the date and the second column as the revenue. I notice that
# the two original files are in time order already. So this script
#takes advantage of that. Future coding needs to be done to deal with csv files that have not been
#sorted by dates.

#To run a another csv file of the same kind, just change the variable
#filename="your_file_name.csv" note the file has to be in the direction PythonBank/raw_data
#for this homework, just change to filename="budget_data_2.csv" to run budget_data_2.csv

#Part I
# ______________________________________________________________
# import csv and convert csv into a list called csv_list
#________________________________________________________________
import csv
import os
filename="budget_data_1.csv"
csvpath=os.path.join("raw_data", filename)
csv_list=[]
with open(csvpath,newline="") as csvfile:
    csvreader=csv.reader(csvfile,delimiter=",")
    #skip the header
    next(csvreader,None)
    for row in csvreader:
        csv_list.append(row)

#Part II
# ____________________________________________________________
# calculate the number of months as well as the total revenue
# ____________________________________________________________
num_of_month=len(csv_list)

#Calculate total revenue
total_revenue=0
for ele in csv_list:
    total_revenue+=int(ele[1])
# # print("debug "+total_revenue)

#Part III
#__________________________________________________________
# create another list to store change of revenue by month
#this list is named as change_by_month_list
#__________________________________________________________
change_by_month_list=[]
i=1
while i<=(len(csv_list)-1):
    change_by_month=int(csv_list[i][1])-int(csv_list[i-1][1])
    change_by_month_list.append(change_by_month)
    i+=1
# # print("debug "+len(change_by_month_list))
# # print("debug "+change_by_month_list[-1])
#
# PartIV
# ________________________________________________________________
# calculate the average_revenue_change
# find the greatest changes of revenue in the change_by_month_list
# and the index number of the greatest changes of the revenue
# use the index number of the greatest increase and decrease to
# access the corresponding date from the csv_list
#________________________________________________________________

#Calculate the average of revenue change by month
average_revenue_change=sum(change_by_month_list)/len(change_by_month_list)

#find the greatest increase revenue and its index number
greatest_increase=max(change_by_month_list)
index_increase=change_by_month_list.index(greatest_increase)

# find the greatest decrease of revenue and its index number
greatest_decrease=min(change_by_month_list)
index_decrease=change_by_month_list.index(greatest_decrease)

# # print("debug {},{},{},{}".format(greatest_increase,index_increase,greatest_decrease,index_decrease))

#retrieve the corresponding dates with greatest increase or decrease from
#csv_list
Date_greatest_increase=csv_list[index_increase+1][0]
Date_greatest_decrease=csv_list[index_decrease+1][0]
# # print("debug "+Date_greatest_increase)
# # print("debug "+Date_greatest_decrease)

#__________________________________________________________________________________________
#PartV output on terminal as well as onto a filename.txt file
# ____________________________________________________________________________________________

message=("\nFinancial Analysis \n"+"__________________________________________\n"+"Total months: "
    +str(num_of_month)+"\nTotal Revenue: $"+str(total_revenue)+
    "\nAverage Revenue Change: $"+str(average_revenue_change)+"\nGreatest Increase in Revenue: "
    +str(Date_greatest_increase) + " ($" + str(greatest_increase) +")\nGreatest Decrease in Revenue: "
    +str(Date_greatest_decrease) + " ($" + str(greatest_decrease) +")\n")
print(message)

output_filename=filename[:-3]+"txt"
with open(output_filename,"w") as text:
    text=text.write(message)


import copy
from unittest.mock import patch

import numpy as np
import torch
from ax.core.data import Data
from ax.core.objective import MultiObjective, Objective
from ax.core.observation import Observation, ObservationData, ObservationFeatures
from ax.core.optimization_config import MultiObjectiveOptimizationConfig
from ax.core.outcome_constraint import ObjectiveThreshold
from ax.core.search_space import SearchSpace
from ax.core.types import ComparisonOp
from ax.exceptions.core import UnsupportedError
from ax.metrics.branin import BraninMetric, NegativeBraninMetric
from ax.modelbridge.registry import Models
from ax.plot.pareto_frontier import interact_multiple_pareto_frontier
from ax.plot.pareto_utils import (
    _extract_observed_pareto_2d,
    compute_posterior_pareto_frontier,
    get_observed_pareto_frontiers,
    infer_reference_point_from_experiment,
    to_nonrobust_search_space,
)

from ax.utils.common.testutils import TestCase
from ax.utils.testing.core_stubs import (
    get_branin_experiment,
    get_branin_experiment_with_multi_objective,
    get_experiment_with_observations,
    get_robust_search_space_environmental,
    get_search_space,
)

class ParetoUtilsTest(TestCase):
    def setUp(self) -> None:
        experiment = get_branin_experiment()
        experiment.add_tracking_metric(
            BraninMetric(name="m2", param_names=["x1", "x2"])
        )
        sobol = Models.SOBOL(experiment.search_space)
        a = sobol.gen(5)
        experiment.new_batch_trial(generator_run=a).run()
        self.experiment = experiment
        self.metrics = list(experiment.metrics.values())

    def testComputePosteriorParetoFrontierByTrial(self) -> None:
        # Experiments with batch trials must specify trial_index or data
        with self.assertRaises(UnsupportedError):
            compute_posterior_pareto_frontier(
                self.experiment,
                self.metrics[0],
                self.metrics[1],
                absolute_metrics=[m.name for m in self.metrics],
            )
        pfr = compute_posterior_pareto_frontier(
            self.experiment,
            self.metrics[0],
            self.metrics[1],
            trial_index=0,
            absolute_metrics=[m.name for m in self.metrics],
            num_points=2,
        )
        self.assertIsNone(pfr.arm_names)

    def testComputePosteriorParetoFrontierByData(self) -> None:
        # Experiments with batch trials must specify trial_index or data
        compute_posterior_pareto_frontier(
            self.experiment,
            self.metrics[0],
            self.metrics[1],
            data=self.experiment.fetch_data(),
            absolute_metrics=[m.name for m in self.metrics],
            num_points=2,
        )

    def testObservedParetoFrontiers(self) -> None:
        experiment = get_branin_experiment_with_multi_objective(
            with_batch=True, has_optimization_config=False, with_status_quo=True
        )

        # Optimization config is not optional
        with self.assertRaises(ValueError):
            get_observed_pareto_frontiers(experiment=experiment, data=Data())

        objectives = [
            Objective(
                metric=BraninMetric(
                    name="m1", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
            Objective(
                metric=NegativeBraninMetric(
                    name="m2", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
            Objective(
                metric=BraninMetric(
                    name="m3", param_names=["x1", "x2"], lower_is_better=True
                ),
                minimize=True,
            ),
        ]
        bounds = [0, -100, 0]
        objective_thresholds = [
            ObjectiveThreshold(
                metric=objective.metric,
                bound=bounds[i],
                relative=True,
                op=ComparisonOp.LEQ,
            )
            for i, objective in enumerate(objectives)
        ]
        objective = MultiObjective(objectives=objectives)
        optimization_config = MultiObjectiveOptimizationConfig(
            objective=objective,
            objective_thresholds=objective_thresholds,
        )
        experiment.optimization_config = optimization_config
        experiment.trials[0].run()

        # For the check below, compute which arms are better than SQ
        df = experiment.fetch_data().df
        df["sem"] = np.nan
        data = Data(df)
        sq_val = df[(df["arm_name"] == "status_quo") & (df["metric_name"] == "m1")][
            "mean"
        ].values[0]
        pareto_arms = sorted(
            df[(df["mean"] <= sq_val) & (df["metric_name"] == "m1")]["arm_name"]
            .unique()
            .tolist()
        )

        pfrs = get_observed_pareto_frontiers(experiment=experiment, data=data)
        # We have all pairs of metrics
        self.assertEqual(len(pfrs), 3)
        true_pairs = [("m1", "m2"), ("m1", "m3"), ("m2", "m3")]
        for i, pfr in enumerate(pfrs):
            self.assertEqual(pfr.primary_metric, true_pairs[i][0])
            self.assertEqual(pfr.secondary_metric, true_pairs[i][1])
            self.assertEqual(pfr.absolute_metrics, [])
            self.assertEqual(list(pfr.means.keys()), ["m1", "m2", "m3"])
            self.assertEqual(len(pfr.means["m1"]), len(pareto_arms))
            self.assertTrue(np.isnan(pfr.sems["m1"]).all())
            # pyre-fixme[6]: For 1st param expected `Sized` but got
            #  `Optional[List[Optional[str]]]`.
            self.assertEqual(len(pfr.arm_names), len(pareto_arms))
            arm_idx = np.argsort(pfr.arm_names)
            for i, idx in enumerate(arm_idx):
                name = pareto_arms[i]
                # pyre-fixme[16]: Optional type has no attribute `__getitem__`.
                self.assertEqual(pfr.arm_names[idx], name)
                self.assertEqual(
                    pfr.param_dicts[idx], experiment.arms_by_name[name].parameters
                )

        pfrs = get_observed_pareto_frontiers(
            experiment=experiment, data=data, arm_names=["0_1"]
        )
        for pfr in pfrs:
            # pyre-fixme[58]: `in` is not supported for right operand type
            #  `Optional[typing.List[typing.Optional[str]]]`.
            self.assertTrue("status_quo" in pfr.arm_names)

    def testPlotMultipleParetoFrontiers(self) -> None:
        experiment = get_branin_experiment_with_multi_objective(
            has_objective_thresholds=True,
        )
        sobol = Models.SOBOL(experiment.search_space)
        a = sobol.gen(5)
        experiment.new_batch_trial(generator_run=a).run()
        pfrs = get_observed_pareto_frontiers(experiment=experiment)
        pfrs2 = copy.deepcopy(pfrs)
        pfr_lists = {"pfrs 1": pfrs, "pfrs 2": pfrs2}
        self.assertIsNotNone(interact_multiple_pareto_frontier(pfr_lists))

    def test_extract_observed_pareto_2d(self) -> None:
        Y = np.array([[1.0, 2.0], [2.1, 1.0], [1.0, 1.0], [2.0, 2.0], [3.0, 0.0]])
        reference_point = (1.5, 0.5)
        minimize = False
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=reference_point, minimize=minimize
        )

        # first and last points are beyond reference point, third point is dominated
        self.assertTrue(np.array_equal(pareto, np.array([[2.1, 1.0], [2.0, 2.0]])))

        # different `minimize` in each direction
        minimize = (True, False)
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=reference_point, minimize=minimize
        )
        self.assertTrue(np.array_equal(pareto, np.array([[1.0, 2.0]])))

        # no `reference_point` provided
        minimize = False
        pareto = _extract_observed_pareto_2d(
            Y=Y, reference_point=None, minimize=minimize
        )
        self.assertTrue(
            np.array_equal(pareto, np.array([[3.0, 0.0], [2.1, 1.0], [2.0, 2.0]]))
        )

    def test_to_nonrobust_search_space(self) -> None:
        # Return non-robust search space as is.
        search_space = get_search_space()
        self.assertIs(to_nonrobust_search_space(search_space), search_space)
        # Prune environmental variables and distributions from RSS.
        rss = get_robust_search_space_environmental()
        transformed_ss = to_nonrobust_search_space(rss)
        # Can't use isinstance here since RSS is also a SearchSpace.
        self.assertEqual(transformed_ss.__class__, SearchSpace)
        self.assertEqual(transformed_ss.parameters, rss._parameters)
        self.assertEqual(
            transformed_ss.parameter_constraints, rss.parameter_constraints
        )

class TestInfereReferencePointFromExperiment(TestCase):
    def test_infer_reference_point_from_experiment(self) -> None:
        for constrained in (True, False):
            observations = [[-1.0, 1.0], [-0.5, 2.0], [-2.0, 0.5], [-0.1, 0.1]]
            if constrained:
                observations = [
                    o + [c] for o, c in zip(observations, [1.0, 0.5, 1.0, 1.0])
                ]
            # Getting an experiment with 2 objectives by the above observations.
            experiment = get_experiment_with_observations(
                observations=observations,
                minimize=True,
                scalarized=False,
                constrained=constrained,
            )

            inferred_reference_point = infer_reference_point_from_experiment(experiment)

            # The nadir point for this experiment is [-0.5, 0.5]. The function actually
            # deducts 0.1*Y_range from each of the objectives. Since the range for each
            # of the objectives is +/-1.5, the inferred reference point would
            # be [-0.35, 0.35].
            self.assertEqual(inferred_reference_point[0].op, ComparisonOp.LEQ)
            self.assertEqual(inferred_reference_point[0].bound, -0.35)
            self.assertEqual(inferred_reference_point[0].metric.name, "m1")
            self.assertEqual(inferred_reference_point[1].op, ComparisonOp.GEQ)
            self.assertEqual(inferred_reference_point[1].bound, 0.35)
            self.assertEqual(inferred_reference_point[1].metric.name, "m2")

    def test_infer_reference_point_from_experiment_shuffled_metrics(self) -> None:
        # Generating an experiment with given data.
        observations = [
            [-1.0, 1.0, 0.1],
            [-0.5, 2.0, 0.2],
            [-2.0, 0.5, 0.3],
            [-0.1, 0.1, 0.4],
        ]
        experiment = get_experiment_with_observations(
            observations=observations,
            minimize=True,
            scalarized=False,
            constrained=True,
        )

        # Constructing fake outputs for `get_pareto_frontier_and_configs` so that
        # the order of metrics `m1`, `m2` and `m3` are reversed.
        frontier_observations_shuffled = [
            Observation(
                features=ObservationFeatures(parameters={"x": 0.0, "y": 0.0}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.1, 1.0, -1.0]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
            Observation(
                features=ObservationFeatures(parameters={"x": 0.1, "y": 0.1}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.2, 2.0, -0.5]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
            Observation(
                features=ObservationFeatures(parameters={"x": 0.2, "y": 0.2}),
                data=ObservationData(
                    metric_names=["m3", "m2", "m1"],
                    means=np.array([0.3, 0.5, -2.0]),
                    covariance=np.diag(np.full(3, float("nan"))),
                ),
            ),
        ]
        f_shuffled = torch.tensor(
            [
                [0.1000, 1.0000, -1.0000],
                [0.2000, 2.0000, -0.5000],
                [0.3000, 0.5000, -2.0000],
            ],
            dtype=torch.float64,
        )
        obj_w_shuffled = torch.tensor([0.0, 1.0, -1.0], dtype=torch.float64)
        obj_t_shuffled = torch.tensor(
            [-torch.inf, -torch.inf, torch.inf], dtype=torch.float64
        )

        # Test the function with these shuffled output for
        # `get_pareto_frontier_and_configs`.
        with patch(
            "ax.plot.pareto_utils.get_pareto_frontier_and_configs",
            return_value=(
                frontier_observations_shuffled,
                f_shuffled,
                obj_w_shuffled,
                obj_t_shuffled,
            ),
        ):
            inferred_reference_point = infer_reference_point_from_experiment(experiment)

            self.assertEqual(inferred_reference_point[0].op, ComparisonOp.LEQ)
            self.assertEqual(inferred_reference_point[0].bound, -0.35)
            self.assertEqual(inferred_reference_point[0].metric.name, "m1")
            self.assertEqual(inferred_reference_point[1].op, ComparisonOp.GEQ)
            self.assertEqual(inferred_reference_point[1].bound, 0.35)
            self.assertEqual(inferred_reference_point[1].metric.name, "m2")


from django import forms

from tinymce.widgets import TinyMCE
from .models import Subject,InfoList

class InfoListForm(forms.ModelForm):
    def __init__(self,department,year,*args,**kwargs):
        super(InfoListForm,self).__init__(*args,**kwargs)
        self.fields['subject'].queryset = Subject.objects.filter(department=department,year=year)

    class Meta:
        model = InfoList
        fields = ['subject','title','content']
        widgets = {
            'content':TinyMCE(attrs={'cols': 80, 'rows': 15})
        }

class SubjectForm(forms.ModelForm):
    class Meta:
        model = Subject
        fields = ['subject']


from os.path import isfile, isdir
from os import listdir, remove
from constants import PATH, AGE, SAFE, BASE_VALUE_PATH, BASE_VALUE_AGE, BASE_VALUE_SAFE, SEPARATOR
import time, os, stat

class FileLoader:
    def remove_file(self, path):
        remove(path)

    def file_as_list(self, path):
        self._secure_file(path)
        text_file = open(path, "r")
        return text_file.readlines()

    def directory_tree(self, path):
        files = []
        for candidate in listdir(path):
            path_to_candidate = path+"/"+candidate
            if isdir(path_to_candidate):
                candidate_files = self.directory_tree(path_to_candidate)
                files = files + candidate_files
            elif isfile(path_to_candidate):
                files.append(path_to_candidate)
        return files

    def file_age_in_seconds(self, pathname):
        return time.time() - os.stat(pathname)[stat.ST_MTIME]

    def list_as_file(self, path, list_to_file):
        file_to_write = open(path, "w")
        for line in list_to_file:
            file_to_write.write(line + "\n")
        file_to_write.close()

    def _secure_file(self, path):
        if not isfile(path):
            text_file = open(path, "w")
            text_file.write(PATH + SEPARATOR + BASE_VALUE_PATH + "\n")
            text_file.write(AGE + SEPARATOR + BASE_VALUE_AGE + "\n")
            text_file.write(SAFE + SEPARATOR + BASE_VALUE_SAFE + "\n")
            text_file.close()