from airflow.models import DAG

from airflow.operators.python import PythonOperator
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.utils.task_group import TaskGroup

from datetime import datetime

from utils.load_data import load_data
from utils.preprocess_data import preprocess_data
from utils.experiment import experiment
from utils.track_experiments_info import track_experiments_info
from utils.fit_best_model import fit_best_model
from utils.save_batch_data import save_batch_data

default_args= {
    'owner': 'Data Scientist',
    'email_on_failure': False,
    'email': ['ds@mymail.com'],
    'start_date': datetime(2021, 12, 1)
}

with DAG(
    "MLOps_Pipeline_Breast",
    description='End-to-end ML pipeline example',
    schedule_interval='@daily',
    default_args=default_args,
    catchup=False
) as dag:

    # Group Task: 1 -> CREATE AUX TABLES IN DB
    with TaskGroup('creating_storage_structures') as creating_storage_structures:

        # task: 1.1 -> Create table experiments in db relate to connection postgres_postgres_local_connection
        creating_experiment_tracking_table = PostgresOperator(
            task_id="creating_experiment_tracking_table",
            postgres_conn_id='postgres_local_connection',
            sql='sql/create_experiments.sql'
        )

        # task: 1.2 -> Create batch data table in db relate to connection postgres_postgres_local_connection
        creating_batch_data_table = PostgresOperator(
            task_id="creating_batch_data_table",
            postgres_conn_id='postgres_local_connection',
            sql='sql/create_batch_data_table.sql'
        )

    # task: 2 -> Execute file load_data
    fetching_data = PythonOperator(
        task_id='fetching_data',
        python_callable=load_data

    )

    # task: 3
    with TaskGroup('preparing_data') as preparing_data:

        # task: 3.1 -> Split dataset in train and test
        preprocessing = PythonOperator(
            task_id='preprocessing',
            python_callable=preprocess_data
        )

        # task: 3.2 -> Save data in database (batch table)
        saving_batch_data = PythonOperator(
            task_id='saving_batch_data',
            python_callable=save_batch_data
        )

    # task: 4 -> Hyperparam tuning
    hyperparam_tuning = PythonOperator(
        task_id='hyperparam_tuning',
        python_callable=experiment
    )

    # task: 5
    with TaskGroup('after_crossvalidation') as after_crossvalidation:

        # =======
        # task: 5.1  --> Save data in experiments table
        saving_results = PythonOperator(
            task_id='saving_results',
            python_callable=track_experiments_info
        )

        # task: 5.2
        fitting_best_model = PythonOperator(
            task_id='fitting_best_model',
            python_callable=fit_best_model
        )

    creating_storage_structures >> fetching_data >> preparing_data >> hyperparam_tuning >> after_crossvalidation


import os

from flask import Flask
import daemon.chat_processor_engine as cpe
from waitress import serve

app = Flask(__name__)

@app.route("/")
def hello_world():
    return "Hello There!"

@app.route("/health-check")
def health_check():
    return {"Status": "Not Implemented"}

@app.route("/load-chat-data")
def run_data_process():
    cpe.data_process_callback()
    return {"Status": "Success"}

if __name__ == '__main__':
    PROD = 1
    if PROD:
        serve(app, listen='*:5000')
    else:
        app.run(debug = True,host='0.0.0.0')


#!/usr/bin/env python

"""
This ROS node is responsible for executing a qualification run.
"""

import roslib; roslib.load_manifest('automow_planning')
import rospy
import tf
from tf.transformations import quaternion_from_euler as qfe
from actionlib import SimpleActionClient

import numpy as np
from math import radians

from geometry_msgs.msg import PolygonStamped, Point, PoseStamped, PointStamped
from std_msgs.msg import ColorRGBA
from visualization_msgs.msg import Marker, MarkerArray
from nav_msgs.msg import Path, Odometry
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
from std_srvs.srv import Empty
from automow_node.srv import Cutters

import shapely.geometry as geo

class PathPlannerNode(object):
    """
    This is a ROS node that is responsible for planning and executing
    the a path through the field.
    """
    def __init__(self):
        # Setup ROS node
        rospy.init_node('path_planner')

        # ROS params
        self.box_size = rospy.get_param("~box_size", 5)

        rospy.Subscriber('/ekf/odom', Odometry, self.odom_callback)
        self.listener = tf.TransformListener()
        set_cutter_states = rospy.ServiceProxy('cutters', Cutters)
        set_cutter_states(False, False)

        # Setup initial variables
        self.robot_pose = None

        # Setup path following
        self.setup_path_following()
        set_cutter_states(True, True)
        self.do_qualification()
        set_cutter_states(False, False)

    def do_qualification(self):
        """
        Executes a qualification run.
        """
        for waypoint in self.path:
            if rospy.is_shutdown(): return
            # Cancel any current goals
            self.move_base_client.cancel_all_goals()
            # Push the goal to the actionlib server
            destination = MoveBaseGoal()
            destination.target_pose.header.frame_id = "odom"
            destination.target_pose.header.stamp = rospy.Time.now()
            # Set the target location
            destination.target_pose.pose.position.x = waypoint[0]
            destination.target_pose.pose.position.y = waypoint[1]
            # Set the heading
            quat = qfe(0, 0, waypoint[2])
            destination.target_pose.pose.orientation.x = quat[0]
            destination.target_pose.pose.orientation.y = quat[1]
            destination.target_pose.pose.orientation.z = quat[2]
            destination.target_pose.pose.orientation.w = quat[3]
            # Send the desired destination to the actionlib server
            rospy.loginfo("Sending waypoint (%f, %f)@%f" % tuple(waypoint))
            self.move_base_client.send_goal(destination)
            # Wait for the goal to finish
            duration = rospy.Duration(1.0)
            for x in range(60):
                self.move_base_client.wait_for_result(duration)
                if rospy.is_shutdown(): return

    def odom_callback(self, msg):
        """
        Watches for the robot's Odometry data, which is used in the path
        planning as the initial robot position.
        """
        self.robot_pose = msg

    def calculate_headings(self, path):
        """
        Calculates the headings between paths and adds them to the waypoints.
        """
        new_path = []
        for index, waypoint in enumerate(path):
            new_path.append(list(path[index]))
            # If the end, copy the previous heading
            if index == len(path)-1:
                new_path[index].append(new_path[index-1][2])
                continue
            # Calculate the angle between this waypoint and the next
            dx = path[index+1][0] - path[index][0]
            dy = path[index+1][1] - path[index][1]
            from math import atan2, pi
            heading = atan2(dy, dx)
            new_path[index].append(heading)
        return new_path

    def plan_path(self):
        """
        Uses the robot's current position to plan a path.
        """
        self.listener.waitForTransform("odom", "base_footprint", rospy.Time(), rospy.Duration(10.0))
        path = []
        indecies = [(0,0), (5,0), (5,5), (0,5), (0,0)]
        now = rospy.Time.now()
        for i, j in indecies:
            ps = PointStamped()
            ps.header.frame_id = "base_footprint"
            ps.header.stamp = now
            ps.point.x = i
            ps.point.y = j
            while True:
                try:
                    tf_ps = self.listener.transformPoint("odom", ps)
                    break
                except tf.ExtrapolationException as e:
                    continue
            path.append([tf_ps.point.x, tf_ps.point.y])
        path = self.calculate_headings(path)
        path.append(path[0])
        self.path = path

    def setup_path_following(self):
        """
        Sets up the node for following the planned path.

        Will wait until the initial robot pose is set and until
        the move_base actionlib service is available.
        """
        # Create the actionlib service
        self.move_base_client = SimpleActionClient('move_base', MoveBaseAction)
        connected_to_move_base = False
        dur = rospy.Duration(1.0)
        # Wait for the robot position
        while self.robot_pose == None:
            # Check to make sure ROS is ok still
            if rospy.is_shutdown(): return
            # Print message about the waiting
            msg = "Qualification: waiting on initial robot pose."
            rospy.loginfo(msg)
            rospy.Rate(1.0).sleep()
        # Now we should plan a path
        self.plan_path()
        rospy.loginfo("Qualification: path planning complete.")
        # Now wait for move_base
        while not connected_to_move_base:
            # Wait for the server for a while
            connected_to_move_base = self.move_base_client.wait_for_server(dur)
            # Check to make sure ROS is ok still
            if rospy.is_shutdown(): return
            # Update the user on the status of this process
            msg = "Qualification: waiting on move_base."
            rospy.loginfo(msg)
        # Now we are ready to start feeding move_base waypoints
        return

if __name__ == '__main__':
    ppn = PathPlannerNode()


# -*- coding: utf-8 -*-

from numpy import *
Nobs = 50
x_true = random.uniform(0,10, size=Nobs)
y_true = random.uniform(-1,1, size=Nobs)
alpha_true = 0.5
beta_x_true = 1.0
beta_y_true = 10.0
eps_true = 0.5
z_true = alpha_true + beta_x_true*x_true + beta_y_true*y_true
z_obs = z_true + random.normal(0, eps_true, size=Nobs)

from matplotlib import pyplot as plt
plt.figure(figsize=(12,6))
plt.subplot(1,2,1)
plt.scatter(x_true, z_obs, c=y_true, marker='o')
plt.colorbar()
plt.xlabel('X')
plt.ylabel('Z')
plt.subplot(1,2,2)
plt.scatter(y_true, z_obs, c=x_true, marker='o')
plt.colorbar()
plt.xlabel('Y')
plt.ylabel('Z')

model = """
data {
   int<lower=4> N; // Number of data points
   real x[N];      // the 1st predictor
   real y[N];      // the 2nd predictor
   real z[N];      // the outcome
}
parameters {
   real alpha;     // intercept
   real betax;     // x-slope
   real betay;     // y-slope
   real<lower=0> eps;       // dispersion
}
model {
   for (i in 1:N)
      z[i] ~ normal(alpha + betax * x[i] + betay * y[i], eps);
}"""

data = {'N':Nobs, 'x':x_true, 'y':y_true, 'z':z_obs}

import pystan
fit = pystan.stan(model_code=model, data=data, iter=10000, chains=4,n_jobs=1)

print(fit)

samples = fit.extract(permuted=True)
alpha = median(samples['alpha'])
beta_x = median(samples['betax'])
beta_y = median(samples['betay'])
eps = median(samples['eps'])

samples = array([samples['alpha'],samples['betax'], samples['betay'], samples['eps']]).T
print(samples.shape)
import corner
tmp = corner.corner(samples[:,:], labels=['alpha','betax','betay','eps'],
                truths=[alpha_true, beta_x_true, beta_y_true, eps_true])


from matplotlib import patches

from .basisFuncs import *
from .part3funcs import computeMostCommonnGrams,getnGramsWithOverMOccurences;
from .favs import loadnGramsWithFavsMinusMeanMedian,calculateAndStorenGramsWithFavsMinusMeanMedian;
from .stats import pearsonCorrelationCoefficient;
import networkx as nx

def compareTwoEpochsByNTramTopo(epoch1, epoch2,allNGrams):
	runningSimilarity = 0;
	for nGram in allNGrams:
		if nGram in epoch1:
			count1, skew1, topo1 = epoch1[nGram]
		else:
			count1, skew1, topo1 = 0,0,0;

		if nGram in epoch2:
			count2, skew2, topo2 = epoch2[nGram]
		else:
			count2, skew2, topo2 = 0,0,0;
		runningSimilarity += np.fabs(topo1-topo2);
	return runningSimilarity;

# def topNWordsOfEpochChangeOverTime(epochCount = 10):
# 	tweets = getTweetsFromDB(purePres=True, returnParams=["favCount", "cleanedText, publishTime"], orderBy="publishTime asc")

def epochSimilarityVisualization(dataFrameDict, epochCount):
	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
	fig.subplots_adjust(left=0.06, right=0.94)
	iterColor = 5592405;
	for epoch in dataFrameDict:
		iterColor += 1001;
		iterColorString = hex(iterColor)[2:]
		c = "#" + (6 - len(iterColorString)) * '0' + iterColorString;
		args = np.argsort(dataFrameDict[epoch])
		plt.plot([e for e in range(epochCount)], dataFrameDict[epoch],color=c);
		plt.scatter([e for e in range(epochCount)], dataFrameDict[epoch], label=str(epoch),color=c);
		plt.scatter(args[1], dataFrameDict[epoch][args[1]], c="red")
		plt.scatter(args[0], dataFrameDict[epoch][args[0]], c="orange")
		for i in range(len(dataFrameDict[epoch])):
			if i == args[0]:
				dataFrameDict[epoch][i] = Fore.RED + str(dataFrameDict[epoch][i]) + Style.RESET_ALL
			elif i == args[1]:
				dataFrameDict[epoch][i] = Fore.BLUE + str(dataFrameDict[epoch][i]) + Style.RESET_ALL
			else:
				dataFrameDict[epoch][i] = Fore.BLACK + str(dataFrameDict[epoch][i]) + Style.RESET_ALL
	plt.xticks([e for e in range(epochCount)]);
	plt.yticks([e for e in range(epochCount)]);
	plt.legend()
	df = pd.DataFrame(data=dataFrameDict, index=[epoch for epoch in range(epochCount)]);

	print(df);
	plt.show()

def computenGramsWithBiggestSkewDifference(nGramsForAllEpochsRegularizedSkews):
	skewsForAllNgrams = {}

	nGramsInAllSets = None
	for epoch in nGramsForAllEpochsRegularizedSkews:
		print(epoch[0])
		nGramsInThisEpoch = set([epoch[i][0] for i in range(len(epoch))])
		if nGramsInAllSets == None:
			nGramsInAllSets = nGramsInThisEpoch;
		else:
			nGramsInAllSets = set.intersection(nGramsInAllSets,nGramsInThisEpoch)

	for epoch in nGramsForAllEpochsRegularizedSkews:
		for nGramTuple in epoch:
			nGram = nGramTuple[0];
			if nGram not in nGramsInAllSets: continue;
			skew = nGramTuple[1]
			if nGram in skewsForAllNgrams:
				skewsForAllNgrams[nGram].append(skew)
			else:
				skewsForAllNgrams[nGram] = [skew]
	allNGrams = []
	skewsDifferences = []
	for nGram in skewsForAllNgrams.keys():
		allNGrams.append(nGram);
		skews = skewsForAllNgrams[nGram];
		difference = max(skews)-min(skews);
		skewsDifferences.append(difference);
	idx = np.argsort(skewsDifferences)[::-1];
	allNGramsInDifferenceOrder = [allNGrams[i] for i in idx]
	skewsDifferences = [skewsDifferences[i] for i in idx]
	print("allNGramsInDifferenceOrder: ",allNGramsInDifferenceOrder, " diffs: ",skewsDifferences)
	return allNGramsInDifferenceOrder

def relativeAbundanceNew(epochCount = 4, globalRegularization = False, minCount = 10, maxNumberOfNGrams = np.infty):

	tweets = getTweetsFromDB(purePres=True, returnParams=["favCount","cleanedText, publishTime"],orderBy="publishTime asc")
	firstPublishTime = tweets[0][2]
	lastPublishTime = tweets[-1][2]
	totalSeconds = (lastPublishTime - firstPublishTime).total_seconds()
	epochTime = totalSeconds / epochCount;
	print(epochTime);

	print("total epoch time in days: ", epochTime / (3600 * 24))
	ns = [1, 2, 3, 4]

	# assign tweets to epochs
	epochs = []
	startingIndex = 0;
	endingTime = firstPublishTime + datetime.timedelta(seconds=epochTime);
	for index, tweet in enumerate(tweets):
		if tweet[2] > endingTime:
			epochs.append(tweets[startingIndex:index]);
			startingIndex = index;
			endingTime = endingTime + datetime.timedelta(seconds=epochTime)
			if len(epochs) == epochCount - 1:
				break;
	epochs.append(tweets[startingIndex:])

	nGramsAndCountsForEachEpoch = []
	nGramsAndCountsForEachEpochRegularized = []
	allNGrams = set()

	for index, epochTweets in enumerate(epochs):
		epochCleanTextAndFavs = [tweet[0:2] for tweet in epochTweets]
		cleanTextsThisEpoch = [t[1] for t in epochCleanTextAndFavs];
		tweetsHash = hashTweets(epochCleanTextAndFavs)
		nGramsAndCountsForEpoch = []
		for n in ns:
			computeMostCommonnGrams(epochTweets, n);
			myNGrams = getnGramsWithOverMOccurences(n, minCount, tweetsHash)

			nGramsAndCountsForEpoch.extend(myNGrams);

		maxOccurencesAcrossEpoch = max([tup[1] for tup in nGramsAndCountsForEpoch])
		nGramsAndCountsForEpochRegularized = [(tup[0],tup[1]/maxOccurencesAcrossEpoch) for tup in nGramsAndCountsForEpoch];
		nGramsAndCountsForEachEpoch.append(nGramsAndCountsForEpoch);
		nGramsAndCountsForEachEpochRegularized.append(nGramsAndCountsForEpochRegularized);
		allNGrams.update([tup[0] for tup in nGramsAndCountsForEpoch])
	allNGrams = list(allNGrams)
	allNGrams.sort()
	for epoch in range(len(nGramsAndCountsForEachEpoch)):
		nGramsForEpoch = [tup[0] for tup in nGramsAndCountsForEachEpoch[epoch]];
		for nGram in allNGrams:
			if nGram not in nGramsForEpoch:
				nGramsAndCountsForEachEpoch[epoch].append((nGram,0));
				nGramsAndCountsForEachEpochRegularized[epoch].append((nGram, 0));
		nGramsAndCountsForEachEpoch[epoch].sort(key = lambda tup: tup[0])
		nGramsAndCountsForEachEpochRegularized[epoch].sort(key=lambda tup: tup[0])

	volatilityPerNgram = []
	for nGramIndex in range(len(allNGrams)):
		volatility = 0;
		for epoch in range(epochCount-1):
			volatility += np.fabs(nGramsAndCountsForEachEpochRegularized[epoch][nGramIndex][1]-nGramsAndCountsForEachEpochRegularized[epoch+1][nGramIndex][1])
		if allNGrams[nGramIndex] == "great" or allNGrams[nGramIndex] == "coronavirus":
			volatility = np.infty
		volatilityPerNgram.append(volatility);
	volatilityIDX = list(np.argsort(volatilityPerNgram))[::-1]
	for epoch in range(epochCount):
		nGramsAndCountsForEachEpochRegularized[epoch] = [nGramsAndCountsForEachEpochRegularized[epoch][i] for i in volatilityIDX][:maxNumberOfNGrams]
		nGramsAndCountsForEachEpoch[epoch] = [nGramsAndCountsForEachEpoch[epoch][i] for i in volatilityIDX][:maxNumberOfNGrams]
	allNGrams = [allNGrams[i] for i in volatilityIDX][:maxNumberOfNGrams]

	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
	fig.subplots_adjust(left=0.08, right=0.94, bottom=0.21, top=0.92)
	fig.tight_layout(pad=3.0)
	axCount = fig.add_subplot(111);
	iterColor = 11;

	ind = np.arange(len(nGramsAndCountsForEachEpochRegularized[0]))
	width = 1 / (epochCount + 1)
	rects = []
	np.random.seed(423)
	for epoch, epochCounts in enumerate(nGramsAndCountsForEachEpochRegularized):
		iterColorString = hex(iterColor)[2:]
		c = randomHexColor()
		rects1 = axCount.bar(ind + width * epoch, [tup[1] for tup in epochCounts], width, color=c)
		rects.append(rects1)

		iterColor += int(16777215 /epochCount)

	for nGramIndex in range(len(allNGrams)):
		acceptableRect = patches.Rectangle((nGramIndex-width*0.5, 0), width*epochCount, 1, linewidth=0.5, alpha=0.5, color=("#aabbcc" if nGramIndex % 2 == 0 else "#bbccaa"))
		plt.gca().add_patch(acceptableRect)

	axCount.set_title("Relative Frequency for each Year of Presidency for a Selection of n-grams",fontsize=25)
	axCount.set_ylabel('Relative Frequency of n-gram Usage',fontsize=20)
	# axCount.set_xlabel('Relative Frequency of n-gram Usage')
	axCount.set_xticks(ind + width)
	axCount.set_xticklabels(["\n"+ nGram.replace(" ","\n") for nGramIndex, nGram in enumerate(allNGrams[:maxNumberOfNGrams])]);
	axCount.tick_params(axis='both', which='major', labelsize=20)
	axCount.xaxis.grid(False)
	for tick in axCount.xaxis.get_major_ticks():
		# tick.label.set_fontsize(14)
		# specify integer or one of preset strings, e.g.
		# tick.label.set_fontsize('x-small')
		tick.label.set_rotation('vertical')
	axCount.legend((rects[i][0] for i in range(len(rects))), ("year " + str(i) for i in range(len(rects))),fontsize=20)
	plt.show()

def skewNew(epochCount = 4, globalRegularization = False, minCount = 10, maxNumberOfNGrams = np.infty):

	tweets = getTweetsFromDB(purePres=True, returnParams=["favCount","cleanedText, publishTime"],orderBy="publishTime asc")
	firstPublishTime = tweets[0][2]
	lastPublishTime = tweets[-1][2]
	totalSeconds = (lastPublishTime - firstPublishTime).total_seconds()
	epochTime = totalSeconds / epochCount;
	print(epochTime);

	print("total epoch time in days: ", epochTime / (3600 * 24))
	ns = [1, 2, 3, 4]

	# assign tweets to epochs
	epochs = []
	startingIndex = 0;
	endingTime = firstPublishTime + datetime.timedelta(seconds=epochTime);
	for index, tweet in enumerate(tweets):
		if tweet[2] > endingTime:
			epochs.append(tweets[startingIndex:index]);
			startingIndex = index;
			endingTime = endingTime + datetime.timedelta(seconds=epochTime)
			if len(epochs) == epochCount - 1:
				break;
	epochs.append(tweets[startingIndex:])

	nGramsAndSkewsForEachEpoch = []
	nGramsAndSkewsForEachEpochRegularized = []
	allNGrams = set()

	for index, epochTweets in enumerate(epochs):
		epochCleanTextAndFavs = [tweet[0:2] for tweet in epochTweets]
		epochMeanFavCount = np.mean([t[0] for t in epochTweets])
		cleanTextsThisEpoch = [t[1] for t in epochCleanTextAndFavs];
		tweetsHash = hashTweets(epochCleanTextAndFavs)
		nGramsAndSkewsForEpoch = []

		for n in ns:
			computeMostCommonnGrams(epochTweets, n);
			myNGrams = getnGramsWithOverMOccurences(n, minCount, tweetsHash)

			for nGram in myNGrams:
				tweetsWithThisNGramInThem = [t for t in epochTweets if nGram[0] in t[1]];
				favsForTweetsWithNGramInThem = [t[0] for t in tweetsWithThisNGramInThem];
				meanFavForTweetsWithNGramInThem = np.mean(favsForTweetsWithNGramInThem)
				skew = meanFavForTweetsWithNGramInThem-epochMeanFavCount
				nGramsAndSkewsForEpoch.append((nGram[0],skew));
		skewsForEpoch = [tup[1] for tup in nGramsAndSkewsForEpoch];
		minSkew = np.min(skewsForEpoch);
		maxSkew = np.max(skewsForEpoch);
		skewsForEpoch = [skew-minSkew for skew in skewsForEpoch];
		skewsForEpoch = [skew/(maxSkew-minSkew) for skew in skewsForEpoch];
		skewsForEpoch = [2*skew-1 for skew in skewsForEpoch];

		nGramsAndSkewsForEpochRegularized = [(tup[0],(2*(tup[1]-minSkew)/(maxSkew-minSkew))-1) for tup in nGramsAndSkewsForEpoch];
		nGramsAndSkewsForEpochRegularized.sort(key = lambda tup: tup[1]);
		nGramsAndSkewsForEachEpoch.append(nGramsAndSkewsForEpoch);
		nGramsAndSkewsForEachEpochRegularized.append(nGramsAndSkewsForEpochRegularized);
		allNGrams.update([tup[0] for tup in nGramsAndSkewsForEpoch])
		print("shwa",list(tup[0] for tup in nGramsAndSkewsForEpoch))
		print("zigadumbe: ",nGramsAndSkewsForEpoch);
	allNGrams = list(allNGrams)
	allNGrams.sort()
	for epoch in range(len(nGramsAndSkewsForEachEpoch)):
		nGramsForEpoch = [tup[0] for tup in nGramsAndSkewsForEachEpoch[epoch]];
		for nGram in allNGrams:
			if nGram not in nGramsForEpoch:
				nGramsAndSkewsForEachEpoch[epoch].append((nGram,0));
				nGramsAndSkewsForEachEpochRegularized[epoch].append((nGram, 0));
		nGramsAndSkewsForEachEpoch[epoch].sort(key = lambda tup: tup[0])
		nGramsAndSkewsForEachEpochRegularized[epoch].sort(key=lambda tup: tup[0])

	volatilityPerNgram = []
	for nGramIndex in range(len(allNGrams)):
		volatility = 0;
		for epoch in range(epochCount-1):
			if nGramsAndSkewsForEachEpochRegularized[epoch][nGramIndex][1] != 0:
				volatility += np.fabs(nGramsAndSkewsForEachEpochRegularized[epoch][nGramIndex][1]-nGramsAndSkewsForEachEpochRegularized[epoch+1][nGramIndex][1])
		print(allNGrams[nGramIndex]," has volatility: ",volatility)
		if allNGrams[nGramIndex] == "invisible" and False:
			volatility = np.infty
		if "prime" in allNGrams[nGramIndex]:
			volatility = 0;
		volatilityPerNgram.append(volatility);
	volatilityIDX = list(np.argsort(volatilityPerNgram))[::-1]
	for epoch in range(epochCount):
		nGramsAndSkewsForEachEpoch[epoch] = [nGramsAndSkewsForEachEpoch[epoch][i] for i in volatilityIDX][:maxNumberOfNGrams]
		nGramsAndSkewsForEachEpochRegularized[epoch] = [nGramsAndSkewsForEachEpochRegularized[epoch][i] for i in volatilityIDX][:maxNumberOfNGrams]
	allNGrams = [allNGrams[i] for i in volatilityIDX][:maxNumberOfNGrams]
	print("allNGrams: ",allNGrams)

	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
	fig.subplots_adjust(left=0.08, right=0.94, bottom=0.21, top=0.92)
	fig.tight_layout(pad=3.0)
	axCount = fig.add_subplot(111);
	iterColor = 11;

	ind = np.arange(len(nGramsAndSkewsForEachEpochRegularized[0]))
	width = 1 / (epochCount + 1)
	rects = []
	np.random.seed(423)
	for epoch, epochSkews in enumerate(nGramsAndSkewsForEachEpochRegularized):
		print("epochSkews: ",epochSkews)
		iterColorString = hex(iterColor)[2:]
		c = randomHexColor()
		print("thingoes: ",[tup[1] for tup in epochSkews]);
		rects1 = axCount.bar(ind + width * epoch, [tup[1] for tup in epochSkews], width, color=c)
		rects.append(rects1)

		iterColor += int(16777215 /epochCount)

	for nGramIndex in range(len(allNGrams)):
		acceptableRect = patches.Rectangle((nGramIndex-width*0.5, 0), width*epochCount, 1, linewidth=0.5, alpha=0.5, color=("#aabbcc" if nGramIndex % 2 == 0 else "#bbccaa"))
		plt.gca().add_patch(acceptableRect)

	axCount.set_title("Skew of Top N-Grams for each Year of Presidency",fontsize=25)
	axCount.set_ylabel('Skew for each n-gram (Regularized)',fontsize=20)
	# axCount.set_xlabel('Relative Frequency of n-gram Usage')
	axCount.set_xticks(ind + width)
	axCount.set_xticklabels(["\n"+ nGram.replace(" ","\n") for nGramIndex, nGram in enumerate(allNGrams[:maxNumberOfNGrams])]);
	axCount.tick_params(axis='both', which='major', labelsize=20)
	axCount.xaxis.grid(False)
	for tick in axCount.xaxis.get_major_ticks():
		# tick.label.set_fontsize(14)
		# specify integer or one of preset strings, e.g.
		# tick.label.set_fontsize('x-small')
		tick.label.set_rotation('vertical')
	axCount.legend((rects[i][0] for i in range(len(rects))), ("year " + str(i) for i in range(len(rects))),fontsize=20)
	plt.show()

	###old
	iterColor = 11;
	for nGramIndex in range(maxNumberOfNGrams):
		nGram = allNGrams[nGramIndex];
		skews = [];
		for epochIndex in range(len(nGramsAndSkewsForEachEpochRegularized)):
			skew = nGramsAndSkewsForEachEpochRegularized[epochIndex][nGramIndex];
			skews.append(skew[1]);
		# c = randomHexColor()
		iterColorString = hex(iterColor)[2:]
		c = "#" + (6 - len(iterColorString)) * '0' + iterColorString;
		print(c)
		print("skews: ",skews)

		plt.plot(list(range(len(skews))), skews, color=c, label=nGram);
		plt.scatter(list(range(len(skews))), skews, color=c);

		iterColor += int(16777215 / maxNumberOfNGrams)

	plt.gca().set_title("Skew for a Selection of Highly Skews n-grams of Each Year of Trump's Presidency")
	plt.gca().set_xticks(list(range(0, epochCount)))
	plt.gca().set_xticklabels(["Year " + str(i + 1) for i in range(0, epochCount)])
	plt.gca().set_yticks([-0.8, 0, 0.8]);
	plt.gca().set_yticklabels(["Least\nPopular", "Neutral", "Most\nPopular"])
	plt.gca().legend(loc="lower middle")  # custom_lines, ['Cold', 'Medium', 'Hot'],
	# axCount.set_ylabel("Relative Abundance of Most Frequent n-grams")

	plt.show()

def compareVocabOverTime(epochCount=5,globalRegularization = False,minCount = 10, maxNumberOfNGrams = np.infty, allGraphsInOnePlot = False):
	# epochCount = 50;
	tweets = getTweetsFromDB(purePres=True, returnParams=["favCount","cleanedText, publishTime"],orderBy="publishTime asc")
	quickieGrams = ["coronavirus", "great", "fake news"]
	quickie = False

	firstPublishTime = tweets[0][2]
	lastPublishTime = tweets[-1][2]
	totalSeconds = (lastPublishTime-firstPublishTime).total_seconds()
	epochTime = totalSeconds/epochCount;
	print(epochTime);

	print("total epoch time in days: ",epochTime/(3600*24))
	ns = [1, 2, 3, 4]

	#assign tweets to epochs
	epochs = []
	startingIndex = 0;
	endingTime = firstPublishTime + datetime.timedelta(seconds = epochTime);
	for index,tweet in enumerate(tweets):
		if tweet[2] > endingTime:
			epochs.append(tweets[startingIndex:index]);
			startingIndex = index;
			endingTime = endingTime + datetime.timedelta(seconds = epochTime)
			if len(epochs) == epochCount-1:
				break;

	epochs.append(tweets[startingIndex:])
	nGramsForAllEpochs = []

	for index, epochTweets in enumerate(epochs):

		epochCleanTextAndFavs = [tweet[0:2] for tweet in epochTweets]
		tweetsHash = hashTweets(epochCleanTextAndFavs)

		nGramsForEpoch = []
		for n in ns:
			computeMostCommonnGrams(epochTweets, n);
			myNGrams = getnGramsWithOverMOccurences(n, minCount, tweetsHash)
			nGramsForEpoch.extend(myNGrams)
		nGramsForEpoch.sort(key=lambda tuple: tuple[1], reverse=True)

		calculateAndStorenGramsWithFavsMinusMeanMedian(ns, epochCleanTextAndFavs, False)
		_,nGramsWithFavsMinusMedian = loadnGramsWithFavsMinusMeanMedian(ns, tweetsHash)
		# if index == len(epochs)-1:
		# 	plt.scatter([i for i in range(len(nGramsWithFavsMinusMedian))],[tup[nGramStorageIndicesSkew["skew"]] for tup in nGramsWithFavsMinusMedian])
		# 	plt.plot([0, len(nGramsWithFavsMinusMedian)],[0,0]);
		# 	plt.show()
		nGramsWithFavsMinusMeanMedianDict = {}
		for nGram in nGramsWithFavsMinusMedian:
			nGramsWithFavsMinusMeanMedianDict[nGram[0]] = nGram
		for i in range(len(nGramsForEpoch)):
			e1 = nGramsForEpoch[i][0]
			e2 = nGramsForEpoch[i][1]
			medianDictEntry = nGramsWithFavsMinusMeanMedianDict[nGramsForEpoch[i][0]]
			e3 = medianDictEntry[nGramStorageIndicesSkew["skew"]]
			if e3 > 0:
				print("n00b")
			nGramsForEpoch[i] = (e1,e2,e3)
		nGramsForAllEpochs.append(nGramsForEpoch);

	print(Fore.RED,"\n\n\nnow sorted by skew\n\n\n",Style.RESET_ALL)

	for epoch in range(len(nGramsForAllEpochs)):
		nGramsForAllEpochs[epoch].sort(key = lambda tup: tup[2], reverse = True)

	minimumCountThreshholds = [5]#[5, 10, 50, 100, 200, 400, 800];
	for minimumCountThreshhold in minimumCountThreshholds:
		print(Fore.MAGENTA, "minimumCountThreshhold: ", minimumCountThreshhold, Style.RESET_ALL, "\n\n")
		for epoch in range(epochCount - 1):
			if epoch == epochCount-2:
				print("n00b:");
			thisEpochsGrams = [tup[0] for tup in nGramsForAllEpochs[epoch]][:minimumCountThreshhold]
			thisEpochsGramsFull = nGramsForAllEpochs[epoch][:minimumCountThreshhold]
			nextEpochsGrams = [tup[0] for tup in nGramsForAllEpochs[epoch + 1]][:minimumCountThreshhold]
			nextEpochsGramsFull = nGramsForAllEpochs[epoch + 1][:minimumCountThreshhold]
			intersectionSet = np.intersect1d(thisEpochsGrams, nextEpochsGrams);

			print("epoch: ", epoch, " has ", len(nGramsForAllEpochs[epoch]), " ngrams of count at least",minCount,"; its most popular minimumCountThreshhold bigrams were shared to ", len(intersectionSet) / len(thisEpochsGrams), " extent with epoch ", epoch + 1);
			for i in range(minimumCountThreshhold):
				try:
					print(thisEpochsGramsFull[i], nextEpochsGramsFull[i])
				except:
					break;
			print()

		intersectionSet = np.intersect1d([tup[0] for tup in nGramsForAllEpochs[0]][:minimumCountThreshhold], [tup[0] for tup in nGramsForAllEpochs[-1]][:minimumCountThreshhold]);
		print("epoch: ", 0, " has ", len(nGramsForAllEpochs[0]), " ngrams of count at least ",minCount,"; its most popular minimumCountThreshhold bigrams were shared to ", len(intersectionSet) / len([tup[0] for tup in nGramsForAllEpochs[0]][:minimumCountThreshhold]), " extent with epoch ", len(epochs) - 1);
	# print(nGramsForAllEpochs);

	'''
	format of nGramsForAllEpochs: [[(nGram, count, skew),...],...]
	'''

	maxCounts = []
	maxAbsSkews = []

	for epochGrams in nGramsForAllEpochs:
		maxCount = max([tup[1] for tup in epochGrams]);
		maxAbsSkew = max([np.fabs(tup[2]) for tup in epochGrams]);
		maxCounts.append(maxCount);
		maxAbsSkews.append(maxAbsSkew)
	maxCount = max(maxCounts);
	maxAbsSkew = max(maxAbsSkews);
	print(maxCount);
	print(maxAbsSkew);

	#TODO: regularize count and skew so that each is between [0,1] and [-1,1] respectively
	nGramsForAllEpochsRegularized = []
	nGramsForAllEpochsRegularizedTopos = []
	nGramsForAllEpochsRegularizedCounts = []
	nGramsForAllEpochsRegularizedSkews = []
	allNgrams = set()
	nGramsInAllEpochs = set([tup[0] for tup in nGramsForAllEpochs[0]]);
	for i in range(len(nGramsForAllEpochs)):
		if i == len(nGramsForAllEpochs)-1:
			print("n00b")
		allNgrams.update([tup[0] for tup in nGramsForAllEpochs[i]])
		nGramsInAllEpochs.intersection_update([tup[0] for tup in nGramsForAllEpochs[i]])

		if globalRegularization:
			nGramsReg = [(tup[0], tup[1] / maxCount, tup[2] / maxAbsSkew, (tup[1] / maxCount) * (tup[2] / maxAbsSkew)) for tup in nGramsForAllEpochs[i]];
		else:
			nGramsReg = [(tup[0], tup[1] / maxCounts[i], tup[2] / maxAbsSkews[i], (tup[1] / maxCounts[i]) * (tup[2] / maxAbsSkews[i])) for tup in nGramsForAllEpochs[i]];

		nGramsForAllEpochsRegularized.append( {tup[0]: tup[1:] for tup in nGramsReg})

		nGramsRegTopos = [(tup[0],np.fabs(tup[3])) for tup in nGramsReg]
		nGramsRegTopos.sort(key = lambda tup: tup[1], reverse=True);
		nGramsForAllEpochsRegularizedTopos.append(nGramsRegTopos)

		nGramsRegCounts = [(tup[0], tup[1]) for tup in nGramsReg]
		nGramsRegCounts.sort(key=lambda tup: tup[1], reverse=True);
		nGramsForAllEpochsRegularizedCounts.append(nGramsRegCounts)

		nGramsRegSkews = [(tup[0], np.fabs(tup[2])) for tup in nGramsReg]
		nGramsRegSkews.sort(key=lambda tup: tup[1], reverse=True);
		nGramsForAllEpochsRegularizedSkews.append(nGramsRegSkews)

	nGramsWithBiggestSkewDifference = computenGramsWithBiggestSkewDifference(nGramsForAllEpochsRegularizedSkews)

	topnFromEachEpoch = 10;
	nGramTopoDict = {nGram:[] for nGram in nGramsInAllEpochs}
	nGramTopoDictAll = {nGram : [] for nGram in allNgrams}
	nGramSkewDictAll = {nGram: [] for nGram in allNgrams}
	nGramCountDictAll = {nGram: [] for nGram in allNgrams}

	for nGram in allNgrams:
		for epoch in range(len(nGramsForAllEpochs)):
			if epoch == len(nGramsForAllEpochs)-1:
				print("n00b")
			topn_nGramsByTopo = [tup[0] for tup in nGramsForAllEpochsRegularizedTopos[epoch]][:topnFromEachEpoch]#
			topn_nGramsByCount = [tup[0] for tup in nGramsForAllEpochsRegularizedCounts[epoch]][:topnFromEachEpoch]#

			topn_nGramsBySkew = [tup[0] for tup in nGramsForAllEpochsRegularizedSkews[epoch]][:topnFromEachEpoch]#
			# topn_nGramsBySkew = nGramsWithBiggestSkewDifference[:topnFromEachEpoch]

			if nGram in nGramsInAllEpochs:
				nGramTopoDict[nGram].append(nGramsForAllEpochsRegularized[epoch][nGram][2])
			if nGram in topn_nGramsByTopo:
				nGramTopoDictAll[nGram].append(nGramsForAllEpochsRegularized[epoch][nGram][2])
			else:
				nGramTopoDictAll[nGram].append(0);

			if nGram in topn_nGramsBySkew:
				nGramSkewDictAll[nGram].append(nGramsForAllEpochsRegularized[epoch][nGram][1])
			else:
				nGramSkewDictAll[nGram].append(0);

			if nGram in topn_nGramsByCount or nGram in nGramsForAllEpochsRegularized[epoch]:
				nGramCountDictAll[nGram].append(nGramsForAllEpochsRegularized[epoch][nGram][0])
			else:
				nGramCountDictAll[nGram].append(0);

	dataFrameDict = {}
	for epoch1 in range(epochCount):
		dataFrameDict[epoch1] = []
		for epoch2 in range(epochCount):
			epoch1Grams = nGramsForAllEpochsRegularized[epoch1]
			epoch2Grams = nGramsForAllEpochsRegularized[epoch2]
			similarity = round(compareTwoEpochsByNTramTopo(epoch1Grams, epoch2Grams, allNgrams), 2);
			dataFrameDict[epoch1].append(similarity)

	if False:
		epochSimilarityVisualization(dataFrameDict,epochCount)

	if True:
		fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
		fig.subplots_adjust(left=0.06, right=0.94, bottom=0.02, top = 0.97)
		fig.tight_layout(pad=3.0)

		if allGraphsInOnePlot:
			axInEveryEpoch = fig.add_subplot(411)
			axTopo = fig.add_subplot(412)
			axCount = fig.add_subplot(413)
			axSkew = fig.add_subplot(414)
			axes = [axInEveryEpoch, axTopo, axSkew, axCount]
		else:
			axCount = fig.add_subplot(111);
			# axInEveryEpoch = fig.add_subplot(111);

		# # iterColor = 11;
		# # for nGram,topos in nGramTopoDict.items():
		# # 	iterColorString = hex(iterColor)[2:]
		# # 	c = "#"+(6-len(iterColorString))*'0'+iterColorString;
		# # 	print(c)
		# #
		# # 	axInEveryEpoch.plot(list(range(len(topos))),topos,color=c,label = nGram);
		# # 	axInEveryEpoch.scatter(list(range(len(topos))), topos, color=c);
		# # 	iterColor += int(16777215 / len(nGramTopoDict))
		# #
		# # if not allGraphsInOnePlot:
		# # 	axInEveryEpoch.set_title("topo for ngrams appearing in every epoch")
		# # 	axInEveryEpoch.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axInEveryEpoch, nGramsForAllEpochsRegularized))  #
		# # 	plt.show()
		# # 	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
		# # 	fig.subplots_adjust(left=0.06, right=0.94, bottom=0.02, top=0.97)
		# # 	fig.tight_layout(pad=3.0)
		# # 	axTopo = fig.add_subplot(111);
		# #
		# #
		# #
		# # iterColor = 11;
		# # for nGram, topos in nGramTopoDictAll.items():
		# # 	if not quickie or nGram in quickieGrams:
		# #
		# # 		# c = randomHexColor()
		# # 		iterColorString = hex(iterColor)[2:]
		# # 		c = "#" + (6 - len(iterColorString)) * '0' + iterColorString;
		# # 		print(c)
		# #
		# #
		# # 		axTopo.plot(list(range(len(topos))), topos, color=c, label=nGram);
		# # 		axTopo.scatter(list(range(len(topos))), topos, color=c);
		# # 		iterColor += int(16777215 / len(nGramTopoDictAll))
		# # if not allGraphsInOnePlot:
		# # 	axTopo.set_title("topo for top " + str(topnFromEachEpoch) + " ngrams of each epoch")
		# # 	axTopo.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axTopo, nGramsForAllEpochsRegularized))  #
		# # 	plt.show()
		# # 	# fig = plt.figure()
		# # 	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
		# # 	fig.subplots_adjust(left=0.08, right=0.94, bottom=0.11, top=0.92)
		# # 	fig.tight_layout(pad=3.0)
		# # 	axCount = fig.add_subplot(111);
		#
		# fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
		# fig.subplots_adjust(left=0.08, right=0.94, bottom=0.11, top=0.92)
		# fig.tight_layout(pad=3.0)
		# axCount = fig.add_subplot(111);
		# iterColor = 11;
		# nGramCountDictItemsList = list(nGramCountDictAll.items())[:maxNumberOfNGrams];
		# nGramCountDictNGrams = [tup[0] for tup in nGramCountDictItemsList]
		# nGramCountDictCounts = [tup[1] for tup in nGramCountDictItemsList]
		# nGramCountDictCountsPerEpoch = []
		# for epoch in range(len(nGramCountDictCounts[0])):
		# 	nGramCountDictCountsThisEpoch = [nGramCounts[epoch] for nGramCounts in nGramCountDictCounts]
		# 	nGramCountDictCountsPerEpoch.append(nGramCountDictCountsThisEpoch)
		# ind = np.arange(len(nGramCountDictNGrams))
		# width = 1/(len(nGramCountDictCountsPerEpoch)+1)
		# rects = []
		# for epoch, epochCounts in enumerate(nGramCountDictCountsPerEpoch):
		# 	rects1 = axCount.bar(ind + width*epoch, epochCounts, width, color='r')
		# 	rects.append(rects1)
		# 	axCount.set_ylabel('Scores')
		# axCount.set_xticks(ind + width)
		# axCount.set_xticklabels(nGramCountDictNGrams)
		# axCount.legend((rects[i][0] for i in range(len(rects))), ("epoch " + str(i) for i in range(len(rects))))
		#
		# if not allGraphsInOnePlot:
		# 	plt.show()
		# 	fig = plt.figure()#num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
		# 	#fig.subplots_adjust(left=0.06, right=0.94, bottom=0.02, top=0.97)
		# 	#fig.tight_layout(pad=3.0)
		# 	axSkew = fig.add_subplot(111);
	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
	fig.subplots_adjust(left=0.08, right=0.94, bottom=0.11, top=0.92)
	fig.tight_layout(pad=3.0)
	axSkew = fig.add_subplot(111);
	iterColor = 11;
	for nGram, skews in list(nGramSkewDictAll.items()):
		if not quickie or nGram in quickieGrams:
			# c = randomHexColor()
			iterColorString = hex(iterColor)[2:]
			c = "#" + (6 - len(iterColorString)) * '0' + iterColorString;
			print(c)

			if not np.any(skews):
				axSkew.plot(list(range(len(skews))), skews, color=c);
				axSkew.scatter(list(range(len(skews))), skews, color=c);
			else:
				axSkew.plot(list(range(len(skews))), skews, color=c, label=nGram);
				axSkew.scatter(list(range(len(skews))), skews, color=c);

			iterColor += int(16777215 / len(nGramCountDictAll))

	if not allGraphsInOnePlot:

		axSkew.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axSkew, nGramsForAllEpochsRegularized))  #
		if epochCount != 4:
			axSkew.set_title("Skew for Highest Skewed " + str(topnFromEachEpoch) + " ngrams of each Month")
			axSkew.set_xticks(list(range(0,epochCount,3)))
			axSkew.set_xticklabels(["Month "+str(i+1) for i in range(0,epochCount,3)])
		else:
			axSkew.set_title("Skew for Highest Skewed " + str(topnFromEachEpoch) + " ngrams of each Year")
			axSkew.set_xticks(list(range(0, epochCount)))
			axSkew.set_xticklabels(["Year " + str(i + 1) for i in range(0, epochCount)])
		axSkew.set_yticks([-0.8, 0,0.8]);
		axSkew.set_yticklabels(["Least\nPopular", "Neutral","Most\nPopular"])
		axSkew.legend(loc="lower middle")#custom_lines, ['Cold', 'Medium', 'Hot'],
		# axCount.set_ylabel("Relative Abundance of Most Frequent n-grams")

		plt.show()

	if allGraphsInOnePlot:
		for ax in axes:
			ax.plot([0, epochCount-1], [0, 0],'--', linewidth=5)
		axInEveryEpoch.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axInEveryEpoch,nGramsForAllEpochsRegularized))#
		axCount.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axCount,nGramsForAllEpochsRegularized))#
		axTopo.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axTopo,nGramsForAllEpochsRegularized))#
		axSkew.figure.canvas.mpl_connect("motion_notify_event", lambda event: on_plot_hover(event, axSkew,nGramsForAllEpochsRegularized))#

		axSkew.set_title("skews top " + str(topnFromEachEpoch) + " ngrams of each epoch")
		axCount.set_title("counts top " + str(topnFromEachEpoch) + " ngrams of each epoch")
		axTopo.set_title("topo for top " + str(topnFromEachEpoch) + " ngrams of each epoch")
		axInEveryEpoch.set_title("topo for ngrams appearing in every epoch")

		plt.show()

	print(nGramsInAllEpochs)

	#TODO: devise a measurement, topo, that combines count and skew to determine the most popular tweets in both count and skew
	#TODO: compare the topo between epochs
	#TODO: visualize the similarities between epochs somehow lol

	#goal: model changing vocabulary over time in order to have a more accurate idea of which terms are in trend.
	#bemerkung: there is great shifting of the tides, words shift in and out of grace
	#conclusion: for training, we should (use only bzw. highly prioritise) nearest training data/last relevant water mark
	#for example: he hasn't tweeted about crooked hillary with as much virility as in 2016 but everytime he does it has a strong skew
	#however: great used to have a great skew, now a shitty one
	#fazit: he is tweeting more and more lol

def tweetCountOverTime(daysPerMonth=7,globalRegularization = False):
	# epochCount = 50;
	tweets = getTweetsFromDB(purePres=True, conditions=["president"],returnParams=["publishTime"],orderBy="publishTime asc")

	fig = plt.figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='k')
	fig.subplots_adjust(left=0.06, right=0.94)

	firstPublishTime = tweets[0][0]
	lastPublishTime = tweets[-1][0]
	totalSeconds = (lastPublishTime-firstPublishTime).total_seconds()
	epochTime = daysPerMonth*60*60*24;
	print(epochTime);
	print("total epoch time in days: ",epochTime/(3600*24))
	ns = [1, 2, 3, 4]

	#assign tweets to epochs
	epochs = []
	startingIndex = 0;
	endingTime = firstPublishTime + datetime.timedelta(seconds = epochTime);
	dates = []

	for index,tweet in enumerate(tweets):
		if tweet[0] > endingTime:
			epochs.append(tweets[startingIndex:index]);
			startingIndex = index;
			endingTime = endingTime + datetime.timedelta(seconds = epochTime)
			dates.append(str(tweet[0])[:4])
		if index == len(tweets)-1:
			break;

	# epochs.append(tweets[startingIndex:])
	dates.append(str(tweets[-1][0])[:4])

	showEveryNthDate = 2;
	newDates = dates.copy()

	dates = [date if index in [5+int(len(dates)/3.3 * x) for x in list(range(4))] else "" for index, date in enumerate(dates[1:])];
	plt.gca().xaxis.grid(False)
	print("dizzle: ",dates)
	epochCount = len(epochs);

	x = list(range(epochCount));
	y = [len(epoch)/daysPerMonth for epoch in epochs];

	# lastYear = 2012
	# firstIndicesOfYear = []
	# ticksIndiciesOfYear = []
	# for index, month in enumerate(dates):
	# 	if month.year >= lastYear:
	# 		firstIndicesOfYear.append(index)
	# 		ticksIndiciesOfYear.append(str(lastYear));
	# 		lastYear += 1;

	plt.xticks([i for i in range(len(dates)) if len(dates[i]) > 1],[date for date in dates if len(date)>1],fontsize=15);
	plt.yticks(fontsize=15)
	fitLineCoefficients = np.polyfit(x, y, 1, full=False)
	slope = fitLineCoefficients[0];
	intercept = fitLineCoefficients[1]
	pearson = pearsonCorrelationCoefficient(y, [slope*i+intercept for i in x])
	plt.ylabel("Tweets per Day",fontsize=15)
	# plt.xlabel("Time")
	plt.plot(x,[slope*i+intercept for i in x],label="ols")
	plt.title("Tweets per Day Increases with Time (Correlation Coefficient "+str(round(pearson,2))+")",fontsize=20)
	plt.scatter(x,y);

	plt.show();
	exit();


from flask import Flask, render_template, request,redirect,url_for
import jsonify
import requests
import pickle
import numpy as np
import sklearn
app = Flask(__name__)
model = pickle.load(open('decision_tree_regression_model.pkl', 'rb'))
@app.route('/',methods=['GET'])
def Home():
    return render_template('home.html')

@app.route('/result',methods=['POST'])
def result():
    return render_template('result.html')

@app.route("/predict", methods=['POST'])
def predict():
	temp_array = list()
	if request.method=='POST':

	  # Batting Team
	  batting_team=request.form['batting_team']
	  if batting_team == 'Chennai Super Kings':
	    temp_array = temp_array + [1,0,0,0,0,0,0,0]
	  elif batting_team == 'Delhi Daredevils':
	    temp_array = temp_array + [0,1,0,0,0,0,0,0]
	  elif batting_team == 'Kings XI Punjab':
	    temp_array = temp_array + [0,0,1,0,0,0,0,0]
	  elif batting_team == 'Kolkata Knight Riders':
	    temp_array = temp_array + [0,0,0,1,0,0,0,0]
	  elif batting_team == 'Mumbai Indians':
	    temp_array = temp_array + [0,0,0,0,1,0,0,0]
	  elif batting_team == 'Rajasthan Royals':
	    temp_array = temp_array + [0,0,0,0,0,1,0,0]
	  elif batting_team == 'Royal Challengers Bangalore':
	    temp_array = temp_array + [0,0,0,0,0,0,1,0]
	  elif batting_team == 'Sunrisers Hyderabad':
	    temp_array = temp_array + [0,0,0,0,0,0,0,1]

	  # Bowling Team
	  bowling_team=request.form['bowling_team']
	  if bowling_team == 'Chennai Super Kings':
	    temp_array = temp_array + [1,0,0,0,0,0,0,0]
	  elif bowling_team == 'Delhi Daredevils':
	    temp_array = temp_array + [0,1,0,0,0,0,0,0]
	  elif bowling_team == 'Kings XI Punjab':
	    temp_array = temp_array + [0,0,1,0,0,0,0,0]
	  elif bowling_team == 'Kolkata Knight Riders':
	    temp_array = temp_array + [0,0,0,1,0,0,0,0]
	  elif bowling_team == 'Mumbai Indians':
	    temp_array = temp_array + [0,0,0,0,1,0,0,0]
	  elif bowling_team == 'Rajasthan Royals':
	    temp_array = temp_array + [0,0,0,0,0,1,0,0]
	  elif bowling_team == 'Royal Challengers Bangalore':
	    temp_array = temp_array + [0,0,0,0,0,0,1,0]
	  elif bowling_team == 'Sunrisers Hyderabad':
	    temp_array = temp_array + [0,0,0,0,0,0,0,1]

	  # Overs, Runs, Wickets, Runs_in_prev_5, Wickets_in_prev_5
	  overs=float(request.form['overs'])
	  runs=int(request.form['runs'])
	  wickets=int(request.form['wickets'])
	  runs_in_prev_5=int(request.form['runs_in_prev_5'])
	  wickets_in_prev_5=int(request.form['wickets_in_prev_5'])
	  temp_array = temp_array + [overs, runs, wickets, runs_in_prev_5, wickets_in_prev_5]

	  # Converting into numpy array
	  temp_array = np.array([temp_array])

	  # Prediction
	  final_score= int(model.predict(temp_array)[0])
	  return render_template('result.html',final_score=final_score)

if __name__=="__main__":
    app.run(debug=True)


import numpy as np
import math

eps = 1e-12

def read_input(filename):
    f = open(filename, 'r')

    p = int(f.readline())
    n = int(f.readline())

    temp = p
    if p > n:
        temp = p - (p - n)

    a = list()
    for i in range(temp):
        temp = [float(elem) for elem in f.readline().strip('\n').split(' ')]
        for j in range(i+1):
            a.append(temp[j])

    return p, n, a

def read_normal(filename):
    f = open(filename, 'r')

    p = int(f.readline())
    n = int(f.readline())

    a = list()
    for i in range(p):
        temp = [float(elem) for elem in f.readline().strip('\n').split(' ')]
        a.append(temp)

    return p, n, a

# def get_matrix_pos(pos, length):
#     k = 0
#     for i in range(1, length):
#         if (i*(i+1))/2 > pos:
#             k = i - 1
#             break
#     return k, int(pos - (k*(k+1))/2)

def v(i, j):
    return int((i*(i+1))/2 + j)

def get_pq(a, n):
    curr = -99999
    i_m = 0
    j_m = 0
    for i in range(n):
        for j in range(i+1):
            new = abs(a[v(i, j)])
            if i != j and new > curr:
                curr = new
                i_m = i
                j_m = j

    return i_m, j_m

def get_tetha(a, p, q):
    alpha = (a[v(p, p)] - a[v(q, q)]) / 2*a[v(p, q)]

    t = 0
    if alpha >= 0:
        t = -alpha + math.sqrt(alpha**2 + 1)
    else:
        t = -alpha - math.sqrt(alpha**2 + 1)

    c = 1 / math.sqrt(1 + t**2)
    s = t / math.sqrt(1 + t**2)

    return c, s

def is_diagonal(a, n):
    for i in range(n):
        for j in range(i+1):
            if i != j and a[v(i, j)] != 0:
                return False
    return True

def rotation_matrix(n, p, q, c, s):
    r = np.identity(n)

    r[p][p] = r[q][q] = c
    r[p][q] = s
    r[q][p] = -s

    return r

def multiply(m, a, n):
    res = np.zeros((n,n))
    for i in range(n):
        for j in range(n):
            for k in range(n):
                # daca e deasupra diagonalei principale, apelam la simetrie
                if j > k:
                    res[i][j] += m[i][k] * a[v(j, k)]
                else:
                    res[i][j] += m[i][k] * a[v(k, j)]
    return res

def recalc_a(a, n, p, q, c, s):
    # b = list()
    # b[:] = a[:]

    # for j in range(n):
    #     if j != p and j != q:
    #         b[v(p, j)] = c*a[v(p, j)] + s*a[v(q, j)]

    # for j in range(n):
    #     if j != p and j != q:
    #         b[v(q, j)] = -s*a[v(j, p)] + c*a[v(q, j)]
    #         b[v(j, q)] = -s*a[v(j, p)] + c*a[v(q, j)]

    # for j in range(n):
    #     if j != p and j != q:
    #         b[v(j, p)] = a[v(p, j)]

    # b[v(p, p)] = a[v(p, p)] + t*a[v(p, q)]
    # b[v(q, q)] = a[v(q, q)] - t*a[v(p, q)]
    # b[v(p, q)] = 0.0
    # b[v(q, p)] = 0.0

    # for i in range(len(b)):
    #     if -eps < b[i] < eps:
    #         b[i] = 0.0

    # return b

    r = rotation_matrix(n, p, q, c, s)
    ra = np.array(multiply(r, a, n))
    rart = np.dot(ra, r.T)

    res = list()
    for i in range(n):
        for j in range(i+1):
            res.append(rart[i][j])

    return res

def recalc_u(u, n, p, q, c, s):
    r = rotation_matrix(n, p, q, c, s)
    u = np.dot(u, r.T)
    # cu = np.zeros((n,n))
    # cu[:,:] = u[:,:]

    # for i in range(n):
    #     u[i][p] = c*u[i][p] + s*u[i][q]

    # for i in range(n):
    #     u[i][q] = -s*u[i][p] + c*u[i][q]

    return u

def jacobi(a, n):
    k = 0
    kmax = 10000
    u = np.identity(n)
    p, q = get_pq(a, n)
    c, s = get_tetha(a, p, q)

    while (not is_diagonal(a, n)) and k <= kmax:
        a = recalc_a(a, n, p, q, c, s)
        u = recalc_u(u, n, p, q, c, s)
        p, q = get_pq(a, n)
        c, s = get_tetha(a, p, q)
        k += 1

    return a, u

def diag(a, n):
    d = list()
    for i in range(n):
        d.append(a[v(i, i)])
    return d

def get_lambdas(values, n):
    res = np.identity(n)
    for i in range(n):
        res[i][i] = values[i]
    return res

def lu_decomp(a):
    n = a.shape[0]
    u = a.copy()
    l = np.eye(n, dtype=np.double)

    for i in range(n):
        factor = u[i+1:, i] / u[i, i]
        l[i+1:, i] = factor
        u[i+1:] -= factor[:, np.newaxis] * u[i]

    return l, u

def cholesky(a):
    a = np.array(a)
    l = np.linalg.cholesky(a)
    print(l, l.T)
    next_a = np.dot(l.T, l)

    while np.linalg.norm(next_a - a) > eps:
        l = np.linalg.cholesky(a)
        next_a = np.dot(l.T, l)
        a = next_a

    return next_a

def get_si(s, p, n):
    si = np.zeros((n, p))
    for i in range(len(s)):
        si[i][i] = 1 / s[i]
    return si

if __name__ == '__main__':
    p, n, a = read_input('input.txt')
    pn, nn, an = read_normal('input.txt')

    if p == n:
        # JACOBI METHOD
        values, vectors = jacobi(a, n)

        values = diag(values, n)
        print('Values:')
        print(values)
        print()

        print('Vectors:')
        for i in range(n):
            print(vectors[:,i])
        print()

        au = multiply(vectors.T, a, n)
        uv = get_lambdas(values, n)

        print(au)
        print()
        print(uv)
        print()

        print('Error:', np.linalg.norm(au - uv))

        # CHOLESKY METHOD
        print(an)
        try:
            chol = cholesky(an)
        except:
            print('Something went wrong')
    elif p > n:
        # SINGULAR VALUE DECOMPOSITION
        an = np.array(an)
        print(an)
        print()

        u, s, vh = np.linalg.svd(an)

        print('Singular values:', s)
        print()

        print('Matrix rank:', len(s))
        print()

        print('Condition number:', max(s) / min(s))
        print()

        print('Moore-Penrose pseudo-inverse:')
        si = get_si(s, pn, nn)
        ai = np.dot(np.dot(vh, si), u.T)
        print(ai)
        print()

        aj = np.dot(np.linalg.inv(np.dot(an.T, an)), an.T)
        print(aj)
        print('Norm:', np.linalg.norm(ai - aj, 1))


#!/usr/bin/python3
from urllib.parse import urljoin
import requests
import random

base_api_dns = 'http://127.0.0.1:8000/api/'
payload = {}
headers = {'Authorization': 'Bearer MWZiYmM5YjE3MWEwNzM0MzlmMjQwYjI0ZDE5MjMzZDMxZDA3YzcyOA=='}
response_halls = requests.get(urljoin(base_api_dns, 'halls/'),
                              headers=headers,
                              data=payload)
halls = random.choice(response_halls.json()['results'])
response_seats = requests.get(urljoin(base_api_dns, 'seats/'),
                              headers=headers,
                              data=payload,
                              params=f"halls__id={halls['id']}")

seats = response_seats.json()['results']
random_nr_of_seats = random.randint(1, len(seats))

for seat in seats[0:random_nr_of_seats]:
    if seat['halls'] == halls['id']:
        payload = {'is_occupied': not seat['is_occupied']}
        requests.put(urljoin(base_api_dns, f"seats/{seat['id']}/"), headers=headers, data=payload)


from pathlib import Path
import pickle
import spike2py_preprocess as s2pp
import spike2py_reflex as s2pr

def study(study_path: Path):
    """Extract reflexes for all trials of all subject for a study

    Parameters
    ----------
    study_path: Path to study folder. Folder is to be structured as indicated in spike2py_preprocess docs
    """
    info = s2pr.info.Info(study_path)
    for subject_ in info.subjects:
        subject(subject_, study_path=study_path)

def subject(subject_: str, study_path: Path):
    """Process all trials and their sections for a given subject

    Parameters
    ----------
    subject_: Subject id, which also has to be the name of their folder
    study_path: Path to study folder
    info: Study-related details; required if called internally
    plot: Whether to generate plots

    """
    print(f"\tsubject: {subject_}")
    subject_folder = study_path / subject_ / "proc"
    for item in subject_folder.iterdir():
        if item.is_file():
            if item.stem.split("_")[-1] == "reflexes":
                print(f"\t\t plotting {item.stem}")
                with open(item, "rb") as f:
                    section = pickle.load(f)

                s2pr.plot.reflexes(section)
                s2pr.plot.outcomes(section)

    subject_path_reflexes = study_path / subject_ / "figures" / "reflexes"
    s2pp.utils.merge_pdfs(subject_path_reflexes)

    subject_path_outcomes = study_path / subject_ / "figures" / "outcomes"
    s2pp.utils.merge_pdfs(subject_path_outcomes)


from django import template
from django.core.mail import send_mail

from config import settings

register = template.Library()
@register.simple_tag
def verification(user):

    send_mail(
        subject="Код активации",
        message=f"Ваш код активации: {user.ver_code}",
        from_email=settings.EMAIL_HOST_USER,
        recipient_list=[user.email],
    )


from util.MysqlUtil import MysqlUtil
from model.person import Person

ip = "localhost"
user = "root"
pwd = ""
database = "momapp"
table = "Persons"
conn = ""

def open_connect_mysql():
    mysql_util = MysqlUtil(ip, user, pwd, database, table, conn)
    mysql_util.mysql_connect()
    return mysql_util

def close_connect_mysql(mysql_util):
    mysql_util.mysql_close()

def get_person_list(start, pagesize):
    mysql_util = open_connect_mysql()
    sql = "select id_p,firstname,lastname,address,city,telephone from Persons limit " + start + "," + pagesize
    rows = mysql_util.mysql_query(sql)
    close_connect_mysql(mysql_util)
    return rows

def get_person(id_p):
    mysql_util = open_connect_mysql()
    sql = "select * from Persons where id_p= " + "'" + id_p + "'"
    person = mysql_util.mysql_query(sql)
    N_Person = Person(person[0][0], person[0][1], person[0][2], person[0][3], person[0][4], person[0][6])
    close_connect_mysql(mysql_util)
    return N_Person.to_string()

def update_person(update_person, id_p):
    mysql_util = open_connect_mysql()
    sql = "update Persons set "
    sql += "where id_p = " + "'" + id_p + "'"
    result = mysql_util.mysql_update_common(sql)
    close_connect_mysql(mysql_util)
    return result

def get_person_list_like(param, start, pagesize):
    mysql_util = open_connect_mysql()
    result = mysql_util.mysql_query_like('firstname', param, start, pagesize)
    close_connect_mysql(mysql_util)
    return result


'''Compares whether two dictionaries containing
nested lists/dictionaries are equal'''

import numpy as np

def dict_equal(dict1, dict2):

    # Check if the keys in both dictionaries match
    if set(dict1.keys()) != set(dict2.keys()):
        return False

    # Recursively compare the values
    for key in dict1.keys():
        value1 = dict1[key]
        value2 = dict2[key]

        # If the values are dictionaries, recursively compare them
        if isinstance(value1, dict) and isinstance(value2, dict):
            if not dict_equal(value1, value2):
                return False

        # If both elements are lists, recursively compare them
        if isinstance(value1, list) and isinstance(value2, list):
            if not deep_list_equals(value1, value2):
                return False

        # If both elements are nparrays, compare them
        if isinstance(value1, np.ndarray) and isinstance(value2, np.ndarray):
            return np.array_equal(value1, value2)

        # For other types, use the default comparison
        else:
            if value1 != value2:
                return False
    return True

def deep_list_equals(list1, list2):
    """
    Test whether two nested lists are equal.

    Args:
    - list1 (list): The first nested list.
    - list2 (list): The second nested list.

    Returns:
    - bool: True if the nested lists are equal, False otherwise.
    """
    # Check if the lengths of the lists are the same
    if len(list1) != len(list2):
        return False

    # Iterate through the elements of the lists
    for elem1, elem2 in zip(list1, list2):

        # If both elements are lists, recursively compare them
        if isinstance(elem1, list) and isinstance(elem2, list):
            if not deep_list_equals(elem1, elem2):
                return False

        # If both elements are nparrays, compare them
        if isinstance(elem1, np.ndarray) and isinstance(elem2, np.ndarray):
            return np.array_equal(elem1, elem2)

        # For other types, use the default comparison
        else:
            if elem1 != elem2:
                return False

    return True


import torch
from torch.utils.data import Dataset
from torchvision import transforms
from torchvision.datasets import VisionDataset
from utils.data_utils import del_theta_quaternion, get_train_mean_and_std, get_normalized_datasets

class DataSample(object):
    def __init__(self, image: torch.Tensor, label: int, pose: torch.Tensor):
        super(DataSample, self).__init__()
        self.image = image
        self.label = label
        self.pose = pose

class Triplet(object):
    def __init__(self, anchor: DataSample, puller: DataSample, pusher: DataSample):
        super(Triplet, self).__init__()
        self.anchor = anchor
        self.puller = puller
        self.pusher = pusher

class TripletDataset(Dataset):
    def __init__(self, data_dir: str, train: bool = True, online: bool = False):
        super(TripletDataset, self).__init__()
        self.data_dir = data_dir
        if online:
            classes = ("ape", "benchvise", "cam", "cat", "duck")
            m, std = get_train_mean_and_std(data_dir, classes)
            db, train, test = get_normalized_datasets(data_dir, classes, m, std)
            if train:
                self.dataset = train
            else:
                self.dataset = test
            self.db = db
        else:
            if train:
                self.dataset = torch.load(data_dir + 'S_train.pt')
            else:
                self.dataset = torch.load(data_dir + 'S_test.pt')
            self.db = torch.load(data_dir + 'S_db.pt')

    def get_puller(self, sample: list):
        puller = None
        min_distance = 50000
        for i in range(len(self.db)):
            if sample[1] == self.db[i][1]:
                distance = del_theta_quaternion(q1=sample[2:6], q2=self.db[i][2:6])
                if distance < min_distance:
                    min_distance = distance
                    puller = self.db[i]
        return puller

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        anchor = self.dataset[item]
        puller = self.get_puller(anchor)
        pusher_idx = torch.randint(len(self.db), (1,)).item()
        pusher = self.db[pusher_idx]
        # anchor = DataSample(image=torch.from_numpy(anchor[0]), label=anchor[1], pose=torch.tensor(anchor[2:]))
        # puller = DataSample(image=torch.from_numpy(puller[0]), label=puller[1], pose=torch.tensor(puller[2:]))
        # pusher = DataSample(image=torch.from_numpy(pusher[0]), label=pusher[1], pose=torch.tensor(pusher[2:]))
        # return Triplet(anchor, puller, pusher)
        return anchor, puller, pusher

class DummyDataset(VisionDataset):
    def __init__(self, data_dir: str, dataset_name: str, transform: transforms = None):
        super(DummyDataset, self).__init__(root=data_dir + dataset_name, transform=transform)
        self.data_dir = data_dir
        self.dataset = torch.load(data_dir + dataset_name)

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        return self.dataset[item]

class TemplateDataset(Dataset):
    def __init__(self, data_dir: str):
        super(TemplateDataset, self).__init__()
        self.data_dir = data_dir
        self.dataset = torch.load(data_dir + 'S_db.pt')

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        return self.dataset[item]

class TestDataset(Dataset):
    def __init__(self, data_dir: str):
        super(TestDataset, self).__init__()
        self.data_dir = data_dir
        self.dataset = torch.load(data_dir + 'S_test.pt')

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        return self.dataset[item]

class TrainDataset(Dataset):
    def __init__(self, data_dir: str):
        super(TrainDataset, self).__init__()
        self.data_dir = data_dir
        self.dataset = torch.load(data_dir + 'S_train.pt')

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        return self.dataset[item]


def plusMinus(arr):
    n = len(arr)
    positive = 0
    negative = 0
    zeros = 0

    for i in arr:
        if i > 0:
            positive += 1
        elif i == 0:
            zeros += 1
        else:
            negative += 1

    print("{:.6f}".format(positive / n))
    print("{:.6f}".format(negative / n))
    print("{:.6f}".format(zeros / n))

if True:
    arr = [-4, 3, -9, 0, 4, 1]
    plusMinus(arr)
    print()

if True:
    arr = [1, 2, 3, -1, -2, -3, 0, 0]
    plusMinus(arr)
    print()


__license__ = """
 File: fluid_dump.py

 BSD 3-Clause License

 Copyright (c) 2020-2022, AFD Group at UIUC
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

 1. Redistributions of source code must retain the above copyright notice, this
    list of conditions and the following disclaimer.

 2. Redistributions in binary form must reproduce the above copyright notice,
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.

 3. Neither the name of the copyright holder nor the names of its
    contributors may be used to endorse or promote products derived from
    this software without specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import copy
import numpy as np

from .defs import Loci
from .grid import Grid

from . import io
from . import variables
from .grmhd.b_field import divB
from .units import get_units

__doc__ = \
"""This file exclusively defines an object representing a GRMHD code snapshot or "dump" file.
The object serves as a cohesive interface for any data or calculations concerning the fluid state data.
"""

def load_dump(fname, **kwargs):
    """Wrapper to create a new FluidDump object using the given file
    """
    return FluidDump(fname, **kwargs)

class FluidDump:
    """Read and cache data from a fluid dump file in any supported format, and allow accessing
    various derived properties directly.
    """

    def __init__(self, fname, tag="", ghost_zones=False, grid_cache=True, cache_conn=False, units=None, add_grid=True, params=None):
        """Attach the fluid dump file 'fname' and make its contents accessible like a dictionary.  For a list of some
        variables and properties accessible this way, see the README.

        Fluid dumps can be sliced like arrays!  That is, ``dump[i,j,k]['var_name']`` will read or compute ``'var_name'`` only for the
        particular index in question, and similarly for slices of any size (e.g., 2D slices for plots).  This is
        *tremendously useful*, so remember to slice first to save time if efficiency is important.  Note, though,
        that slicing preserves the dimensionality -- that is, ``dump[i,j,k][var].shape`` will be (1,1,1).  Being able to assume
        every dump has three dimensions makes a lot of internal logic easier.  If you need 2D arrays, just use
        ``np.squeeze``, or the internal functions ``flatten_xz`` and ``flatten_xy``, which return common 2D slices or averages.

        Also, note that slicing does not support strides, and that slices may be *views* rather than *copies* --
        if you're going to modify array contents yourself within a slice (for some reason...), it may affect the global array.
        Generally this will just behave how you want, but it can be confusing if you're really digging around.  If you have the memory,
        you can use ``copy.copy`` or ``copy.deepcopy`` to be certain.

        :param fname: file name or path to dump
        :param tag: any string, usually long name of dump/model for plotting
        :param ghost_zones: Load ghost zones when reading from a dump file
        :param grid_cache: Cache geometry values in the grid file.  These are *not* yet automatically added,
                           so keep this True unless plotting a very simple variable
        :param cache_conn: Cache the connection coefficients at zone centers. Default off as memory-intensive and rarely needed
        :param units: a 'Units' object representing a physical scale for the dump (density M_unit and BH mass MBH)
        :param add_grid: Whether to construct a Grid object at all.  Only used for copy construction.
        :param params: dictionary of parameters. Only used for copy construction.
        """
        self.fname = fname
        if tag == "":
            self.tag = fname
        else:
            self.tag = tag
        self.units = units

        # Choose an importer based on what we know of filenames
        self.reader = io.file_reader(fname, params=params, ghost_zones=ghost_zones)
        if params is None:
            self.params = self.reader.params
        else:
            self.params = params
        self.cache = {}
        self.slice = ()
        if add_grid:
            self.grid = Grid(self.params, caches=grid_cache, cache_conn=cache_conn)
        else:
            self.grid = None

    def __del__(self):
        # Try to clean up what we can. Anything that may possibly not be a simple ref
        for cache in ('cache', 'units', 'params', 'grid'):
            if cache in self.__dict__:
                del self.__dict__[cache]

    def set_units(self, MBH, M_unit):
        """Associate a scale & units with this dump, for calculating scale-dependent quantities in CGS.
        :param MBH: Black hole mass in solar masses
        :param M_unit: Density unit in grams, as fit by imaging with e.g. ``ipole``
        """
        self.units = get_units(MBH, M_unit, gam=self.params['gam'])

    def __getitem__(self, key):
        """Get any of a number of different things from the backing dump file, or from a cached version.
        The full list of keys is covered in depth in the documentation at :ref:`keys`.

        Also allows slicing FluidDump objects to get just a section, and read/operate on just that section
        thereafter. This supports only a small subset of slicing operations:  you must pass a tuple of three
        elements, all of which must either be integers or slice objects (not None).
        Due to overloading, it is thus impossible to allow requesting lists of variables at once.
        I have no idea why you'd want that.  Just, don't.
        """
        if type(key) in (list, tuple):
            slc = key
            # TODO handle further slicing after this is a 2D object?
            relevant = [False, False, False]
            new_slc = list(slc)
            for i in range(3):
                if isinstance(slc[i], slice):
                    new_slc[i] = slc[i]
                else:
                    new_slc[i] = slice(slc[i], slc[i]+1) # For gauging relevance later
                relevant[i] = ((new_slc[i].start is not None) or (new_slc[i].stop is not None))

            if not (relevant[0] or relevant[1] or relevant[2]):
                return self

            # TODO somehow proper copy constructor
            slc = tuple(new_slc)
            #print("FluidDump slice copy: ", self.cache, key)
            out = FluidDump(self.fname, add_grid=False, params=self.params)
            #out = copy.deepcopy(self) # In case this proves faster
            for c in self.cache:
                out.cache[c] = self.cache[c][(Ellipsis,) + slc]
            out.grid = self.grid[slc]
            out.slice = slc
            return out

        # Return things from the cache if we can
        elif key in self.cache:
            return self.cache[key]
        elif key in self.params:
            return self.params[key]

        # Otherwise run functions and cache the result
        # Putting this before reading lets us translate & standardize reads/caches
        elif key in variables.fns_dict:
            self.cache[key] = variables.fns_dict[key](self)
            return self.cache[key]

        # Return coordinates and things from the grid
        # Default to centers when returning multi-location vars, to avoid location madness
        # TODO allow _mesh generally?
        elif self.grid.can_provide(key):
            if key in ('gcon', 'gcov', 'gdet', 'lapse'):
                return self.grid[key][Loci.CENT.value]
            else:
                return self.grid[key]

        # Prefixes for a few common 1:1 math operations.
        # Most math should be done by reductions.py
        # Don't bother to cache these, they aren't intensive to calculate
        elif key[:5] == "sqrt_":
            return np.sqrt(self[key[5:]])
        elif key[:4] == "abs_":
            return np.abs(self[key[4:]])
        elif key[:4] == "log_":
            return np.log10(self[key[4:]])
        elif key[:3] == "ln_":
            return np.log(self[key[3:]])
        elif key[:4] == "inv_":
            return 1/self[key[4:]]
        elif key[:4] == "neg_":
            return -self[key[4:]]

        # Return MHD tensor components: don't cache
        elif ((key[-2:] == "_0" or key[-2:] == "_1" or key[-2:] == "_2" or key[-2:] == "_3")
              and (key[-4:-2] == "_0" or key[-4:-2] == "_1" or key[-4:-2] == "_2" or key[-4:-2] == "_3")):
            i, j = int(key[-3]), int(key[-1])
            if key[-5:-4] == "T":
                return variables.T_cov(self, i, j)
            elif key[-5:-4] == "F":
                return variables.F_cov(self, i, j)

        elif ((key[-2:] == "_0" or key[-2:] == "_1" or key[-2:] == "_2" or key[-2:] == "_3")
              and (key[-4:-2] == "^0" or key[-4:-2] == "^1" or key[-4:-2] == "^2" or key[-4:-2] == "^3")):
            i, j = int(key[-3]), int(key[-1])
            if key[-5:-4] == "T":
                return variables.T_mixed(self, i, j)
            elif key[-7:-4] == "TEM":
                return variables.TEM_mixed(self, i, j)
            elif key[-9:-4] == "TPAKE":
                return variables.TPAKE_mixed(self, i, j)
            elif key[-7:-4] == "TEN":
                return variables.TEN_mixed(self, i, j)
            elif key[-7:-4] == "TFl":
                return variables.TFl_mixed(self, i, j)

        elif ((key[-2:] == "^0" or key[-2:] == "^1" or key[-2:] == "^2" or key[-2:] == "^3")
              and (key[-4:-2] == "^0" or key[-4:-2] == "^1" or key[-4:-2] == "^2" or key[-4:-2] == "^3")):
            i, j = int(key[-3]), int(key[-1])
            if key[-5:-4] == "T":
                return variables.T_con(self, i, j)
            elif key[-5:-4] == "F":
                return variables.F_con(self, i, j)

        # Return vector components: do cache
        elif key[-2:] == "_0" or key[-2:] == "_1" or key[-2:] == "_2" or key[-2:] == "_3":
            return self[key[:-2]+"cov"][int(key[-1])]
        elif key[-2:] == "^0" or key[-2:] == "^1" or key[-2:] == "^2" or key[-2:] == "^3":
            return self[key[:-2]+"con"][int(key[-1])]

        # Return transformed vector components
        elif key[-2:] == "_t" or key[-2:] == "_r" or key[-3:] == "_th" or key[-4:] == "_phi":
            return self[key[0]+"cov_base"][["t", "r", "th", "phi"].index(key.split("_")[-1])]
        elif key[-2:] == "^t" or key[-2:] == "^r" or key[-3:] == "^th" or key[-4:] == "^phi":
            return self[key[0]+"con_base"][["t", "r", "th", "phi"].index(key.split("^")[-1])]
        elif key[-2:] == "_x" or key[-2:] == "_y" or key[-2:] == "_z":
            return self[key[0]+"cov_cart"][["t", "x", "y", "z"].index(key.split("_")[-1])]
        elif key[-2:] == "^x" or key[-2:] == "^y" or key[-2:] == "^z":
            return self[key[0]+"con_cart"][["t", "x", "y", "z"].index(key.split("^")[-1])]

        # Return an array of the correct size filled with just zero or one
        # Don't cache these
        elif key in ('zero', '0'):
            return np.zeros_like(self['rho'])
        elif key in ('one', '1'):
            return np.ones_like(self['rho'])
        else:
            # Read things that we haven't cached and absolutely can't calculate
            # The reader keeps its own cache, so we don't add its items to ours
            if "flag" in key:
                out = self.reader.read_var(key, astype=np.int32, slc=self.slice)
            else:
                # TODO Option for double
                out = self.reader.read_var(key, astype=np.float64, slc=self.slice)
            if out is None:
                raise ValueError("FluidDump cannot find or compute {}".format(key))
            else:
                return out

        raise RuntimeError("Reached the end of FluidDump.__getitem__, should have returned a value!")


import gym
import numpy as np
import torch

from policy_gradient.ai import Network
from policy_gradient.reward_utils import normalize_rewards

max_episode = 1000
num_steps = 500
gamma = 0.95

environment = gym.make('CartPole-v0')
input_shape = environment.observation_space.shape
output_shape = environment.action_space.n

network = Network(input_shape[0], output_shape)
optimizer = torch.optim.Adam(network.parameters(), lr=0.01)

def learn(states, actions, discounted_rewards):
    action_probs = network(states)

    discounted_rewards = np.expand_dims(discounted_rewards, -1).astype(np.float32)
    actions = np.expand_dims(actions, -1)
    actions = torch.from_numpy(actions)
    action_probs = torch.gather(action_probs, dim=1, index=actions)

    loss = -torch.log(action_probs) * torch.from_numpy(discounted_rewards)
    loss = loss.mean()
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

max_reward = 0
for current_episode in range(max_episode):

    episode_rewards = []
    states = []
    actions = []
    state = environment.reset()
    states.append(state)
    state = np.expand_dims(state, 0)
    for current_step in range(num_steps):
        action_prob = network(state)
        action = np.random.choice(range(action_prob.shape[1]), p=action_prob.data.numpy().ravel())
        actions.append(action)

        next_state, reward, done, _ = environment.step(action)
        if not done: states.append(next_state)
        next_state = np.expand_dims(next_state, 0)
        state = next_state

        episode_rewards.append(reward)

        if done:
            discounted_rewards = normalize_rewards(episode_rewards, gamma)
            states = np.array(states)
            actions = np.array(actions)
            learn(states, actions, discounted_rewards)
            if sum(episode_rewards) > max_reward:
                max_reward = sum(episode_rewards)
            print(f'episode {current_episode}, reward: {sum(episode_rewards)}')
            break

print(f"max reward: {max_reward}")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun May 22 13:24:53 2022

@author: Sydney
"""

import unittest
import Person

class TestPerson(unittest.TestCase):

    def test_create_person(self):
        """
        Test Person object is created
        """
        lastName = "testLastName"
        firstName = "testFirstName"
        gender = "Female"
        dateOfBirth = "5/22/2022"
        favoriteColor = "Yellow"
        person = Person.Person(lastName, firstName, gender, dateOfBirth, favoriteColor)

        self.assertEqual(person.lastName, lastName)
        self.assertEqual(person.firstName, firstName)
        self.assertEqual(person.gender, gender)
        self.assertEqual(person.dateOfBirth, dateOfBirth)
        self.assertEqual(person.favoriteColor, favoriteColor)

    def test_create_person_format_birth_date(self):
        """
        Test date of birth is formatted when Person object is created
        """
        expectedDateOfBirth = "5/22/2022"
        lastName = "testLastName"
        firstName = "testFirstName"
        gender = "Female"
        dateOfBirth = "5-22-2022"
        favoriteColor = "Yellow"
        person = Person.Person(lastName, firstName, gender, dateOfBirth, favoriteColor)

        self.assertEqual(person.dateOfBirth, expectedDateOfBirth)

    def test_create_person_format_gender_f(self):
        """
        Test gender is formatted when Person object is created (Female)
        """
        expectedGender = "Female"
        lastName = "testLastName"
        firstName = "testFirstName"
        gender = "F"
        dateOfBirth = "5-22-2022"
        favoriteColor = "Yellow"
        person = Person.Person(lastName, firstName, gender, dateOfBirth, favoriteColor)

        self.assertEqual(person.gender, expectedGender)

    def test_create_person_format_gender_m(self):
        """
        Test gender is formatted when Person object is created (Male)
        """
        expectedGender = "Male"
        lastName = "testLastName"
        firstName = "testFirstName"
        gender = "M"
        dateOfBirth = "5-22-2022"
        favoriteColor = "Yellow"
        person = Person.Person(lastName, firstName, gender, dateOfBirth, favoriteColor)

        self.assertEqual(person.gender, expectedGender)

if __name__ == '__main__':
    unittest.main()


"""Arcos GUI Plugin for Napari."""

__version__ = "0.1.1"

from arcos_gui._helper_functions import (
    filter_data,
    get_arcos_output,
    get_current_arcos_plugin,
    load_dataframe,
    load_dataframe_with_columnpicker,
    load_sample_data,
    open_plugin,
    run_arcos,
    run_binarization_only,
)
from napari.utils.notifications import show_info

show_info("Loading Plugin")

__all__ = [
    "open_plugin",
    "load_dataframe",
    "load_sample_data",
    "load_dataframe_with_columnpicker",
    "filter_data",
    "run_binarization_only",
    "run_arcos",
    "get_arcos_output",
    "get_current_arcos_plugin",
]


import os
import logging
import glob

import yaml
import joblib
import numpy as np

from mathtools import utils
from kinemparse import airplanecorpus

logger = logging.getLogger(__name__)

def makeBinLabels(action_labels, part_idxs_to_bins, num_samples):
    no_bin = part_idxs_to_bins[0]  # 0 is the index of the null part
    bin_labels = np.full(num_samples, no_bin, dtype=int)

    for part_index, start_index, end_index in action_labels:
        bin_index = part_idxs_to_bins[part_index]
        bin_labels[start_index:end_index + 1] = bin_index

    return bin_labels

def main(
        out_dir=None, hand_detections_dir=None, labels_dir=None,
        plot_output=None, results_file=None, sweep_param_name=None):

    hand_detections_dir = os.path.expanduser(hand_detections_dir)
    labels_dir = os.path.expanduser(labels_dir)

    out_dir = os.path.expanduser(out_dir)
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    out_data_dir = os.path.join(out_dir, 'data')
    if not os.path.exists(out_data_dir):
        os.makedirs(out_data_dir)

    fig_dir = os.path.join(out_dir, 'figures')
    if not os.path.exists(fig_dir):
        os.makedirs(fig_dir)

    def saveVariable(var, var_name):
        joblib.dump(var, os.path.join(out_data_dir, f'{var_name}.pkl'))

    logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt'))

    logger.info(f"Reading from: {hand_detections_dir}")
    logger.info(f"Writing to: {out_dir}")

    if results_file is None:
        results_file = os.path.join(out_dir, 'results.csv')
        # write_mode = 'w'
    else:
        results_file = os.path.expanduser(results_file)
        # write_mode = 'a'

    fig_dir = os.path.join(out_dir, 'figures')
    if not os.path.exists(fig_dir):
        os.makedirs(fig_dir)

    out_data_dir = os.path.join(out_dir, 'data')
    if not os.path.exists(out_data_dir):
        os.makedirs(out_data_dir)

    part_names, part_names_to_idxs, part_idxs_to_bins = airplanecorpus.loadParts()

    for i, fn in enumerate(glob.glob(os.path.join(hand_detections_dir, '*.txt'))):
        video_id = utils.stripExtension(fn).split('.handsdetections')[0]

        logger.info(f"Processing video {video_id}")

        hand_detections = airplanecorpus.loadHandDetections(
            video_id, dir_name=hand_detections_dir, unflatten=True
        )
        hand_detections = hand_detections.reshape(hand_detections.shape[0], -1)
        mean_detection = np.nanmean(hand_detections)
        hand_detections[np.isnan(hand_detections)] = mean_detection

        action_labels = airplanecorpus.loadLabels(
            video_id, dir_name=labels_dir,
            part_names_to_idxs=part_names_to_idxs
        )

        bin_labels = makeBinLabels(action_labels, part_idxs_to_bins, hand_detections.shape[0])

        fig_fn = os.path.join(fig_dir, f"{video_id}.png")
        utils.plot_array(hand_detections.T, (bin_labels,), ('bin',), fn=fig_fn)

        video_id = video_id.replace('_', '-')
        saveVariable(hand_detections, f'trial={video_id}_feature-seq')
        saveVariable(bin_labels, f'trial={video_id}_label-seq')

if __name__ == "__main__":
    # Parse command-line args and config file
    cl_args = utils.parse_args(main)
    config, config_fn = utils.parse_config(cl_args, script_name=__file__)

    # Create output directory, instantiate log file and write config options
    out_dir = os.path.expanduser(config['out_dir'])
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)
    with open(os.path.join(out_dir, config_fn), 'w') as outfile:
        yaml.dump(config, outfile)
    utils.copyFile(__file__, out_dir)

    main(**config)