from collections import namedtuple
import os
import shlex
import subprocess
import tempfile
import time

ConfigDmgtools = namedtuple('Config', 'dmg hfsplus newfs_hfs verbose')
ConfigHdiutil = namedtuple('Config', 'hdiutil verbose')

def chunks(l, n):
	"""
	Yield successive n-sized chunks from l.

	From https://stackoverflow.com/a/312464.
	"""
	for i in range(0, len(l), n):
		yield l[i:i + n]

class Dmg:
	def __init__(self, config):
		self.config = config

	def _check_call(self, process_args, *args, **kwargs):
		if self.config.verbose >= 1:
			print("EXECUTING {}".format(" ".join(shlex.quote(x) for x in process_args)))
		if not (self.config.verbose >= 2 and "stdout" not in kwargs):
			kwargs["stdout"] = open(os.devnull, 'wb')
		subprocess.check_call(process_args, *args, **kwargs)

class Dmgtools(Dmg):
	def _mkfs_hfs(self, *args):
		self._check_call((self.config.newfs_hfs,) + args)
	def _hfs(self, *args):
		self._check_call((self.config.hfsplus,) + args)
	def _dmg(self, *args):
		self._check_call((self.config.dmg,) + args)

	def _create_hfs(self, hfs, volume_name, size):
		if self.config.verbose >= 1:
			print("TRUNCATING {} to {} bytes".format(hfs, size))
		with open(hfs, 'wb') as f:
			f.truncate(size)
		self._mkfs_hfs('-v', volume_name, hfs)

	def _symlink(self, hfs, target, link_name):
		self._hfs(hfs, 'symlink', link_name, target)

	def _add(self, hfs, directory):
		self._hfs(hfs, 'addall', directory)

	def _finish(self, hfs, dmg):
		self._dmg('build', hfs, dmg)

	def create(self, dmg, volume_name, directory, symlinks):
		input_size = sum(os.stat(os.path.join(path, f)).st_size for path, dirs, files in os.walk(directory) for f in files)
		output_size = max(input_size * 2, 1024**2)
		hfs = tempfile.mktemp(prefix=dmg + '.', suffix='.hfs')
		self._create_hfs(hfs, volume_name, output_size)
		self._add(hfs, directory)
		for target, link_name in symlinks:
			self._symlink(hfs, target, link_name)
		self._finish(hfs, dmg)
		if self.config.verbose >= 1:
			print("REMOVING {}".format(hfs))
		os.remove(hfs)

class Hdiutil(Dmg):
	def _hdiutil(self, *args):
		self._check_call((self.config.hdiutil,) + args)

	def create(self, dmg, volume_name, directory, symlinks):
		if symlinks:
			raise NotImplementedError("symlinks are not yet implemented")
		for i in range(5):
			if os.path.exists(volume_name + '.dmg'):
				os.remove(volume_name + '.dmg')
			try:
				self._hdiutil('create', '-volname', volume_name, '-srcdir', directory, dmg)
			except subprocess.CalledProcessError as e:
				if i == 4:
					raise e
				print("Retrying hdiutil create")
				time.sleep(5)
			else:
				break

def main():
	import argparse
	p = argparse.ArgumentParser(description="Manipulate dmg archives")

	subcommands = p.add_subparsers(help="Subcommand", dest='command', metavar="COMMAND")
	subcommands.required = True

	create = subcommands.add_parser("create", help="Create a dmg archive from files or directories")
	create.add_argument('-v', '--verbose', action='count', help="Verbose output")
	createx = create.add_mutually_exclusive_group(required=True)
	createx.add_argument('--dmgtools', nargs=3, help="Paths to the dmg and hfsplus executable (https://github.com/mozilla/libdmg-hfsplus) and the newfs_hfs executable (http://pkgs.fedoraproject.org/repo/pkgs/hfsplus-tools/diskdev_cmds-540.1.linux3.tar.gz/0435afc389b919027b69616ad1b05709/diskdev_cmds-540.1.linux3.tar.gz)")
	createx.add_argument('--hdiutil', help="Path to the hdiutil (only exists for macOS at time of writing)")
	create.add_argument('output', metavar="OUTPUT", help="Filename of the output dmg archive")
	create.add_argument('volume_name', metavar="VOLUME_NAME", help="Name of the dmg archive")
	create.add_argument('directory', metavar="DIR", help="Directory to create the archive from")
	create.add_argument('--symlink', metavar="SYMLINK", nargs=2, action="append", help="Symlink the first argument under the second name")
	args = p.parse_args()

	verbose = args.verbose or 0
	symlinks = args.symlink or []
	if args.dmgtools:
		dmg, hfsplus, newfs_hfs = args.dmgtools
		dmg = Dmgtools(ConfigDmgtools(dmg=dmg, hfsplus=hfsplus, newfs_hfs=newfs_hfs, verbose=verbose))
	elif args.hdiutil:
		dmg = Hdiutil(ConfigHdiutil(hdiutil=args.hdiutil, verbose=verbose))
	else:
		raise RuntimeError("unreachable")
	dmg.create(volume_name=args.volume_name, directory=args.directory, dmg=args.output, symlinks=symlinks)

if __name__ == '__main__':
	main()


import pandas as pd
import nltk
import os
PROXY = ""
os.environ["HTTP_PROXY"]  = PROXY
os.environ["HTTPS_PROXY"] = PROXY

# redact proper names and company names using nltk
def names(df):
    nltk.set_proxy(None)
    #nltk.download()

    # print('\nTag each row sentences\n')
    tagged_sentences = []
    for text in range(len(df)):
        sentence = df.iloc[text]
        # print('sentence\n', sentence)
        tagged_sentences.append(nltk.tag.pos_tag(sentence.split()))
        # print('tagged_sentences\n', tagged_sentences[text])

    # print('\nEdit each row sentences\n')
    edited_sentences = []
    for sentence in range(len(tagged_sentences)):
        edited_sentence = []
        for word, tag in tagged_sentences[sentence]:
            if tag == 'NNP' or tag == 'NNPS' or tag == 'NNS':
                edited_sentence.append('xxx')
            else:
                edited_sentence.append(word)

        edited_sentences.append(str(' '.join(edited_sentence)))
        # print('edited_sentences\n', edited_sentences)

    return pd.Series(edited_sentences)

######################### Test
##############################

# Texts = ["they work at Microsoft, and my name is Sami", "Google CEO is Sunder Pichai"]
# data = pd.Series(Texts)
# print(names(data))


import abc
import json
import threading
import time
from sched import scheduler
from .debug import Debug
from framework.utils.service_logging import logger

class DependencyState(object):
    DEPS_STARTING = 0
    DEPS_FAILED = 2
    DEPS_SUCCESS = 4

class InitState(object):
    INITIALIZING = 0
    INIT_FAILED = 1
    INIT_SUCCESS = 2

class SensorThreadState(object):
    WAITING = 0
    RUNNING = 1
    FAILED = 3

class ModuleThread(metaclass=abc.ABCMeta):
    """Base Class for all Module Threads"""

    @abc.abstractmethod
    def initialize(self):
        """Initialize the module thread"""
        raise NotImplementedError("Subclasses should implement this!")

    @abc.abstractmethod
    def run(self):
        """Periodically run the module thread"""
        raise NotImplementedError("Subclasses should implement this!")

class ScheduledModuleThread(ModuleThread, Debug):
    """A module thread with an internal scheduler"""

    # Module thread states
    ACTIVE = 1
    SUSPENDED = 2
    HALTED = 3

    def __init__(self, module_name, priority):
        super(ScheduledModuleThread, self).__init__()

        self._scheduler   = scheduler(time.time, time.sleep)
        self._module_name = module_name
        self._priority    = priority
        self._running     = False

    def initialize(self, conf_reader):
        """Initialize the monitoring thread"""
        # Set the configuration file reader located in /etc/sspl-ll.conf
        self._conf_reader = conf_reader

        # Set the scheduler to fire the thread right away
        self._scheduler.enter(1, self._priority, self.run, ())

    def start(self):
        """Run the scheduler"""
        self._running = True
        self._scheduler.run()

    def start_thread(self, conf_reader, msgQlist, product):
        self.initialize(conf_reader, msgQlist, product)
        self.start()

    def _cleanup_and_stop(self):
        """Clean out the remainder of events from the scheduler queue."""
        self._log_debug("last module calling _cleanup_and_stop thread scheduler")
        for event in self._scheduler.queue:
            try:
                self._scheduler.cancel(event)
            except ValueError:
                # Being shut down so ignore
                pass
        self._log_debug("Thread scheduler cancelled successfully")

    def shutdown(self):
        """Clean up and shut down this monitor."""
        self._running = False

        # Give the module a few seconds to close down
        self._scheduler.enter(10, self._priority, self._cleanup_and_stop, ())
        self._log_debug("Scheduling shut down")

    def _getConf_reader(self):
        return self._conf_reader

    def is_running(self):
        return self._running

    def suspend(self):
        logger.debug("suspend() of {0} is called".format(self.name()))

    def resume(self):
        """Resumes the module thread. It should be non-blocking"""
        logger.debug("resume() of {0} is called".format(self.name()))

    def get_state(self):
        """Returns the current state of module thread"""
        current_state = None
        if self.is_running() and not self.is_suspended():
            current_state = ScheduledModuleThread.ACTIVE
        elif self.is_running() and self.is_suspended():
            current_state = ScheduledModuleThread.SUSPENDED
        elif not self.is_running():
            current_state = ScheduledModuleThread.HALTED
        return current_state

    def is_suspended(self):
        """Returns True if the module thread is suspended. False otherwise"""
        logger.debug("is_suspended() of {0} is called".format(self.name()))

class SensorThread(ScheduledModuleThread):
    SENSORTHREAD_INIT_WAIT_TIMEOUT = 300
    SENSORTHREAD_INIT_WAIT_INTERVAL = 10

    def __init__(self, module_name, priority):
        super(SensorThread, self).__init__(module_name, priority)

        self.lock = threading.Lock()

        self.remaining_dependees = set()
        self.num_failed_dependees = 0
        self.waiting_dependers = set()
        self.deps_status = DependencyState.DEPS_STARTING

        self.init_status = InitState.INITIALIZING

        self.has_timed_out = False
        self.current_timeout_wait = 0

        self.status = SensorThreadState.WAITING

    def initialize(self, conf_reader):
        """Initialize the monitoring thread"""
        # Set the configuration file reader located in /etc/sspl-ll.conf
        self._conf_reader = conf_reader

        # Set the scheduler to fire the thread right away
        self._scheduler.enter(1, self._priority, self.check_and_run, ())

    def get_thread_init_status(self):
        self.lock.acquire()
        s = self.status
        self.lock.release()

        return s

    def prepare(self, dependencies):
        logger.debug("{} depends on {}".format(
            self.__class__.__name__, [d.__class__.__name__ for d in dependencies]))

        self.remaining_dependees |= set(dependencies)
        for d in self.remaining_dependees:
            d.my_register(self)

    def my_register(self, depender):
        logger.debug("{} registering dependency on {}".format(
            depender.__class__.__name__, self.__class__.__name__))

        self.lock.acquire()
        self.waiting_dependers |= {depender}
        self.lock.release()

    def check_and_conclude_initialization(self):
        logger.debug("Begin {}.conclude_initialization()".format(
            self.__class__.__name__))

        # Check that self.lock is held by the caller
        if self.lock.acquire(blocking=False):
            self.lock.release()
            logger.error("SensorThread.check_and_conclude_initialization() called"\
                " without acquiring lock. Returning immediately")
            return

        if self.status != SensorThreadState.WAITING:
            return

        # It is possible that self.event() is not called at all.
        if self.num_failed_dependees:
            self.deps_status = DependencyState.DEPS_FAILED
        elif not self.remaining_dependees:
            self.deps_status = DependencyState.DEPS_SUCCESS

        definitely_failed = \
                self.deps_status == DependencyState.DEPS_FAILED or \
                self.init_status == InitState.INIT_FAILED or \
                self.has_timed_out
        definitely_succeeded = \
                self.deps_status == DependencyState.DEPS_SUCCESS and \
                self.init_status == InitState.INIT_SUCCESS

        if (definitely_failed):
            self.status = SensorThreadState.FAILED
        elif (definitely_succeeded):
            self.status = SensorThreadState.RUNNING
        # else it remains as waiting

        if self.status != SensorThreadState.WAITING:
            for d in self.waiting_dependers:
                d.event(self, self.status == SensorThreadState.RUNNING)

        logger.debug("End {}.conclude_initialization() with state {}".format(
            self.__class__.__name__, self.status))

    def event(self, dependee, running):
        logger.debug("event in {} by {} calling start() with status {}".format(
            self.__class__.__name__, dependee.__class__.__name__, running))

        self.lock.acquire()

        self.remaining_dependees -= {dependee}
        if running:
            pass
        else:
            self.num_failed_dependees += 1

        self.check_and_conclude_initialization()

        self.lock.release()

    def start_thread(self, conf_reader, msgQlist, product):
        logger.debug("Begin {}.start_thread()".format(self.__class__.__name__))
        self.init_status = InitState.INITIALIZING
        try:
            self.initialize(conf_reader, msgQlist, product)
        except Exception as err:
            self.status = self.init_status = InitState.INIT_FAILED
            raise Exception("initialize() failed. {}".format(err))

        self.init_status = InitState.INIT_SUCCESS

        self.lock.acquire()

        self.check_and_conclude_initialization()

        self.lock.release()

        self.start()
        logger.debug("End {}.start_thread()".format(self.__class__.__name__))

    def check_and_run(self):
        logger.debug("Begin {}.should_run()".format(self.__class__.__name__))
        self.lock.acquire()

        self.current_timeout_wait += self.SENSORTHREAD_INIT_WAIT_INTERVAL
        if self.current_timeout_wait > self.SENSORTHREAD_INIT_WAIT_TIMEOUT:
            self.has_timed_out = True
        self.check_and_conclude_initialization()

        should_run = self.status == SensorThreadState.RUNNING
        self.lock.release()

        if not should_run:
            self._scheduler.enter(self.SENSORTHREAD_INIT_WAIT_INTERVAL,
                    self._priority, self.check_and_run, ())
        else:
            self._scheduler.enter(1, self._priority, self.run, ())

class ThreadException(Exception):
    """Generic Exception to handle Threads errors."""

    def __init__(self, module, message):
        """Handle error msg from thread modules."""
        self._module = module
        self._desc = message

    def __str__(self):
        """Returns formated error msg."""
        return "%s: error: %s" %(self._module, self._desc)


# 必要なライブラリをインポートします。
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.preprocessing import StandardScaler
import lightgbm as lgb

# データの読み込みと前処理を行う関数を定義します。
def load_and_preprocess_data():
    # CSVファイルからデータを読み込みます。
    df_mc = pd.read_csv("out/mc_append.csv", header=None)
    df_hc = pd.read_csv("out/hc_append.csv", header=None)

    # データを標準化します。
    scaler_hc = StandardScaler()
    scaler_mc = StandardScaler()
    scaler_hc.fit(df_hc)
    scaler_mc.fit(df_mc)
    df_hc = scaler_hc.transform(df_hc)
    X = scaler_mc.transform(df_mc)
    y = df_hc[:, 3]

    # データを学習用と評価用に分割します。
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

    return X_train, X_test, y_train, y_test

# モデルの学習と評価を行う関数を定義します。
def train_and_evaluate_model(X_train, X_test, y_train, y_test):
    # LightGBMのモデルを作成するためのパラメータを設定します。
    params = {
        "boosting_type": "gbdt",
        "objective": "binary",
        "metric": "auc",
        "num_leaves": 16,
        "learning_rate": 0.1,
        "n_estimators": 100000,
        "random_state": 0,
    }
    model = lgb.LGBMRegressor(**params)

    # モデルを学習させます。学習中に評価用データでのAUCを計算し、早期停止の条件を設定します。
    model.fit(
        X_train,
        y_train,
        eval_set=[(X_train, y_train), (X_test, y_test)],
        early_stopping_rounds=100,
    )

    # モデルの最適なイテレーション数を表示します。
    print(model.best_iteration_)

    # 評価用データでの予測値を生成します。
    y_pred = model.predict(X_test)

    # 予測値と真の値の決定係数（R2スコア）を計算します。
    print(r2_score(y_test, y_pred))

    return model, y_pred

# メインの処理を実行します。
if __name__ == "__main__":
    # データの読み込みと前処理を行います。
    X_train, X_test, y_train, y_test = load_and_preprocess_data()

    # モデルの学習と評価を行います。
    model, y_pred = train_and_evaluate_model(X_train, X_test, y_train, y_test)

    # 予測値と真の値の散布図をプロットします。
    plt.scatter(y_test, y_pred)
    plt.xlabel("True value")
    plt.ylabel("Predicted value")
    plt.show()


import logging
import time
import unittest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait
from TestBaseSetUp.Variable import ConifigVariable
from TestPages.SettingPage import Setting
from UserTestData.UserTestDataVar import TestData

class DropBoxConnTest(unittest.TestCase):
    logging.basicConfig(filename="../LoggerReport/report.log",
                        format='%(asctime)s - %(message)s',
                        level=logging.DEBUG,
                        filemode='w')

    def setUp(self):
        self.driver = webdriver.Chrome(ConifigVariable.chromePathWin)
        print("Chrome driver opened for testing dropBox conn test")
        self.driver.maximize_window()
        logging.info("Browser launched and maximize")
        print("Chrome browser launched and maximized")
        self.driver.get(ConifigVariable.StaggingURL)
        logging.info("I2chain Application Opened")
        print("i2chain application running in backend")

    def testDropBoxCon(self):
        setting = Setting(self.driver)
        logging.info("=========Setting page initialized=====")
        print("setting page initialised")
        window_before = self.driver.window_handles[0]
        setting.clickOnSetting()
        print("clicked on setting")
        time.sleep(5)
        setting.clickDropBoxButton()
        time.sleep(10)
        window_After = self.driver.window_handles[1]

        if window_After != window_before:
            self.driver.switch_to_window(window_After)
            logging.info("getting window details" + window_After)
            print("new window found for dropBox connection user need to provide credentials")
            setting.clickOnSignWithGoogle()
            self.driver.implicitly_wait(40)
            setting.enterDropBoxEmailID(TestData.setUserName)
            time.sleep(10)
            logging.info("==entering gmail user name ==")
            print("entering drop box user name")
            setting.clickOnDropBoxUserNextButton()
            time.sleep(10)
            logging.info("==Clicking on next button==")
            print("==clicking on next button==")
            setting.enterDropBoxPassword(TestData.dropBoxPass)
            logging.info("==Entering gmail password==")
            print("entering gmail password")
            time.sleep(5)
            setting.clickOnDropBoxPassNextButton()
            time.sleep(15)

            self.driver.switch_to_window(window_before)

            dropBoxSucMessage = WebDriverWait(self.driver, 30).until(
                expected_conditions.presence_of_element_located(
                    (By.XPATH, "//div[@class='alert alert-success alert-dismissible fade show']")))

            logging.info(dropBoxSucMessage.text)

            text = "Successfully connected to dropbox!"

            if text in dropBoxSucMessage.text:
                logging.info("==dropBox drive connected successfully==")
                time.sleep(15)
                setting.clickDropBoxButton()
                logging.info("==clicking on gdrive button for logout connected drive==")
                print("==clicking on gdrive button for logout connected drive==")
                time.sleep(10)
                setting.clickOnDropBoxLogOutButton()
                logging.info("==clicking on logout button to log our==")
                print("==clicking on logout button to log our==")
                time.sleep(10)

                setting.clickOnGdriveLogOutyesButton()
                logging.info("==clicking on yes button==")
                print("==clicking on yes button ==")
                time.sleep(10)

                logoutSuccessMessage = WebDriverWait(self.driver, 30).until(
                    expected_conditions.presence_of_element_located(
                        (By.XPATH, "//div[@class='alert alert-success alert-dismissible fade show']")))

                logging.info(logoutSuccessMessage.text)

                text = "Successfully logout from dropbox!!!"
                if text in logoutSuccessMessage.text:
                    print("==Log Out Successfully=="+text)
                    logging.info("====logout successfully====")

            else:
                logging.info("==========alert Message DoesNot Match drive not connected============")

    def tearDown(self):
        self.driver.close()

if __name__ == '__main__':
    unittest.main()


import datetime
import hashlib
import hmac
import json
import platform
from learnosity_sdk._version import __version__

from learnosity_sdk.exceptions import ValidationException

def format_utc_time():
    "Get the current UTC time, formatted for a security timestamp"
    now = datetime.datetime.utcnow()
    return now.strftime("%Y%m%d-%H%M")

class Init(object):
    """
    This class is used to generate the necessary security and request data (in
    the correct format) to integrate with any of the Learnosity API services.
    """

    services = [
        'annotations', 'assess', 'author', 'data', 'events', 'items', 'questions',
        'reports'
    ]

    security_keys = [
        'consumer_key', 'domain', 'timestamp', 'expires', 'user_id'
    ]

    __telemetry_enabled = True

    def __init__(
            self, service, security, secret,
            request=None, action=None):
        self.service = service
        self.security = security.copy()
        self.secret = secret
        self.request = request
        if hasattr(request, 'copy'):
            self.request = request.copy()
        self.action = action

        self.validate()
        self.add_telemetry_data()
        self.request_string = self.generate_request_string()
        self.sign_request_data = True
        self.set_service_options()
        self.security['signature'] = self.generate_signature()

    def is_telemetry_enabled(self):
        return self.__telemetry_enabled

    def generate(self, encode=True):
        """
        Generate the data necessary to make a request to one of the Learnosity
        products/services.

        If encode is True, the result is a JSON string. Otherwise, it's a
        dictionary. If self.service == data, encode is ignored.
        """
        output = {}

        if self.service == 'questions':
            # Add the security packet to the root of output
            output.update(self.security)

            # Remove the domain key from the security packet
            if 'domain' in output.keys():
                del output['domain']

            # Stringify the request packet if necessary
            if self.request is not None:
                output.update(self.request)

        elif self.service == 'events':
            output['security'] = self.security
            output['config'] = self.request
        elif self.service == 'assess':
            if self.request is not None:
                output.update(self.request)
        elif self.service == 'data':
            # We ignore the encode param for data API
            output['security'] = json.dumps(self.security)

            if self.request_string is not None:
                output['request'] = self.request_string

            if self.action is not None:
                output['action'] = self.action

            return output
        else:
            output['security'] = self.security

            if self.request is not None:
                output['request'] = self.request

            if self.action is not None:
                output['action'] = self.action

        if encode or self.request_passed_as_string:
            return json.dumps(output)
        else:
            return output

    def get_sdk_meta(self):
        return {
            'version': self.get_sdk_version(),
            'lang': 'python',
            'lang_version': platform.python_version(),
            'platform': platform.system(),
            'platform_version': platform.release()
        }

    def get_sdk_version(self):
        return __version__

    def generate_request_string(self):
        if self.request is None:
            return None
        return json.dumps(self.request, separators=(',', ':'), ensure_ascii=False)

    def generate_signature(self):

        vals = []

        # Add each valid security field.
        # The order is signifcant.
        for key in self.security_keys:
            if key in self.security:
                vals.append(self.security[key])

        # Add the request if necessary
        if self.sign_request_data and self.request_string is not None:
            vals.append(self.request_string)

        if self.action is not None:
            vals.append(self.action)

        return self.hash_list(vals)

    def validate(self):
        # Parse the security packet if the user provided it as a string
        if isinstance(self.security, str):
            self.security = json.loads(self.security)

        # Parse the request packet if the user provided it as a string
        self.request_passed_as_string = False
        if isinstance(self.request, str):
            self.request = json.loads(self.request)
            self.request_passed_as_string = True

        # Validate field lengths and types
        if len(self.service) == 0:
            raise ValidationException(
                "The `service` argument wasn't found or was empty")

        if len(self.secret) == 0:
            raise ValidationException(
                "The `secret` argument wasn't found or  was empty")

        if self.action is not None and not isinstance(self.action, str):
            raise ValidationException("The action parameter must be a string")

        # Check that service is valid
        if self.service not in self.services:
            raise ValidationException(
                "Service not valid: {}".format(self.service))

        # Check that security keys are valid
        for k in self.security.keys():
            if k not in self.security_keys:
                raise ValidationException(
                    "Invalid key found in the security packet: {}".format(k))

        # Add timestamp if missing
        if 'timestamp' not in self.security:
            self.security['timestamp'] = format_utc_time()

        # Special case for Questions API
        if self.service == 'questions' and \
                'user_id' not in self.security:
            raise ValidationException("questions API requires a user id")

    def set_service_options(self):
        if self.service == 'questions':
            self.sign_request_data = False
        elif self.service == 'assess':
            self.sign_request_data = False

            if 'questionsApiActivity' in self.request:
                questionsApi = self.request['questionsApiActivity']

                if 'domain' in self.security:
                    domain = self.security['domain']
                elif 'domain' in questionsApi:
                    domain = questionsApi['domain']
                else:
                    domain = 'assess.learnosity.com'

                self.request['questionsApiActivity'] = {
                    'consumer_key': self.security['consumer_key'],
                    'timestamp': self.security['timestamp'],
                    'user_id': self.security['user_id'],
                    'signature': self.hash_list({
                        'consumer_key': self.security['consumer_key'],
                        'domain': domain,
                        'timestamp': self.security['timestamp'],
                        'user_id': self.security['user_id'],
                        'secret': self.secret
                    }.values())
                }

                for key in [ 'consumer_key', 'domain', 'timestamp', 'user_id', 'signature']:
                    if key in questionsApi.keys():
                        del questionsApi[key]

                self.request['questionsApiActivity'].update(questionsApi)

        elif self.service == 'items' or self.service == 'reports':
            if 'user_id' not in self.security and \
                    'user_id' in self.request:
                self.security['user_id'] = self.request['user_id']

        elif self.service == 'events':
            self.sign_request_data = False
            hashed_users = {}
            for user in self.request.get('users', []):
                concat = "{}{}".format(user, self.secret)
                hashed_users[user] = hashlib.sha256(concat.encode('utf-8')).hexdigest()

            if len(hashed_users) > 0:
                self.security['users'] = hashed_users

    def hash_list(self, l):
        "Hash a list by concatenating values with an underscore"
        concatValues = "_".join(l)
        signature =  hmac.new(bytes(str(self.secret),'utf_8'), msg = bytes(str(concatValues) , 'utf-8'), digestmod = hashlib.sha256).hexdigest()
        return '$02$' + signature

    def add_telemetry_data(self):
        if self.__telemetry_enabled:
            if 'meta' in self.request:
                self.request['meta']['sdk'] = self.get_sdk_meta()
            else:
                self.request['meta'] = {
                    'sdk': self.get_sdk_meta()
                }

    """
    We use telemetry to enable better support and feature planning. It is
    however not advised to disable it, and it will not interfere with any usage.
    """

    @classmethod
    def disable_telemetry(cls):
        cls.__telemetry_enabled = False

    @classmethod
    def enable_telemetry(cls):
        cls.__telemetry_enabled = True


from typing import List

class Solution:
    def findDuplicate(self, nums: List[int]) -> int:

        def nextSlow(index: int):
            if index + 1 == len(nums):
                return 0
            else:
                return index + 1

        slow = 0
        fast = 0
        intersection = -1

        while True:

            slow = nums[slow]
            fast = nums[nums[fast]]

            if nums[slow] == nums[fast]:
                intersection = slow
                break

        slow = 0

        while slow != intersection:
            slow = nums[slow]
            intersection = nums[intersection]

        return intersection


import numpy as np
import torch
from model import Net

def chr2float(c):
    return ord(c) / 255. * 2. - 1

def float2chr(f):
    return f

def verify(flag_path):

    def verify_flag(flag_tensor):
        checkpoint = torch.load('_Checkpoint.pth')
        net = Net()
        net.load_state_dict(checkpoint['model'])
        base_input = checkpoint['input']
        output = net(base_input + flag_tensor)
        return torch.equal(torch.round(output.detach(), decimals=5), torch.tensor([2., 3., 3.]))

    def show_flag(flag_tensor):
        flag = ''
        for f in np.array(flag_tensor).ravel():
            flag += float2chr(f)
        print('moectf{' + flag + '}')

    try:
        flag_tensor = torch.load(flag_path).detach()
    except:
        print('Invalid flag path.')
        return
    if verify_flag(flag_tensor):
        print('You made this little MLP happy, here\'s his reward:')
        show_flag(flag_tensor)
    else:
        print('Sad :(')

if __name__ == '__main__':
    verify('_Flag.pth')


from mpi4py import MPI
import subprocess
import os

comet_tools_dir = '/lustre/orion/syb111/proj-shared/Personal/jmerlet/tools/comet/genomics_gpu/tools'

def tped_to_bin(path, validate=False):
    # drop '.tped from filename to create prefix
    prefix = path[:-5]
    subprocess.run([os.path.join(comet_tools_dir, 'line_labels.sh'), path, prefix + '_line_labels.txt'])
    subprocess.run([os.path.join(comet_tools_dir, 'line_indices'), path, prefix + '_line_indices.bin'])
    out_file_path = prefix + '_allele_labels.txt'
    out_file = open(out_file_path, 'w')
    subprocess.run(['sed', '-e', 's/.*/AT/', prefix + '_line_labels.txt'], stdout=out_file)
    out_file.close()
    subprocess.run([os.path.join(comet_tools_dir, 'preprocess'), 'duo', path, prefix + '.bin'])
    if validate:
        subprocess.run([os.path.join(comet_tools_dir, 'preprocess_validate'), 'duo', prefix + '.bin', path, prefix + '_allele_labels.txt'])

data_dir = '/lustre/orion/syb111/proj-shared/Projects/scrna-seq/data/human'
tped_paths = []
for r, d, f in os.walk(data_dir):
    for tped in f:
        if '.tped' in tped:
            if 'heart' not in tped:
                tped_paths.append(os.path.join(r, tped))

tped_paths.sort()

comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()

for i, tped_path in enumerate(tped_paths):
    # distribute filepaths to ranks
    # this assumes the number of files is gve to the number of ranks
    if i != rank: continue
    head, filename = os.path.split(tped_path)
    print(f'preprocessing tped {filename} ({rank}/{size})', flush=True)
    tped_to_bin(tped_path)


from statsmodels.tsa.ar_model import AR
from sklearn.metrics import mean_absolute_error
import pandas as pd

from agents.agent import Agent
from agents.prediction_market_adapter import ACCOUNT_0, NUM_PREDICTIONS, TOP_TIER_THRESHOLD

class ArRetrainDecisionAgent(Agent):
    """Agent that uses autoregression with public data only.
       It fits its model again after each day.
       It only bets if its new model returns similar results to the last
       successful model.

    Attributes:
        models: list of dictionaries of models, their train amount, and prediction for
                their respective betting round (in contrast to prediction_history, regardless
                of whether or not the agent bet). The list only contains models trained
                to predict for days that haven't passed yet. Newest first.
        last_successful_model: last model that would have won top tier.
        to_predict_for: first time slice to predict for.
    """
    MODEL_SIMILARITY_THRESHOLD = 70

    def __init__(self, account=ACCOUNT_0, logging=True, **kwargs):
        super(ArRetrainDecisionAgent, self).__init__(account, logging, **kwargs)
        # None as we are not predicting for first day or day before
        self.models = [None, None]
        self.last_successful_model = None
        self.to_predict_for = len(self.aggregate_history) + NUM_PREDICTIONS
        self.log('ArRetrainDecisionAgent')

    def predict_for_tomorrow(self):
        self.update_models()

        # corresponds to first time slice of next prediciton range
        self.to_predict_for += NUM_PREDICTIONS

        # predict all but only take last NUM_PREDICTIONS
        predictions = self.models[0]['model'].predict(
                                    start=self.models[0]['train_amt'],
                                    end=self.to_predict_for-1,
                                    dynamic=False)[-NUM_PREDICTIONS:]
        self.models[0]['predictions'] = predictions

        if self.last_successful_model is None:
            return None

        # predict using last successful model for comparision
        last_successful_model_predictions = self.last_successful_model['model'].predict(
                                         start=self.last_successful_model['train_amt'],
                                         end=self.to_predict_for-1,
                                         dynamic=False)[-NUM_PREDICTIONS:]

        mae = mean_absolute_error(predictions, last_successful_model_predictions)
        if mae > ArRetrainDecisionAgent.MODEL_SIMILARITY_THRESHOLD:
            return None

        return list(map(int, predictions))

    def update_models(self):
        """
        Updates current model and last successful model.
        """
        # only None at the start
        if self.models[-1] is not None:
            mae = mean_absolute_error(self.aggregate_history[-NUM_PREDICTIONS:],
                                      self.models[-1]['predictions'])
            if mae <= TOP_TIER_THRESHOLD:
                self.last_successful_model = self.models[-1]

        self.models = [{'model': AR(self.aggregate_history).fit(),
                        'train_amt': len(self.aggregate_history)}] + self.models[:-1]


import json

import boto3

from src.relation import DynamoRelation
from src.search import findPath

class SearchAction:
    def __init__(self, start: str, end: str, ):
        self.start = start
        self.end = end

resource = boto3.resource('dynamodb')
dynamoRelation = DynamoRelation(resource, 'Person')

def lambda_handler(event, _):
    search_act = SearchAction(**event)

    path = findPath(search_act.start, search_act.end, dynamoRelation.neighbors)
    return {
        'body': json.dumps(path) if path else "None"
    }


import numpy as np
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Activation, Flatten, Embedding, Reshape
from keras.optimizers import Adam
from keras.callbacks import TensorBoard
from collections import deque
import time
import random
from tqdm import tqdm
import os
from PIL import Image
import cv2
import gym
import pyglet

DISCOUNT = 0.99
REPLAY_MEMORY_SIZE = 50_000  # How many last steps to keep for model training
MIN_REPLAY_MEMORY_SIZE = 1_000  # Minimum number of steps in a memory to start training
MINIBATCH_SIZE = 64  # How many steps (samples) to use for training
UPDATE_TARGET_EVERY = 5  # Terminal states (end of episodes)
MODEL_NAME = 'pacman'
MIN_REWARD = 0  # For model save
AVG_REWARD = 0
MEMORY_FRACTION = 0.20

# Environment settings
EPISODES = 100

# Exploration settings
epsilon = 1  # not a constant, going to be decayed
EPSILON_DECAY = 0.99975
MIN_EPSILON = 0.001

#  Stats settings
AGGREGATE_STATS_EVERY = 1  # episodes
SHOW_PREVIEW = True

OBSERVATION_SPACE_DIMS = (210, 160, 3)
ACTION_SPACE_SIZE = 5
env = gym.make('MsPacman-v0')

# For stats
ep_rewards = [0]

# For more repetitive results
random.seed(1)
np.random.seed(1)
tf.set_random_seed(1)

# Memory fraction, used mostly when trai8ning multiple agents
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION)
#backend.set_session(tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)))

# Create models folder
if not os.path.isdir('models'):
	os.makedirs('models')

if not os.path.isdir('logs'):
	os.makedirs('logs')

# Own Tensorboard class
class ModifiedTensorBoard(TensorBoard):

		# Overriding init to set initial step and writer (we want one log file for all .fit() calls)
		def __init__(self, **kwargs):
			super().__init__(**kwargs)
			self.step = 1
			self.writer = tf.summary.FileWriter(self.log_dir)

		# Overriding this method to stop creating default log writer
		def set_model(self, model):
			pass

		# Overrided, saves logs with our step number
		# (otherwise every .fit() will start writing from 0th step)
		def on_epoch_end(self, epoch, logs=None):
			self.update_stats(**logs)

		# Overrided
		# We train for one batch only, no need to save anything at epoch end
		def on_batch_end(self, batch, logs=None):
			pass

		# Overrided, so won't close writer
		def on_train_end(self, _):
			pass

		# Custom method for saving own metrics
		# Creates writer, writes custom metrics and closes writer
		def update_stats(self, **stats):
			self._write_logs(stats, self.step)

class DQNAgent:
		def __init__(self):

				# Main model
				self.model = self.create_model()

				# Target network
				self.target_model = self.create_model()
				self.target_model.set_weights(self.model.get_weights())

				# An array with last n steps for training
				self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE)

				# Custom tensorboard object
				self.tensorboard = ModifiedTensorBoard(log_dir="logs/{}-{}".format(MODEL_NAME, int(time.time())))

				# Used to count when to update target network with main network's weights
				self.target_update_counter = 0

		def create_model(self):
				model = Sequential()

				model.add(Conv2D(256, (4, 4), input_shape=OBSERVATION_SPACE_DIMS))  # OBSERVATION_SPACE_VALUES = (210, 160, 3) a 210 x 160 RGB image.
				model.add(Activation('relu'))
				model.add(MaxPooling2D(pool_size=(2, 2)))
				model.add(Dropout(0.2))

				model.add(Conv2D(256, (3, 3)))
				model.add(Activation('relu'))
				model.add(MaxPooling2D(pool_size=(2, 2)))
				model.add(Dropout(0.2))

				model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors
				model.add(Dense(64))

				model.add(Dense(ACTION_SPACE_SIZE, activation='linear'))  # ACTION_SPACE_SIZE = how many choices (9)
				model.compile(loss="mse", optimizer=Adam(lr=0.001), metrics=['accuracy'])
				return model

		def update_replay_memory(self, transition):
				self.replay_memory.append(transition)

		def get_qs(self, state):
				return self.model.predict(np.array([state]))

		def train(self, terminal_state, step):
				if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE:
						return
				minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE)

				current_states = np.array([transition[0] for transition in minibatch])/255
				current_qs_list = self.model.predict(current_states)

				new_current_states = np.array([transition[3] for transition in minibatch])/255
				future_qs_list = self.target_model.predict(new_current_states)

				X = []	#feature sets (iamges from the game)
				y = []	#label sets (actions we can possibly take)

				for index, (current_state, action, reward, new_current_state, done) in enumerate(minibatch):
						if not done:
								max_future_q = np.max(future_qs_list[index])	#highest Q(s', a') value where s' is the next state
								new_q = reward + DISCOUNT * max_future_q
						else:
								new_q = reward

							#update Q value for given state
						current_qs = current_qs_list[index]    #current Q(s, a) values for current state
						current_qs[action] = new_q	#update how good the given action at the given current state is

						X.append(current_state)
						y.append(current_qs)

				self.model.fit(np.array(X)/255, np.array(y), batch_size = MINIBATCH_SIZE, verbose=0,
								shuffle=False, callbacks=[self.tensorboard] if terminal_state else None)

				#updating to determine if we want to update target_model yet
				if terminal_state:
						self.target_update_counter += 1

				if self.target_update_counter > UPDATE_TARGET_EVERY:
						self.target_model.set_weights(self.model.get_weights())
						self.target_update_counter = 0

agent = DQNAgent()

for episode in tqdm(range(1, EPISODES+1), ascii=True, unit="episode"):
		agent.tensorboard.step = episode

		episode_reward = 0
		step = 1
		current_state = env.reset()

		done = False

		while not done:
				#exploitation vs exploration (explore more at first and eventually ease into exploiting)
				if np.random.random() > epsilon:
						action = np.argmax(agent.get_qs(current_state))
				else:
						action = np.random.randint(0, ACTION_SPACE_SIZE)

				new_state, reward, done, info = env.step(action)

				episode_reward += reward

				if SHOW_PREVIEW and not episode % AGGREGATE_STATS_EVERY:
						env.render()

				agent.update_replay_memory((current_state, action, reward, new_state, done))
				agent.train(done, step)

				current_state = new_state
				step += 1

		# Append episode reward to a list and log stats (every given number of episodes)
		ep_rewards.append(episode_reward)
		if not episode % AGGREGATE_STATS_EVERY or episode == 1:
				average_reward = sum(ep_rewards[-AGGREGATE_STATS_EVERY:])/len(ep_rewards[-AGGREGATE_STATS_EVERY:])
				min_reward = min(ep_rewards[-AGGREGATE_STATS_EVERY:])
				max_reward = max(ep_rewards[-AGGREGATE_STATS_EVERY:])
				agent.tensorboard.update_stats(reward_avg=average_reward, reward_min=min_reward, reward_max=max_reward, epsilon=epsilon)

				# Save model, but only when min reward is greater or equal a set value
				if average_reward >= AVG_REWARD or min_reward >= MIN_REWARD:
						agent.model.save(f'pacman_models/{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model')

		# Decay epsilon
		if epsilon > MIN_EPSILON:
				epsilon *= EPSILON_DECAY
				epsilon = max(MIN_EPSILON, epsilon)


from itertools import permutations

def solution(k, dungeons):
    answer = 0
    lst = list(permutations(dungeons))

    for i in lst:
        cnt = 0
        energy = k
        for j in range(len(i)):
            if i[j][0] <= energy:
                energy -= i[j][1]
                cnt += 1
        if cnt > answer:
            answer = cnt
        if answer == len(dungeons):
            break

    return answer


from datetime import datetime

from setuptools._vendor import more_itertools

class Apriori:
    def __init__(self, transactions, min_support, min_confidence, len_max):
        self.transactions = transactions
        self.min_support = min_support
        self.min_confidence = min_confidence
        self.len_max = len_max
        self.sum_tran = len(transactions)
        self.list_frequent_itemsets = []
        self.l_conf = []
        self.temp = []
        self.l_support = []

    def time(self, s1, s2):
        s1, s2 = str(s1), str(s2)
        hour1 = int(s1[0:2])
        hour2 = int(s2[0:2])
        min1 = int(s1[3:5])
        min2 = int(s2[3:5])
        second1 = int(s1[6:8])
        second2 = int(s2[6:8])
        return (hour2 - hour1) * 3600 + (min2 - min1) * 60 + (second2 - second1)

    #toileanh
    def init_itemsets(self):
        itemsets = set()
        for i in self.transactions:
            for item in i:
                itemsets.add(item)
        results = []
        for i in list(itemsets):
            results.append([i])
        return results

    def create_new_candidate(self, itemsets):
        len_itemset = len(itemsets[0]) + 1
        items = set()
        for i in itemsets:
            for item in i:
                items.add(item)
        items = list(items)
        sinhtohop = more_itertools.distinct_combinations(items, len_itemset)
        candidate_itemsets = []
        for i in sinhtohop:
            candidate_itemsets.append(list(i))
        return candidate_itemsets

    def frequent_itemset(self, candidate_itemsets):
        frequent_itemsets = []
        for i in candidate_itemsets:
            if self.support_itemset(i) >= self.min_support:
                frequent_itemsets.append(i)
        return frequent_itemsets

    def create_frequent_itemsets(self):
        if (self.len_max < 1):
            print("Số item trong Frequent set phải lớn hơn 1 :) ")
        else:
            time1 = datetime.now().time()
            frequent_itemset = []
            itemset = self.init_itemsets()
            frequent_itemsets = self.frequent_itemset(itemset)
            for i in range(self.len_max - 1):
                new_candidates = self.create_new_candidate(
                    frequent_itemsets)
                frequent_itemsets = self.frequent_itemset(
                    new_candidates)
                if (len(frequent_itemsets) == 0):
                    print("Độ dài tối đa của  : %d" % (i + 1))
                    time2 = datetime.now().time()
                    print("Thời gian supported_itemsets len %d: " % (i + 2), self.time(time1, time2))
                    return frequent_itemset
                    break
                else:
                    frequent_itemset.append(frequent_itemsets)
                    time2 = datetime.now().time()
                    print("Time supported_itemsets len %d: " % (i + 2), self.time(time1, time2))
            return frequent_itemset

    def create_frequent_itemset_max(self):
        itemset = self.init_itemsets()
        frequent_itemsets = self.frequent_itemset(itemset)
        print("Len new_item 1 :", len(itemset))
        print("new_items 1: ", itemset)
        print("Len supported_itemset 1: ", len(frequent_itemsets))
        print("frequent_itemset 1: ", frequent_itemsets)
        i = 0
        while (True):
            new_candidates = self.create_new_candidate(frequent_itemsets)
            frequent_itemsets = self.frequent_itemset(new_candidates)
            if len(frequent_itemsets) > 0:
                print(
                    "================================================================================================")
                print("Len new_item %d :" % (i + 2), len(new_candidates))
                print("new_items %d : " % (i + 2), new_candidates)
                print("Len supported_itemset %d : " % (i + 2), len(frequent_itemsets))
                print("frequent_itemset %d : " % (i + 2), frequent_itemsets)
                i += 1
            else:
                return i + 1, frequent_itemsets

    def support_itemset(self, itemset):
        dem = 0
        for i in self.transactions:
            if all(itemm in i for itemm in itemset):
                dem += 1
        return dem / self.sum_tran

    def confident_itemset(self, X, Y):
        XY = []
        XY.extend(X)
        XY.extend(Y)
        return round(float(self.support_itemset(XY) / self.support_itemset(X)), 3)

    def confident_supported_itemset(self, left, right):
        self.rules(left, right)
        if len(left) == 1:
            pass
        else:
            for i in more_itertools.distinct_combinations(left, 1):
                x, y = left.copy(), right.copy()
                y.append(list(i)[0])
                x.remove(list(i)[0])
                if self.confident_itemset(x, y) >= self.min_confidence:
                    self.confident_supported_itemset(x, y)

    def rules(self, left, right):
        if len(right) > 0:
            for i in self.temp:
                if all(itemm in i for itemm in left) and len(left) == len(i):
                    return
            self.list_frequent_itemsets.append([left, right])
            self.l_conf.append(self.confident_itemset(left, right))
            c = []
            c.extend(left)
            c.extend(right)
            self.l_support.append(self.support_itemset(c))
            self.temp.append(left)

    def gen_rules(self, frequent_itemsets):
        for itemset in frequent_itemsets:
            self.temp = []
            left = itemset.copy()
            right = []
            self.confident_supported_itemset(left, right)
        return self.list_frequent_itemsets, self.l_support, self.l_conf


#########################################################################
## Train EmoCatcher for Speech Emotion Recognition on RAVDESS dataset  ##
#########################################################################

import torch
from torch.utils.data import DataLoader
from trainer import train, evaluate
from adabelief_pytorch import AdaBelief
from utils.env import create_config, save_model, create_folder
from utils.criterion import LabelSmoothingLoss
from model import EmoCatcher
from dataset import RAVDESSMelS
from dataloader import get_meta, split_meta, mels2batch
import pickle
import matplotlib.pyplot as plt
import os
import warnings
warnings.simplefilter(action = "ignore", category = FutureWarning)

config = create_config()

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') ## I tested on the CPU

# Extract meta information from .wav file name
meta = get_meta(config.dataset_dir)

# Mel spectrogram parameters
mel_kwargs = {"n_mels":64, "fmax": 6000, "win_length": 2048, "hop_length" :2048//3*2, "n_fft": 2048*2}

if config.kfcv:
    # Stratified 5-Fold CV
    splitted_meta= split_meta(meta, kfcv=True, n_splits=5, target='emotion', random_state=config.random_state)
else:
    # Hold-out 8:2
    splitted_meta= split_meta(meta=meta, kfcv = False, test_ratio=0.2, stratify=True, target='emotion', random_state=config.random_state)

print('device: {}'.format(device))
print('OUTPUT_PATH: {}'.format(config.out_dir))

for k, (tr, ts) in enumerate(splitted_meta):
    if config.kfcv:
        # K-Fold CV
        print(f'----------- Fold {k+1} -----------')
        train_meta = meta.iloc[tr, :]
        test_meta = meta.iloc[ts, :]
    else:
        # Hold-out
        train_meta, test_meta = splitted_meta
    train_dset = RAVDESSMelS(dataset_dir= config.dataset_dir , mel_specs_kwargs=mel_kwargs, speech_list = train_meta.speech_list.tolist())
    test_dset = RAVDESSMelS(dataset_dir= config.dataset_dir , mel_specs_kwargs=mel_kwargs,speech_list=test_meta.speech_list.tolist())

    # Train/test dataloader
    train_dloader = DataLoader(train_dset, batch_size=config.batch_size, collate_fn=mels2batch, shuffle=True)
    test_dloader = DataLoader(test_dset, batch_size=64, collate_fn=mels2batch, shuffle=False)

    # Define model
    model = EmoCatcher(input_dim = mel_kwargs['n_mels'], hidden_dim = 128,kernel_size= 3, num_classes=8).to(device)
    crit = LabelSmoothingLoss(n_classes=8, smoothing=.1, dim=-1)
    optimizer = AdaBelief(model.parameters(), lr=1e-3, eps=1e-8, betas=(0.9,0.999), weight_decay=1e-7, weight_decouple = False, rectify = False, print_change_log=False)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=.7, patience=2, cooldown = 3,  verbose=True, min_lr=1e-6, mode = 'min', threshold=0.001)

    MODEL_SAVE_DIR = os.path.join(config.out_dir, f'model/fold{k+1}/' if config.kfcv else 'model/')
    LOG_OUT_DIR = os.path.join(config.out_dir, 'logs/')
    LOG_FILENAME = os.path.join(LOG_OUT_DIR, f'fold{k+1}_{config.log}' if config.kfcv else config.log)
    N_EPOCHS = config.n_epochs
    create_folder(config.out_dir)
    create_folder(MODEL_SAVE_DIR)
    create_folder(LOG_OUT_DIR)
    print('LOG_OUT_PATH: {}'.format(LOG_FILENAME))
    print('MODEL_OUT_PATH: {}'.format(os.path.join(MODEL_SAVE_DIR, config.model_name)))

    # Training
    print('Training {} Epochs'.format(N_EPOCHS))
    train_loss_list = [] ; test_loss_list = []
    train_acc_list = [] ; test_acc_list = []

    best_test_loss = torch.inf
    best_test_acc = 0

    for e in range(N_EPOCHS):
        train_loss, train_acc = train(train_dloader, model, optimizer, crit,e+1, device = device)
        test_loss, test_acc = evaluate(test_dloader, model, crit, e+1, device = device)

        f = open(f"{LOG_FILENAME}", "a")
        epoch_result = "[Epoch {e}] train Loss: {trainL:.5f} Accuracy: {trainAcc:.4f} | test Loss: {testL:.5f} Accuracy: {testAcc:.4f} ".format(e = e+1, trainL = train_loss, trainAcc = train_acc, testL = test_loss, testAcc = test_acc)
        print(epoch_result, file = f)
        print(epoch_result)

        scheduler.step(train_loss)

        train_loss_list.append(train_loss); test_loss_list.append(test_loss)
        train_acc_list.append(train_acc); test_acc_list.append(test_acc)
        f.close()

        if (test_acc > best_test_acc):
            save_model(MODEL_SAVE_DIR, test_loss = test_loss, test_acc = test_acc, model = model, model_name=config.model_name)
            best_test_acc = test_acc
            best_test_loss = test_loss

        elif ((best_test_acc - test_acc) <1e-3) & (test_loss < best_test_loss):
            save_model(MODEL_SAVE_DIR, test_loss = test_loss, test_acc = test_acc, model = model, model_name=config.model_name)
            best_test_acc = test_acc
            best_test_loss = test_loss
        else:
            pass

    # Save training log
    metric_log_dict = {'train': {
        'acc': train_acc_list,
        'loss': train_loss_list
    },
    'test': {
        'acc': test_acc_list,
        'loss': test_loss_list
    }
    }

    with open(os.path.join(LOG_OUT_DIR, (f'fold{k+1}_' if config.kfcv else '') + 'metric_log_dict.pkl'), 'wb') as f:
        pickle.dump(metric_log_dict, f)

    # Last epoch
    save_model(MODEL_SAVE_DIR, test_loss = test_loss, test_acc = test_acc, model = model, model_name="last_epoch", init=False)
    if config.kfcv == False:
        break


from django.urls import path
from django.views.decorators.csrf import csrf_exempt
from .views import (PathsHomeView,
                    PathwayCreate,
                    PathwayContentCreate,
                    PathwayView,
                    PathwayDevelopView,
                    PathwayEdit,
                    get_dev_pathway_content,
                    edit_dev_pathway_content,
                    get_pathway_content_obj,
                    submit_content_setting_changes,
                    submit_content_delete,
                    UserPathways_ajax_get_pathway_costs,
                    UserPathways_ajax_check_pathway_invites,
                    UserPathways_ajax_submit_pathway_invite,
)

urlpatterns = [
    path('pathways/user', PathsHomeView.as_view(), name="pathways"),
    path('pathway/create', PathwayCreate.as_view(), name="pathway-create"),
    path('pathway/add_content/<int:pathway_id>/', PathwayContentCreate.as_view(), name="pathway-content-create"),
    path('pathway/<int:pathway_id>/', PathwayView.as_view(), name="pathway-view"),
    path('pathway/develop/<int:pathway_id>/', PathwayDevelopView.as_view(), name="pathway-develop"),
    path('pathway/update/<int:pk>/', PathwayEdit.as_view(), name="pathway-edit"),
    path('get_dev_pathway_content/', get_dev_pathway_content, name="pathway-dev-content-get"),
    path('dev_pathway_edit/', edit_dev_pathway_content, name="pathway-dev-content-edit"),
    path('ajax_get_pathway_content_obj/', get_pathway_content_obj, name="pathway-dev-content-modal-edit"),
    path('ajax_submit_content_setting_changes/', submit_content_setting_changes, name="content-setting-changes-submit"),
    path('ajax_submit_delete_pathway_content/', submit_content_delete, name="submit-content-delete"),
    path("UserPathways_ajax_get_pathway_costs/", UserPathways_ajax_get_pathway_costs, name="get-pathway-costs"),
    path("UserPathways_ajax_check_pathway_invites/", UserPathways_ajax_check_pathway_invites, name="check-pathway-invites"),
    path("UserPathways_ajax_submit_pathway_invite/", UserPathways_ajax_submit_pathway_invite, name="submit-pathway-invite"),

]


# coding=utf-8
import pytest
from zentropi.spaces import Space
from zentropi.spaces import Spaces

def test_space():
    space = Space('test-space')
    assert space.name == 'test-space'
    assert space.agents == set()
    space.join('test-agent')
    assert space.agents == {'test-agent'}
    space.leave('test-agent')
    assert space.agents == set()

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_space_fails_on_double_join():
    space = Space('test-space')
    space.join('test-agent')
    space.join('test-agent')

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_space_fails_on_double_leave():
    space = Space('test-space')
    space.join('test-agent')
    space.leave('test-agent')
    space.leave('test-agent')

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_space_fails_on_leave_without_join():
    space = Space('test-space')
    space.leave('test-agent')

def test_spaces():
    spaces = Spaces()
    assert spaces.agents() == []
    assert spaces.spaces() == []

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_spaces_agents_fails_on_invalid_space():
    spaces = Spaces()
    _ = spaces.agents('whoa')

def test_spaces_connect():
    agent_name = 'test-agent'
    spaces = Spaces()
    assert spaces.agents() == []
    spaces.agent_connect(agent_name, connection=None)
    assert spaces.agents() == [agent_name]

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_spaces_connect_fails_on_duplicate_name():
    agent_name = 'test-agent'
    spaces = Spaces()
    assert spaces.agents() == []
    spaces.agent_connect(agent_name, connection=None)
    assert spaces.agents() == [agent_name]
    spaces.agent_connect(agent_name, connection=None)

@pytest.mark.xfail(raises=ValueError, strict=True)
def test_spaces_connect_fails_on_invalid_connection():
    agent_name = 'test-agent'
    spaces = Spaces()
    assert spaces.agents() == []
    spaces.agent_connect(agent_name, connection="lol wut")

def test_spaces_join():
    agent_name = 'test-agent'
    space_name = 'test-space'
    spaces = Spaces()
    assert spaces.agents() == []
    assert spaces.spaces() == []
    spaces.agent_connect(agent_name, connection=None)
    cmd = spaces.join(agent_name, space_name)
    assert cmd.name == 'join'
    assert cmd.data == {'space': space_name}
    assert spaces.agents() == [agent_name]
    assert spaces.agents(space_name) == [agent_name]
    assert spaces.spaces() == [space_name]
    cmd2 = spaces.join(agent_name, space_name + '2')
    assert cmd2.data == {'space': space_name + '2'}
    assert spaces.spaces() == [space_name, space_name + '2']
    cmd3 = spaces.join(agent_name + '3', space_name)
    assert agent_name in spaces.agents(space_name)
    assert agent_name + '3' in spaces.agents(space_name)

def test_spaces_join_fails():
    agent_name = 'test-agent'
    space_name = 'test-space'
    spaces = Spaces()
    spaces.agent_connect(agent_name, connection=None)
    cmd = spaces.join(agent_name, space_name)
    assert cmd.name == 'join'
    cmd = spaces.join(agent_name, space_name)
    assert cmd.name == 'join-failed'


from __future__ import print_function
import subprocess
import json
import datetime
import time
import os

from push_to_datastores import Announcement

DEBUG = False
PENALTY = .18

import paho.mqtt.client as mqtt
from influxdb import InfluxDBClient

# InfluxDB connections settings
host = '10.0.80.30'
dbname = 'energy'

INFLUX_CLIENT = InfluxDBClient(host, database=dbname)

MQTT_CLIENT = mqtt.Client()
MQTT_CLIENT.connect("10.0.80.30", 1883, 60)

shunt_announcement = Announcement("shunt_readings", INFLUX_CLIENT, MQTT_CLIENT, mqtt_topic_prefix="house/power/")

transmit_time = time.time() + 2

while True:
    reader = subprocess.Popen(os.path.expanduser('~/git/pmcomm/libpmcomm/example/example'), stdout=subprocess.PIPE)
    json_power_dict, error = reader.communicate()
    try:
        power_dict = json.loads(json_power_dict)
    except ValueError:
            if DEBUG:
                print("Unable to read JSON: %s" % json_power_dict, error)
            continue

    a1_reading = power_dict['a1']
    a2_reading = power_dict['a2']
    a3_reading = power_dict['a3']

    if a1_reading > 90:
        a1_reading = 0

    if a2_reading > 90:
        a2_reading = 0

    if a3_reading > 90:
        a3_reading = 0

    shunt_announcement.include_for_average("shunt_1", a1_reading)
    shunt_announcement.include_for_average("shunt_2", a2_reading)
    shunt_announcement.include_for_average("shunt_3", a3_reading)

    net = a1_reading + a2_reading + a3_reading
    if net < 0:
        penalty = net * PENALTY
    else:
        penalty = 0

    shunt_announcement.include_for_average("penalty", penalty)

    if time.time() > transmit_time:
        shunt_announcement.do_averages()
        shunt_announcement.transmit()
        for topic, value in shunt_announcement.mqtt_payload.items():
            MQTT_CLIENT.publish("house/power/" + topic.replace("shunt", "watts"), round(value * 12, 3))
        MQTT_CLIENT.publish("house/power/net", round(sum(shunt_announcement.mqtt_payload.values()) * 12, 3))
        shunt_announcement = Announcement("shunt_readings", INFLUX_CLIENT, MQTT_CLIENT, mqtt_topic_prefix="house/power/")
        transmit_time = time.time() + 2


#!/usr/bin/env python3
import numpy as np
from scipy import interpolate
from matplotlib import pyplot as plt

from tqdm import tqdm
import argparse
import sys
import os

import NSencounter as NE
import perturbations as PB
import mass_function
import params
import dirs
import tools
from tools import r_AS

import MilkyWay
import Andromeda

if not os.path.exists(dirs.data_dir + "distributions/"):
    os.makedirs(dirs.data_dir + "distributions/")

# constants
hbar = 6.582e-16  # GeV/GHz
Tage = 4.26e17
RSun = 8.33e3  # pc
pc = 3.086e16  # pc in m
cs = 3.0e8  # speed of light in m/s
vrel0 = 1.0e-3
vrel = vrel0 * cs / pc  # relative velocity in pc/s
u_dispersion = 1.0e-11  # velocity dispersion in pc/s

M_cut = 1.0e-29

## Neutron Star characteristics
MNS = 1.4  # MSun
RNS = 3.24e-13  # pc -- This corresponds to 10km

Nbins_mass = 300

# ---------------------------------
# -------- OPTIONS ----------------
# ---------------------------------

def sample_encounters(Ne, m_a, profile,  AMC_MF, galaxyID = "MW", circular=False, AScut = False, unperturbed=False, IDstr=""):

    Ne = int(Ne)

    file_suffix_in = tools.generate_suffix(profile, AMC_MF, circular=circular,
                                        AScut=False, unperturbed=unperturbed, IDstr=IDstr, verbose=False)

    #We need different labels for the input and output files, because
    #we may also need to label the outputs with "AScut" if we use that option
    file_suffix_out = tools.generate_suffix(profile, AMC_MF, circular=circular,
                                        AScut=AScut, unperturbed=unperturbed, IDstr=IDstr, verbose=True)

    #Set up the mass function
    M0 = AMC_MF.M0

    if (galaxyID == "MW"):
        Galaxy = MilkyWay
    elif (galaxyID == "M31"):
        Galaxy = Andromeda
    else:
        raise ValueError("Invalid galaxyID.")

    plt_path = dirs.plot_dir
    dist_path = dirs.data_dir + "distributions/"

    #Select which columns we want, depending on whether we're including the AS cut
    if (AScut):
        col = 2
    else:
        col = 1

    # Load in survival probabilities
    R_surv_file = dirs.data_dir + "SurvivalProbability_R_" + file_suffix_in + ".txt"# List of survival probabilities
    R_list, psurv_R_list = np.loadtxt(R_surv_file, delimiter=",", dtype="f8", usecols=(0, col), unpack=True)

    # Load in encounter rates
    encounter_file = dirs.data_dir + "EncounterRate_" + file_suffix_in + ".txt"
    R_list, dGammadR_list = np.loadtxt(encounter_file, delimiter=",", dtype="f8", usecols=(0, col), unpack=True)

    # Generate some interpolation functions
    # psurv_a     = interpolate.interp1d(a_list, psurv_a_list) # survival probability (as a function of a)
    # psurv_R     = interpolate.interp1d(R_list, psurv_R_list) # survival probability (as a function of R)
    dGammadR = interpolate.interp1d(R_list, dGammadR_list)  # PDF of the galactic radius

    #Set up dictionaries, which will hold the M, R, rho distributions of AMCs at each radius
    dist_r_list = []
    dist_rho_list = []

    dict_interp_r = dict()
    dict_interp_r_corr = dict()
    dict_interp_rho = dict()
    dict_interp_z = dict()
    dict_interp_mass = dict()

    # --------------------- First we prepare the sampling distributions and total interactions

    if unperturbed:
        dist_r, dist_Pr, dist_Pr_sigu = np.loadtxt( dist_path + "distribution_radius_" + file_suffix_in + ".txt", delimiter=", ", dtype="f8", usecols=(0, 1, 2), unpack=True)

        interp_r = interpolate.interp1d(dist_r, dist_Pr)
        interp_r_corr = interpolate.interp1d(dist_r, dist_Pr_sigu)

    else:

        # Loop through distances
        for i, R in enumerate(R_list):
            #R_kpc = R / 1e3

            # distRX is AMC radius
            # distRY is the PDF dP/dR (normalised as int dP/dR dR = 1) where R is the AMC radius
            # distRC is the PDF <sigma u>*dP/dR where R is the AMC radius
            # distDX is the list of densities for dPdrho
            # distDY is the dPdrho

            try:
                #print("Loading file for radius %.4f " % (R,))
                fname = dist_path + f"distribution_radius_{R:.4f}_{file_suffix_out}.txt"
                #print(fname)
                distRX, distRY, distRC = np.loadtxt(
                    fname,
                    delimiter=", ",
                    dtype="f8",
                    usecols=(0, 1, 2),
                    unpack=True,
                )
                # distDX, distDY = np.loadtxt(dist_path + 'distribution_rho_%.2f_%s.txt'%(R_kpc, PROFILE), delimiter =', ', dtype='f8', usecols=(0,1), unpack=True)

                dist_r_list.append(distRX)
                dict_interp_r[i] = interpolate.interp1d(
                    distRX, distRY
                )  # Radius distribution
                dict_interp_r_corr[i] = interpolate.interp1d(
                    distRX, distRC
                )  # Corrected radius distr

                fname = dist_path + f"distribution_mass_{R:.4f}_{file_suffix_out}.txt"
                #print(fname)
                distMX, distMY = np.loadtxt(
                    fname,
                    delimiter=", ",
                    unpack=True,
                )
                dict_interp_mass[i] = interpolate.interp1d(
                    distMX, distMY, bounds_error=False, fill_value=0.0
                )

            except:
                #print("Warning: File for radius %.4f does not exist" % (R,))
                dist_r_list.append(None)
                dist_rho_list.append(None)
                dict_interp_r[i] = None
                dict_interp_r_corr[i] = None
                # dict_interp_rho[i] = None

                # if (PROFILE == "NFW"):
                dict_interp_mass[i] = None

                continue

    GammaTot = np.trapz(dGammadR_list, R_list)  # s^-1
    #print("Gamma = ", GammaTot, "s^-1 [for f_AMC = 1]")

    # --------------------- Below we are computing the Signal

    # BJK: This still needs lots of cleaning up! A lot of this may be left-over useless stuff.

    #BJK: GOT TO HERE!!!
    Interactions = []

    R_sample = tools.inverse_transform_sampling(
        dGammadR, [np.min(R_list[psurv_R_list > 2e-6]), np.max(R_list)], n_samples=Ne, logarithmic=True
    )  # Galactocentric radius of interaction in pc
    R_sample = np.array(R_sample)

    # Draw the radii and densities of the axion MCs
    Z_gal = np.zeros(Ne)
    MC_r = np.zeros(Ne)
    MC_rho = np.zeros(Ne)

    for l, R in enumerate(tqdm(R_sample, desc="> Sampling encounters")):

        # Draw a value of z for the encounter
        dpdz = lambda Z: Galaxy.dPdZ(Z)
        Z_gal[l] = tools.inverse_transform_sampling(dpdz, np.array([-R, R]), n_samples=1, nbins=1000)

        Pr_check = 0
        if not unperturbed:
            # Find the GC radius that is closer to what is drawn from distribution
            abs_val = np.abs(R_list - R)
            smallest = abs_val.argmin()
            while (Pr_check <= 0):
                R_orb = R_list[smallest]  # Closest orbit to the r drawn
                dist_r = dist_r_list[smallest]  # List of MC radii
                # dist_rho  = dist_rho_list[smallest] # List of MC densities
                # interp_rho = dict_interp_rho[smallest]
                interp_r_corr = dict_interp_r_corr[smallest]

                # if (PROFILE == "NFW"):
                interp_M = dict_interp_mass[smallest]
                Pr_check = np.sum(interp_r_corr(dist_r))
                if (Pr_check <= 0):
                    #print("Trying next radius...")
                    smallest -= 1

        alpha_AS = r_AS(1.0, m_a)
        k_AMC = (3 / (4 * np.pi)) ** (1 / 3)

        # radius in pc
        MC_r[l] = tools.inverse_transform_sampling(
            interp_r_corr, dist_r, n_samples=1, nbins=10000, logarithmic=True
        )

        rho_max = 3 * AMC_MF.mmax / (4 * np.pi * MC_r[l] ** 3)

        if AScut:
            rho_min_AS = (alpha_AS * k_AMC / MC_r[l] ** 2) ** 3
        else:
            rho_min_AS = 1e-30

        # Sample AMC density rho given the AMC radius R
        if unperturbed:

            dPdM = lambda x: AMC_MF.dPdlogM(x) / x
            # (M_f/rho)*P(M_f)
            P_rho_given_r = lambda rho: ((4 * np.pi / 3) * MC_r[l] ** 3) * dPdM(
                (4 * np.pi / 3) * rho * MC_r[l] ** 3
            )
            rho_min = np.maximum(3 * AMC_MF.mmin / (4 * np.pi * MC_r[l] ** 3), rho_min_AS)

        else:
            P_rho_given_r = lambda rho: ((4 * np.pi / 3) * MC_r[l] ** 3) * interp_M(
                (4 * np.pi / 3) * rho * MC_r[l] ** 3
            )
            #rho_min = np.maximum(
            #    3 * (1e-6 * AMC_MF.mmin) / (4 * np.pi * MC_r[l] ** 3), rho_min_AS
            #)
            rho_min = np.maximum(
                3 * (1e-10 * M0) / (4 * np.pi * MC_r[l] ** 3), rho_min_AS
            )
        # P_rho_given_r = lambda rho: NE.P_r_given_rho(MCrad[l], rho, mmin, mmax, gg)*dict_interp_rho[smallst](rho)/dict_interp_rad[smallst](MCrad[l])

        MC_rho[l] = tools.inverse_transform_sampling(
            P_rho_given_r, [1e-6*rho_min, rho_max], n_samples=1, nbins=100000, logarithmic=True
        )

        if (np.isnan(MC_rho[l])):
            print(R, R_orb, MC_r[l], MC_rho[l])
            print(dist_r)
            print(interp_r_corr(dist_r))

        #if (1 < 2):
            plt.figure()
            rho_list = np.geomspace(rho_min, rho_max, 100000)
            plt.loglog(rho_list, P_rho_given_r(rho_list))
            plt.show()

        # print(rho_min, MC_rho[l])

    psi = np.random.uniform(-np.pi, np.pi, Ne)  # Longitude
    xi = np.arcsin(Z_gal / R_sample)  # Latitude

    z0 = Z_gal
    x0 = R_sample * np.cos(xi) * np.cos(psi)
    y0 = R_sample * np.cos(xi) * np.sin(psi)
    R_spherical = np.sqrt(x0 ** 2 + y0 ** 2 + z0 ** 2)

    s0 = np.sqrt((RSun + x0) ** 2 + y0 ** 2 + z0 ** 2)  # Distance from events in pc
    bG = (
        np.arctan(z0 / np.sqrt((RSun + x0) ** 2 + y0 ** 2)) * 180.0 / np.pi
    )  # Galactic Latitude
    lG = np.arctan2(y0, (RSun + x0)) * 180.0 / np.pi  # Galactic Longitude

    # Relative velocity between NS & AMC
    vrel = np.sqrt(2) * Galaxy.sigma(R_sample) * 3.24078e-14
    ux = np.random.normal(0, vrel, Ne)  # pc/s
    uy = np.random.normal(0, vrel, Ne)  # pc/s
    uz = np.random.normal(0, vrel, Ne)  # pc/s
    # ut    = np.abs(np.sqrt((ux + VNSX)**2 + (uy + VNSY)**2 + uz**2)) # pc/s
    ut = np.sqrt(ux ** 2 + uy ** 2 + uz ** 2)  # pc/s

    # BJK: Here, we need to make sure that we implement the same cut on the
    # impact parameter as we did for the radii of very diffuse AMCs
    rho_loc = Galaxy.rhoNFW(R_sample)
    rho_crit = rho_loc * params.min_enhancement

    if profile == "PL":
        r_cut = MC_r * np.minimum(
            (MC_rho / (4 * rho_crit)) ** (4 / 9), np.ones_like(MC_rho)
        )

    elif profile == "NFW":
        c = 100
        rho_s = MC_rho * (
            c ** 3 / (3 * NE.f_NFW(c))
        )  # Convert mean density rhoi to AMC density
        r_cut = MC_r * np.minimum(NE.x_of_rho(rho_crit / rho_s), np.ones_like(MC_rho))

    # Make sure everything is fine
    r_cut[r_cut < 0.0] = 0.0 * r_cut[r_cut < 0.0] + MC_r[r_cut < 0.0]
    r_cut[np.isnan(r_cut)] = MC_r[np.isnan(r_cut)]
    r_cut[np.isinf(r_cut)] = MC_r[np.isinf(r_cut)]

    # Impact parameter
    b = np.sqrt(np.random.uniform(0.0, r_cut ** 2, Ne))

    MC_M = (4*np.pi*MC_r**3/3)*MC_rho

    lablist = ["x0", "y0", "z0", "rho", "r", "M"]

    for k in range(Ne):
        #Select from either the GC population or the CMZ population
        if (R_spherical[k] > 10):
            NS_dict = Galaxy.CMZ_dict
            len_NS = Galaxy.len_CMZ
        else:
            NS_dict = Galaxy.GC_dict
            len_NS = Galaxy.len_GC

        iNS = np.random.randint(0, len_NS)

        # SJW version
        interaction_params = [
            NS_dict["B0"][iNS],
            NS_dict["T"][iNS],
            NS_dict["theta"][iNS],
            NS_dict["t"][iNS],
            x0[k],
            y0[k],
            z0[k],
            MC_rho[k],
            MC_r[k],
            MC_M[k],
            b[k],
            ut[k]
        ]
        Interactions.append(interaction_params)

    #Interactions = np.array(Interactions)
    Interactions = np.array(Interactions)
    # print(Interactions, Interactions.shape)

    int_file = dirs.data_dir + "Interaction_params_" + file_suffix_out + ".txt.gz"

    print("> Outputting encounters to file:", int_file)

    np.savetxt(
        int_file,
        Interactions,
        header="B0 [G], Period [s], Misalignment angle [radians], NS Age [Myr], x [pc], y [pc], z [pc], AMC density [Msun/pc^3], AMC radius [pc], AMC mass [Msun],  Impact parameter [pc], Relative velocity [pc/s]",
        fmt="%.5e",
    )
    return b/ut

def getOptions(args=sys.argv[1:]):
    # Parse the arguments!
    parser = argparse.ArgumentParser(description="...")

    parser.add_argument("-Ne", "--Ne", help="Number of encounters to sample", type=int, default=1000)
    parser.add_argument("-profile", "--profile", help="Density profile for AMCs - `NFW` or `PL`", type=str, default="PL")
    parser.add_argument("-m_a", "--m_a", type=float, help="Axion mass in eV", default = 50e-6)
    parser.add_argument("-unperturbed", "--unperturbed", help="Calculate for unperturbed profiles?", type=bool, default=False)
    parser.add_argument("-galaxyID", "--galaxyID", type=str, help="ID of galaxy - 'MW' or 'M31'", default="MW")
    parser.add_argument("-circ", "--circular", dest="circular", action="store_true", help="Use the circular flag to force e = 0 for all orbits.")
    parser.add_argument("-AScut", "--AScut", dest="AScut", action="store_true", help="Include an axion star cut on the AMC properties.")
    parser.add_argument("-MF_ID", "--mass_function_ID", help="...", type=str, default="delta_c")
    parser.add_argument("-IDstr", "--IDstr", type=str, help = "ID string to label the output files.", default="")
    parser.set_defaults(circular=False)
    parser.set_defaults(AScut=False)

    options = parser.parse_args(args)
    return options

if __name__ == '__main__':
    opts = getOptions(sys.argv[1:])

    #Create a mass function based on the input "mass function ID"
    AMC_MF = get_mass_function(opts.MF_ID, opts.m_a, opts.profile)
    AMC_MF.label = opts.MF_ID

    sample_encounters(opts.Ne, opts.m_a, opts.profile, AMC_MF,  opts.galaxyID,  opts.circular, opts.AScut, opts.unperturbed, opts.IDstr)


import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import pandas as pd

from sklearn.preprocessing import LabelEncoder
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

# read data set from cvs file

def get_dataset():
    df = pd.read_csv("dataset.cvs")

    x = df[df.columns[0:60]].values
    y = df[df.columns[60]]

    # Encode the dependant variables

    encoder = LabelEncoder()
    encoder.fit(y)
    y = encoder.transform(y)
    _y = one_hot_encode(y)
    print(x.shape)
    return (x, _y)

def one_hot_encode(labels):
    n_labels = len(labels)
    n_unique_labels = len(np.unique(labels))
    one_hot_encoded_matrix = np.zeros((n_labels, n_unique_labels))
    one_hot_encoded_matrix[np.arange(n_labels), labels] = 1
    return one_hot_encoded_matrix

# Read processed data set from file
X, Y = get_dataset()

# shuffle row wise to mix up as its in order
X, Y = shuffle(X, Y, random_state=1)

# print shapes
train_x, test_x, train_y, test_y = train_test_split(X, Y, test_size=0.20, random_state=42)

print("Train data shape : \t", train_x.shape)
print("Train labels shape : \t", train_y.shape)
print("Test data shape : \t", test_x.shape)
print("Test data shape : \t", test_y.shape)

# Define the parameter

lr = 0.3
train_epochs = 1000
cost_history = np.empty(shape=[1], dtype=float)
each_sample_dim = X.shape[1]

print("Each sample is of size : ", each_sample_dim)

n_output_classes = test_y.shape[1]

print("Number of output classes : ", n_output_classes)

model_path = "model\\NMI"

# Define number of hidden layer and number of neurons for each layer

n_hidden_1 = 60
n_hidden_2 = 60
n_hidden_3 = 60
n_hidden_4 = 60

# define your model

# last layer
x = tf.placeholder(tf.float32, [None, each_sample_dim])
_y = tf.placeholder(tf.float32, [None, n_output_classes])

# Hidden layer

def multilayer_perceptron(x, weights, biases):
    # Hidden layer + sigmoid
    layer1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer1 = tf.nn.sigmoid(layer1)

    # Hidden layer + sigmoid
    layer2 = tf.add(tf.matmul(layer1, weights['h2']), biases['b2'])
    layer2 = tf.nn.sigmoid(layer2)

    # Hidden layer + sigmoid
    layer3 = tf.add(tf.matmul(layer2, weights['h3']), biases['b3'])
    layer3 = tf.nn.sigmoid(layer3)

    # Hidden layer + RELU
    layer4 = tf.add(tf.matmul(layer3, weights['h4']), biases['b4'])
    layer4 = tf.nn.relu(layer4)

    # last layer
    out_layer = tf.matmul(layer4, weights['out']) + biases['out']
    return out_layer

# define the weights and biases
weights = {
    'h1': tf.Variable(tf.truncated_normal([each_sample_dim, n_hidden_1])),
    'h2': tf.Variable(tf.truncated_normal([n_hidden_1, n_hidden_2])),
    'h3': tf.Variable(tf.truncated_normal([n_hidden_2, n_hidden_3])),
    'h4': tf.Variable(tf.truncated_normal([n_hidden_3, n_hidden_4])),
    'out': tf.Variable(tf.truncated_normal([n_hidden_4, n_output_classes])),
}

biases = {
    'b1': tf.Variable(tf.truncated_normal([n_hidden_1])),
    'b2': tf.Variable(tf.truncated_normal([n_hidden_2])),
    'b3': tf.Variable(tf.truncated_normal([n_hidden_3])),
    'b4': tf.Variable(tf.truncated_normal([n_hidden_4])),
    'out': tf.Variable(tf.truncated_normal([n_output_classes])),
}

# Initialize variables
init = tf.global_variables_initializer()
save_model = tf.train.Saver()

# call model
our_model = multilayer_perceptron(x, weights, biases)

# loss function and back propagation
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=our_model, labels=_y))
back_prop = tf.train.GradientDescentOptimizer(lr).minimize(loss)

sess = tf.Session()
sess.run(init)

# Maintain loss and accuracy at each epoch

mse_history = []
accuracy_history = []

for epoch in range(train_epochs):
    sess.run(back_prop, feed_dict={x: train_x, _y: train_y})
    cost = sess.run(loss, feed_dict={x: train_x, _y: train_y})
    cost_history = np.append(cost_history, cost)
    correct_prediction = tf.equal(tf.argmax(our_model, 1), tf.argmax(_y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    # run on test data
    pred_y = sess.run(our_model, feed_dict={x: test_x})
    mse = tf.reduce_mean(tf.square(pred_y - test_y))
    mse_ = sess.run(mse)
    mse_history.append(mse_)

    accuracy = sess.run(accuracy, feed_dict={x: train_x, _y: train_y})
    accuracy_history.append(accuracy)

    print('epoch ', epoch, '\t\t- ', 'cost', cost, '\t\t- mse_: ', mse_, '\t\t- Train Accuracy : ', accuracy)

save_path = save_model.save(sess, model_path)
print("Model is saved at %s" % save_path)

plt.plot(mse_history[5:], 'r')
plt.show()

plt.plot(accuracy_history)
plt.show()

# print the final accuracy

correct_prediction = tf.equal(tf.argmax(our_model, 1), tf.argmax(_y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("Test Accuracy : ", sess.run(accuracy, feed_dict={x: test_x, _y: test_y}))

# Print the final mean square error

pred_y = sess.run(our_model, feed_dict={x: test_x})
mse = tf.reduce_mean(tf.square(pred_y - test_y))
print('MSE : %.f' % sess.run(mse))


import numpy as np
import torch

import geometry_util

def _integrated_angle_metric(angles):
    angles_rolled = np.roll(angles, 1, axis=-1)
    return np.abs(angles - angles_rolled)

def _densely_sample_angles(angles, n_samples):
    angles_rolled = np.roll(angles, 1, axis=1)
    return np.linspace(start=angles, stop=angles_rolled, num=n_samples, axis=-1)

def _resample_points_ring(points, n_samples):
    angles = torch.atan2(points[:, :, 1], points[:, :, 0])
    sampled_angles = torch.transpose(torch.transpose(geometry_util.torch_linspace(angles, torch.roll(angles, 1, dims=-1), n_samples), 0, 2), 0, 1)
    points = geometry_util.torch_angles_to_ring(sampled_angles)
    return angles, points

def test_integrated_metric():
    n_resamples = 200
    angles = np.sort(np.random.uniform(-np.pi, np.pi, (1, 4)), axis=1)
    angular_distance = _integrated_angle_metric(angles)
    resampled_angles = _densely_sample_angles(angles, n_resamples)
    points = torch.from_numpy(geometry_util.angles_to_ring(resampled_angles))
    point_metric = geometry_util.integrated_point_metric(points)
    np.testing.assert_allclose(point_metric, angular_distance, rtol=1e-3)

def test_ring_interpolation():
    n_samples = 100
    actual_angles = torch.from_numpy(np.random.uniform(-np.pi, np.pi, (1, 10)))
    points = geometry_util.torch_angles_to_ring(actual_angles)
    pred_angles, interpolated_points = _resample_points_ring(points, n_samples)
    np.testing.assert_allclose(actual_angles, pred_angles)
    point_metric = geometry_util.integrated_point_metric(interpolated_points)
    angular_metric = _integrated_angle_metric(actual_angles)
    np.testing.assert_allclose(point_metric, angular_metric, rtol=1e-3)

def test_minimum_periodic_dist():
    test_1d_1 = torch.tensor([[2 * np.pi - 0.5], [1]])
    test_1d_2 = torch.tensor([[0.5], [1.2]])
    with torch.no_grad():
        min_dist, p2_remap = geometry_util.minimum_periodic_distance(
            test_1d_1, test_1d_2)
    np.testing.assert_allclose(np.squeeze(p2_remap), [2 * np.pi + 0.5, 1.2], rtol=1e-6)
    np.testing.assert_allclose(min_dist, [1, 0.2], rtol=1e-6)

    test_2d_1 = torch.tensor([[0.5, 0.5], [0, 0]])
    test_2d_2 = torch.tensor([[2 * np.pi - 0.5, 2 * np.pi - 0.5], [1, 1]])
    with torch.no_grad():
        min_dist, p2_remap = geometry_util.minimum_periodic_distance(
            test_2d_1, test_2d_2)
    np.testing.assert_allclose(p2_remap, [[-0.5, -0.5], [1, 1]])
    np.testing.assert_allclose(min_dist, [np.sqrt(2), np.sqrt(2)], rtol=1e-6)

def test_closest_points_periodic():
    points = torch.from_numpy(np.array([[0, 0.1], [0, 2 * np.pi - 0.1], [0.2, 2 * np.pi - 0.2], [0.2, 0.2]]))
    with torch.no_grad():
        min_dist,  matched, matches = geometry_util.closest_points_periodic(points)
        matched_ring = geometry_util.torch_angles_to_ring(matched)
        matches_ring = geometry_util.angles_to_ring(torch.gather(points, 0, torch.tile(torch.unsqueeze(matches, -1), [points.size(-1)])))
    np.testing.assert_allclose(matches, [1, 0, 1, 0])
    np.testing.assert_allclose(min_dist, [0.2, 0.2, np.sqrt(0.2**2 + 0.1**2), np.sqrt(0.2**2 + 0.1**2)], rtol=1e-6)
    np.testing.assert_allclose(matched_ring, matches_ring, atol=1e-6)


from __future__ import print_function
import torch
import numpy as np
import torch.utils.data as data
import pandas as pd
from nn.common import DBType
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.utils import shuffle

# https://www.kaggle.com/c/mercari-price-suggestion-challenge/data

MIN_CATEGORY_NAMES = 500
MIN_BRAND_NAMES = 100
MIN_DESC_DF = 1000

def init_dataset(config, db_type, name_vectorizer, raw_data):
    dataset = MercariDataset(config, db_type, name_vectorizer, raw_data)

    if not (db_type == DBType.Train):
        data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=config.test.batch_size, shuffle=False,
                                                  num_workers=8, drop_last=False)
    else:
        data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=config.train.batch_size, shuffle=True,
                                                  num_workers=8, drop_last=True)

    return data_loader, dataset

def limit_categorical_field(data, field_name, min_occur, raw_data):
    value_counts = raw_data[field_name].value_counts()
    categories = [k for k, v in value_counts.items() if v > min_occur]
    categories_dict = {k: i + 1 for i, k in enumerate(categories)}

    # Convert missing values to a separate category - 0
    categories_dict[np.nan] = 0

    # Convert all the infrequent values to a separate category
    unknown_val = len(categories_dict)

    def gen_category_id(cat):
        if cat in categories_dict:
            return categories_dict[cat]
        else:
            return unknown_val

    data[field_name + '_id'] = data[field_name].apply(gen_category_id)

    return data.drop(field_name, axis=1), raw_data.drop(field_name, axis=1)

def preprocess_category_name(data, raw_data):
    for i in range(4):
        def get_part(x):
            # Handle missing values (np.nan)
            if type(x) != str:
                return np.nan

            parts = x.split('/')
            if i >= len(parts):
                return np.nan
            else:
                return parts[i]

        field_name = 'category_part_' + str(i)
        data[field_name] = data['category_name'].apply(get_part)
        raw_data[field_name] = raw_data['category_name'].apply(get_part)
        data, raw_data = limit_categorical_field(data, field_name, MIN_CATEGORY_NAMES, raw_data)

    return data.drop('category_name', axis=1)

def preprocess_brand_name(data, raw_data):
    data, raw_data = limit_categorical_field(data, 'brand_name', MIN_BRAND_NAMES, raw_data)
    return data

def train_tf_idf(kn, data, v):
    X = v.transform(data[kn])
    df = pd.DataFrame(X.toarray())
    return df

def preprocess_data(data, name_vectorizer, raw_data):
    print("Limiting category and brand names...")
    data = preprocess_category_name(data, raw_data)
    data = preprocess_brand_name(data, raw_data)

    print("Categorizing features...")
    data = pd.get_dummies(data, columns=['category_part_0_id', 'category_part_1_id',
                                         'category_part_2_id', 'brand_name_id'])

    names = train_tf_idf('name', data, name_vectorizer)

    return data, names

class MercariDataset(data.Dataset):
    def __init__(self, config, db_type, name_vectorizer, raw_data):
        self.config = config
        self.db_type = db_type
        self.longest_sent = 245
        self.vocab = np.load(config.dataset.vocab_path).item()
        self.pad_str = '<PAD>'
        self.pad_token = int(self.vocab[self.pad_str])

        self.embedding_dim = 50
        self.nb_vocab_words = len(self.vocab)

        if db_type == DBType.Test:
            db_path = config.dataset.test_path
            self.batch_size = config.test.batch_size
        elif db_type == DBType.Validation:
            db_path = config.dataset.validation_path
            self.batch_size = config.test.batch_size
        elif db_type == DBType.Train:
            db_path = config.dataset.train_path
            self.batch_size = config.train.batch_size
        else:
            raise Exception('Invalid db type')

        df = pd.read_csv(db_path, sep="\t")

        if db_type == DBType.Train:
            df = shuffle(df)

        self.df, self.names = preprocess_data(df, name_vectorizer, raw_data)
        self.dataset_size = len(df.values)
        # self.glove_model = self.load_glove_model(config.dataset.glove_path)

    def __len__(self):
        return self.dataset_size // self.batch_size * self.batch_size

    @staticmethod
    def load_glove_model(glove_file):
        print("Loading Glove Model")

        with open(glove_file, encoding="utf8") as f:
            content = f.readlines()

        model = {}
        for line in content:
            split_line = line.split()
            word = split_line[0]
            embedding = np.array([float(val) for val in split_line[1:]])
            model[word] = embedding

        print("Done.", len(model), " words loaded!")

        return model

    def __getitem__(self, index):
        record = self.df.iloc[[index]]

        name = self.names.iloc[[index]].values[0]
        name = torch.tensor(name, dtype=torch.float)

        assert torch.isnan(name).max().detach().cpu().numpy() == 0

        # name = [self.vocab[word] for word in (record[1].split() if not pd.isnull(record[1]) else [self.pad_str])]
        # name_len = len(name)
        # padded_name = np.ones(self.longest_sent) * self.pad_token
        # padded_name[0:name_len] = name
        # padded_name = torch.tensor(padded_name, dtype=torch.long)
        # padded_name = self.words_embedding(padded_name)

        c_id = record.ix[:, 2].values[0]
        c_id = torch.tensor(c_id, dtype=torch.float)

        assert torch.isnan(c_id).max().detach().cpu().numpy() == 0

        price = record.ix[:, 3].values[0]
        price = torch.tensor(np.log1p(price), dtype=torch.float)

        assert torch.isnan(price).max().detach().cpu().numpy() == 0

        shipping = record.ix[:, 4].values[0]
        shipping = torch.tensor(shipping, dtype=torch.float)

        assert torch.isnan(shipping).max().detach().cpu().numpy() == 0

        tmp_desc = record.ix[:, 5].values[0]
        desc = [self.vocab[word] for word in (tmp_desc.split() if not pd.isnull(tmp_desc) else [self.pad_str])]
        desc_len = len(desc)
        padded_desc = np.ones(self.longest_sent) * self.pad_token
        padded_desc[0:desc_len] = desc
        padded_desc = torch.tensor(padded_desc, dtype=torch.long)

        assert torch.isnan(padded_desc).max().detach().cpu().numpy() == 0
        # padded_desc = self.words_embedding(padded_desc)

        # c_name = [self.vocab[word] for word in (record[3].split() if not pd.isnull(record[3]) else [self.pad_str])]
        # c_name_len = len(c_name)
        # padded_c_name = np.ones(self.longest_sent) * self.pad_token
        # padded_c_name[0:c_name_len] = c_name
        # padded_c_name = torch.tensor(padded_c_name, dtype=torch.long)

        c_name = record.ix[:, 6:353].values[0]
        c_name = torch.tensor(c_name, dtype=torch.float)

        assert torch.isnan(c_name).max().detach().cpu().numpy() == 0

        # b_name = [self.vocab[word] for word in (record[4].split() if not pd.isnull(record[4]) else [self.pad_str])]
        # b_name_len = len(b_name)
        # padded_b_name = np.ones(self.longest_sent) * self.pad_token
        # padded_b_name[0:b_name_len] = b_name
        # padded_b_name = torch.tensor(padded_b_name, dtype=torch.long)

        b_name = record.ix[:, 353:].values[0]
        b_name = torch.tensor(b_name, dtype=torch.float)

        assert torch.isnan(b_name).max().detach().cpu().numpy() == 0

        input_dict = {'name': name,
                      'cid': c_id,
                      'c_name': c_name,
                      'b_name': b_name,
                      'price': price,
                      'shipping': shipping,
                      'desc': padded_desc,
                      'desc_len': desc_len}

        return input_dict

# def word2vec(self, word):
#     if self.glove_model.__contains__(word.lower()):
#         return self.glove_model[word.lower()]
#     else:
#         return np.zeros(50)

# def __getitem__(self, index):
#     record = self.values[index]
#
#     name = [self.word2vec(x) for x in record[1].split(' ')]
#     name = torch.tensor(name, dtype=torch.float)
#
#     cid = record[2]
#     cid = torch.tensor(cid, dtype=torch.int)
#
#     c_name = [self.word2vec(x) for x in (record[3].split(' ') if not pd.isnull(record[3]) else ' ')]
#     c_name = torch.tensor(c_name, dtype=torch.float)
#
#     b_name = [self.word2vec(x) for x in (record[4].split(' ') if not pd.isnull(record[4]) else ' ')]
#     b_name = torch.tensor(b_name, dtype=torch.float)
#
#     price = record[5]
#     price = torch.tensor(price, dtype=torch.int)
#
#     shipping = record[6]
#     shipping = torch.tensor(shipping, dtype=torch.int)
#
#     desc = [self.word2vec(x) for x in (record[7].split(' ') if not pd.isnull(record[7]) else ' ')]
#     desc = torch.tensor(desc, dtype=torch.float)
#
#     input_dict = {'name': name,
#                   'cid': cid,
#                   'c_name': c_name,
#                   'b_name': b_name,
#                   'price': price,
#                   'shipping': shipping,
#                   'desc': desc}
#
#     return input_dict