text:
// Copyright 2008 by BBN Technologies Corp.
// All Rights Reserved.

#include "common/leak_detection.h"

#include "docRelationsEvents/RelationTimexArgFinder.h"
#include "docRelationsEvents/RelationTimexArgFeatureTypes.h"
#include "docRelationsEvents/RelationTimexArgFeatureType.h"
#include "docRelationsEvents/RelationTimexArgObservation.h"
#include "common/UnexpectedInputException.h"
#include "common/StringTransliterator.h"
#include "theories/RelMention.h"
#include "theories/RelMentionSet.h"
#include "theories/Token.h"
#include "common/ParamReader.h"
#include "common/HeapChecker.h"
#include "maxent/MaxEntModel.h"
#include "discTagger/DTFeatureTypeSet.h"
#include "discTagger/DTTagSet.h"
#include "theories/TokenSequence.h"
#include "theories/Parse.h"
#include "theories/NPChunkTheory.h"
#include "theories/MentionSet.h"
#include "theories/ValueMentionSet.h"
#include "theories/PropositionSet.h"
#include "theories/RelMentionSet.h"
#include "wordClustering/WordClusterTable.h"
#include "state/TrainingLoader.h"
#include "morphAnalysis/MorphologicalAnalyzer.h"
#include <boost/scoped_ptr.hpp>

RelationTimexArgFinder::RelationTimexArgFinder(int mode_) : MODE(mode_),
	_tagSet(0),	_tagScores(0), _featureTypes(0), _weights(0),
    _model(0), _secondPassWeights(0), _secondPassModel(0), _morphAnalysis(0)
{
	_observation = _new RelationTimexArgObservation();
	RelationTimexArgFeatureTypes::ensureFeatureTypesInstantiated();
	WordClusterTable::ensureInitializedFromParamFile();

	if (MODE == TRAIN && ParamReader::isParamTrue("train_second_pass_relation_time_model"))
		MODE = TRAIN_SECOND_PASS;

	// DEBUGGING
	std::string buffer = ParamReader::getParam("relation_time_debug_file");
	DEBUG = false;
	if (!buffer.empty()) {
		DEBUG = true;
		_debugStream.open(buffer.c_str());
	}

	// TAG SET
	std::string tag_set_file = ParamReader::getRequiredParam("relation_time_tag_set_file");
	_tagSet = _new DTTagSet(tag_set_file.c_str(), false, false);
	_tagScores = _new double[_tagSet->getNTags()];

	// FEATURES
	std::string features_file = ParamReader::getRequiredParam("relation_time_features_file");
	_featureTypes = _new DTFeatureTypeSet(features_file.c_str(), RelationTimexArgFeatureType::modeltype);

	_use_two_pass_model = ParamReader::isParamTrue("use_two_pass_time_arg_model");
	if (_use_two_pass_model || MODE == TRAIN_SECOND_PASS) {
		std::string second_pass_features_file =	ParamReader::getRequiredParam("relation_time_second_pass_features_file");
		_secondPassFeatureTypes = _new DTFeatureTypeSet(second_pass_features_file.c_str(),
			RelationTimexArgFeatureType::modeltype);
	}

	if (MODE == DECODE) {

		// RECALL THRESHOLD
		_recall_threshold = ParamReader::getRequiredFloatParam("relation_time_recall_threshold");

		// MODEL FILE
		std::string model_file = ParamReader::getParam("relation_time_model_file");
		if (!model_file.empty()) {
			_weights = _new DTFeature::FeatureWeightMap(50000);
			DTFeature::readWeights(*_weights, model_file.c_str(), RelationTimexArgFeatureType::modeltype);
			_model = _new MaxEntModel(_tagSet, _featureTypes, _weights);

			if (_use_two_pass_model) {
				std::string buffer = model_file + ".second-pass";
				_secondPassWeights = _new DTFeature::FeatureWeightMap(50000);
				DTFeature::readWeights(*_secondPassWeights, buffer.c_str(), RelationTimexArgFeatureType::modeltype);
				_secondPassModel = _new MaxEntModel(_tagSet, _secondPassFeatureTypes, _secondPassWeights);
			}

		} else {
			model_file = ParamReader::getParam("relation_time_round_robin_setup");
			if (model_file.empty()) {
				throw UnexpectedInputException("RelationTimexArgFinder::RelationTimexArgFinder()",
					"Neither 'relation_time_model_file' nor 'event_round_robin_setup' specified");
			} else {
				_weights = 0;
				_model = 0;
				_secondPassWeights = 0;
				_secondPassModel = 0;
			}
		}

	} else if (MODE == TRAIN || MODE == TRAIN_SECOND_PASS) {

		// PERCENT HELD OUT
		int percent_held_out =
			ParamReader::getRequiredIntParam("relation_time_maxent_trainer_percent_held_out");
		if (percent_held_out < 0 || percent_held_out > 50)
			throw UnexpectedInputException("RelationTimexArgFinder::RelationTimexArgFinder()",
			"Parameter 'relation_time_maxent_trainer_percent_held_out' must be between 0 and 50");

		// MAX NUMBER OF ITERATIONS (STOPPING CONDITION)
		int max_iterations =
			ParamReader::getRequiredIntParam("relation_time_maxent_trainer_max_iterations");

		// GAUSSIAN PRIOR VARIANCE
		double variance = ParamReader::getRequiredFloatParam("relation_time_maxent_trainer_gaussian_variance");

		// MIN CHANGE IN LIKELIHOOD (STOPPING CONDITION)
		double likelihood_delta = ParamReader::getOptionalFloatParamWithDefaultValue("relation_time_maxent_trainer_gaussian_variance", .0001);

		// FREQUENCY OF STOPPING CONDITION CHECKS (NUM ITERATIONS)
		int stop_check_freq = ParamReader::getOptionalIntParamWithDefaultValue("relation_time_maxent_trainer_stop_check_frequency",1);

		if (MODE == TRAIN) {
			_weights = _new DTFeature::FeatureWeightMap(50000);
			_model = _new MaxEntModel(_tagSet, _featureTypes, _weights,
				MaxEntModel::SCGIS, percent_held_out, max_iterations, variance,
				likelihood_delta, stop_check_freq, "", "");
		} else {
			// load first-pass model
			std::string model_file = ParamReader::getRequiredParam("relation_time_1p_model_file");
			_weights = _new DTFeature::FeatureWeightMap(50000);
			DTFeature::readWeights(*_weights, model_file.c_str(), RelationTimexArgFeatureType::modeltype);
			_model = _new MaxEntModel(_tagSet, _featureTypes, _weights);

			// create empty second-pass model
			_secondPassWeights = _new DTFeature::FeatureWeightMap(50000);
			_secondPassModel = _new MaxEntModel(_tagSet, _secondPassFeatureTypes, _secondPassWeights,
				MaxEntModel::SCGIS, percent_held_out, max_iterations, variance,
				likelihood_delta, stop_check_freq, "", "");
		}
	}

	_morphAnalysis = MorphologicalAnalyzer::build();
}

RelationTimexArgFinder::~RelationTimexArgFinder() {
	delete _tagSet;
	delete _featureTypes;
	delete _weights;
	delete _model;
	delete _secondPassWeights;
	delete _secondPassModel;
	delete _morphAnalysis;

	delete [] _tagScores;
	delete _observation;
}

void RelationTimexArgFinder::cleanup() {
	delete _observation;
	_observation = 0;
}

void RelationTimexArgFinder::train() {
	std::string file_list = ParamReader::getRequiredParam("relation_time_training_file_list");

	TrainingLoader *trainingLoader = _new TrainingLoader(file_list.c_str(), L"doc-relations-events");
	HeapChecker::checkHeap("TL created");
	int max_sentences = trainingLoader->getMaxSentences();

	_tokenSequences = _new TokenSequence *[max_sentences];
	_parses = _new Parse *[max_sentences];
	_secondaryParses = _new Parse *[max_sentences];
	_npChunks = _new NPChunkTheory* [max_sentences];
	_mentionSets = _new MentionSet * [max_sentences];
	_valueMentionSets = _new ValueMentionSet * [max_sentences];
	_propSets = _new PropositionSet * [max_sentences];
	_relationMentionSets = _new RelMentionSet * [max_sentences];

	int num_sentences = loadTrainingData(trainingLoader);
	delete trainingLoader;

	SessionLogger::info("SERIF") << "done with loading\n";

	train(_tokenSequences, _parses, _valueMentionSets, _mentionSets,
		 _propSets, _relationMentionSets, num_sentences);

	for (int i = 0; i < num_sentences; i++) {
		delete _tokenSequences[i];
		delete _parses[i];
		delete _secondaryParses[i];
		delete _npChunks[i];
		delete _mentionSets[i];
		delete _valueMentionSets[i];
		delete _propSets[i];
		delete _relationMentionSets[i];
	}

	delete[] _tokenSequences;
	delete[] _parses;
	delete[] _secondaryParses;
	delete[] _npChunks;
	delete[] _mentionSets;
	delete[] _valueMentionSets;
	delete[] _propSets;
	delete[] _relationMentionSets;

	SessionLogger::info("SERIF") << "done with training\n";
}

void RelationTimexArgFinder::roundRobin() {

	std::string setup_file = ParamReader::getRequiredParam("relation_time_round_robin_setup");
	std::string output_file = ParamReader::getRequiredParam("relation_time_round_robin_results");

	UTF8OutputStream resultStream;

	std::string str(output_file.c_str());
	str += ".html";
	resultStream.open(str.c_str());

	UTF8Token modelToken;
	UTF8Token messageToken;
	int max_sentences = 0;

	boost::scoped_ptr<UTF8InputStream> countStream_scoped_ptr(UTF8InputStream::build(setup_file.c_str()));
	UTF8InputStream& countStream(*countStream_scoped_ptr);
	while (!countStream.eof()) {
		countStream >> messageToken;
		if (wcscmp(messageToken.chars(), L"") == 0)
			break;
		countStream >> modelToken;
		if (wcscmp(modelToken.chars(), L"") == 0)
			break;
		TrainingLoader *trainingLoader = _new TrainingLoader(messageToken.chars(),L"doc-relations-events", true);
		int batch_max_sentences = trainingLoader->getMaxSentences();
		delete trainingLoader;
		max_sentences = (batch_max_sentences > max_sentences) ? batch_max_sentences : max_sentences;
	}
	countStream.close();

	_tokenSequences = _new TokenSequence *[max_sentences];
	_parses = _new Parse *[max_sentences];
	_secondaryParses = _new Parse *[max_sentences];
	_npChunks = _new NPChunkTheory* [max_sentences];
	_mentionSets = _new MentionSet * [max_sentences];
	_valueMentionSets = _new ValueMentionSet * [max_sentences];
	_propSets = _new PropositionSet * [max_sentences];
	_relationMentionSets = _new RelMentionSet * [max_sentences];

	boost::scoped_ptr<UTF8InputStream> setupFileStream_scoped_ptr(UTF8InputStream::build(setup_file.c_str()));
	UTF8InputStream& setupFileStream(*setupFileStream_scoped_ptr);

	resetRoundRobinStatistics();

	while (!setupFileStream.eof()) {
		setupFileStream >> messageToken;
		if (wcscmp(messageToken.chars(), L"") == 0)
			break;
		setupFileStream >> modelToken;
		if (wcscmp(modelToken.chars(), L"") == 0)
			break;

		TrainingLoader *trainingLoader = _new TrainingLoader(messageToken.chars(),L"doc-relations-events", true);
		int num_sentences = loadTrainingData(trainingLoader);
		delete trainingLoader;

		char char_str[501];
		StringTransliterator::transliterateToEnglish(char_str, modelToken.chars(), 500);
		replaceModel(char_str);
		decode(_tokenSequences, _parses, _valueMentionSets, _mentionSets,
  			_propSets, _relationMentionSets, num_sentences, resultStream);

		for (int i = 0; i < num_sentences; i++) {
			delete _tokenSequences[i];
			delete _parses[i];
			delete _secondaryParses[i];
			delete _npChunks[i];
			delete _valueMentionSets[i];
			delete _mentionSets[i];
			delete _propSets[i];
			delete _relationMentionSets[i];
		}
	}

	printRoundRobinStatistics(resultStream);
	resultStream.close();

	delete[] _tokenSequences;
	delete[] _parses;
	delete[] _secondaryParses;
	delete[] _npChunks;
	delete[] _mentionSets;
	delete[] _valueMentionSets;
	delete[] _propSets;
	delete[] _relationMentionSets;

}

void RelationTimexArgFinder::devtest() {
	std::string file_list = ParamReader::getRequiredParam("relation_time_devtest_file_list");

	std::string output_stream = ParamReader::getRequiredParam("relation_time_devtest_output");
	UTF8OutputStream outputStream;
	outputStream.open(output_stream.c_str());

	TrainingLoader *trainingLoader = _new TrainingLoader(file_list.c_str(), L"doc-relations-events");
	HeapChecker::checkHeap("TL created");
	int max_sentences = trainingLoader->getMaxSentences();

	_tokenSequences = _new TokenSequence *[max_sentences];
	_parses = _new Parse *[max_sentences];
	_secondaryParses = _new Parse *[max_sentences];
	_npChunks = _new NPChunkTheory* [max_sentences];
	_mentionSets = _new MentionSet * [max_sentences];
	_valueMentionSets = _new ValueMentionSet * [max_sentences];
	_propSets = _new PropositionSet * [max_sentences];
	_relationMentionSets = _new RelMentionSet * [max_sentences];

	int num_sentences = loadTrainingData(trainingLoader);
	delete trainingLoader;

	SessionLogger::info("SERIF") << "done with loading\n";

	decode(_tokenSequences, _parses, _valueMentionSets, _mentionSets,
		 _propSets, _relationMentionSets, num_sentences, outputStream);

	printRoundRobinStatistics(outputStream);

	for (int i = 0; i < num_sentences; i++) {
		delete _tokenSequences[i];
		delete _parses[i];
		delete _secondaryParses[i];
		delete _npChunks[i];
		delete _mentionSets[i];
		delete _valueMentionSets[i];
		delete _propSets[i];
		delete _relationMentionSets[i];
	}

	delete[] _tokenSequences;
	delete[] _parses;
	delete[] _secondaryParses;
	delete[] _npChunks;
	delete[] _mentionSets;
	delete[] _valueMentionSets;
	delete[] _propSets;
	delete[] _relationMentionSets;

	outputStream.close();

	SessionLogger::info("SERIF") << "done with devtest\n";
}

void RelationTimexArgFinder::replaceModel(char *model_file) {
	delete _model;
	delete _weights;
	_weights = _new DTFeature::FeatureWeightMap(50000);
	DTFeature::readWeights(*_weights, model_file, RelationTimexArgFeatureType::modeltype);
	_model = _new MaxEntModel(_tagSet, _featureTypes, _weights);

	if (_use_two_pass_model) {
		delete _secondPassWeights;
		delete _secondPassModel;
		char buffer[550];
		sprintf(buffer, "%s.second-pass", model_file);
		_secondPassWeights = _new DTFeature::FeatureWeightMap(50000);
		DTFeature::readWeights(*_secondPassWeights, buffer, RelationTimexArgFeatureType::modeltype);
		_secondPassModel = _new MaxEntModel(_tagSet, _secondPassFeatureTypes, _secondPassWeights);
	}
}

// this is for stand-alone use only -- essentially for development use
void RelationTimexArgFinder::decode(TokenSequence **tokenSequences, Parse **parses,
		ValueMentionSet **valueMentionSets, MentionSet **mentionSets, PropositionSet **propSets,
		RelMentionSet **relsets, int nsentences, UTF8OutputStream& resultStream)
{
	if (MODE != DECODE || _model == 0)
		return;

	for (int i = 0; i < nsentences; i++) {
		for (int relment = 0; relment < relsets[i]->getNRelMentions(); relment++) {
			RelMention *key_rm = relsets[i]->getRelMention(relment);
			RelMention *test_rm = _new RelMention(*key_rm);
			test_rm->resetTimeArgument();
			attachTimeArgument(test_rm, tokenSequences[i], valueMentionSets[i],
				parses[i], mentionSets[i], propSets[i]);

			const ValueMention *key_ment = key_rm->getTimeArgument();
			const ValueMention *test_ment = test_rm->getTimeArgument();

			if (key_ment != 0 || test_ment != 0) {
				// print relation
				resultStream << L"
";
				resultStream << key_rm->getLeftMention()->getHead()->toString(0);
				resultStream << L" and ";
				resultStream << key_rm->getRightMention()->getHead()->toString(0);
				resultStream << L" (" << key_rm->getType().to_string() << L")
\n";

				for (int tok = 0; tok < tokenSequences[i]->getNTokens(); tok++) {
					if (tok == key_rm->getLeftMention()->getHead()->getStartToken())
						resultStream << L"";
					if (tok == key_rm->getRightMention()->getHead()->getStartToken())
						resultStream << L"";
					resultStream << tokenSequences[i]->getToken(tok)->getSymbol().to_string();
					if (tok == key_rm->getLeftMention()->getHead()->getEndToken())
						resultStream << L"";
					if (tok == key_rm->getRightMention()->getHead()->getEndToken())
						resultStream << L"";
					resultStream << L" ";
				}
				resultStream << L"

\n";

				bool matched = false;

				if (test_ment == key_ment) {
					matched = true;
					if (key_rm->getTimeRole() == test_rm->getTimeRole()) {
						resultStream << L"CORRECT: ";
						resultStream << key_ment->toString(tokenSequences[i]) << L" (";
						resultStream << key_rm->getTimeRole().to_string() << L")
\n";
						printDebugScores(key_rm, key_ment, resultStream);
						resultStream << "
\n";
						_correct_args++;
					} else {
						resultStream << L"WRONG TYPE (should be ";
						resultStream << key_rm->getTimeRole().to_string();
						resultStream << "): ";
						resultStream << key_ment->toString(tokenSequences[i]);
						resultStream << L" (" << test_rm->getTimeRole() << L")
\n";
						printDebugScores(key_rm, key_ment, resultStream);
						resultStream << L"
\n";
						_wrong_type++;
					}
				}
				if (!matched) {
					if (key_ment != 0 && test_ment == 0) {
						resultStream << L"MISSED: ";
						resultStream << key_ment->toString(tokenSequences[i]) << L" (";
						resultStream << key_rm->getTimeRole() << L")
\n";
						printDebugScores(key_rm, key_ment, resultStream);
						resultStream << "
\n";
						_missed++;
					}
					else if (key_ment == 0 && test_ment != 0) {
						resultStream << L"SPURIOUS: ";
						resultStream << test_ment->toString(tokenSequences[i]) << L" (";
						resultStream << test_rm->getTimeRole() << L")
\n";
						printDebugScores(test_rm, test_ment, resultStream);
						resultStream << "
\n";
						_spurious++;
					}
					else if (key_ment != 0 && test_ment != 0) {
						resultStream << L"WRONG ARG (should be ";
						resultStream << key_ment->toString(tokenSequences[i]);
						resultStream << L" " << key_rm->getTimeRole() << L"): ";
						resultStream << test_ment->toString(tokenSequences[i]);
						resultStream << L" (" << test_rm->getTimeRole() << L")
\n";
						printDebugScores(key_rm, key_ment, resultStream);
						resultStream << L"
\n";
						_wrong_type++;
					}
					resultStream << L"
";
				}
			}
		}
	}

}

void RelationTimexArgFinder::printDebugScores(RelMention *rm, const ValueMention *arg,
										   UTF8OutputStream& stream)
{
	if (!_observation)
		_observation = _new RelationTimexArgObservation();
	_observation->setRelation(rm);
	_observation->setCandidateArgument(arg);

	int best_tag;
	_model->decodeToDistribution(_observation, _tagScores, _tagSet->getNTags(), &best_tag);

	int best = 0;
	int second_best = 0;
	int third_best = 0;
	double best_score = -100000;
	double second_best_score = -100000;
	double third_best_score = -100000;
	for (int i = 0; i < _tagSet->getNTags(); i++) {
		if (_tagScores[i] > best_score) {
			third_best = second_best;
			third_best_score = second_best_score;
			second_best = best;
			second_best_score = best_score;
			best = i;
			best_score = _tagScores[i];
		} else if (_tagScores[i] > second_best_score) {
			third_best = second_best;
			third_best_score = second_best_score;
			second_best = i;
			second_best_score = _tagScores[i];
		} else if (_tagScores[i] > third_best_score) {
			third_best = i;
			third_best_score = _tagScores[i];
		}
	}
	stream << _tagSet->getTagSymbol(best).to_string() << L": " << _tagScores[best] << L"
\n";
	stream << "\n";
	_model->printHTMLDebugInfo(_observation, best, stream);
	stream << "\n";
	stream << _tagSet->getTagSymbol(second_best).to_string() << L": " << _tagScores[second_best] << L"
\n";
	stream << "\n";
	_model->printHTMLDebugInfo(_observation, second_best, stream);
	stream << "\n";
	stream << _tagSet->getTagSymbol(third_best).to_string() << L": " << _tagScores[third_best] << L"
\n";
	stream << "\n";
	_model->printHTMLDebugInfo(_observation, third_best, stream);
	stream << "\n";
}

void RelationTimexArgFinder::train(TokenSequence **tokenSequences, Parse **parses,
		ValueMentionSet **valueMentionSets, MentionSet **mentionSets, PropositionSet **propSets,
		RelMentionSet **relsets, int nsentences)
{
	if (!_observation)
		_observation = _new RelationTimexArgObservation();
	if (MODE != TRAIN && MODE != TRAIN_SECOND_PASS)
		return;

	// PRUNING
	int pruning = ParamReader::getRequiredIntParam("relation_time_maxent_trainer_pruning_cutoff");

	// MODEL FILE
	std::string model_file = ParamReader::getRequiredParam("relation_time_model_file");

	for (int i = 0; i < nsentences; i++) {
		_observation->resetForNewSentence(tokenSequences[i], parses[i],
			mentionSets[i], valueMentionSets[i], propSets[i]);
		for (int rment = 0; rment < relsets[i]->getNRelMentions(); rment++) {
			RelMention *rm = relsets[i]->getRelMention(rment);
			RelMention *new_rm = rm;
			if (MODE == TRAIN_SECOND_PASS) {
				new_rm = _new RelMention (*rm);
				new_rm->resetTimeArgument();
				attachTimeArgument(_model, new_rm, tokenSequences[i], valueMentionSets[i],
					parses[i], mentionSets[i], propSets[i]);
			}
			_observation->setRelation(new_rm);

			const ValueMention *keyMention = rm->getTimeArgument();

			for (int val_id = 0; val_id < valueMentionSets[i]->getNValueMentions(); val_id++) {
				ValueMention *valueMention = valueMentionSets[i]->getValueMention(val_id);
				// we only care about timex values
				if (!valueMention->isTimexValue())
					continue;
				_observation->setCandidateArgument(valueMention);

				Symbol roleSym = Symbol();
				int role = _tagSet->getNoneTagIndex();

				if (keyMention == valueMention)
					roleSym = rm->getTimeRole();
				if (!roleSym.is_null()) {
					role = _tagSet->getTagIndex(roleSym);
					if (role == -1) {
						char c[500];
						sprintf(c, "ERROR: Invalid time argument role in training file: %s",
							roleSym.to_debug_string());
						throw UnexpectedInputException("RelationTimexArgumentFinder::train()", c);
					}
				}
				if (MODE == TRAIN_SECOND_PASS) {
					if (new_rm->getTimeArgument() == valueMention) {
						Symbol newRoleSym = new_rm->getTimeRole();
						_secondPassModel->addToTraining(_observation, role);
					}
				} else _model->addToTraining(_observation, role);

			}
		}
	}

	char output_file[550];

	if (MODE == TRAIN_SECOND_PASS) {
		_secondPassModel->deriveModel(pruning);
		sprintf(output_file, "%s.second-pass", model_file.c_str());
	} else {
        _model->deriveModel(pruning);
		sprintf(output_file, "%s", model_file.c_str());
	}

	UTF8OutputStream out;
	out.open(output_file);

	if (out.fail()) {
		throw UnexpectedInputException("RelationTimexArgFinder::train()",
			"Could not open model file for writing");
	}

	dumpTrainingParameters(out);
	if (MODE == TRAIN_SECOND_PASS)
		DTFeature::writeWeights(*_secondPassWeights, out);
	else DTFeature::writeWeights(*_weights, out);
	out.close();

	// we are now set to decode
	MODE = DECODE;

}

void RelationTimexArgFinder::attachTimeArguments(RelMentionSet *relMentionSet, const TokenSequence *tokens,
		ValueMentionSet *valueMentionSet, Parse *parse,
		MentionSet *mentionSet, PropositionSet *propSet)
{
	for (int i = 0; i < relMentionSet->getNRelMentions(); i++) {
		attachTimeArgument(relMentionSet->getRelMention(i), tokens,
							valueMentionSet, parse, mentionSet, propSet);
	}
}

void RelationTimexArgFinder::attachTimeArgument(RelMention *relMention, const TokenSequence *tokens,
		ValueMentionSet *valueMentionSet, Parse *parse,
		MentionSet *mentionSet, PropositionSet *propSet)
{
	attachTimeArgument(_model, relMention, tokens, valueMentionSet, parse, mentionSet, propSet);
	if (_use_two_pass_model)
		attachTimeArgument(_secondPassModel, relMention, tokens, valueMentionSet,
		parse, mentionSet, propSet);
}

void RelationTimexArgFinder::attachTimeArgument(MaxEntModel *model,
		RelMention *relMention, const TokenSequence *tokens,
		ValueMentionSet *valueMentionSet, Parse *parse,
		MentionSet *mentionSet, PropositionSet *propSet)
{
	if (!_observation)
		_observation = _new RelationTimexArgObservation();
	_observation->resetForNewSentence(tokens, parse, mentionSet, valueMentionSet, propSet);
	_observation->setRelation(relMention);

	if (DEBUG) _debugStream << relMention->toString() << L"\n";

	_potentialTimeArgument = 0;
	double best_score = 0.0;

	// attach all possible time arguments to the relation mention
	for (int val_id = 0; val_id < valueMentionSet->getNValueMentions(); val_id++) {
		ValueMention *valueMention = valueMentionSet->getValueMention(val_id);
		if (!valueMention->isTimexValue())
			continue;
		_observation->setCandidateArgument(valueMention);
		addTagsForCandidateValue(model, valueMention);
	}

	// filter those arguments
	// how about for now we allow only one for each label (and of course,
	//  one for each mention!)
	while (_potentialTimeArgument != 0) {

		// only add this if we don't already have one
		if (relMention->getTimeArgument() == 0) {
			relMention->setTimeArgument(_potentialTimeArgument->role,
										_potentialTimeArgument->valueMention,
										_potentialTimeArgument->score);

			if (DEBUG) {
				_debugStream << "Adding \"";
				_debugStream << _potentialTimeArgument->valueMention->toString(tokens);
				_debugStream << "\" as " << _potentialTimeArgument->role << "\n";
			}

		}
		Symbol role = _potentialTimeArgument->role;
		const ValueMention *valueMention = _potentialTimeArgument->valueMention;
		PotentialTimeArgument *iter = _potentialTimeArgument;
		while (iter->next != 0) {
			if (DEBUG) {
				_debugStream << "Dumping \"";
				_debugStream << valueMention->toString(tokens);
				_debugStream  << "\" as " << iter->next->role << "\n";
			}
			PotentialTimeArgument *temp = iter->next;
			iter->next = iter->next->next;
			temp->next = 0;
			delete temp;
		}
		PotentialTimeArgument *temp = _potentialTimeArgument;
		_potentialTimeArgument = _potentialTimeArgument->next;
		temp->next = 0;
		delete temp;
	}

}

int RelationTimexArgFinder::addTagsForCandidateValue(MaxEntModel *model, const ValueMention *value) {
	int ntags_found = 0;
	int best_tag;
	model->decodeToDistribution(_observation, _tagScores, _tagSet->getNTags(), &best_tag);
	for (int tag = 0; tag < _tagSet->getNTags(); tag++) {
		if (tag == _tagSet->getNoneTagIndex())
			continue;
		if (_tagScores[tag] > _recall_threshold) {
			PotentialTimeArgument *newArgument =
				_new PotentialTimeArgument(_tagSet->getTagSymbol(tag), value,
				(float) _tagScores[tag]);
			addPotentialArgument(newArgument);
			ntags_found++;
			if (DEBUG) {
				_debugStream << _tagSet->getTagSymbol(tag) << L": ";
				_debugStream << L"(UNPRINTED TIMEX)" << L" (";
				_debugStream << _tagScores[tag] << L")\n";
			}
		}
	}
	return ntags_found;
}

void RelationTimexArgFinder::addPotentialArgument(PotentialTimeArgument *newArgument) {
	if (_potentialTimeArgument == 0) {
		_potentialTimeArgument = newArgument;
		return;
	} else if (newArgument->score > _potentialTimeArgument->score) {
		newArgument->next = _potentialTimeArgument;
		_potentialTimeArgument = newArgument;
		return;
	} else {
		PotentialTimeArgument *iter = _potentialTimeArgument;
		while (iter->next != 0) {
			if (newArgument->score > iter->next->score) {
				newArgument->next = iter->next;
				iter->next = newArgument;
				return;
			}
			iter = iter->next;
		}
		iter->next = newArgument;
		return;
	}
}

void RelationTimexArgFinder::dumpTrainingParameters(UTF8OutputStream &out) {

	DTFeature::recordDate(out);

	out << L"Parameters:\n";
	DTFeature::recordParamForConsistency(Symbol(L"relation_time_tag_set_file"), out);
	DTFeature::recordParamForConsistency(Symbol(L"relation_time_features_file"), out);
	DTFeature::recordParamForConsistency(Symbol(L"word_cluster_bits_file"), out);
	DTFeature::recordParamForConsistency(Symbol(L"lc_word_cluster_bits_file"), out);

	DTFeature::recordParamForReference(Symbol(L"relation_time_training_file_list"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_percent_held_out"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_max_iterations"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_gaussian_variance"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_min_likelihood_delta"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_stop_check_frequency"), out);
	DTFeature::recordParamForReference(Symbol(L"relation_time_maxent_trainer_pruning_cutoff"), out);
	out << L"\n";

}

void RelationTimexArgFinder::resetRoundRobinStatistics() {
	_correct_args = 0;
	_correct_non_args = 0;
	_wrong_type = 0;
	_missed = 0;
	_spurious = 0;
}

void RelationTimexArgFinder::printRoundRobinStatistics(UTF8OutputStream &out) {

	double recall = (double) _correct_args / (_missed + _wrong_type + _correct_args);
	double precision = (double) _correct_args / (_spurious + _wrong_type + _correct_args);

	out << L"CORRECT: " << _correct_args + _correct_non_args << L" ";
	out << L"(arguments: " << _correct_args << L", non-arguments: ";
	out << _correct_non_args << L")
\n";
	out << L"MISSED: " << _missed << L"
\n";
	out << L"SPURIOUS: " << _spurious << L"
\n";
	out << L"WRONG TYPE: " << _wrong_type << L"

\n\n";
	out << L"RECALL: " << recall << L"
\n";
	out << L"PRECISION: " << precision << L"

\n\n";

}

int RelationTimexArgFinder::loadTrainingData(TrainingLoader *trainingLoader) {

	int max_sentences = trainingLoader->getMaxSentences();
	int i;
	for (i = 0; i < max_sentences; i++) {
		SentenceTheory *theory = trainingLoader->getNextSentenceTheory();
		if (theory == 0)
			return i;
		_tokenSequences[i] = theory->getTokenSequence();
		_tokenSequences[i]->gainReference();
		_parses[i] = theory->getPrimaryParse();
		_parses[i]->gainReference();
		//have to add a reference to the np chunk theory, because it may be
		//the source of the parse, if the reference isn't gained, the parse
		//will be deleted when result is deleted
		_npChunks[i] = theory->getNPChunkTheory();
		if(_npChunks[i] != 0){
			_npChunks[i]->gainReference();
			// is this what we want?
			_secondaryParses[i] = theory->getFullParse();
			_secondaryParses[i]->gainReference();
		} else _secondaryParses[i] = 0;
		_mentionSets[i] = theory->getMentionSet();
		_mentionSets[i]->gainReference();
		_valueMentionSets[i] = theory->getValueMentionSet();
		_valueMentionSets[i]->gainReference();
		_propSets[i] = theory->getPropositionSet();
		_propSets[i]->gainReference();
		_propSets[i]->fillDefinitionsArray();
		_relationMentionSets[i] = theory->getRelMentionSet();
		_relationMentionSets[i]->gainReference();
	}

	return i;
}

max_stars_repo_name: BBN-E/serif

max_issues_repo_name: BBN-E/serif

max_forks_repo_name: BBN-E/serif


text:
/*
 * Copyright 2020 OmniSci, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * @file DashboardTest.cpp
 * @brief Test suite for dashboard commands
 */

#include <gtest/gtest.h>
#include <boost/range/combine.hpp>

#include "DBHandlerTestHelpers.h"
#include "TestHelpers.h"

#ifndef BASE_PATH
#define BASE_PATH "./tmp"
#endif

class ShareDashboardsTest : public DBHandlerTestFixture {
 public:
  static void SetUpTestSuite() {
    createDBHandler();
    createUsersAndRoles();
  }

  static void TearDownTestSuite() {
    loginAdmin();
    dropUsersAndRoles();
  }

 protected:
  void SetUp() override {
    DBHandlerTestFixture::SetUp();
    setupDashboards();
  }
  void TearDown() override {
    teardownDashboards();
    DBHandlerTestFixture::TearDown();
  }

  std::vector<int32_t> getTestDashboardIds() const {
    std::vector<int32_t> ids;
    for (const auto& test_dashboard : test_dashboards_) {
      ids.push_back(test_dashboard.dashboard_id);
    }
    return ids;
  }

  struct TestPermissions {
    TDashboardPermissions permissions;
    std::map<std::string, AccessPrivileges> privileges;
    std::map<std::string, AccessPrivileges> all_privileges;

    TestPermissions(bool create_permission,
                    bool delete_permission,
                    bool edit_permission,
                    bool view_permission) {
      permissions.create_ = create_permission;
      permissions.delete_ = delete_permission;
      permissions.edit_ = edit_permission;
      permissions.view_ = view_permission;
      if (create_permission) {
        privileges.insert({"CREATE_DASHBOARD", AccessPrivileges::CREATE_DASHBOARD});
      }
      if (delete_permission) {
        privileges.insert({"DELETE_DASHBOARD", AccessPrivileges::DELETE_DASHBOARD});
      }
      if (edit_permission) {
        privileges.insert({"EDIT_DASHBOARD", AccessPrivileges::EDIT_DASHBOARD});
      }
      if (view_permission) {
        privileges.insert({"VIEW_DASHBOARD", AccessPrivileges::VIEW_DASHBOARD});
      }
      all_privileges.insert({"CREATE_DASHBOARD", AccessPrivileges::CREATE_DASHBOARD});
      all_privileges.insert({"DELETE_DASHBOARD", AccessPrivileges::DELETE_DASHBOARD});
      all_privileges.insert({"EDIT_DASHBOARD", AccessPrivileges::EDIT_DASHBOARD});
      all_privileges.insert({"VIEW_DASHBOARD", AccessPrivileges::VIEW_DASHBOARD});
    }

    TestPermissions getComplement() {
      return TestPermissions{!permissions.create_,
                             !permissions.delete_,
                             !permissions.edit_,
                             !permissions.view_};
    }

    AccessPrivileges getPrivileges() {
      AccessPrivileges priv;
      if (permissions.create_) {
        priv.add(AccessPrivileges::CREATE_DASHBOARD);
      }
      if (permissions.delete_) {
        priv.add(AccessPrivileges::DELETE_DASHBOARD);
      }
      if (permissions.edit_) {
        priv.add(AccessPrivileges::EDIT_DASHBOARD);
      }
      if (permissions.view_) {
        priv.add(AccessPrivileges::VIEW_DASHBOARD);
      }
      return priv;
    }
  };

  void shareOrUnshareDashboards(const TestPermissions& test_permissions,
                                const std::vector<std::string>& groups,
                                const std::vector<int32_t>& dashboard_ids,
                                const bool do_share = true) {
    auto [handler, session_id] = DBHandlerTestFixture::getDbHandlerAndSessionId();
    if (do_share) {
      handler->share_dashboards(
          session_id, dashboard_ids, groups, test_permissions.permissions);
    } else {
      handler->unshare_dashboards(
          session_id, dashboard_ids, groups, test_permissions.permissions);
    }
  }

  void shareOrUnshareDashboards(const TestPermissions& test_permissions,
                                const std::vector<std::string>& groups,
                                const bool do_share = true) {
    shareOrUnshareDashboards(test_permissions, groups, getTestDashboardIds(), do_share);
  }

  virtual void setupNewDashboards(int count) {
    auto [handler, session_id] = DBHandlerTestFixture::getDbHandlerAndSessionId();
    for (int i = 0; i < count; ++i) {
      test_dashboards_.emplace_back(i, handler, session_id);
    }
  }

  void teardownLastDashboard() { test_dashboards_.pop_back(); }

  void teardownDashboards() { test_dashboards_.clear(); }

  void assertExpectedDashboardPrivilegesForUsers(const TestPermissions& test_permissions,
                                                 const std::vector<std::string>& users) {
    auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
    auto& catalog = getCatalog();
    for (auto const& test_dashboard : test_dashboards_) {
      DBObject object(test_dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
      object.loadKey(catalog);
      for (auto const& user : users) {
        for (auto const& [privilege_name, privilege] : test_permissions.all_privileges) {
          object.setPrivileges(privilege);
          bool privilege_allowed = sys_catalog.checkPrivileges(user, {object});
          const auto& allowed_privileges = test_permissions.privileges;
          bool should_privilege_be_allowed =
              allowed_privileges.find(privilege_name) != allowed_privileges.end();
          ASSERT_EQ(privilege_allowed, should_privilege_be_allowed)
              << " with user " << user << ", dashboard id " << test_dashboard.dashboard_id
              << " and privilege_name " << privilege_name << " being "
              << privilege_allowed << " but expected to be "
              << should_privilege_be_allowed;
        }
      }
    }
  }

  static void createUsersAndRoles() {
    sql("CREATE USER test_user_1 (password = '');");
    sql("CREATE USER test_user_2 (password = '');");
    sql("CREATE USER test_user_3 (password = '');");
    sql("CREATE ROLE test_role_1;");
    sql("GRANT test_role_1 TO test_user_2, test_user_3;");
  }

  static void dropUsersAndRoles() {
    sql("DROP USER test_user_1;");
    sql("DROP USER test_user_2;");
    sql("DROP USER test_user_3;");
    sql("DROP ROLE test_role_1;");
  }

  const static inline std::string dashboard_name_prefix_ = "test_dashboard";
  const static inline std::string dashboard_state_prefix_ = "test_state";
  const static inline std::string dashboard_image_hash_prefix_ = "test_image_hash";
  const static inline std::string dashboard_metadata_prefix_ = "test_metadata";
  const static inline int num_test_dashboards_ = 3;

  struct TestDashboard {
    int dashboard_id;
    DBHandler* handler;
    const TSessionId& session;

    std::string get_suffix(const int unique_id) {
      std::stringstream suffix;
      suffix << "_" << std::setfill('0') << std::setw(2) << unique_id;
      return suffix.str();
    }

    TestDashboard(const int unique_id, DBHandler* handler, const TSessionId& session)
        : handler(handler), session(session) {
      const auto suffix = get_suffix(unique_id);
      dashboard_id = handler->create_dashboard(session,
                                               dashboard_name_prefix_ + suffix,
                                               dashboard_state_prefix_ + suffix,
                                               dashboard_image_hash_prefix_ + suffix,
                                               dashboard_metadata_prefix_ + suffix);
    }

    ~TestDashboard() { handler->delete_dashboard(session, dashboard_id); }
  };

  std::list<TestDashboard> test_dashboards_;
  void setupDashboards() { setupNewDashboards(num_test_dashboards_); }
};

struct PermissionParam {
  bool create_p, delete_p, view_p, edit_p;
};

namespace {
struct PrintToStringParamName {
  std::string operator()(const ::testing::TestParamInfo<PermissionParam>& info) const {
    std::stringstream ss;
    const auto& perm = info.param;
    ss << "create_permission_" << perm.create_p << "_delete_permission_" << perm.delete_p
       << "_view_permission_" << perm.view_p << "_edit_permission_" << perm.edit_p;
    return ss.str();
  }

  std::string operator()(const ::testing::TestParamInfo<bool>& info) const {
    return info.param ? "share" : "unshare";
  }
};
}  // namespace

class DashboardsTestWithInitialPermissions : public ShareDashboardsTest {
 protected:
  void grantInitialPermissions() {
    for (auto const& test_dashboard : test_dashboards_) {
      DBObject object(test_dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
      object.setPrivileges(getInitialPermissions().getPrivileges());
      auto entities =
          std::vector<std::string>{"test_user_1", "test_user_2", "test_user_3"};
      if (grantInitialPermissionsToGroupOnly()) {
        entities = {"test_role_1"};
      }
      Catalog_Namespace::SysCatalog::instance().grantDBObjectPrivilegesBatch(
          entities, {object}, getCatalog());
    }
  }

  void setupNewDashboards(int count) override {
    ShareDashboardsTest::setupNewDashboards(count);
    grantInitialPermissions();
  }

  virtual TestPermissions getInitialPermissions() = 0;
  virtual bool grantInitialPermissionsToGroupOnly() { return false; }
};

class ShareDashboardsGroupTest : public ShareDashboardsTest {
 protected:
  TestPermissions getInitialPermissions() {
    return TestPermissions{false, false, false, false};
  }
};

class UnshareDashboardsGroupTest : public DashboardsTestWithInitialPermissions {
 protected:
  TestPermissions getInitialPermissions() override {
    return TestPermissions{true, true, true, true};
  }

  bool grantInitialPermissionsToGroupOnly() override { return true; }
};

class ShareDashboardsRegularTest : public ShareDashboardsTest {
 protected:
  TestPermissions getInitialPermissions() {
    return TestPermissions{false, false, false, false};
  }
};

class UnshareDashboardsRegularTest : public DashboardsTestWithInitialPermissions {
 protected:
  TestPermissions getInitialPermissions() override {
    return TestPermissions{true, true, true, true};
  }
};

class ShareAndUnshareDashboardsTest : public DashboardsTestWithInitialPermissions,
                                      public testing::WithParamInterface<bool> {
 protected:
  bool getDoShare() { return GetParam(); }

  TestPermissions getInitialPermissions() override {
    return TestPermissions{true, false, true, false};
  }
};

INSTANTIATE_TEST_SUITE_P(ShareAndUnshare,
                         ShareAndUnshareDashboardsTest,
                         ::testing::Values(true, false),
                         PrintToStringParamName());

class SharePermissionPermutationsTest
    : public ShareDashboardsTest,
      public testing::WithParamInterface<PermissionParam> {
 protected:
  const PermissionParam& getPermissionParam() { return GetParam(); }
};

class UnsharePermissionPermutationsTest
    : public DashboardsTestWithInitialPermissions,
      public testing::WithParamInterface<PermissionParam> {
 protected:
  const PermissionParam& getPermissionParam() { return GetParam(); }

  TestPermissions getInitialPermissions() override {
    return TestPermissions{true, true, true, true};
  }
};

INSTANTIATE_TEST_SUITE_P(AllLegalPermutations,
                         SharePermissionPermutationsTest,
                         ::testing::Values(PermissionParam{true, true, true, true},
                                           PermissionParam{true, true, true, false},
                                           PermissionParam{true, true, false, false},
                                           PermissionParam{true, false, true, false},
                                           PermissionParam{false, true, true, false},
                                           PermissionParam{true, false, false, false},
                                           PermissionParam{false, true, false, false}),
                         PrintToStringParamName());

INSTANTIATE_TEST_SUITE_P(AllLegalPermutations,
                         UnsharePermissionPermutationsTest,
                         ::testing::Values(PermissionParam{true, true, true, true},
                                           PermissionParam{true, true, true, false},
                                           PermissionParam{true, true, false, false},
                                           PermissionParam{true, false, true, false},
                                           PermissionParam{false, true, true, false},
                                           PermissionParam{true, false, false, false},
                                           PermissionParam{false, true, false, false}),
                         PrintToStringParamName());

TEST_P(SharePermissionPermutationsTest, ShareAllLegalPermutations) {
  auto const& param = getPermissionParam();
  TestPermissions permissions(param.create_p, param.delete_p, param.edit_p, param.view_p);
  shareOrUnshareDashboards(permissions, {"test_user_1"}, true);
  assertExpectedDashboardPrivilegesForUsers(permissions, {"test_user_1"});
}

TEST_P(UnsharePermissionPermutationsTest, UnshareAllLegalPermutations) {
  auto const& param = getPermissionParam();
  TestPermissions permissions(param.create_p, param.delete_p, param.edit_p, param.view_p);
  shareOrUnshareDashboards(permissions, {"test_user_1"}, false);
  assertExpectedDashboardPrivilegesForUsers(permissions.getComplement(), {"test_user_1"});
}

TEST_P(ShareAndUnshareDashboardsTest, NoPermissions) {
  bool do_share = getDoShare();
  TestPermissions no_permissions(false, false, false, false);
  executeLambdaAndAssertException(
      [&, this] { shareOrUnshareDashboards(no_permissions, {"test_user_1"}, do_share); },
      "At least one privilege should be assigned for " +
          std::string(do_share ? "grants" : "revokes"));
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(), {"test_user_1"});
}

TEST_P(ShareAndUnshareDashboardsTest, NonExistentGroupOrUser) {
  bool do_share = getDoShare();
  TestPermissions permissions(true, true, true, true);
  executeLambdaAndAssertException(
      [&, this] {
        shareOrUnshareDashboards(permissions, {"non_existent_user"}, do_share);
      },
      "Exception: User/Role 'non_existent_user' does not exist");
}

TEST_P(ShareAndUnshareDashboardsTest, NonExistentDashboards) {
  bool do_share = getDoShare();
  TestPermissions permissions(true, true, true, true);
  auto dashboard_ids = getTestDashboardIds();
  teardownLastDashboard();
  teardownLastDashboard();
  std::stringstream error_stream, id_list;
  id_list << dashboard_ids[dashboard_ids.size() - 2] << ", "
          << dashboard_ids[dashboard_ids.size() - 1];
  error_stream << "Share/Unshare dashboard(s) failed with error(s)\n"
               << "Dashboard ids " << id_list.str() << ": Dashboard id does not exist\n";
  executeLambdaAndAssertException(
      [&, this] {
        shareOrUnshareDashboards(permissions, {"test_user_1"}, dashboard_ids, do_share);
      },
      error_stream.str());
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(), {"test_user_1"});
}

TEST_F(ShareDashboardsGroupTest, Group) {
  TestPermissions permissions(true, true, true, true);
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(),
                                            {"test_user_2", "test_user_3"});
  shareOrUnshareDashboards(permissions, {"test_role_1"}, true);
  assertExpectedDashboardPrivilegesForUsers(permissions, {"test_user_2", "test_user_3"});
}

TEST_F(UnshareDashboardsGroupTest, Group) {
  TestPermissions permissions(true, true, true, true);
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(),
                                            {"test_user_2", "test_user_3"});
  shareOrUnshareDashboards(permissions, {"test_role_1"}, false);
  assertExpectedDashboardPrivilegesForUsers(permissions.getComplement(),
                                            {"test_user_2", "test_user_3"});
}

TEST_P(ShareAndUnshareDashboardsTest, AsUnauthorizedUser) {
  bool do_share = getDoShare();
  sql("GRANT ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1");
  login("test_user_1", "");
  std::stringstream error_stream;
  error_stream << "Share/Unshare dashboard(s) failed with error(s)\n";
  error_stream
      << "Dashboard ids " << join(getTestDashboardIds(), ", ") << ": "
      << "User should be either owner of dashboard or super user to share/unshare it\n";
  TestPermissions permissions(true, true, true, true);
  executeLambdaAndAssertException(
      [&, this] { shareOrUnshareDashboards(permissions, {"test_user_1"}, do_share); },
      error_stream.str());
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(), {"test_user_1"});
  loginAdmin();
  sql("REVOKE ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1");
}

TEST_P(ShareAndUnshareDashboardsTest, AsUnauthorizedUserForSomeDashboards) {
  bool do_share = getDoShare();
  sql("GRANT ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1, test_user_2;");
  sql("GRANT CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB +
      " TO test_user_1, test_user_2;");
  teardownDashboards();
  login("test_user_1", "");
  setupNewDashboards(3);
  login("test_user_2", "");
  setupNewDashboards(1);
  auto dashboard_ids = getTestDashboardIds();
  std::stringstream error_stream, dashboard_ids_stream;
  error_stream << "Share/Unshare dashboard(s) failed with error(s)\n";
  for (int i = 0; i < 3; ++i) {
    dashboard_ids_stream << (!dashboard_ids_stream.str().empty() ? ", " : "")
                         << dashboard_ids[i];
  }
  error_stream
      << "Dashboard ids " << dashboard_ids_stream.str() << ": "
      << "User should be either owner of dashboard or super user to share/unshare it\n";
  TestPermissions permissions(true, true, true, true);
  executeLambdaAndAssertException(
      [&, this] { shareOrUnshareDashboards(permissions, {"test_user_3"}, do_share); },
      error_stream.str());
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(), {"test_user_3"});
  loginAdmin();
  sql("REVOKE CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB +
      " FROM test_user_1, test_user_2;");
  sql("REVOKE ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB +
      " FROM test_user_1, test_user_2;");
}

TEST_F(ShareDashboardsRegularTest, AsNonSuperUser) {
  sql("GRANT ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1");
  sql("GRANT CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1;");
  teardownDashboards();
  login("test_user_1", "");
  setupNewDashboards(3);
  TestPermissions permissions(true, true, true, true);
  shareOrUnshareDashboards(permissions, {"test_user_2"}, true);
  assertExpectedDashboardPrivilegesForUsers(permissions, {"test_user_2"});
  loginAdmin();
  sql("REVOKE CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1;");
  sql("REVOKE ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1");
}

TEST_F(UnshareDashboardsRegularTest, AsNonSuperUser) {
  sql("GRANT ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1");
  sql("GRANT CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1;");
  teardownDashboards();
  login("test_user_1", "");
  setupNewDashboards(3);
  TestPermissions permissions(true, true, true, true);
  shareOrUnshareDashboards(permissions, {"test_user_2"}, false);
  assertExpectedDashboardPrivilegesForUsers(permissions.getComplement(), {"test_user_2"});
  loginAdmin();
  sql("REVOKE CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1;");
  sql("REVOKE ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1");
}

TEST_P(ShareAndUnshareDashboardsTest, MixedErrors) {
  bool do_share = getDoShare();
  sql("GRANT ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1");
  sql("GRANT CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " TO test_user_1;");
  login("test_user_1", "");
  setupNewDashboards(3);
  auto dashboard_ids = getTestDashboardIds();
  teardownLastDashboard();
  teardownLastDashboard();
  std::stringstream error_stream, do_not_exist_dashboard_ids_stream,
      unauthorized_user_dashboard_ids_stream;
  for (size_t i = 0; i < dashboard_ids.size() - 3; ++i) {
    unauthorized_user_dashboard_ids_stream
        << (!unauthorized_user_dashboard_ids_stream.str().empty() ? ", " : "")
        << dashboard_ids[i];
  }
  for (size_t i = dashboard_ids.size() - 2; i < dashboard_ids.size(); ++i) {
    do_not_exist_dashboard_ids_stream
        << (!do_not_exist_dashboard_ids_stream.str().empty() ? ", " : "")
        << dashboard_ids[i];
  }
  error_stream << "Share/Unshare dashboard(s) failed with error(s)\n";
  error_stream << "Dashboard ids " << do_not_exist_dashboard_ids_stream.str()
               << ": Dashboard id does not exist\n";
  error_stream
      << "Dashboard ids " << unauthorized_user_dashboard_ids_stream.str() << ": "
      << "User should be either owner of dashboard or super user to share/unshare it\n";
  TestPermissions permissions(true, true, true, true);
  executeLambdaAndAssertException(
      [&, this] {
        shareOrUnshareDashboards(permissions, {"test_user_2"}, dashboard_ids, do_share);
      },
      error_stream.str());
  assertExpectedDashboardPrivilegesForUsers(getInitialPermissions(), {"test_user_2"});
  loginAdmin();
  sql("REVOKE CREATE DASHBOARD ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1;");
  sql("REVOKE ACCESS ON DATABASE " + OMNISCI_DEFAULT_DB + " FROM test_user_1");
}

class DashboardBulkDeleteTest : public DBHandlerTestFixture {
 public:
  static void SetUpTestSuite() {
    createDBHandler();
    test_user_1_id = createTestUser("test_user", "test_pass");
  }

  static void TearDownTestSuite() {
    loginAdmin();
    dropTestUser("test_user");
  }

  size_t getNumDashboards() {
    const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
    std::vector<TDashboard> dashboards;
    db_handler->get_dashboards(dashboards, session_id);
    return dashboards.size();
  }

 protected:
  void SetUp() override {
    DBHandlerTestFixture::SetUp();
    const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
    // Remove any dashboards that may be left by previous tests
    std::vector<TDashboard> dashboards;
    db_handler->get_dashboards(dashboards, session_id);
    if (dashboards.size()) {
      std::vector<int32_t> db_ids;
      for (const auto& dashboard : dashboards) {
        db_ids.push_back(dashboard.dashboard_id);
      }
      db_handler->delete_dashboards(session_id, db_ids);
    }
  }
  void TearDown() override { DBHandlerTestFixture::TearDown(); }

  static int32_t createTestUser(const std::string& user_name, const std::string& pass) {
    sql("CREATE USER " + user_name + " (password = '" + pass + "');");
    sql("GRANT ACCESS ON DATABASE omnisci TO " + user_name + ";");
    Catalog_Namespace::UserMetadata user_metadata{};
    Catalog_Namespace::SysCatalog::instance().getMetadataForUser(user_name,
                                                                 user_metadata);
    return user_metadata.userId;
  }

  static void dropTestUser(const std::string& user_name) {
    try {
      sql("DROP USER " + user_name + ";");
    } catch (const std::exception& e) {
      // Swallow and log exceptions that may occur, since there is no "IF EXISTS" option.
      LOG(WARNING) << e.what();
    }
  }

  inline static int32_t test_user_1_id;
};

TEST_F(DashboardBulkDeleteTest, CreateDelete) {
  const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
  auto db_id_1 =
      db_handler->create_dashboard(session_id, "test1", "state", "image", "metadata");
  auto db_id_2 =
      db_handler->create_dashboard(session_id, "test2", "state", "image", "metadata");
  ASSERT_EQ(getNumDashboards(), size_t(2));
  db_handler->delete_dashboards(session_id, {db_id_1, db_id_2});
  ASSERT_EQ(getNumDashboards(), size_t(0));
}

TEST_F(DashboardBulkDeleteTest, SomeInvalidIDs) {
  const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
  auto db_id =
      db_handler->create_dashboard(session_id, "test1", "state", "image", "metadata");

  try {
    db_handler->delete_dashboards(session_id, {db_id, 0});
    FAIL() << "An exception should have been thrown for this test case.";
  } catch (const TOmniSciException& e) {
    assertExceptionMessage(e,
                           "Exception: Delete dashboard(s) failed with "
                           "error(s):\nDashboard id: 0 - Dashboard id does not exist");
  }
  ASSERT_EQ(getNumDashboards(), size_t(1));
  db_handler->delete_dashboards(session_id, {db_id});
  ASSERT_EQ(getNumDashboards(), size_t(0));
}

TEST_F(DashboardBulkDeleteTest, NoDeleteDashboardPrivilege) {
  const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
  auto db_id_1 =
      db_handler->create_dashboard(session_id, "test1", "state", "image", "metadata");
  auto db_id_2 =
      db_handler->create_dashboard(session_id, "test2", "state", "image", "metadata");
  TSessionId unprivileged_session;
  login("test_user", "test_pass", "omnisci", unprivileged_session);
  sql("GRANT DELETE ON DASHBOARD " + std::to_string(db_id_1) + " to test_user");
  try {
    db_handler->delete_dashboards(unprivileged_session, {db_id_1, db_id_2});
    FAIL() << "An exception should have been thrown for this test case.";
  } catch (const TOmniSciException& e) {
    assertExceptionMessage(
        e,
        "Exception: Delete dashboard(s) failed with error(s):\nDashboard id: " +
            std::to_string(db_id_2) +
            " - User should be either owner of dashboard or super user to delete it");
  }
  ASSERT_EQ(getNumDashboards(), size_t(2));
  db_handler->delete_dashboards(session_id, {db_id_1, db_id_2});
  ASSERT_EQ(getNumDashboards(), size_t(0));
  logout(unprivileged_session);
}

TEST_F(DashboardBulkDeleteTest, NonSuperDeleteDashboardPrivilege) {
  const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
  auto db_id_1 =
      db_handler->create_dashboard(session_id, "test1", "state", "image", "metadata");
  auto db_id_2 =
      db_handler->create_dashboard(session_id, "test2", "state", "image", "metadata");
  TSessionId non_super_session;
  login("test_user", "test_pass", "omnisci", non_super_session);
  sql("GRANT DELETE ON DASHBOARD " + std::to_string(db_id_1) + " to test_user");
  sql("GRANT DELETE ON DASHBOARD " + std::to_string(db_id_2) + " to test_user");
  db_handler->delete_dashboards(non_super_session, {db_id_1, db_id_2});
  ASSERT_EQ(getNumDashboards(), size_t(0));
  logout(non_super_session);
}

TEST_F(DashboardBulkDeleteTest, InvalidNoPrivilegeMix) {
  const auto& [db_handler, session_id] = getDbHandlerAndSessionId();
  auto db_id =
      db_handler->create_dashboard(session_id, "test1", "state", "image", "metadata");
  TSessionId non_super_session;
  login("test_user", "test_pass", "omnisci", non_super_session);
  try {
    db_handler->delete_dashboards(non_super_session, {0, db_id});
    FAIL() << "An exception should have been thrown for this test case.";
  } catch (const TOmniSciException& e) {
    assertExceptionMessage(
        e,
        std::string("Exception: Delete dashboard(s) failed with error(s):\n") +
            "Dashboard id: 0 - Dashboard id does not exist\n" +
            "Dashboard id: " + std::to_string(db_id) +
            " - User should be either owner of dashboard or super user to delete it");
  }
  ASSERT_EQ(getNumDashboards(), size_t(1));
  db_handler->delete_dashboards(session_id, {db_id});
  ASSERT_EQ(getNumDashboards(), size_t(0));

  logout(non_super_session);
}

int main(int argc, char** argv) {
  TestHelpers::init_logger_stderr_only(argc, argv);
  testing::InitGoogleTest(&argc, argv);

  int err{0};
  try {
    err = RUN_ALL_TESTS();
  } catch (const std::exception& e) {
    LOG(ERROR) << e.what();
  }

  return err;
}

ext: cpp


text:
#define BOOST_TEST_MODULE
#include <boost/test/unit_test.hpp>
#include <util/test_macros.hpp>
#include <stdlib.h>
#include <vector>
#include <string>
#include <functional>
#include <random>
#include <cfenv>
#include <cmath>

#include <ml_data/ml_data.hpp>
#include <optimization/optimization_interface.hpp>
#include <optimization/utils.hpp>
#include <toolkits/supervised_learning/logistic_regression.hpp>
#include <toolkits/supervised_learning/logistic_regression_opt_interface.hpp>
#include <sframe/testing_utils.hpp>

using namespace turi;
using namespace turi::supervised;

void run_logistic_regression_test(std::map<std::string, flexible_type> opts) {

  size_t examples = opts.at("examples");
  size_t features = opts.at("features");
  std::string target_column_name = "target";

  // Answers
  // -----------------------------------------------------------------------
  DenseVector coefs(features+1);
  coefs.randn();

  // Feature names
  std::vector<std::string> feature_names;
  std::vector<flex_type_enum> feature_types;
  for(size_t i=0; i < features; i++){
    feature_names.push_back(std::to_string(i));
    feature_types.push_back(flex_type_enum::FLOAT);
  }

  // Data
  std::vector<std::vector<flexible_type>> y_data;
  std::vector<std::vector<flexible_type>> X_data;
  for(size_t i=0; i < examples; i++){
    DenseVector x(features);
    x.randn();
    std::vector<flexible_type> x_tmp;
    for(size_t k=0; k < features; k++){
      x_tmp.push_back(x(k));
    }

    // Compute the prediction for this
    double t = dot(x, coefs.subvec(0, features-1)) + coefs(features);
    t = 1.0/(1.0+exp(-1.0*t));
    int c = turi::random::bernoulli(t);
    if (i == 0) c = 0; // Make sure category 0 is category 0 (for testing)
    std::vector<flexible_type> y_tmp;
    y_tmp.push_back(c);

    X_data.push_back(x_tmp);
    y_data.push_back(y_tmp);
  }

  // Options
  std::map<std::string, flexible_type> options = {
    {"convergence_threshold", 1e-2},
    {"step_size", 1.0},
    {"lbfgs_memory_level", 3},
    {"max_iterations", 10},
    {"l1_penalty", 0.0},
    {"l2_penalty", 1e-2}
  };

  // Make the data
  sframe X = make_testing_sframe(feature_names, feature_types, X_data);
  sframe y = make_testing_sframe({"target"}, {flex_type_enum::STRING}, y_data);
  std::shared_ptr<logistic_regression> model;
  model.reset(new logistic_regression);
  model->init(X,y);
  model->init_options(options);
  model->train();

  // Construct the ml_data
  ml_data data = model->construct_ml_data_using_current_metadata(X, y);

  // Check coefficients & options
  // ----------------------------------------------------------------------
  DenseVector _coefs(features+1);
  model->get_coefficients(_coefs);
  TS_ASSERT(_coefs.size() == features + 1);

  std::map<std::string, flexible_type> _options;
  _options = model->get_current_options();
  for (auto& kvp: options){
    TS_ASSERT(_options[kvp.first] == kvp.second);
  }
  TS_ASSERT(model->is_trained() == true);

  // Check predictions
  // ----------------------------------------------------------------------
  std::vector<flexible_type> pred_margin;
  std::shared_ptr<sarray<flexible_type>> _pred_margin
    = model->predict(data, "margin");
  std::vector<flexible_type> pred_class;
  std::shared_ptr<sarray<flexible_type>> _pred_class
    = model->predict(data, "class");
  std::vector<flexible_type> pred_prob;
  std::shared_ptr<sarray<flexible_type>> _pred_prob
    = model->predict(data, "probability");

  // Save predictions made by the model
  auto reader = _pred_margin->get_reader();
  reader->read_rows(0, examples, pred_margin);
  reader = _pred_class->get_reader();
  reader->read_rows(0, examples, pred_class);
  reader = _pred_prob->get_reader();
  reader->read_rows(0, examples, pred_prob);

  // Check that the predictions made by the model are right!
  for(size_t i=0; i < examples; i++){
    DenseVector x(features + 1);
    for(size_t k=0; k < features; k++){
      x(k) = X_data[i][k];
    }
    x(features) = 1;
    double t = arma::dot(x, _coefs);
    double p = pred_margin[i];
    TS_ASSERT(abs(p - t) < 1e-5);
    t = 1.0/(1.0+exp(-1.0*t));
    p = pred_prob[i];
    TS_ASSERT(abs(p - t) < 1e-5);
    int c = t >= 0.5;
    TS_ASSERT_EQUALS(pred_class[i], std::to_string(c));
  }

  // Check save and load
  // ----------------------------------------------------------------------
  dir_archive archive_write;
  archive_write.open_directory_for_write("regr_logistic_regression_tests");
  turi::oarchive oarc(archive_write);
  oarc << *model;
  archive_write.close();

  // Load it
  dir_archive archive_read;
  archive_read.open_directory_for_read("regr_logistic_regression_tests");
  turi::iarchive iarc(archive_read);
  iarc >> *model;

  // Check coefficients after saving and loading.
  // ----------------------------------------------------------------------
  DenseVector _coefs_after_load(features+1);
  model->get_coefficients(_coefs_after_load);
  TS_ASSERT(_coefs_after_load.size() == features + 1);
  TS_ASSERT(arma::approx_equal(_coefs_after_load, _coefs,"absdiff", 1e-5));
  _options = model->get_current_options();
  for (auto& kvp: options){
    TS_ASSERT(_options[kvp.first] == kvp.second);
  }
  TS_ASSERT(model->is_trained() == true);

  // Check coefficients after saving and loading.
  // ----------------------------------------------------------------------
  _pred_margin = model->predict(data, "margin");
  _pred_class = model->predict(data, "class");
  _pred_prob = model->predict(data, "probability");
  reader = _pred_margin->get_reader();
  reader->read_rows(0, examples, pred_margin);
  reader = _pred_class->get_reader();
  reader->read_rows(0, examples, pred_class);
  reader = _pred_prob->get_reader();
  reader->read_rows(0, examples, pred_prob);

  // Check that the predictions made by the model are right!
  for(size_t i=0; i < examples; i++){
    DenseVector x(features + 1);
    for(size_t k=0; k < features; k++){
      x(k) = X_data[i][k];
    }
    x(features) = 1;
    double t = arma::dot(x, _coefs);
    double p = pred_margin[i];
    TS_ASSERT(abs(p - t) < 1e-5);

    t = 1.0/(1.0+exp(-1.0*t));
    p = pred_prob[i];
    TS_ASSERT(abs(p - t) < 1e-5);
    int c = t > 0.5;
    TS_ASSERT_EQUALS(pred_class[i], std::to_string(c));
  }

  model->get_coefficients(_coefs);
  TS_ASSERT(_coefs.size() == features + 1);
  model.reset();

  // Check that we can train a model when providing a validation set
  model.reset(new logistic_regression);
  logprogress_stream << "Training with a validation set" << std::endl;
  model->init(X, y, X, y);
  model->init_options(options);
  model->train();

}

/**
 *  Check logistic regression
*/
struct logistic_regression_test  {

  public:

  void test_logistic_regression_basic_2d() {
    std::map<std::string, flexible_type> opts = {
      {"examples", 100},
      {"features", 1}};
    run_logistic_regression_test(opts);
  }

  void test_logistic_regression_small() {
    std::map<std::string, flexible_type> opts = {
      {"examples", 1000},
      {"features", 10}};
    run_logistic_regression_test(opts);
  }

};

void run_logistic_regression_opt_interface_test(std::map<std::string,
    flexible_type> opts) {

  size_t examples = opts.at("examples");
  size_t features = opts.at("features");
  std::string target_column_name = "target";
  std::vector<std::string> column_names = {"user", "item"};

  // Answers
  // -----------------------------------------------------------------------
  DenseVector coefs(features+1);
  coefs.randn();

  // Feature names
  std::vector<std::string> feature_names;
  std::vector<flex_type_enum> feature_types;
  for(size_t i=0; i < features; i++){
    feature_names.push_back(std::to_string(i));
    feature_types.push_back(flex_type_enum::FLOAT);
  }

  // Data
  std::vector<std::vector<flexible_type>> y_data;
  std::vector<std::vector<flexible_type>> X_data;
  for(size_t i=0; i < examples; i++){
    DenseVector x(features);
    x.randn();
    std::vector<flexible_type> x_tmp;
    for(size_t k=0; k < features; k++){
      x_tmp.push_back(x(k));
    }

    // Compute the prediction for this
    double t = dot(x, coefs.subvec(0, features-1)) + coefs(features);
    t = 1.0/(1.0+exp(-1.0*t));
    int c = turi::random::bernoulli(t);
    std::vector<flexible_type> y_tmp;
    y_tmp.push_back(c);

    X_data.push_back(x_tmp);
    y_data.push_back(y_tmp);
  }

  // Options
  std::map<std::string, flexible_type> options = {
    {"convergence_threshold", 1e-2},
    {"step_size", 1.0},
    {"lbfgs_memory_level", 3},
    {"mini_batch_size", 1},
    {"max_iterations", 10},
    {"solver", "auto"},
  };

  // Construct the ml_data
  // Make the data
  sframe X = make_testing_sframe(feature_names, feature_types, X_data);
  sframe y = make_testing_sframe({"target"}, {flex_type_enum::STRING}, y_data);
  std::shared_ptr<logistic_regression> model;
  model.reset(new logistic_regression);
  model->init(X, y);

  // Construct the ml_data
  ml_data data = model->construct_ml_data_using_current_metadata(X, y);
  ml_data valid_data;

  std::shared_ptr<logistic_regression_opt_interface> lr_interface;
  lr_interface.reset(new logistic_regression_opt_interface(data, valid_data, *model));

  // Check examples & variables.
  TS_ASSERT(lr_interface->num_variables() == features + 1);
  TS_ASSERT(lr_interface->num_examples() == examples);

  size_t variables = lr_interface->num_variables();
  for(size_t i=0; i < 10; i++){

    DenseVector point(variables);
    point.randn();

    // Check gradients, functions and hessians.
    DenseVector gradient(variables);
    double func_value;
    DenseMatrix hessian(variables, variables);

    func_value = lr_interface->compute_function_value(point);
    lr_interface->compute_gradient(point, gradient);
    lr_interface->compute_hessian(point, hessian);
    TS_ASSERT(check_gradient(*lr_interface, point, gradient));
    if( variables <= 2){
      TS_ASSERT(check_hessian(*lr_interface, point, hessian));
    }

    // Check first order & second order computations
    DenseVector _gradient(variables);
    double _func_value;
    DenseMatrix _hessian(variables, variables);

    lr_interface->compute_first_order_statistics(point, _gradient,
      _func_value);
    TS_ASSERT(abs(func_value - _func_value) < 1e-5);
    TS_ASSERT(arma::approx_equal(gradient, _gradient,"absdiff", 1e-10));
    lr_interface->compute_second_order_statistics(point, _hessian, _gradient,
      _func_value);
    TS_ASSERT(abs(func_value - _func_value) < 1e-5);
    TS_ASSERT(arma::approx_equal(gradient, _gradient,"absdiff", 1e-10));
    TS_ASSERT(arma::approx_equal(hessian, _hessian,"absdiff", 1e-10));

  }

  model.reset();
  lr_interface.reset();
}

/**
 *  Check logistic regression opt interface
*/
struct logistic_regression_opt_interface_test  {

  public:

  void test_logistic_regression_opt_interface_basic_2d() {
    std::map<std::string, flexible_type> opts = {
      {"examples", 100},
      {"features", 1}};
    run_logistic_regression_opt_interface_test(opts);
  }

  void test_logistic_regression_opt_interface_small() {
    std::map<std::string, flexible_type> opts = {
      {"examples", 1000},
      {"features", 10}};
    run_logistic_regression_opt_interface_test(opts);
  }

};

BOOST_FIXTURE_TEST_SUITE(_logistic_regression_test, logistic_regression_test)
BOOST_AUTO_TEST_CASE(test_logistic_regression_basic_2d) {
  logistic_regression_test::test_logistic_regression_basic_2d();
}
BOOST_AUTO_TEST_CASE(test_logistic_regression_small) {
  logistic_regression_test::test_logistic_regression_small();
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_FIXTURE_TEST_SUITE(_logistic_regression_opt_interface_test, logistic_regression_opt_interface_test)
BOOST_AUTO_TEST_CASE(test_logistic_regression_opt_interface_basic_2d) {
  logistic_regression_opt_interface_test::test_logistic_regression_opt_interface_basic_2d();
}
BOOST_AUTO_TEST_CASE(test_logistic_regression_opt_interface_small) {
  logistic_regression_opt_interface_test::test_logistic_regression_opt_interface_small();
}
BOOST_AUTO_TEST_SUITE_END()

ext: cxx


text:
/*
** The MIT License (MIT)
**
** Copyright (c) 2020, National Cybersecurity Agency of France (ANSSI)
**
** Permission is hereby granted, free of charge, to any person obtaining a copy
** of this software and associated documentation files (the "Software"), to deal
** in the Software without restriction, including without limitation the rights
** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
** copies of the Software, and to permit persons to whom the Software is
** furnished to do so, subject to the following conditions:
**
** The above copyright notice and this permission notice shall be included in
** all copies or substantial portions of the Software.
**
** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
** THE SOFTWARE.
**
** Author:
**   - Guillaume Bouffard <guillaume.bouffard@ssi.gouv.fr>
*/

#ifdef PC_VERSION

#include "debug.hpp"
#include "ffi.h"
#include "interpretor.hpp"
#include "jc_config.h"
#include "jni.hpp"
#include "types.hpp"

#ifdef DEBUG
#include <iomanip>
#include <iostream>
#endif /* DEBUG */

#include "jc_handlers/flashmemory.hpp"
#include <algorithm>
#include <boost/program_options.hpp>
#include <fstream>
#include <iterator>
#include <string>
#include <vector>

bool isSaving = false;
uint8_t *flash = nullptr;
uint32_t flash_length = 0;
std::string flash_filename;

int main_pc(int argc, char *argv[]) {

  /** Define and parse the program options
   */
  boost::program_options::options_description desc("Options");
  desc.add_options()
#ifdef DEBUG
      ("help,h", "Print help messages")
#endif /* DEBUG */
          ("memory,m",
           boost::program_options::value<std::string>(&flash_filename)
               ->required()
               ->value_name("MEMORY_FILENAME"),
           "Flash Memory")("save,s", "Save modifications on MEMORY_FILENAME");

  boost::program_options::variables_map parameters;

  try {
    boost::program_options::store(
        boost::program_options::parse_command_line(argc, argv, desc),
        parameters); // can throw

#ifdef DEBUG

    /**
     * --help option
     */
    if (parameters.count("help")) {
      std::cout
          << "USAGE: " << argv[0] << " [OPTION] -m MEMORY_FILENAME" << std::endl
          << std::endl
          << "CHOUPI is a secure Java Card open-source implementation."
          << std::endl
          << "This proof of concept is powered by ANSSI (www.ssi.gouv.fr)."
          << std::endl
          << "Last update: " << __DATE__ << " - " << __TIME__ << std::endl
          << std::endl
          << desc << std::endl;
      return EXIT_SUCCESS;
    }

#endif /* DEBUG */

    boost::program_options::notify(parameters);

  } catch (boost::program_options::error &e) {
#ifdef DEBUG
    std::cerr << "ERROR: " << e.what() << std::endl
              << std::endl
              << "USAGE: " << argv[0] << " [OPTION] -m MEMORY_FILENAME"
              << std::endl
              << std::endl
              << "CHOUPI is a secure Java Card open-source implementation."
              << std::endl
              << "This proof of concept is powered by ANSSI (www.ssi.gouv.fr)."
              << std::endl
              << "Last update: " << __DATE__ << " - " << __TIME__ << std::endl
              << std::endl
              << desc << std::endl;
#endif /* DEBUG */
    return EXIT_FAILURE;
  }

#ifdef DEBUG
  std::cout << "Welcome to CHOUPI implementation!" << std::endl
            << "CHOUPI is a secure Java Card open-source implementation."
            << std::endl
            << "This proof of concept is powered by ANSSI (www.ssi.gouv.fr)."
            << std::endl
            << "Last update: " << __DATE__ << " - " << __TIME__ << std::endl
            << std::endl
            << "CHOUPI is starting :)" << std::endl
            << std::endl;
#endif /* DEBUG */

  auto file_memory_in = std::ifstream(flash_filename, std::ifstream::binary);
  flash = flash_pointer();

  if (file_memory_in.is_open()) {
    uint8_t *last = std::copy(std::istreambuf_iterator<char>(file_memory_in),
                              std::istreambuf_iterator<char>(), flash);
    flash_length = (last - flash);

#ifdef DEBUG
    TRACE_JCVM_ERR("Flash length = %u Byte", flash_length);
#endif /* DEBUG */
  } else {
#ifdef DEBUG
    TRACE_JCVM_ERR("ERROR: Unable to open %s", flash_filename);
#endif /* DEBUG */
    return EXIT_FAILURE;
  }

  file_memory_in.close();

  isSaving = parameters.count("save");

  // running emulator
  run_emulator();

  return 0;
}

#ifdef __cplusplus
extern "C" {
#endif

//  The runtime_main function is called in the context 0.
int starting_jcre() {

  TRACE_JCVM_DEBUG("Starting JCRE");

  //  Call GP applet => main security domain
  uint32_t arg = (STARTING_JAVACARD_PACKAGE << 16) |
                 (STARTING_JAVACARD_CLASS << 8) | (STARTING_JAVACARD_METHOD);
  remote_call(2, arg, 0);

  if (isSaving) {
    TRACE_JCVM_DEBUG("Saving memory in FLASH_MEMORY file");

    auto flash_memory_out =
        std::ofstream(flash_filename, std::ofstream::binary);
    std::ostream_iterator<char> output(flash_memory_out);
    std::copy(flash, flash + flash_length, output);
    flash_memory_out.close();
  }

  return 0;
}
#ifdef __cplusplus
}
#endif

#endif /* PC_VERSION */

max_stars_repo_name: gavz/choupi

max_issues_repo_name: gavz/choupi

max_forks_repo_name: gavz/choupi