text:
// Copyright 2009 by BBN Technologies Corp.
// All Rights Reserved.

#include "Generic/common/leak_detection.h"

#include "Generic/common/limits.h"
#include "Generic/common/ParamReader.h"
#include "Generic/common/UTF8InputStream.h"
#include "Generic/common/Sexp.h"
#include "Generic/theories/Lexicon.h"
#include "Generic/common/FileSessionLogger.h"
#include "Generic/common/UnrecoverableException.h"
#include "Generic/morphAnalysis/MorphologicalAnalyzer.h"
#include "Generic/morphAnalysis/SessionLexicon.h"
#include "Generic/morphSelection/MorphModel.h"
#include "Generic/morphSelection/MorphDecoder.h"
#include "Generic/morphSelection/MorphSelector.h"
#include "Generic/normalizer/MTNormalizer.h"
#include "Generic/tokens/Tokenizer.h"
#include "Generic/tokens/SymbolSubstitutionMap.h"
#include "Generic/theories/TokenSequence.h"
#include "Generic/theories/Token.h"
#include "Generic/theories/LexicalTokenSequence.h"
#include "Generic/theories/LexicalToken.h"
#include "Generic/names/NameClassTags.h"
#include "Generic/common/version.h"
#include "Generic/common/FeatureModule.h"
#include <boost/scoped_ptr.hpp>

#define MAX_LINE_LENGTH 15000
wchar_t line[MAX_LINE_LENGTH];

void print_usage() {
	std::cout << "Training Usage: MorphologyTrainer -t PARAM_FILE TRAIN_FILE MULT (MODEL_PREFIX)" << std::endl;
	std::cout << "Decode Usage (Untokenized Input): MorphologyTrainer -d PARAM_FILE [<input-file> <output-file>]" << std::endl;
	std::cout << "Decode Usage (for name-training sexp input): MorphologyTrainer -dsexp PARAM_FILE [<input-file> <output-file>]" << std::endl;
	std::cout << "Decode Usage (Pre-Tokenized and Analyzed Input): MorphologyTrainer -a PARAM_FILE" << std::endl;
	std::cout << "Tokenizer Usage: MorphologyTrainer -tok PARAM_FILE" << std::endl;
}

bool readSentence(UTF8InputStream& uis, LocatedString* loc) {
	int len, start, end;
	len = 0;

	while (!uis.eof()) {
		uis.getLine(line, MAX_LINE_LENGTH);
		len = static_cast<int>(wcslen(line));
		if (len == 0)
			continue;
		start = 0;
		end = len-1;
		while((start < len) && iswspace(line[start])) start++;
		while((end > 0) && iswspace(line[end])) end--;
		if (start != end)
			break;
	}
	if (len == 0) {
		loc = 0;
		return false;
	}
	if (line[start] != L'(') {
		throw UnexpectedInputException("MorphologyTrainerMain::readSentence()",
									   "Test sentence doesn't start with '('");
	}
	if (line[end] != L')') {
		throw UnexpectedInputException("MorphologyTrainerMain::readSentence()",
										"Test sentence doesn't end with ')'");
	}
	start++;
	end++;
	LocatedString* temp = _new LocatedString(line);
	temp->trim();
	start = temp->indexOf(L"(");
	end = temp->indexOf(L")");
	loc = temp->substring(start+1, end);
	loc->replace(L"-LRB-",L"(");
	loc->replace(L"-RRB-",L")");

	delete temp;
	return true;
}

bool readTokenizedAndAnalyzedSentence(UTF8InputStream& uis, TokenSequence** tokens, LocatedString &tokenString) {
	FeatureValueStructure *fvs = 0;
	Token **origTokens = _new Token*[MAX_SENTENCE_TOKENS];
	Token **morphTokens = _new Token*[MAX_SENTENCE_TOKENS];
	Sexp *sentence = _new Sexp(uis);
	int n_lex = 0;

	if (sentence->isVoid())
		return false;

	if (sentence->getNumChildren() != 2)
		throw UnexpectedInputException("MorphologyTrainerMain::readTokenizedAndAnalyzedSentence()",
									   "Encountered improper sexp format.");

	Sexp *tokenList = sentence->getFirstChild();
	Sexp *morphList = sentence->getSecondChild();
	tokenString = LocatedString(tokenList->to_token_string().c_str());
	int n_tokens = tokenList->getNumChildren() < MAX_SENTENCE_TOKENS ? tokenList->getNumChildren() : MAX_SENTENCE_TOKENS;

	if (morphList->getNumChildren() != tokenList->getNumChildren())
		throw UnexpectedInputException("MorphologyTrainerMain::readTokenizedAndAnalyzedSentence()",
									   "Encountered improper sexp format.");

	int curr_char = 0;
	for (int i = 0; i < n_tokens; i++) {

		Sexp *analysisList = morphList->getNthChild(i);
		int n_analyses = analysisList->getNumChildren();
		LexicalEntry **analyses = _new LexicalEntry*[n_analyses];

		for (int j = 0; j < n_analyses; j++) {
			Sexp *analysis = analysisList->getNthChild(j);
			int n_morphs = analysis->getNumChildren();
			LexicalEntry **morphEntries = _new LexicalEntry*[n_morphs];
			for (int k = 0; k < n_morphs; k++) {
				Sexp *morph = analysis->getNthChild(k);
				fvs = FeatureValueStructure::build(); // TODO: fix this
				morphEntries[k] = _new LexicalEntry(n_lex++, morph->getValue(), fvs, NULL, 0);
			}
			fvs = FeatureValueStructure::build(); // TODO: fix this
			analyses[j] = _new LexicalEntry(n_lex++, tokenList->getNthChild(i)->getValue(),
							  fvs, morphEntries, n_morphs);
		}

		std::wstring token_str = wstring(tokenList->getNthChild(i)->getValue().to_string());
		size_t token_len = token_str.length();
		OffsetGroup startGroup = OffsetGroup(CharOffset(curr_char), EDTOffset(curr_char));
		OffsetGroup endGroup = OffsetGroup(CharOffset(curr_char + token_len - 1), EDTOffset(curr_char + token_len - 1));

//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
		if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
			origTokens[i] = _new LexicalToken(startGroup, endGroup, tokenList->getNthChild(i)->getValue(), i);
			morphTokens[i] = _new LexicalToken(startGroup, endGroup, tokenList->getNthChild(i)->getValue(), i, n_analyses, analyses);
		}
//#endif
		curr_char += token_len + 1; // add an extra character for whitespace
		delete [] analyses;
	}

//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
	if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
		tokens[0] = _new LexicalTokenSequence(0, n_tokens, origTokens);
	} else {
//#else
		tokens[0] = _new TokenSequence(0, n_tokens, origTokens);
	}
//#endif
	tokens[0]->retokenize(n_tokens, morphTokens);

	delete [] origTokens;
	delete [] morphTokens;
	delete sentence;
	return true;
}

void runTraining(int argc, char* argv[]) {
	try {
		char mprefix[500];

		if (argc != 5 && argc != 6) {
			print_usage();
			return;
		}

		const char* pfile = argv[2];
		const char* tfile = argv[3];
		const char* mbuffer = argv[4];
		int mult = atoi(mbuffer);

		ParamReader::readParamFile(pfile);
		FeatureModule::load();

		if (argc == 6) {
			strncpy(mprefix, argv[5], 500);
		}
		else if	(argc == 5)	{
			ParamReader::getRequiredParam("MorphModel", mprefix, 500);
		}
		std::cout << "Model Prefix: " << mprefix << " Mult: " << mult << std::endl;

		//Create a default session logger since the lexicon assumes its there
		std::wstringstream buffer;
		buffer << mprefix << ".trainSession.log";
		SessionLogger* sessionLogger = _new FileSessionLogger(buffer.str().c_str(), 0, 0);
		SessionLogger::logger = sessionLogger;

		MorphModel *m = MorphModel::build();
		m->setMultiplier(mult);
		m->trainOnSingleFile(tfile, mprefix);

		delete m;
		delete sessionLogger;

	} catch (UnrecoverableException e) {
		std::cout << "Exception: " << e.getSource() << " " << e.getMessage() << std::endl;
	}
}
void runTokenizer(int argc, char* argv[]) {
	try {

		const char* pfile = argv[2];
		ParamReader::readParamFile(pfile);
		FeatureModule::load();

		std::string dfile = ParamReader::getRequiredParam("input_file");
		std::cout << "Input File: " << dfile << std::endl;

		std::string ofile = ParamReader::getRequiredParam("output_file");
		std::cout << "Output File: " << ofile << std::endl;

		std::string map = ParamReader::getRequiredParam("reverse_subst_map");
		SymbolSubstitutionMap* reverseSubs = _new SymbolSubstitutionMap(map.c_str());

		LocatedString *sentenceString = 0;
		Tokenizer *tokenizer = Tokenizer::build();
		TokenSequence *tokens[1];

		//Create a default session logger since the lexicon assumes its there
		std::wstring log(ofile.begin(), ofile.end());
		log.append(L"-tokenize.log");
		SessionLogger* sessionLogger = _new FileSessionLogger(log.c_str(), 0, 0);
		SessionLogger::logger = sessionLogger;

        boost::scoped_ptr<UTF8InputStream> uis_scoped_ptr(UTF8InputStream::build(dfile.c_str()));
        UTF8InputStream& uis(*uis_scoped_ptr);
		UTF8OutputStream uos(ofile.c_str());
		int linecount = 0;

		while (!uis.eof()) {
			linecount++;
			if ((linecount % 250) == 0) {
				std::cout << "Read Line " << linecount << std::endl;
			}

			uis.getLine(line, MAX_LINE_LENGTH);
			if (wcslen(line) == 0)
				continue;
			sentenceString = _new LocatedString(line);

			tokenizer->getTokenTheories(tokens, 1, sentenceString);

			for (int j = 0; j < tokens[0]->getNTokens(); j++) {
				Symbol out = reverseSubs->replace(tokens[0]->getToken(j)->getSymbol());
				uos << out.to_string() << " ";
			}
			uos << L"\n";

			delete tokens[0];
			delete sentenceString;
		}

		uis.close();
		uos.close();

		delete tokenizer;
		delete sessionLogger;
		delete reverseSubs;

	} catch (UnrecoverableException e) {
		std::cout << "Exception: " << e.getSource() << " " << e.getMessage() << std::endl;
	}

}

void runDecoderNameTraining(int argc, char* argv[]){
	// Expects input of the form ((word1 NONE-ST) (word2 GPE-ST) ... )
	// Outputs the same form.
	try {
		if (argc != 3 && argc != 5) {
			print_usage();
			return;
		}

		const char* pfile = argv[2];
		ParamReader::readParamFile(pfile);
		FeatureModule::load();

		char dfile[600];
		char ofile[600];
		if (argc == 3) {
			ParamReader::getRequiredParam("input_file", dfile, 600);
			ParamReader::getRequiredParam("output_file", ofile, 600);
		}
		else {
			strncpy(dfile, argv[3], 600);
			strncpy(ofile, argv[4], 600);
		}

		std::cout << "Input File: " << dfile << std::endl;
		std::cout << "Output File: " << ofile << std::endl;

		//Create a default session logger since the lexicon assumes its there
		std::wstringstream buffer;
		buffer << ofile << "-tokenize.log";
		SessionLogger* sessionLogger = _new FileSessionLogger(buffer.str().c_str(), 0, 0);
		SessionLogger::logger = sessionLogger;

		Symbol *origTags = _new Symbol[MAX_SENTENCE_TOKENS];

		MorphologicalAnalyzer *morphAnalysis = MorphologicalAnalyzer::build();
		std::cout << "Initialized Morphological Analyzer" << std::endl;

		MorphSelector *morphSelector = _new MorphSelector();
		std::cout << "Initialized Morphological Selector" << std::endl;

		NameClassTags *nameClassTags = _new NameClassTags();

		MTNormalizer *mtNormalizer;
		bool do_mt_normalization = ParamReader::isParamTrue("do_mt_normalization");
		if(do_mt_normalization)
			mtNormalizer = MTNormalizer::build();

        boost::scoped_ptr<UTF8InputStream> uis_scoped_ptr(UTF8InputStream::build(dfile));
        UTF8InputStream& uis(*uis_scoped_ptr);
		UTF8OutputStream uos(ofile);
		int linecount = 0;

		TokenSequence *tokenSequenceBuf;
		Token **origTokens;

		origTokens = _new Token*[MAX_SENTENCE_TOKENS];

		while (!uis.eof()) {
			linecount++;
			if ((linecount % 250) == 0) {
				std::cout << "Read Line " << linecount << std::endl;
				// This line appears to fix a memory leak that causes a crash after about 23000 sentences.
				SessionLexicon::getInstance().getLexicon()->clearDynamicEntries();
			}

			Sexp sentence(uis);
			if(sentence.isVoid())
				continue;
			int n_tokens = sentence.getNumChildren() < MAX_SENTENCE_TOKENS ? sentence.getNumChildren() : MAX_SENTENCE_TOKENS;
			int curr_char = 0;
			for (int i = 0; i < n_tokens; i++) {
				Sexp *pair = sentence.getNthChild(i);
				std::wstring token_str = wstring(pair->getFirstChild()->getValue().to_string());
				size_t token_len = token_str.length();
				OffsetGroup startGroup = OffsetGroup(CharOffset(curr_char), EDTOffset(curr_char));
				OffsetGroup endGroup = OffsetGroup(CharOffset(curr_char + token_len - 1), EDTOffset(curr_char + token_len - 1));
//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
				if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
					origTokens[i] = _new LexicalToken(startGroup, endGroup, pair->getFirstChild()->getValue(), i);
				}
//#endif
				origTags[i] = pair->getSecondChild()->getValue();
				curr_char += token_len + 1; // add an extra character for whitespace
			}

//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
			if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
				tokenSequenceBuf = _new LexicalTokenSequence(0, n_tokens, origTokens);
			} else {
//#else
				tokenSequenceBuf = _new TokenSequence(0, n_tokens, origTokens);
			}
//#endif
			LocatedString *sentenceString = _new LocatedString(sentence.to_token_string().c_str());
			morphAnalysis->getMorphTheories(tokenSequenceBuf);
			morphSelector->selectTokenization(sentenceString, tokenSequenceBuf);
			if(do_mt_normalization){
				int numTokensBeforeNormalization = tokenSequenceBuf->getNTokens();

				mtNormalizer->normalize(tokenSequenceBuf);

				int numTokensAfterNormalization = tokenSequenceBuf->getNTokens();
				if(numTokensBeforeNormalization != numTokensAfterNormalization){
					// In the rare event that a token is deleted, it could cause a problem like the first tag being NONE-CO.
					// Rather than fixing the tag, we just skip the sentence.
					std::cout << "Number of tokens changed during normalization. Skipping line.\n";
					delete tokenSequenceBuf;
					morphAnalysis->resetForNewSentence();
					morphAnalysis->resetDictionary();
					morphSelector->resetForNewSentence();
					mtNormalizer->resetForNewSentence();
					continue;
				}
			}

//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
			if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
				uos << "( " ;
				for (int i = 0; i < tokenSequenceBuf->getNTokens(); i++) {
					const LexicalToken *out_token = dynamic_cast<const LexicalToken*>(tokenSequenceBuf->getToken(i));
					Symbol out_tag = origTags[out_token->getOriginalTokenIndex()];

					// If out_token is not at the beginning of an original token, replace out_tag with it's -CO form
					if(i > 0)
						if(out_token->getOriginalTokenIndex() == dynamic_cast<const LexicalToken*>(tokenSequenceBuf->getToken(i-1))->getOriginalTokenIndex())
							out_tag = nameClassTags->getTagSymbol(nameClassTags->getContinueForm(nameClassTags->getIndexForTag(out_tag)));

					uos << "( " << out_token->getSymbol().to_string() << " " << out_tag.to_string()  << " )";
				}
				uos << " )" << L"\n";
			}
//#endif

			delete tokenSequenceBuf;
			delete sentenceString;
			morphAnalysis->resetForNewSentence();
			morphAnalysis->resetDictionary();
			morphSelector->resetForNewSentence();
			if(do_mt_normalization){
				mtNormalizer->resetForNewSentence();
			}
		}

		uis.close();
		uos.close();

		delete sessionLogger;
		delete [] origTokens;
		delete [] origTags;
		delete morphAnalysis;
		delete morphSelector;
		delete nameClassTags;
		if(do_mt_normalization)
			delete mtNormalizer;

	} catch (UnrecoverableException e) {
		std::cout << "Exception: " << e.getSource() << " " << e.getMessage() << std::endl;
	}
}
void runDecoder(int argc, char* argv[]) {
	try {
		if (argc != 3 && argc != 5) {
			print_usage();
			return;
		}

		const char* pfile = argv[2];
		ParamReader::readParamFile(pfile);
		FeatureModule::load();

		char dfile[600];
		char ofile[600];
		if (argc == 3) {
			ParamReader::getRequiredParam("input_file", dfile, 600);
			ParamReader::getRequiredParam("output_file", ofile, 600);
		}
		else {
			strncpy(dfile, argv[3], 600);
			strncpy(ofile, argv[4], 600);
		}

		std::cout << "Input File: " << dfile << std::endl;
		std::cout << "Output File: " << ofile << std::endl;

		// Cluster training doesn't need sexp, just text.
		bool output_for_cluster_training = ParamReader::isParamTrue("output_for_cluster_training");

		LocatedString *sentenceString = 0;
		Tokenizer *tokenizer = Tokenizer::build();
		TokenSequence *tokens[1];
		Symbol words[MAX_SENTENCE_TOKENS];
		Symbol sentence[MAX_SENTENCE_TOKENS];

		//Create a default session logger since the lexicon assumes its there
		std::string expt_dir = ParamReader::getRequiredParam("experiment_dir");
		std::wstring log_file(expt_dir.begin(), expt_dir.end());
		log_file.append(L"/decode.log");
		SessionLogger* sessionLogger = _new FileSessionLogger(log_file.c_str(), 0, 0);
		SessionLogger::logger = sessionLogger;

		MorphologicalAnalyzer *morphAnalyzer = MorphologicalAnalyzer::build();
		std::cout << "Initialized Morphological Analyzer" << std::endl;

		MorphSelector *morphSelector = _new MorphSelector();
		std::cout << "Initialized Morphological Selector" << std::endl;

		MTNormalizer *mtNormalizer;
		bool do_mt_normalization = ParamReader::isParamTrue("do_mt_normalization");
		if(do_mt_normalization)
			mtNormalizer = MTNormalizer::build();

		boost::scoped_ptr<UTF8InputStream> uis_scoped_ptr(UTF8InputStream::build(dfile));
		UTF8InputStream& uis(*uis_scoped_ptr);
		UTF8OutputStream uos(ofile);
		int linecount = 0;

		while (!uis.eof()) {
			linecount++;
			if ((linecount % 250) == 0) {
				std::cout << "Read Line " << linecount << std::endl;
				// This line fixes a memory leak in runDecoderNameTraining(), and was added here out of caution.
				SessionLexicon::getInstance().getLexicon()->clearDynamicEntries();
			}

			uis.getLine(line, MAX_LINE_LENGTH);
			if (wcslen(line) == 0){
				std::cout << "Blank line " << linecount << std::endl;
				continue;
			}
			sentenceString = _new LocatedString(line);

			tokenizer->getTokenTheories(tokens, 1, sentenceString);
			morphAnalyzer->getMorphTheories(tokens[0]);
			morphSelector->selectTokenization(sentenceString, tokens[0]);
			if(do_mt_normalization)
				mtNormalizer->normalize(tokens[0]);

			int num_tokens = tokens[0]->getNTokens();

			if(!output_for_cluster_training)
				uos<< "( ";
			for (int j = 0; j < num_tokens; j++) {
				uos << tokens[0]->getToken(j)->getSymbol().to_string() << " ";
			}
			if(!output_for_cluster_training)
				uos << L" )";
			uos << "\n";

			delete tokens[0];
			delete sentenceString;

			morphAnalyzer->resetDictionary();
			morphSelector->resetForNewSentence();
			if(do_mt_normalization)
				mtNormalizer->resetForNewSentence();
		}

		uis.close();
		uos.close();

		delete tokenizer;
		delete sessionLogger;
		delete morphAnalyzer;
		delete morphSelector;
		if(do_mt_normalization)
			delete mtNormalizer;

	} catch (UnrecoverableException e) {
		std::cout << "Exception: " << e.getSource() << " " << e.getMessage() << std::endl;
	}
}
void runDecoderPretokenized(int argc, char* argv[]) {
	try {
		char buffer[700];

		const char* pfile = argv[2];
		ParamReader::readParamFile(pfile);
		FeatureModule::load();

		ParamReader::getParam("reverse_subst_map", buffer, 700);
		SymbolSubstitutionMap* reverseSubs = _new SymbolSubstitutionMap(buffer);

		//Create a default session logger since the lexicon assumes its there
		std::string log_file = ParamReader::getRequiredParam("log_file");
		SessionLogger* sessionLogger = _new FileSessionLogger(std::wstring(log_file.begin(), log_file.end()).c_str(), 0, 0);
		SessionLogger::logger = sessionLogger;

		TokenSequence **tokens = _new TokenSequence*[1];
		LocatedString sentenceString(L"");
		Symbol words[MAX_SENTENCE_TOKENS];
		int map[MAX_SENTENCE_TOKENS];

		MorphDecoder *morphDecoder = MorphDecoder::build();
		std::cout << "Initialized Morphological Decoder" << std::endl;

		if (ParamReader::isParamTrue("list_mode")) {
			char file_list[600];
			char dfile[600];
			char ofile[600];

			ParamReader::getRequiredParam("input_file_list", file_list, 600);
			std::cout << "Input File List: " << file_list << std::endl;

			std::basic_ifstream<char> in;
			in.open(file_list);
			if (in.fail()) {
				throw UnexpectedInputException("MorphologyTrainerMain::main()",
												"could not open input file list");
			}

			while (!in.eof()) {
				in.getline(dfile, 600);
				sprintf(ofile, "%s.morph", dfile);

				if (strcmp(dfile, "") == 0)
					break;

				std::cout << "Input File: " << dfile << std::endl;
				std::cout << "Output File: " << ofile << std::endl;

				boost::scoped_ptr<UTF8InputStream> uis_scoped_ptr(UTF8InputStream::build(dfile));
				UTF8InputStream& uis(*uis_scoped_ptr);
				UTF8OutputStream uos(ofile);
				int linecount = 0;

				while (readTokenizedAndAnalyzedSentence(uis, tokens, sentenceString)) {
					linecount++;
					if ((linecount % 250) == 0) {
						std::cout << "Read Line " << linecount << std::endl;
					}

					int num_tokens = morphDecoder->getBestWordSequence(sentenceString, tokens[0], words, map, MAX_SENTENCE_TOKENS);

					//morphDecoder->printTrellis(uos);

					int map_idx = 0;
					uos << "( (";
					for (int j = 0; j < num_tokens; j++) {
						Symbol out = reverseSubs->replace(words[j]);
						if (map_idx != map[j]) {
							uos << ") (";
							// mark tokens where decoder had >1 analysis to choose from
//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
							if (SerifVersion::isArabic() || SerifVersion::isKorean()) {

								if (dynamic_cast<const LexicalToken*>(tokens[0]->getToken(map[j]))->getNLexicalEntries() > 1)
									uos << "*";
							}
//#endif
							map_idx = map[j];
						}
						uos << words[j].to_string();
						if ((j != num_tokens - 1) && (map[j] == map[j+1]))
							uos << "+";
					}
					uos << L") )\n";

					delete tokens[0];
				}

				uis.close();
				uos.close();
			}

			in.close();
		}
		else {
			char dfile[600];
			ParamReader::getRequiredParam("input_file", dfile, 600);
			std::cout << "Input File: " << dfile << std::endl;

			char ofile[600];
			ParamReader::getRequiredParam("output_file", ofile, 600);
			std::cout << "Output File: " << ofile << std::endl;

			boost::scoped_ptr<UTF8InputStream> uis_scoped_ptr(UTF8InputStream::build(dfile));
			UTF8InputStream& uis(*uis_scoped_ptr);
			UTF8OutputStream uos(ofile);
			int linecount = 0;

			while (readTokenizedAndAnalyzedSentence(uis, tokens, sentenceString)) {
				linecount++;
				if ((linecount % 250) == 0) {
					std::cout << "Read Line " << linecount << std::endl;
				}

				int num_tokens = morphDecoder->getBestWordSequence(sentenceString, tokens[0], words, map, MAX_SENTENCE_TOKENS);

				//morphDecoder->printTrellis(uos);

				int map_idx = 0;
				uos << "( (";
				for (int j = 0; j < num_tokens; j++) {
					Symbol out = reverseSubs->replace(words[j]);
					if (map_idx != map[j]) {
						uos << ") (";
						// mark tokens where decoder had >1 analysis to choose from
//#if defined(ARABIC_LANGUAGE) || defined(KOREAN_LANGUAGE)
						if (SerifVersion::isArabic() || SerifVersion::isKorean()) {
							if (dynamic_cast<const LexicalToken*>(tokens[0]->getToken(map[j]))->getNLexicalEntries() > 1)
								uos << "*";
						}
//#endif
						map_idx = map[j];
					}
					uos << words[j].to_string();
					if ((j != num_tokens - 1) && (map[j] == map[j+1]))
						uos << "+";
				}
				uos << L") )\n";

				delete tokens[0];
			}

			uis.close();
			uos.close();
		}

		delete sessionLogger;
		delete reverseSubs;
		delete morphDecoder;

	} catch (UnrecoverableException e) {
		std::cout << "Exception: " << e.getSource() << " " << e.getMessage() << std::endl;
	}
}
int main(int argc, char* argv[]) {

	std::cout << "MorphologyTrainerMain v2.0" << std::endl;
	if (argc != 3 && argc != 5 && argc != 6) {
		print_usage();
		return 0;
	}

	if (strcmp(argv[1], "-t") == 0) {
		runTraining(argc, argv);
	} else if (strcmp(argv[1], "-tok") == 0) {
		runTokenizer(argc, argv);
	} else if (strcmp(argv[1], "-dsexp") == 0) {
		runDecoderNameTraining(argc, argv);
	} else if (strcmp(argv[1], "-d") == 0) {
		runDecoder(argc, argv);
	} else if (strcmp(argv[1], "-a") == 0) {
		runDecoderPretokenized(argc, argv);
	} else {
		print_usage();
		return 0;
	}

	return 0;
}

max_stars_repo_name: BBN-E/serif

max_issues_repo_name: BBN-E/serif

max_forks_repo_name: BBN-E/serif


text:
/*
 * Copyright (c) 2016 Steemit, Inc., and contributors.
 */

// clang-format off

#pragma once

#include <scorum/protocol/types.hpp>
#include <boost/preprocessor/cat.hpp>

namespace scorum {
namespace protocol {

namespace detail {

    struct config
    {
        static std::unique_ptr<config> instance;

        const uint32_t blockid_pool_size;

        const uint32_t vesting_withdraw_intervals;
        const uint32_t vesting_withdraw_interval_seconds;

        const uint32_t cashout_window_seconds;

        const fc::microseconds upvote_lockout;

        const fc::microseconds active_sp_holders_reward_period;

        const uint32_t advertising_cashout_period_sec;

        const fc::microseconds reverse_auction_window_seconds;

        const fc::microseconds vote_regeneration_seconds;

        const fc::microseconds owner_auth_recovery_period;
        const fc::microseconds account_recovery_request_expiration_period;
        const fc::microseconds owner_update_limit;

        const fc::microseconds recent_rshares_decay_rate;

        const uint32_t rewards_initial_supply_period_in_days;
        const uint32_t guaranted_reward_supply_period_in_days;
        const uint32_t reward_increase_threshold_in_days;

        const uint32_t budgets_limit_per_owner;

        const uint32_t atomicswap_initiator_refund_lock_secs;
        const uint32_t atomicswap_participant_refund_lock_secs;

        const uint32_t atomicswap_limit_requested_contracts_per_owner;
        const uint32_t atomicswap_limit_requested_contracts_per_recipient;

        const uint32_t min_vote_interval_sec;

        const uint32_t db_free_memory_threshold_mb;

        const fc::time_point_sec initial_date;

        const fc::time_point_sec blogging_start_date;

        const fc::time_point_sec fifa_world_cup_2018_bounty_cashout_date;

        const fc::microseconds expiraton_for_registration_bonus;

        const fc::time_point_sec witness_reward_migration_date;

        const uint8_t scorum_max_witnesses;

        const uint8_t scorum_max_voted_witnesses;

        const uint8_t scorum_hardfork_required_witnesses;

        enum test_mode { test };

        explicit config(test_mode);
        config();
    };

    const config& get_config();

    void override_config(std::unique_ptr<config> new_config);
}
}
}

#define DAYS_TO_SECONDS(X)                     (60u*60u*24u*X)

#define SCORUM_BLOCKCHAIN_VERSION              ( version(0, 4, 1) )

#define SCORUM_BLOCKCHAIN_HARDFORK_VERSION     ( hardfork_version( SCORUM_BLOCKCHAIN_VERSION ) )

#define SCORUM_ADDRESS_PREFIX                  "SCR"

#define SCORUM_CURRENCY_PRECISION  9

#define SCORUM_DEFAULT_BUDGETS_AUCTION_SET  {100, 85, 75, 65}

// Scorum Coin = SCR with 9 digits of precision
#define SCORUM_SYMBOL  (uint64_t(SCORUM_CURRENCY_PRECISION) | (uint64_t('S') << 8) | (uint64_t('C') << 16) | (uint64_t('R') << 24))
// Scorum Power = SP with 9 digits of precision
#define SP_SYMBOL   (uint64_t(SCORUM_CURRENCY_PRECISION) | (uint64_t('S') << 8) | (uint64_t('P') << 16))

#define SCORUM_MAX_SHARE_SUPPLY                share_value_type(100000000e+9) //100 million

#define SCORUM_VOTE_DUST_THRESHOLD             share_type(50)

#define SCORUM_ATOMICSWAP_CONTRACT_METADATA_MAX_LENGTH  10*1024
#define SCORUM_ATOMICSWAP_SECRET_MAX_LENGTH             1024

//Got only minimum for transactions bandwidth. Required spend SCR to enlarge up to SCORUM_VOTE_DUST_THRESHOLD
#define SCORUM_MIN_ACCOUNT_CREATION_FEE        asset(SCORUM_VOTE_DUST_THRESHOLD/2, SCORUM_SYMBOL)

#define SCORUM_MIN_COMMENT_PAYOUT_SHARE        share_type(5)

#define SCORUM_MIN_PER_BLOCK_REWARD            share_type(1)

#define SCORUM_CREATE_ACCOUNT_WITH_SCORUM_MODIFIER  30

#define SCORUM_MIN_DELEGATE_VESTING_SHARES_MODIFIER 10

#define SCORUM_START_WITHDRAW_COEFFICIENT           10

#define SCORUM_ACTIVE_SP_HOLDERS_REWARD_PERIOD           (scorum::protocol::detail::get_config().active_sp_holders_reward_period)

#define SCORUM_MIN_BET_STAKE_FOR_MATCHING      share_type(1)

////////////////////////////////////////////////////////////////////////////////////////////////////
#define SCORUM_BLOCKID_POOL_SIZE                (scorum::protocol::detail::get_config().blockid_pool_size)

#define SCORUM_CASHOUT_WINDOW_SECONDS           (scorum::protocol::detail::get_config().cashout_window_seconds)

#define SCORUM_UPVOTE_LOCKOUT                   (scorum::protocol::detail::get_config().upvote_lockout)

#define SCORUM_OWNER_AUTH_RECOVERY_PERIOD                   (scorum::protocol::detail::get_config().owner_auth_recovery_period)
#define SCORUM_ACCOUNT_RECOVERY_REQUEST_EXPIRATION_PERIOD   (scorum::protocol::detail::get_config().account_recovery_request_expiration_period)
#define SCORUM_OWNER_UPDATE_LIMIT                           (scorum::protocol::detail::get_config().owner_update_limit)

#define SCORUM_REWARDS_INITIAL_SUPPLY_PERIOD_IN_DAYS        (scorum::protocol::detail::get_config().rewards_initial_supply_period_in_days)

#define SCORUM_GUARANTED_REWARD_SUPPLY_PERIOD_IN_DAYS       (scorum::protocol::detail::get_config().guaranted_reward_supply_period_in_days)
#define SCORUM_REWARD_INCREASE_THRESHOLD_IN_DAYS            (scorum::protocol::detail::get_config().reward_increase_threshold_in_days)

#define SCORUM_BUDGETS_LIMIT_PER_OWNER                      (scorum::protocol::detail::get_config().budgets_limit_per_owner)

#define SCORUM_ATOMICSWAP_INITIATOR_REFUND_LOCK_SECS        (scorum::protocol::detail::get_config().atomicswap_initiator_refund_lock_secs)
#define SCORUM_ATOMICSWAP_PARTICIPANT_REFUND_LOCK_SECS      (scorum::protocol::detail::get_config().atomicswap_participant_refund_lock_secs)

#define SCORUM_ATOMICSWAP_LIMIT_REQUESTED_CONTRACTS_PER_OWNER       (scorum::protocol::detail::get_config().atomicswap_limit_requested_contracts_per_owner)
#define SCORUM_ATOMICSWAP_LIMIT_REQUESTED_CONTRACTS_PER_RECIPIENT   (scorum::protocol::detail::get_config().atomicswap_limit_requested_contracts_per_recipient)

#define SCORUM_VESTING_WITHDRAW_INTERVALS                           (scorum::protocol::detail::get_config().vesting_withdraw_intervals)
#define SCORUM_VESTING_WITHDRAW_INTERVAL_SECONDS                    (scorum::protocol::detail::get_config().vesting_withdraw_interval_seconds) /// 1 week per interval

#define SCORUM_MIN_VOTE_INTERVAL_SEC            (scorum::protocol::detail::get_config().min_vote_interval_sec)

#define SCORUM_DB_FREE_MEMORY_THRESHOLD_MB      (scorum::protocol::detail::get_config().db_free_memory_threshold_mb)

////////////////////////////////////////////////////////////////////////////////////////////////////

#define SCORUM_REGISTRATION_BONUS_LIMIT_PER_MEMBER_PER_N_BLOCK    100 /// * registration_bonus

#define SCORUM_REGISTRATION_BONUS_LIMIT_PER_MEMBER_N_BLOCK        2

#define SCORUM_REGISTRATION_COMMITTEE_MAX_MEMBERS_LIMIT        30
#define SCORUM_DEVELOPMENT_COMMITTEE_MAX_MEMBERS_LIMIT         30

#define SCORUM_BLOCK_INTERVAL                  3
#define SCORUM_BLOCKS_PER_YEAR                 (365*24*60*60/SCORUM_BLOCK_INTERVAL)
#define SCORUM_BLOCKS_PER_DAY                  (24*60*60/SCORUM_BLOCK_INTERVAL)
#define SCORUM_BLOCKS_PER_HOUR                 (60*60/SCORUM_BLOCK_INTERVAL)
#define SCORUM_START_MINER_VOTING_BLOCK        (SCORUM_BLOCKS_PER_DAY * 30)

#define SCORUM_MAX_VOTED_WITNESSES              (scorum::protocol::detail::get_config().scorum_max_voted_witnesses)
#define SCORUM_MAX_RUNNER_WITNESSES             (scorum::protocol::detail::get_config().scorum_max_witnesses - scorum::protocol::detail::get_config().scorum_max_voted_witnesses)
#define SCORUM_MAX_WITNESSES                    (scorum::protocol::detail::get_config().scorum_max_witnesses)
#define SCORUM_MAX_WITNESSES_LIMIT               21
#define SCORUM_WITNESS_MISSED_BLOCKS_THRESHOLD  SCORUM_BLOCKS_PER_DAY/2
#define SCORUM_HARDFORK_REQUIRED_WITNESSES      (scorum::protocol::detail::get_config().scorum_hardfork_required_witnesses)

#define SCORUM_MAX_TIME_UNTIL_EXPIRATION       (60*60) // seconds,  aka: 1 hour
#define SCORUM_MAX_MEMO_SIZE                   2048
#define SCORUM_MAX_PROXY_RECURSION_DEPTH       4

#define SCORUM_MAX_WITHDRAW_ROUTES             10
#define SCORUM_SAVINGS_WITHDRAW_TIME           (fc::days(3))
#define SCORUM_SAVINGS_WITHDRAW_REQUEST_LIMIT  100
#define SCORUM_VOTE_REGENERATION_SECONDS       (scorum::protocol::detail::get_config().vote_regeneration_seconds)
#define SCORUM_MAX_VOTE_CHANGES                3
#define SCORUM_REVERSE_AUCTION_WINDOW_SECONDS  (scorum::protocol::detail::get_config().reverse_auction_window_seconds)

#define SCORUM_MIN_ROOT_COMMENT_INTERVAL       (fc::minutes(5))
#define SCORUM_MIN_REPLY_INTERVAL              (fc::seconds(20))

#define SCORUM_MAX_ACCOUNT_WITNESS_VOTES       30

#define SCORUM_100_PERCENT                     10000
#define SCORUM_1_PERCENT                       (SCORUM_100_PERCENT/100)
#define SCORUM_1_TENTH_PERCENT                 (SCORUM_100_PERCENT/1000)
#define SCORUM_PERCENT(X)                      (uint16_t)(X*SCORUM_1_PERCENT)

#define SCORUM_DEV_TEAM_PER_BLOCK_REWARD_PERCENT            SCORUM_PERCENT(50)
#define SCORUM_WITNESS_PER_BLOCK_REWARD_PERCENT             SCORUM_PERCENT(10)
#define SCORUM_ACTIVE_SP_HOLDERS_PER_BLOCK_REWARD_PERCENT   SCORUM_PERCENT(10)
#define SCORUM_CURATION_REWARD_PERCENT                      SCORUM_PERCENT(25)
#define SCORUM_PARENT_COMMENT_REWARD_PERCENT        		SCORUM_PERCENT(50)

#define SCORUM_ADJUST_REWARD_PERCENT                        SCORUM_PERCENT(5)

#define SCORUM_BANDWIDTH_AVERAGE_WINDOW_SECONDS (DAYS_TO_SECONDS(7))
#define SCORUM_BANDWIDTH_PRECISION              (uint64_t(1000000)) ///< 1 million
#define SCORUM_MAX_COMMENT_DEPTH                (6)
#define SCORUM_SOFT_MAX_COMMENT_DEPTH           (6)

#define SCORUM_MAX_RESERVE_RATIO                (20000)

#define SCORUM_CREATE_ACCOUNT_DELEGATION_RATIO     5
#define SCORUM_CREATE_ACCOUNT_DELEGATION_TIME      fc::days(30)
#define SCORUM_CREATE_ACCOUNT_REG_COMMITTEE_DELEGATION_MAX      asset(10'000'000'000, SP_SYMBOL)

#define SCORUM_RECENT_RSHARES_DECAY_RATE       (scorum::protocol::detail::get_config().recent_rshares_decay_rate)
// note, if redefining these constants make sure calculate_claims doesn't overflow

#define SCORUM_MIN_ACCOUNT_NAME_LENGTH          3
#define SCORUM_MAX_ACCOUNT_NAME_LENGTH         16

#define SCORUM_MIN_PERMLINK_LENGTH             0
#define SCORUM_MAX_PERMLINK_LENGTH             256
#define SCORUM_MAX_WITNESS_URL_LENGTH          2048

#define SCORUM_MAX_SIG_CHECK_DEPTH             2

#define SCORUM_MAX_TRANSACTION_SIZE            (1024*64)
#define SCORUM_MIN_BLOCK_SIZE_LIMIT            (SCORUM_MAX_TRANSACTION_SIZE)
#define SCORUM_MAX_BLOCK_SIZE                  (SCORUM_MAX_TRANSACTION_SIZE*SCORUM_BLOCK_INTERVAL*2000)

#define SCORUM_MIN_UNDO_HISTORY                 10
#define SCORUM_MAX_UNDO_HISTORY                 10000

#define SCORUM_MIN_TRANSACTION_EXPIRATION_LIMIT (SCORUM_BLOCK_INTERVAL * 5) // 5 transactions per block

#define SCORUM_IRREVERSIBLE_THRESHOLD           (75 * SCORUM_1_PERCENT)

#define VIRTUAL_SCHEDULE_LAP_LENGTH             ( fc::uint128::max_value() )

#define SCORUM_COMMITTEE_QUORUM_PERCENT         (60u)

#define SCORUM_COMMITTEE_ADVERTISING_MODERATOR_QUORUM_PERCENT       (50u)
#define SCORUM_COMMITTEE_BETTING_MODERATOR_QUORUM_PERCENT       (50u)
#define SCORUM_COMMITTEE_BETTING_RESOLVE_DELAY_QUORUM_PERCENT   (50u)
#define SCORUM_COMMITTEE_TRANSFER_QUORUM_PERCENT            (50u)
#define SCORUM_COMMITTEE_ADD_EXCLUDE_QUORUM_PERCENT         (60u)

#define SCORUM_VOTING_POWER_DECAY_PERCENT 5

#define SCORUM_MIN_QUORUM_VALUE_PERCENT         (50u)
#define SCORUM_MAX_QUORUM_VALUE_PERCENT         (100u)

#define SCORUM_PROPOSAL_LIFETIME_MIN_SECONDS    (DAYS_TO_SECONDS(1u))
#define SCORUM_PROPOSAL_LIFETIME_MAX_SECONDS    (DAYS_TO_SECONDS(10u))

#define SCORUM_BETTING_RESOLVE_DELAY_SEC        (DAYS_TO_SECONDS(1u))
#define SCORUM_BETTING_START_TIME_DIFF_MAX      (fc::hours(12))

#define SCORUM_BETTING_THRESHOLD_FACTOR         1000

/**
 *  Reserved Account IDs with special meaning
 */
/// Represents the canonical account for specifying you will vote for directly (as opposed to a proxy)
#define SCORUM_PROXY_TO_SELF_ACCOUNT           (scorum::protocol::account_name_type())
/// Represents the canonical root post parent account
#define SCORUM_ROOT_POST_PARENT_ACCOUNT        (scorum::protocol::account_name_type())

#define SCORUM_MISSING_MODERATOR_ACCOUNT       (scorum::protocol::account_name_type())

#define SCORUM_BLOGGING_START_DATE                     (scorum::protocol::detail::get_config().blogging_start_date)
#define SCORUM_FIFA_WORLD_CUP_2018_BOUNTY_CASHOUT_DATE (scorum::protocol::detail::get_config().fifa_world_cup_2018_bounty_cashout_date)
#define SCORUM_EXPIRATON_FOR_REGISTRATION_BONUS        (scorum::protocol::detail::get_config().expiraton_for_registration_bonus)
#define SCORUM_WITNESS_REWARD_MIGRATION_DATE           (scorum::protocol::detail::get_config().witness_reward_migration_date)

#define SCORUM_ADVERTISING_CASHOUT_PERIOD_SEC          (scorum::protocol::detail::get_config().advertising_cashout_period_sec)

#define SCORUM_MIN_ODDS                                (scorum::protocol::odds(1001, 1000)) // min: 1.001  &&  max: 1001
#define SCORUM_MIN_STAKE_FOR_MATCHING                  (share_type(1))
#define SCORUM_MIN_BET_STAKE                           (scorum::protocol::asset(1'000'000, SCORUM_SYMBOL)) // 0.001'000'000 SCR

#define SCORUM_MAX_BET_STAKE                           (scorum::protocol::asset(9214157878975800, SCORUM_SYMBOL))

#ifdef IS_LOW_MEM
#define SCORUM_LOW_MEMORY_NODE (true)
#else
#define SCORUM_LOW_MEMORY_NODE (false)
#endif

#ifdef CLEAR_VOTES
#define SCORUM_CLEAR_VOTES (true)
#else
#define SCORUM_CLEAR_VOTES (false)
#endif

#ifdef SKIP_BY_TX_ID
#define SCORUM_SKIP_BY_TX_ID (true)
#else
#define SCORUM_SKIP_BY_TX_ID (false)
#endif

#ifdef LIVE_TESTNET
#define SCORUM_LIVE_TESTNET (true)
#else
#define SCORUM_LIVE_TESTNET (false)
#endif

///@}

// clang-format on

ext: hpp

max_stars_repo_name: scorum/scorum

max_issues_repo_name: scorum/scorum

max_forks_repo_name: scorum/scorum


text:
//----------------------------------------------------------------------------
/// \file  list.hpp
//----------------------------------------------------------------------------
/// \brief A class implementing a list object of Erlang external
///        term format. The list can contain any heterogeneous eterm's.
//----------------------------------------------------------------------------
// Copyright (c) 2010 Serge Aleynikov <saleyn@gmail.com>
// Created: 2010-09-20
//----------------------------------------------------------------------------
/*
***** BEGIN LICENSE BLOCK *****

Copyright 2010 Serge Aleynikov <saleyn at gmail dot com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

***** END LICENSE BLOCK *****
*/
#pragma once

#include <list>
#include <boost/static_assert.hpp>
#include <eixx/marshal/defaults.hpp>
#include <eixx/marshal/visit_encode_size.hpp>
#include <initializer_list>

namespace eixx {
namespace marshal {

template <typename Alloc>
struct cons {
    eterm<Alloc> node;
    cons<Alloc>* next;
};

template <typename Alloc>
class list : protected alloc_base<cons<Alloc>, Alloc> {
public:
    typedef cons<Alloc> cons_t;
private:
    typedef alloc_base<cons_t, Alloc> base_t;

    struct header_t {
        header_t()          {}
        header_t(std::nullptr_t)
            : initialized(true)
            , alloc_size (0)
            , size       (0)
            , tail       (nullptr)
        {}
        bool            initialized;
        unsigned int    alloc_size;
        unsigned int    size;
        cons_t*         tail;
        #pragma GCC diagnostic push
        #pragma GCC diagnostic ignored "-Wpedantic"
        cons_t          head[0];
        #pragma GCC diagnostic pop
    };

    typedef blob<char, Alloc> blob_t;

    blob_t* m_blob;

    /// Returns a pointer to a singleton empty list
    static blob_t* empty_list() {
        auto creator = []() {
            auto p = new blob_t(sizeof(header_t));
            auto h = reinterpret_cast<header_t*>(p->data());
            new (h) header_t(nullptr);
            return p;
        };
        static std::unique_ptr<blob_t> s_empty(creator());
        return s_empty.get();
    }

    header_t* header() {
        BOOST_ASSERT(m_blob); return reinterpret_cast<header_t*>(m_blob->data());
    }
    const header_t* header() const {
        BOOST_ASSERT(m_blob); return reinterpret_cast<const header_t*>(m_blob->data());
    }
    const cons_t* head() const    { return header()->head; }
    cons_t*       head()          { return header()->head; }
    cons_t*       tail()          { return header()->tail; }

    void release() {
        if (!m_blob || m_blob == empty_list())
           return;
        if (m_blob->release(false)) {
            header_t* l_header = header();
            if (l_header->size > 0) {
                for (cons_t* p = head(); p; p = p->next)
                    p->node.~eterm();
                // If there were any allocations after the original
                // construction of the list head descriptor, deallocate
                // all the following cons.
                if (l_header->alloc_size > 0 && l_header->size > l_header->alloc_size)
                    for (cons_t* p = (l_header->head + l_header->alloc_size - 1)->next, *q; p; p = q) {
                        q = p->next;
                        this->get_t_allocator().deallocate(p, 1);
                    }
            }
            m_blob->free();
        }
    }

    // For use only from constructors.
    void init(const eterm<Alloc> items[], size_t a_size, const Alloc& alloc);
public:
    class iterator;
    typedef const iterator const_iterator;

    iterator       begin()       { iterator it(empty() ? NULL : head()); return it; }
    iterator       end()         { return iterator::end(); }

    const_iterator begin() const { const_iterator it(empty() ? NULL : head()); return it; }
    const_iterator end()   const { return iterator::end(); }

    explicit list(const Alloc& alloc = Alloc())
        : base_t(alloc)
        , m_blob(NULL)
    {}

    /// Construct an NIL list (initialized list with no elements)
    explicit list(std::nullptr_t) : m_blob(empty_list()) {}

    /// Construct a list with a given estimated size.
    ///
    /// When a_estimated_size is 0, an empty initialized list is created. Otherwise,
    /// the list is not initialized.
    explicit list(int a_estimated_size, const Alloc& alloc = Alloc())
        : base_t(alloc)
    {
        if (a_estimated_size == 0)
            m_blob = empty_list();
        else {
            m_blob = new blob_t(sizeof(header_t) + a_estimated_size*sizeof(cons_t), alloc);
            header_t* hdr      = header();
            hdr->initialized   = a_estimated_size == 0;
            hdr->alloc_size    = a_estimated_size;
            hdr->size          = 0;
            hdr->tail          = NULL;
        }
    }

    list(const list<Alloc>& a) : base_t(a.get_allocator()), m_blob(a.m_blob) {
        BOOST_ASSERT(a.initialized());
        if (m_blob) m_blob->inc_rc();
    }

    list(list<Alloc>&& a) : base_t(a.get_allocator()), m_blob(a.m_blob) {
        a.m_blob = nullptr;
    }

    explicit list(const cons_t* a_head, int a_len = -1, const Alloc& alloc = Alloc());

    template <int N>
    list(const eterm<Alloc> (&items)[N], const Alloc& alloc = Alloc())
        : list(items, N, alloc) {}

    list(const eterm<Alloc>* items, size_t a_size, const Alloc& alloc = Alloc())
        : base_t(alloc) { init(items, a_size, alloc); }

    list(std::initializer_list<eterm<Alloc>> items, const Alloc& alloc = Alloc())
        : list(items.begin(), items.size(), alloc) {}

    /**
     * Decode the list from a binary buffer.
     */
    explicit list(const char* buf, uintptr_t& idx, size_t size, const Alloc& a_alloc = Alloc());

    ~list() { release(); }

    /**
     * Add a term to list. Tuples added must be fully filled
     * with all elements, and lists added must be properly closed (initialized).
     */
    void push_back(const eterm<Alloc>& a);

    template <typename G>
    void push_back(const G& a) {
        eterm<Alloc> t(a);
        push_back(t);
    }

    /**
     * Closes the list.
     * A list must be closed before it can be copied or included into other terms.
     */
    void    close() {
        if (!m_blob || m_blob == empty_list()) return;
        header()->initialized = true;
    }

    /// Return list length. This method has O(1) complexity.
    size_t  length()        const { return  m_blob ?  header()->size :  0; }
    bool    empty()         const { return !m_blob || header()->size == 0; }
    bool    initialized()   const { return  m_blob && header()->initialized; }

    /// Return pointer to the N'th element in the list. This method has
    /// O(N) complexity.
    const eterm<Alloc>& nth(size_t n) const {
        if (n > length())
            throw err_bad_argument("Index out of bounds", n);
        size_t i = 0;
        auto it = begin();
        for(auto endit = end(); it != endit && i < n; ++it, ++i);
        return *it;
    }

    list<Alloc> tail(size_t idx) const;

    list<Alloc>& operator= (const list<Alloc>& rhs) {
        BOOST_ASSERT(rhs.initialized());
        release();
        m_blob = rhs.m_blob;
        if (m_blob) m_blob->inc_rc();
        return *this;
    }

    bool operator== (const list<Alloc>& rhs) const {
        const_iterator it1  = begin(), it2  = rhs.begin(),
                       end1 = end(),   end2 = rhs.end();
        for(; it1 != end1 && it2 != end2; ++it1, ++it2) {
            if (!(*it1 == *it2))
                return false;
        }
        return it1 == end1 && it2 == end2;
    }

    bool operator< (const list<Alloc>& rhs) const {
        const_iterator it1  = begin(), it2  = rhs.begin(),
                       end1 = end(),   end2 = rhs.end();
        for(; it1 != end1 && it2 != end2; ++it1, ++it2) {
            if (!(*it1 < *it2))
                return false;
        }
        return it1 == end1 && it2 != end2;
    }

    size_t encode_size() const {
        if (length() == 0)
            return 1;
        size_t result = 5 + 1 /* 1 byte for ERL_NIL_EXT */;
        const header_t* hd = header();
        BOOST_ASSERT(hd->initialized);
        for (const cons_t* it=hd->head; it != NULL; it = it->next) {
            visit_eterm_encode_size_calc<Alloc> visitor;
            result += visitor.apply_visitor(it->node);
        }
        return result;
    }

    void encode(char* buf, uintptr_t& idx, size_t size) const;

    bool subst(eterm<Alloc>& out, const varbind<Alloc>* binding) const;

    bool match(const eterm<Alloc>& pattern, varbind<Alloc>* binding) const;

    std::ostream& dump(std::ostream& out, const varbind<Alloc>* vars = NULL) const;

    static list<Alloc> make(const Alloc& a = Alloc()) {
        return list<Alloc>(0, a);
    }

    template <class T1>
    static list<Alloc> make(T1 t1, const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2>
    static list<Alloc> make(T1 t1, T2 t2,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3, class T4>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3, T4 t4,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3),
                             eterm<Alloc>::cast(t4) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3, class T4, class T5>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3, T4 t4, T5 t5,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3),
                             eterm<Alloc>::cast(t4),
                             eterm<Alloc>::cast(t5) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3, class T4, class T5, class T6>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3),
                             eterm<Alloc>::cast(t4),
                             eterm<Alloc>::cast(t5),
                             eterm<Alloc>::cast(t6) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3, class T4, class T5, class T6, class T7>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3),
                             eterm<Alloc>::cast(t4),
                             eterm<Alloc>::cast(t5),
                             eterm<Alloc>::cast(t6),
                             eterm<Alloc>::cast(t7) };
        return list<Alloc>(l, a);
    }

    template <class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8>
    static list<Alloc> make(T1 t1, T2 t2, T3 t3, T4 t4, T5 t5, T6 t6, T7 t7, T8 t8,
        const Alloc& a = Alloc()) {
        eterm<Alloc> l[] = { eterm<Alloc>::cast(t1),
                             eterm<Alloc>::cast(t2),
                             eterm<Alloc>::cast(t3),
                             eterm<Alloc>::cast(t4),
                             eterm<Alloc>::cast(t5),
                             eterm<Alloc>::cast(t6),
                             eterm<Alloc>::cast(t7),
                             eterm<Alloc>::cast(t8) };
        return list<Alloc>(l, a);
    }
};

template <typename Alloc>
class list<Alloc>::iterator {
    mutable cons_t* cursor;
public:
    iterator(const cons_t* a_cur) : cursor(const_cast<cons_t*>(a_cur)) {}
    static iterator end() { iterator it(NULL); return it; }
    iterator&       operator++()        { cursor = cursor->next; return *this; }
    const iterator& operator++() const  { cursor = cursor->next; return *this; }
    const iterator  operator++(int)const{ iterator it(cursor); cursor = cursor->next; return it; }
    iterator        operator++(int)     { iterator it(cursor); cursor = cursor->next; return it; }
    const eterm<Alloc>& operator*()  const { BOOST_ASSERT(cursor); return cursor->node; }
    eterm<Alloc>&   operator*()         { BOOST_ASSERT(cursor); return cursor->node; }
    eterm<Alloc>*   operator->()        { BOOST_ASSERT(cursor); return &cursor->node; }
    const eterm<Alloc>* operator->() const { BOOST_ASSERT(cursor); return &cursor->node; }
    bool            operator==(const iterator& rhs) const { return cursor == rhs.cursor; }
    bool            operator!=(const iterator& rhs) const { return cursor != rhs.cursor; }
};

} // namespace marshal
} // namespace eixx

namespace std {

    template <class Alloc>
    ostream& operator<< (ostream& out, const eixx::marshal::list<Alloc>& a) {
        return a.dump(out);
    }

} // namespace std

#include <eixx/marshal/list.hxx>


text:
// See LICENSE for license details.

#include <string>
#include <iostream>
#include <iterator>
#include <algorithm>
#include <future>
#include <boost/program_options.hpp>

#include "mmapped_file.hpp"
#include "bin.hpp"
#include "dims_create.hpp"

bool divisible(ssize_t v, int d)
{
    return (v / d) * d == v;
}

using bins_type = Bins<3>;

bins_type bin_all(bins_type::index_type nbins, bins_type::point_type bounding_box, double const*first, double const*last)
{
    if (!divisible(last - first, 3)) {
        throw std::runtime_error("Range not a multiple of 3.");
    }

    auto b = bins_type{nbins, bounding_box};
    for (; first != last; first += 3)
        b.insert({first[0], first[1], first[2]});
    return b;
}

namespace statistics {
template <typename T, typename R = double>
R mean(const std::vector<T>& v)
{
    T sum = std::accumulate(v.begin(), v.end(), T{0}, std::plus<T>{});
    return static_cast<R>(sum) / v.size();
}

template <typename T, typename R = double>
R var(const std::vector<T>& v)
{
    T sqsum = std::accumulate(v.begin(), v.end(), T{0}, [](T acc, T val){ return acc + val * val; });
    R m = mean(v);
    return static_cast<R>(sqsum) / v.size() - m * m;
}
}

// For boost::program_options
namespace streamable {
template <int N>
struct NDoubles {
    typename Bins<N>::point_type data;
};
template <int N>
std::istream& operator>>(std::istream& is, NDoubles<N>& ti)
{
    char c;
    for (int i = 0; i < N; ++i) {
        is >> ti.data[i];
        if (i < N - 1)
            is.read(&c, 1);
    }
    return is;
}
}

int main(int argc, char **argv)
{
    const int nthreads = 4;
    using namespace std::string_literals;
    namespace po = boost::program_options;

    po::options_description desc("Allowed options");
    desc.add_options()
        ("help", "produce help message")
        ("file", po::value<std::string>(), "MPI-IO position file")
        ("box", po::value<streamable::NDoubles<3>>(), "Bounding box of the simulation")
        ("nproc", po::value<int>(), "Number of processes")
    ;

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);

    if (vm.count("help") || !vm.count("file") || !vm.count("box") || !vm.count("nproc")) {
        std::cout << desc << std::endl;
        return 1;
    }

    const auto fn = vm["file"].as<std::string>();
    const auto nbins = dims_create(vm["nproc"].as<int>());
    const auto bbox = vm["box"].as<streamable::NDoubles<3>>().data;

    std::cout << "File : " << fn << std::endl;
    std::cout << "NProc: " << vm["nproc"].as<int>() << " = " << nbins[0] << " x " << nbins[1] << " x " << nbins[2] << std::endl;
    std::cout << "Box  : " << bbox[0] << " " << bbox[1] << " " << bbox[2] << "\n" << std::endl;

    auto f = MFile<double>{fn.c_str()};
    std::cout << "File has " << f.size() << " elemets." << std::endl;

    auto data = f.data();
    auto b = bin_all(nbins, bbox, data, data + f.size());

    /* Sanity check */
    int i = std::accumulate(b.bins.begin(), b.bins.end(), 0, std::plus<int>{});
    std::cout << "Binned   " << i << " particles." << std::endl;
    if (3 * i != f.size()) {
        throw std::runtime_error("Particles disappeared...");
    }
    /* End */

    std::cout << std::endl;
    std::cout << "Min: " << *std::min_element(b.bins.begin(), b.bins.end()) << std::endl;
    std::cout << "Max: " << *std::max_element(b.bins.begin(), b.bins.end()) << std::endl;

    auto dmean = statistics::mean(b.bins);
    auto dsdev = std::sqrt(statistics::var(b.bins));

    std::cout << "Mean: " << dmean << std::endl;
    std::cout << "SDev: " << dsdev << " ( = " << std::floor(dsdev / dmean * 1000.)/10. << " %)" << std::endl;
}

ext: cc

max_stars_repo_name: hirschsn/imba-eval

max_issues_repo_name: hirschsn/imba-eval

max_forks_repo_name: hirschsn/imba-eval


text:
#include <iostream>
#include <boost/algorithm/minmax_element.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/container/vector.hpp>
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>

#include "part_two.h"

part_two::part_two(const boost::container::string& input) : abstract_part(input)
{
}

part_two::~part_two()
{
}

int part_two::sum_of_divisible_pair(boost::container::vector<int> numbers)
{
	for (size_t a = 0; a < numbers.size(); ++a)
	{
		for (size_t b = 0; b < numbers.size(); ++b)
		{
			if (a == b)
				continue;

			if (numbers[a] % numbers[b] == 0)
				return numbers[a] / numbers[b];
		}
	}

	return 0;
}

void part_two::run()
{
	boost::filesystem::ifstream ifs;

	ifs.open(input_path, boost::filesystem::ifstream::in);

	try
	{
		std::string line;
		int sum = 0;

		while (std::getline(ifs, line))
		{
			boost::container::vector<std::string> tokens;
			boost::container::vector<int> numbers;

			split(tokens, line, boost::is_any_of("\t"));
			numbers.resize(tokens.size());

			std::transform(tokens.begin(), tokens.end(), numbers.begin(), boost::lexical_cast<int, std::string>);

			sum += sum_of_divisible_pair(numbers);
		}

		std::cout << "sum: " << sum << std::endl;
	}
	catch (const std::ios_base::failure& e)
	{
		std::cout << e.what() << std::endl;
	}
}

ext: cpp


text:
/*
 * Copyright (C) 2020 ZMP Inc info@zmp.co.jp
 *
 * Distributed under the MIT License (http://opensource.org/licenses/MIT)
 *
 * Written by Andrei Pak
 */
#include "rc110_behavior.hpp"

#include <ackermann_msgs/AckermannDriveStamped.h>
#include <sensor_msgs/point_cloud2_iterator.h>

#include <Eigen/Geometry>

namespace zmp
{
namespace
{
constexpr float DEG_TO_RAD = M_PI / 180;
constexpr float INF = std::numeric_limits<float>::infinity();

// coordinates relative to robot origin, i.e. center of rear wheels axis
const Eigen::AlignedBox2f nearBox = {Eigen::Vector2f(0.2, -0.15f), Eigen::Vector2f(0.5f, 0.15f)};
const Eigen::AlignedBox2f leftBox = {Eigen::Vector2f(0.5f, 0), Eigen::Vector2f(1.0f, 0.35f)};
const Eigen::AlignedBox2f rightBox = {Eigen::Vector2f(0.5f, -0.35f), Eigen::Vector2f(1.0f, 0)};
}  // namespace

Rc110Behavior::Rc110Behavior() :
        handle(),
        cloudSubscriber(handle.subscribe("points2", 1, &Rc110Behavior::onCloud, this)),
        drivePublisher(handle.advertise<ackermann_msgs::AckermannDriveStamped>("drive_ad", 1))
{
    ros::NodeHandle privateHandle("~");
    privateHandle.param("forward_command", forwardCommand, forwardCommand);
    privateHandle.param("left_command", leftCommand, leftCommand);
    privateHandle.param("right_command", rightCommand, rightCommand);
}

void Rc110Behavior::onCloud(const sensor_msgs::PointCloud2& cloud)
{
    float closestLeftX = INF;
    float closestRightX = INF;
    auto result = forwardCommand;

    using Iter = sensor_msgs::PointCloud2ConstIterator<float>;
    for (Iter it = {cloud, "x"}; it != it.end(); ++it) {
        float x = it[0], y = it[1];
        Eigen::Vector2f point = {x, y};

        if (nearBox.contains(point)) {
            result = stopCommand;
            break;
        } else if (leftBox.contains(point)) {
            closestLeftX = std::min(x, closestLeftX);
        } else if (rightBox.contains(point)) {
            closestRightX = std::min(x, closestRightX);
        }
    }

    if (result == forwardCommand) {
        if (closestLeftX != INF || closestRightX != INF) {
            result = closestLeftX < closestRightX ? rightCommand : leftCommand;
        }
    }
    if (result.size() != 2) {
        ROS_ERROR_STREAM("Command should have 2 values: speed and steering! Current number of values: " << result.size());
    }
    publishCommand(result[0], result[1]);
}

void Rc110Behavior::publishCommand(float speed, float steering)
{
    ackermann_msgs::AckermannDriveStamped message;
    message.drive.speed = speed;
    message.drive.steering_angle = steering * DEG_TO_RAD;

    drivePublisher.publish(message);
}

}  // namespace zmp


text:
/*
 * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
 */

#include "base/os.h"
#include <sys/socket.h>

#include <net/if.h>

#ifdef __linux__
#include <linux/netlink.h>
#include <linux/if_tun.h>
#include <linux/if_packet.h>
#endif

#ifdef __FreeBSD__
#include <sys/sockio.h>
#include <ifaddrs.h>
#endif

#include "testing/gunit.h"

#include <base/logging.h>
#include <io/event_manager.h>
#include <tbb/task.h>
#include <base/task.h>

#include <cmn/agent_cmn.h>

#include "cfg/cfg_init.h"
#include "oper/operdb_init.h"
#include "controller/controller_init.h"
#include "pkt/pkt_init.h"
#include "services/services_init.h"
#include "vrouter/ksync/ksync_init.h"
#include "oper/interface_common.h"
#include "oper/nexthop.h"
#include "route/route.h"
#include "oper/vrf.h"
#include "oper/mpls.h"
#include "oper/vm.h"
#include "oper/vn.h"
#include "oper/path_preference.h"
#include "filter/acl.h"
#include "test_cmn_util.h"
#include "vr_types.h"
#include <controller/controller_export.h>
#include <ksync/ksync_sock_user.h>
#include <boost/assign/list_of.hpp>

using namespace boost::assign;

#define NULL_VRF ""
#define ZERO_IP "0.0.0.0"

IpamInfo ipam_info[] = {
    {"1.1.1.0", 24, "1.1.1.10"},
    {"2.2.2.0", 24, "2.2.2.10"},
    {"10.1.1.0", 24, "10.1.1.10"}
};

IpamInfo fabric_ipam_info[] = {
    {"10.1.1.0", 24, "10.1.1.254", true}
};

class FabricVmiTest : public ::testing::Test {
public:
    virtual void SetUp() {
        agent = Agent::GetInstance();
        client->WaitForIdle();
        peer_ = CreateBgpPeer(Ip4Address(1), "BGP Peer 1");
        client->WaitForIdle();
        AddIPAM("vn1", ipam_info, 2);
        AddIPAM("vn2", ipam_info, 2);
        client->WaitForIdle();
        AddVn(agent->fabric_vn_name().c_str(), 100);
        AddVrf(agent->fabric_policy_vrf_name().c_str(), 100);
        AddLink("virtual-network", agent->fabric_vn_name().c_str(),
                "routing-instance", agent->fabric_policy_vrf_name().c_str());
        client->WaitForIdle();
        AddIPAM(agent->fabric_vn_name().c_str(), fabric_ipam_info, 1);
        client->WaitForIdle();
    }

    virtual void TearDown() {
        DelIPAM("vn1");
        DelIPAM(agent->fabric_vn_name().c_str());
        DelIPAM("vn2");
        DelNode("virtual-network", agent->fabric_vn_name().c_str());
        DelLink("virtual-network", agent->fabric_vn_name().c_str(),
                "routing-instance", agent->fabric_policy_vrf_name().c_str());
        client->WaitForIdle();
        WAIT_FOR(100, 1000, (agent->vrf_table()->Size() == 2U));
        WAIT_FOR(100, 1000, (agent->vm_table()->Size() == 0U));
        WAIT_FOR(100, 1000, (agent->vn_table()->Size() == 0U));
        DeleteBgpPeer(peer_);
    }

    Agent *agent;
    BgpPeer *peer_;
};

TEST_F(FabricVmiTest, Vhost) {
    Ip4Address ip = Ip4Address::from_string("10.1.1.1");

    EXPECT_TRUE(RouteFind(agent->fabric_policy_vrf_name(), ip, 32));
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_policy_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::INTERFACE);

    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::RECEIVE);

    const VmInterface *vhost =
        static_cast<const VmInterface *>(agent->vhost_interface());
    EXPECT_TRUE(vhost->policy_enabled() == false);
}

TEST_F(FabricVmiTest, basic_1) {
    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    Ip4Address server_ip = Ip4Address::from_string("10.1.1.3");

    PathPreference path_preference(1, PathPreference::LOW, false, false);
    TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE |
                                 1 << TunnelType::NATIVE);
    Inet4TunnelRouteAdd(peer_,
                        agent->fabric_policy_vrf_name().c_str(),
                        ip, 32, server_ip, bmap, 16, "vn1",
                        SecurityGroupList(), TagList(), path_preference);
    client->WaitForIdle();

    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::ARP);
    EXPECT_TRUE(rt->GetActivePath()->gw_ip() == server_ip);

    DeleteRoute(agent->fabric_policy_vrf_name().c_str(), "1.1.1.1", 32,
                peer_);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    DelVrf(agent->fabric_policy_vrf_name().c_str());
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, basic_2) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportEnv(input1, 1);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    const VmInterface *vm_intf = static_cast<const VmInterface *>(VmPortGet(2));
    EXPECT_TRUE(vm_intf->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name());
    const InterfaceNH *intf_nh =
        dynamic_cast<const InterfaceNH *>(vm_intf->flow_key_nh());
    EXPECT_TRUE(intf_nh->GetVrf()->GetName() == agent->fabric_vrf_name());
    EXPECT_TRUE(vm_intf->proxy_arp_mode() == VmInterface::PROXY_ARP_UNRESTRICTED);

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::INTERFACE);
    EXPECT_TRUE(rt->GetActivePath()->peer() == vm_intf->peer());
    EXPECT_TRUE(rt->GetActivePath()->native_vrf_id() ==
                (uint32_t)(VrfGet("vrf1")->rd()));
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) != 0);

    agent->oper_db()->route_preference_module()->
        EnqueueTrafficSeen(ip, 32, vm_intf->id(),
                           vm_intf->forwarding_vrf()->vrf_id(),
                           vm_intf->vm_mac());
    client->WaitForIdle();

    EXPECT_TRUE(rt->GetActivePath()->path_preference().preference() ==
                PathPreference::HIGH);
    rt = RouteGet("vrf1", ip, 32);
    EXPECT_TRUE(rt->GetActivePath()->path_preference().preference() ==
                PathPreference::HIGH);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf() == NULL);

    EXPECT_TRUE(vm_intf->forwarding_vrf()->GetName() == "vrf1");
    EXPECT_TRUE(intf_nh->GetVrf()->GetName() == "vrf1");

    DeleteVmportEnv(input1, 1, true);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    client->WaitForIdle();
}

//Export route in ip-fabric VRF with no forwarding VRF
//Verify that NH is published with local peer and not
//route export peer
TEST_F(FabricVmiTest, basic_3) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportEnv(input1, 1);
    client->WaitForIdle();

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    VnListType vn_list;
    agent->fabric_inet4_unicast_table()->AddLocalVmRouteReq(peer_,
            agent->fabric_policy_vrf_name(), ip, 32, MakeUuid(2),
            vn_list, 10, SecurityGroupList(),
            TagList(), CommunityList(), false, PathPreference(),
            Ip4Address(0), EcmpLoadBalance(), false, false, true);
    client->WaitForIdle();

    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    const VmInterface *vm_intf = static_cast<const VmInterface *>(VmPortGet(2));
    EXPECT_TRUE(vm_intf->forwarding_vrf()->GetName() == "vrf1");

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::INTERFACE);
    EXPECT_TRUE(rt->GetActivePath()->peer() == agent->local_peer());

    //Add a local peer route
    agent->fabric_inet4_unicast_table()->AddLocalVmRouteReq(vm_intf->peer(),
            agent->fabric_policy_vrf_name(), ip, 32, MakeUuid(2),
            vn_list, 10, SecurityGroupList(),
            TagList(), CommunityList(), false, PathPreference(),
            Ip4Address(0), EcmpLoadBalance(), false, false, true);
    client->WaitForIdle();

    EXPECT_TRUE(rt->GetActivePath()->peer() == vm_intf->peer());

    //Delete local peer path route should we with local Peer now
    agent->fabric_inet4_unicast_table()->DeleteReq(vm_intf->peer(),
            agent->fabric_policy_vrf_name(), ip, 32, NULL);
    client->WaitForIdle();
    EXPECT_TRUE(rt->GetActivePath()->peer() == agent->local_peer());

    agent->fabric_inet4_unicast_table()->DeleteReq(peer_,
            agent->fabric_policy_vrf_name(), ip, 32, NULL);
    client->WaitForIdle();
    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    DeleteVmportEnv(input1, 1, true);
    client->WaitForIdle();
}

//Add gateway if tunnel nh and addr mismatch
TEST_F(FabricVmiTest, GatewayRoute) {
    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    Ip4Address server_ip = Ip4Address::from_string("10.1.1.254");

    PathPreference path_preference(1, PathPreference::LOW, false, false);
    TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE |
                                 1 << TunnelType::NATIVE);
    Inet4TunnelRouteAdd(peer_,
                        agent->fabric_policy_vrf_name().c_str(),
                        ip, 32, ip, bmap, 16, "vn1",
                        SecurityGroupList(), TagList(), path_preference);
    client->WaitForIdle();

    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::ARP);
    EXPECT_TRUE(rt->GetActivePath()->gw_ip() == server_ip);

    DeleteRoute(agent->fabric_policy_vrf_name().c_str(), "1.1.1.1", 32,
                peer_);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    DelVrf(agent->fabric_policy_vrf_name().c_str());
    client->WaitForIdle();
}

//Verify FIP route gets exported with Native encap, if FIP VN
//is using fabric for forwarding
TEST_F(FabricVmiTest, FIP_Native_Encap) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportEnv(input1, 1);
    client->WaitForIdle();

    AddVn("default-project:vn2", 10);
    AddVrf("default-project:vn2:vn2");
    AddLink("virtual-network", "default-project:vn2",
            "routing-instance", "default-project:vn2:vn2");
    client->WaitForIdle();

    AddFloatingIpPool("fip-pool1", 1);
    AddFloatingIp("fip1", 1, "2.1.1.100");
    AddLink("floating-ip", "fip1", "floating-ip-pool", "fip-pool1");
    AddLink("floating-ip-pool", "fip-pool1", "virtual-network",
            "default-project:vn2");
    AddLink("virtual-machine-interface", "intf2", "floating-ip", "fip1");
    client->WaitForIdle();

    AddLink("virtual-network", "default-project:vn2", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    EXPECT_TRUE(VrfGet("default-project:vn2:vn2")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());

    Ip4Address ip = Ip4Address::from_string("2.1.1.100");
    //Route should be leaked to fabric VRF
    WAIT_FOR(1000, 1000, RouteFind(agent->fabric_vrf_name(), ip, 32));

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet("default-project:vn2:vn2", ip, 32);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) != 0);

    DelLink("virtual-network", "default-project:vn2", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    WAIT_FOR(1000, 1000, RouteFind(agent->fabric_vrf_name(), ip, 32) == false);

    DelFloatingIpPool("fip-pool1");
    DelFloatingIp("fip1");
    DelLink("virtual-network", "default-project:vn2",
            "routing-instance", "default-project:vn2:vn2");
    DelLink("virtual-machine-interface", "intf2", "floating-ip", "fip1");
    DelVn("default-project:vn2");
    DelVrf("default-project:vn2:vn2");
    client->WaitForIdle();
    DeleteVmportEnv(input1, 1, true);
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, PolicyVrfDelete) {
    DelVrf(agent->fabric_policy_vrf_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet(agent->fabric_policy_vrf_name().c_str())->
                forwarding_vrf()->GetName() == agent->fabric_vrf_name().c_str());
}

//Add a route in default vrf via fabric peer
//and another path via local peer(route leaking scenario)
//Verify that both path are added and deleted appropriatly
TEST_F(FabricVmiTest, FabricAndLocalPeerRoute) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportEnv(input1, 1);
    client->WaitForIdle();
    const VmInterface *vm_intf = static_cast<const VmInterface *>(VmPortGet(2));

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");

    AddLocalVmRoute(agent, agent->fabric_policy_vrf_name(), "1.1.1.1",
                    32, "vn1", 2, peer_);
    client->WaitForIdle();

    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->FindPath(agent->local_peer()) != NULL);
    EXPECT_TRUE(rt->FindPath(vm_intf->peer()) != NULL);

    DeleteRoute(agent->fabric_policy_vrf_name().c_str(), "1.1.1.1", 32,
                peer_);
    client->WaitForIdle();
    EXPECT_TRUE(rt->FindPath(agent->local_peer()) == NULL);
    EXPECT_TRUE(rt->FindPath(vm_intf->peer()) != NULL);

    DeleteVmportEnv(input1, 1, true);
    client->WaitForIdle();
    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, v6) {
    client->Reset();
    struct PortInfo input[] = {
        {"vnet1", 1, "1.1.1.1", "00:00:00:01:01:01", 1, 1, "fd11::2"},
    };

    IpamInfo ipam_info[] = {
        {"1.1.1.0", 24, "1.1.1.10"},
        {"fd11::", 96, "fd11::1"},
    };

    CreateV6VmportEnv(input, 1, 0);
    client->WaitForIdle();
    AddIPAM("vn1", ipam_info, 2);
    client->WaitForIdle();

    boost::system::error_code ec;
    Ip6Address addr = Ip6Address::from_string(input[0].ip6addr, ec);
    InetUnicastRouteEntry* rt = RouteGetV6("vrf1", addr, 128);
    EXPECT_TRUE(rt != NULL);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) == 0);

    DeleteVmportEnv(input, 1, 1, 0, NULL, NULL, true, true);
    client->WaitForIdle();
    DelIPAM("vn1");
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, DefaultRoute) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportEnv(input1, 1);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    Ip4Address ip = Ip4Address::from_string("0.0.0.0");
    //Route should be leaked to fabric VRF
    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 0));

    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet("vrf1", ip, 0);
    EXPECT_TRUE(rt->FindPath(agent->local_peer()) != NULL);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                TunnelType::NativeType()) != 0);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    rt = RouteGet("vrf1", ip, 0);
    EXPECT_TRUE(rt == NULL);

    DeleteVmportEnv(input1, 1, true);
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, NonOverlayRoute) {
    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    Ip4Address server_ip = Ip4Address::from_string("10.1.1.3");

    PathPreference path_preference(1, PathPreference::LOW, false, false);
    TunnelType::TypeBmap bmap = (1 << TunnelType::MPLS_GRE);
    Inet4TunnelRouteAdd(peer_,
                        agent->fabric_policy_vrf_name().c_str(),
                        ip, 32, server_ip, bmap, 16, "vn1",
                        SecurityGroupList(), TagList(), path_preference);
    client->WaitForIdle();

    //Route should be leaked to fabric VRF
    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    bmap = (1 << TunnelType::MPLS_GRE | 1 << TunnelType::NATIVE);
    Inet4TunnelRouteAdd(peer_, agent->fabric_policy_vrf_name().c_str(),
                       ip, 32, server_ip, bmap, 16, "vn1",
                       SecurityGroupList(), TagList(), path_preference);
    client->WaitForIdle();

    EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    //Verify that nexthop is ARP nexthop fpr server_ip
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(), ip, 32);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::ARP);
    EXPECT_TRUE(rt->GetActivePath()->gw_ip() == server_ip);

    bmap = (1 << TunnelType::MPLS_GRE);
    Inet4TunnelRouteAdd(peer_, agent->fabric_policy_vrf_name().c_str(),
                        ip, 32, server_ip, bmap, 16, "vn1",
                        SecurityGroupList(), TagList(), path_preference);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));

    DeleteRoute(agent->fabric_policy_vrf_name().c_str(), "1.1.1.1", 32,
                peer_);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    DelVrf(agent->fabric_policy_vrf_name().c_str());
    client->WaitForIdle();
}

//Subnet and default route in fabric VRF should not be
//overwritten
TEST_F(FabricVmiTest, IntfStaticRoute) {
    struct PortInfo input[] = {
        {"vnet1", 1, "1.1.1.1", "00:00:00:01:01:01", 1, 1},
    };

    client->Reset();
    CreateVmportEnv(input, 1);
    client->WaitForIdle();
    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    EXPECT_TRUE(VmPortActive(input, 0));

   //Add a static route
   struct TestIp4Prefix static_route[] = {
       { Ip4Address::from_string("0.0.0.0"), 0},
       { Ip4Address::from_string("10.1.1.0"), 24},
   };

   AddInterfaceRouteTable("static_route", 1, static_route, 2);
   AddLink("virtual-machine-interface", "vnet1",
           "interface-route-table", "static_route");
   client->WaitForIdle();

   InetUnicastRouteEntry *rt = RouteGet(agent->fabric_vrf_name(),
                                        static_route[0].addr_,
                                        static_route[0].plen_);
   EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::ARP);

   rt = RouteGet(agent->fabric_vrf_name(),
                 static_route[1].addr_,
                 static_route[1].plen_);
   EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::RESOLVE);

   //Delete the link between interface and route table
   DelLink("virtual-machine-interface", "vnet1",
           "interface-route-table", "static_route");
   client->WaitForIdle();

   rt = RouteGet(agent->fabric_vrf_name(),
                 static_route[0].addr_,
                 static_route[0].plen_);
   EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::ARP);

   rt = RouteGet(agent->fabric_vrf_name(),
           static_route[1].addr_,
           static_route[1].plen_);
   EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::RESOLVE);

   DelLink("virtual-machine-interface", "vnet1",
           "interface-route-table", "static_route");
   DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
   DeleteVmportEnv(input, 1, true);
   client->WaitForIdle();

   EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(),
                         static_route[0].addr_,
                         static_route[0].plen_));
   EXPECT_TRUE(RouteFind(agent->fabric_vrf_name(),
                         static_route[1].addr_,
                         static_route[1].plen_));
   EXPECT_FALSE(VmPortFind(1));
}

TEST_F(FabricVmiTest, GwRoute) {
    struct PortInfo input[] = {
        {"vnet1", 1, "1.1.1.1", "00:00:00:01:01:01", 1, 1},
    };

    CreateVmportEnv(input, 1);
    client->WaitForIdle();
    AddIPAM("vn1", ipam_info, 2);
    client->WaitForIdle();
    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    boost::system::error_code ec;
    Ip4Address addr = Ip4Address::from_string("1.1.1.10", ec);
    InetUnicastRouteEntry* rt = RouteGet(agent->fabric_vrf_name(), addr, 32);
    EXPECT_TRUE(rt != NULL);

    DeleteVmportEnv(input, 1, true);
    client->WaitForIdle();
    DelIPAM("vn1");
    client->WaitForIdle();

    rt = RouteGet(agent->fabric_vrf_name(), addr, 32);
    EXPECT_TRUE(rt == NULL);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, VmWithVhostIp) {
    struct PortInfo input[] = {
        {"vnet1", 1, "10.1.1.1", "00:00:00:01:01:01", 1, 1},
    };

    CreateVmportEnv(input, 1);
    client->WaitForIdle();
    AddIPAM("vn1", ipam_info, 3);
    client->WaitForIdle();
    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    boost::system::error_code ec;
    Ip4Address addr = Ip4Address::from_string("10.1.1.1", ec);
    InetUnicastRouteEntry* rt = RouteGet(agent->fabric_vrf_name(), addr, 32);
    EXPECT_TRUE(rt != NULL);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::RECEIVE);

    DeleteVmportEnv(input, 1, true);
    client->WaitForIdle();
    DelIPAM("vn1");
    client->WaitForIdle();

    rt = RouteGet(agent->fabric_vrf_name(), addr, 32);
    EXPECT_TRUE(rt != NULL);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::RECEIVE);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
}

TEST_F(FabricVmiTest, default_route) {
    Ip4Address ip(0);
    EXPECT_TRUE(RouteFind(agent->fabric_policy_vrf_name(), ip, 0));
    InetUnicastRouteEntry *rt = RouteGet(agent->fabric_policy_vrf_name(), ip, 0);
    EXPECT_TRUE(rt->GetActivePath()->tunnel_bmap() & TunnelType::NativeType());

    AddEncapList("MPLSoGRE", "MPLSoUDP", "VXLAN");
    client->WaitForIdle();

    EXPECT_TRUE(RouteFind(agent->fabric_policy_vrf_name(), ip, 0));
    EXPECT_TRUE(rt->GetActivePath()->tunnel_bmap() & TunnelType::NativeType());
}

//Add 2 VMI with same IP address verify ECMP peer
//creates a path with both nexthop
TEST_F(FabricVmiTest, Ecmp1) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
        {"intf3", 3, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportWithEcmp(input1, 2);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());
    const VmInterface *vm_intf1 = static_cast<const VmInterface *>(VmPortGet(2));
    const VmInterface *vm_intf2 = static_cast<const VmInterface *>(VmPortGet(3));

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    InetUnicastRouteEntry *rt =
        static_cast<InetUnicastRouteEntry*>(RouteGet(agent->fabric_vrf_name(),
                                                     ip, 32));
    EXPECT_TRUE(rt->FindPath(vm_intf1->peer()) != NULL);
    EXPECT_TRUE(rt->FindPath(vm_intf2->peer()) != NULL);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) != 0);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::COMPOSITE);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf() == NULL);
    EXPECT_TRUE(RouteGet(agent->fabric_vrf_name(), ip, 32) == NULL);

    DeleteVmportEnv(input1, 2, true);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    client->WaitForIdle();
}

//Add same IP address in 2 different virtual network
//Verify ECMP path gets created for the same
TEST_F(FabricVmiTest, Ecmp2) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
        {"intf3", 3, "1.1.1.1", "00:00:00:01:01:01", 2, 2},
    };
    CreateVmportWithEcmp(input1, 2);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());
    const VmInterface *vm_intf1 = static_cast<const VmInterface *>(VmPortGet(2));
    const VmInterface *vm_intf2 = static_cast<const VmInterface *>(VmPortGet(3));

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
    InetUnicastRouteEntry *rt =
        static_cast<InetUnicastRouteEntry*>(RouteGet(agent->fabric_vrf_name(),
                                                     ip, 32));
    EXPECT_TRUE(rt->FindPath(vm_intf1->peer()) != NULL);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) != 0);

    AddLink("virtual-network", "vn2", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    EXPECT_TRUE(rt->FindPath(vm_intf1->peer()) != NULL);
    EXPECT_TRUE(rt->FindPath(vm_intf2->peer()) != NULL);
    EXPECT_TRUE((rt->GetActivePath()->tunnel_bmap() &
                 TunnelType::NativeType()) != 0);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::COMPOSITE);

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(rt->FindPath(vm_intf1->peer()) == NULL);
    EXPECT_TRUE(rt->FindPath(vm_intf2->peer()) != NULL);
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::INTERFACE);

    DelLink("virtual-network", "vn2", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(RouteGet(agent->fabric_vrf_name(), ip, 32) == NULL);

    DeleteVmportEnv(input1, 2, true);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    client->WaitForIdle();
}

//Add a ECMP path
//Add a BGP path
//Verify that BGP path added has local ECMP path also
TEST_F(FabricVmiTest, Ecmp3) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
        {"intf3", 3, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportWithEcmp(input1, 2);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
        InetUnicastRouteEntry *rt =
        static_cast<InetUnicastRouteEntry*>(RouteGet(agent->fabric_vrf_name(),
                    ip, 32));

    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::COMPOSITE);

    Ip4Address remote_address = Ip4Address::from_string("10.10.10.100");
    ComponentNHKeyPtr nh_data1(new ComponentNHKey(30, agent->fabric_vrf_name(),
                agent->router_id(), remote_address, false, TunnelType::NativeType()));
    ComponentNHKeyList comp_nh_list;
    //Insert new NH first and then existing route NH
    comp_nh_list.push_back(nh_data1);

    SecurityGroupList sg_list;
    EcmpTunnelRouteAdd(peer_, agent->fabric_vrf_name(), ip, 32,
                       comp_nh_list, -1, "vn1",
                       sg_list, TagList(), PathPreference(), true);
    client->WaitForIdle();
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::COMPOSITE);
    const CompositeNH *comp_nh =
        dynamic_cast<const CompositeNH *>(rt->GetActiveNextHop());
    EXPECT_TRUE(comp_nh->ComponentNHCount() == 3);

    agent->fabric_inet4_unicast_table()->DeleteReq(peer_,
           agent->fabric_vrf_name().c_str(), ip, 32,
           new ControllerVmRoute(peer_));
    client->WaitForIdle();

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf() == NULL);
    EXPECT_TRUE(RouteGet(agent->fabric_vrf_name(), ip, 32) == NULL);

    DeleteVmportEnv(input1, 2, true);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    client->WaitForIdle();
}

//Add one local VM path
//Add a BGP path verify BGP path is ECMP with right no. of nexthop
//Add one more local path, verify BGP path now has 3 path
//Delete one local VM path and verify same gets reflected in
//BGP path also
TEST_F(FabricVmiTest, Ecmp4) {
    struct PortInfo input1[] = {
        {"intf2", 2, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
        {"intf3", 3, "1.1.1.1", "00:00:00:01:01:01", 1, 2},
    };
    CreateVmportWithEcmp(input1, 1);
    client->WaitForIdle();

    AddLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();

    Ip4Address ip = Ip4Address::from_string("1.1.1.1");
        InetUnicastRouteEntry *rt =
        static_cast<InetUnicastRouteEntry*>(RouteGet(agent->fabric_vrf_name(),
                    ip, 32));

    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf()->GetName() ==
                agent->fabric_vrf_name().c_str());
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() != NextHop::COMPOSITE);

    Ip4Address remote_address = Ip4Address::from_string("10.10.10.100");
    ComponentNHKeyPtr nh_data1(new ComponentNHKey(30, agent->fabric_vrf_name(),
                agent->router_id(), remote_address, false, TunnelType::NativeType()));
    ComponentNHKeyList comp_nh_list;
    //Insert new NH first and then existing route NH
    comp_nh_list.push_back(nh_data1);

    SecurityGroupList sg_list;
    EcmpTunnelRouteAdd(peer_, agent->fabric_vrf_name(), ip, 32,
                       comp_nh_list, -1, "vn1",
                       sg_list, TagList(), PathPreference(), true);
    client->WaitForIdle();
    EXPECT_TRUE(rt->GetActiveNextHop()->GetType() == NextHop::COMPOSITE);
    const CompositeNH *comp_nh =
        dynamic_cast<const CompositeNH *>(rt->GetActiveNextHop());
    EXPECT_TRUE(comp_nh->ComponentNHCount() == 2);

    CreateVmportWithEcmp(input1, 2);
    client->WaitForIdle();

    comp_nh =
        dynamic_cast<const CompositeNH *>(rt->GetActiveNextHop());
    EXPECT_TRUE(comp_nh->ComponentNHCount() == 3);

    DeleteVmportEnv(input1, 1, false);
    client->WaitForIdle();

    comp_nh =
        dynamic_cast<const CompositeNH *>(rt->GetActiveNextHop());
    EXPECT_TRUE(comp_nh->ActiveComponentNHCount() == 2);
    EXPECT_TRUE(comp_nh->GetNH(0) == NULL);
    EXPECT_TRUE(comp_nh->PolicyEnabled() == true);

    Ip4Address aap_ip = Ip4Address::from_string("10.10.10.10");
    MacAddress mac("0a:0b:0c:0d:0e:0f");
    AddAapWithMacAndDisablePolicy("intf3", 3, aap_ip, mac.ToString(), true);
    client->WaitForIdle();

    comp_nh =
        dynamic_cast<const CompositeNH *>(rt->GetActiveNextHop());
    EXPECT_TRUE(comp_nh->ActiveComponentNHCount() == 2);
    EXPECT_TRUE(comp_nh->GetNH(0) == NULL);
    EXPECT_TRUE(comp_nh->PolicyEnabled() == false);

    agent->fabric_inet4_unicast_table()->DeleteReq(peer_,
            agent->fabric_vrf_name().c_str(), ip, 32,
            new ControllerVmRoute(peer_));
    client->WaitForIdle();

    DelLink("virtual-network", "vn1", "virtual-network",
            agent->fabric_vn_name().c_str());
    client->WaitForIdle();
    EXPECT_TRUE(VrfGet("vrf1")->forwarding_vrf() == NULL);
    EXPECT_TRUE(RouteGet(agent->fabric_vrf_name(), ip, 32) == NULL);

    DeleteVmportEnv(input1, 2, true);
    client->WaitForIdle();

    EXPECT_FALSE(RouteFind(agent->fabric_vrf_name(), ip, 32));
    client->WaitForIdle();
}

int main(int argc, char **argv) {
    GETUSERARGS();
    client = TestInit(init_file, ksync_init, false, false, false);
    int ret = RUN_ALL_TESTS();
    usleep(10000);
    client->WaitForIdle();
    usleep(10000);
    TestShutdown();
    delete client;
    return ret;
}

max_stars_repo_name: jnpr-pranav/contrail-controller

max_issues_repo_name: jnpr-pranav/contrail-controller

max_forks_repo_name: jnpr-pranav/contrail-controller


text:
//==================================================================================================
/**
  Copyright 2017 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
**/
//==================================================================================================

#ifndef BOOST_SIMD_FUNCTION_SCALAR_IFIX_HPP_INCLUDED
#define BOOST_SIMD_FUNCTION_SCALAR_IFIX_HPP_INCLUDED

#include <boost/simd/function/definition/ifix.hpp>
#include <boost/simd/arch/common/scalar/function/ifix.hpp>

#endif

max_stars_repo_name: SylvainCorlay/pythran

max_issues_repo_name: SylvainCorlay/pythran

max_forks_repo_name: SylvainCorlay/pythran


text:
/**
 * Copyright (c) 2018, University Osnabrück
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the University Osnabrück nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL University Osnabrück BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * ReductionAlgorithms.hpp
 *
 * All these algorithms are based on/inspired by:
 *
 * Melax, Stan. "A simple, fast, and effective polygon reduction algorithm."
 * Game Developer 11 (1998): 44-49.
 */

#ifndef LVR2_ALGORITHM_REDUCTIONALGORITHMS_H_
#define LVR2_ALGORITHM_REDUCTIONALGORITHMS_H_

#include <boost/optional.hpp>

using boost::optional;

namespace lvr2
{

/**
 * @brief Collapses `count` many edges of `mesh` for which `collapseCost`
 *        returns the smallest values.
 *
 * This algorithm collapses edges in the given mesh. The order of edge
 * collapses is determined by `collapseCost`: the edge with the smallest cost
 * is collapsed first, if that is possible. In case the edge is not
 * collapsable, it is ignored and the next edge is choosen. After each
 * collapse, the cost function is called again to update the costs for all
 * edges that could have been affected, which are the edges of all faces which
 * touch the vertex the edge was collapsed to.
 *
 * This algorithm stops when either `count` many edges have been collapsed or
 * if there are no collapsable edges left.
 *
 * @param[in] count Number of edges to collapse
 * @param[in, out] faceNormals A face map storing valid normals of all faces in
 *                             the mesh. This map is altered by this algorithm
 *                             according to the changes done in the mesh.
 * @param[in] collapseCost Function which is called with an edge handle and a
 *                         FaceMap containing normals; it is expected to return
 *                         an optional float. `boost::none` means that this
 *                         edge cannot be collapsed.
 *
 * @return The number of edges actually collapsed.
 */
template<typename BaseVecT, typename CostF>
size_t iterativeEdgeCollapse(
    BaseMesh<BaseVecT>& mesh,
    const size_t count,
    FaceMap<Normal<typename BaseVecT::CoordType>>& faceNormals,
    CostF collapseCost
);

/**
 * @brief Like `iterativeEdgeCollapse` but with a fixed cost function.
 */
template<typename BaseVecT>
size_t simpleMeshReduction(
    BaseMesh<BaseVecT>& mesh,
    const size_t count,
    FaceMap<Normal<typename BaseVecT::CoordType>>& faceNormals
);

} // namespace lvr2

#include <lvr2/algorithm/ReductionAlgorithms.tcc>

#endif /* LVR2_ALGORITHM_REDUCTIONALGORITHMS_H_ */


text:
{{{header}}}
{{#includes}}
#include <{{{.}}}>
{{/includes}}

#include <boost/python.hpp>
#include <cmath>
#include <iostream>

/* main postinclude */

BOOST_PYTHON_MODULE({{module.name}})
{
::boost::python::docstring_options options(true, true, false);

{{{precontent}}}
{{#module.namespaces}}{{#name}}
  ::boost::python::scope(){{!
    }}{{#scope}}{{#name}}.attr("{{name}}"){{/name}}{{/scope}}.attr("{{name}}") = {{!
    }}::boost::python::object(::boost::python::handle<>({{!
        }}::boost::python::borrowed(::PyImport_AddModule({{!
            }}"{{module.name}}{{#scope}}{{#name}}.{{name}}{{/name}}{{/scope}}.{{name}}"))));
{{/name}}{{/module.namespaces}}

{{#module.bindings}}
  void {{.}}();
  try {
    {{.}}();
  } catch(...) {
    std::cerr << "Exception in: {{.}}" << std::endl;
    throw;
  }

{{/module.bindings}}
{{{postcontent}}}
}
{{{footer}}}