text:
/** This library is under the 3-Clause BSD License

Copyright (c) 2017, Orange S.A.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

  2. Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.

  3. Neither the name of the copyright holder nor the names of its contributors
     may be used to endorse or promote products derived from this software without
     specific prior written permission.

 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 Author: Johannes Heinecke
 Version:  1.1 as of 24th July 2017
*/

#include "segmenter.h"
#include "normalisations.h"

#include <iostream>
#include <fstream>
#include <sstream>
#include <boost/regex.hpp>

#include "unicode_tools.h"

// TODO: resolve decomposition

using namespace std;
using namespace Segmenteur;

SegDef::SegDef(const string &re, const string &name, unsigned int id, bool ignore, bool endsentence, bool endpara, wchar_t *(*func)(const wchar_t *) = 0) :
    name(name),
    id(id),
    ignore(ignore),
    endSent(endsentence),
    endPara(endpara) {
    normalisation = func;
    wchar_t *dest = Unicode::fromUTF8(re.c_str());
    this->re = new boost::wregex((wchar_t *)dest, boost::wregex::optimize);
    delete [] dest;
}

SegDef::~SegDef() {
    delete re;
}

ostream& operator<<(ostream& out, const SegDef& s) {
    char *utf8 = Unicode::toUTF8(s.re->str().c_str());
    out << "\t" <<
	//s.re->str() << "\t" <<
            utf8 << "\t" <<
	s.name << "\t" <<
	s.ignore << "\t" <<
	s.endSent << "\t" <<
	s.endPara << "\t";
    delete [] utf8 ;
    return out;
}

Segmenter::Segmenter(const string &filename
#ifdef TILTMETHODS
		     , Recodeur *recodeur
#endif
		     )
#ifdef TILTMETHODS
: recodeur(recodeur)
#endif
{
    if (sizeof(wchar_t) != 4) {
        throw SegmenterException("Needs system where wchar_t has 4 bytes");
    }
    ifstream ifp(filename.c_str());

    //DecrypteurFichier decr(filename.c_str());
    //istream *ifp = decr.getFlux();

    if (!ifp) {
        throw SegmenterException("cannot open segmentation rules " + filename);
    }

    string line;
    unsigned int ct = 0;
    bool mostGeneralRuleSeen = false;
    while(!ifp.eof()) {
        ct++;
	getline(ifp, line);
	if (line.empty()) continue;
	if (line[0] == '#') continue;
	//cerr << line << endl;

	vector<string>elems;
	Unicode::SplitNoEmptyFields(elems, line, "\t");
        if (elems[0].empty()) {
	    cerr << "Empty RegEx in segmenter rules in line " <<  ct << "; " << line << endl;
            continue;
        }
	bool ignore = false;
	bool es = false;
	bool ep = false;
	string normalisation_name;
	switch(elems.size()) {
	    case 6: normalisation_name = elems[5];
	    case 5: if (elems[4] == "yes") ep = true;
	    case 4: if (elems[3] == "yes") es = true;
	    case 3: if (elems[2] == "yes") ignore = true;
	    case 2: break;
	    default:
                ostringstream msg;
                msg << "invalid line (" << ct << "): " << line << endl;
                throw SegmenterException(msg.str());
                //continue;
	}

	wchar_t *(*normalisation)(const wchar_t *) = 0;

	if (!normalisation_name.empty()) {
	    if (normalisation_name == "DELETE_SPACES") {
		normalisation = deleteSpaces;
	    }
	    else if (normalisation_name == "DELETE_POINTS") {
		normalisation = deletePoints;
	    }
	    else {
                ostringstream msg;
                msg << "invalid normalisation (" << ct << "): " << line << endl;
                throw SegmenterException(msg.str());

	    }
	}

        if (elems[0] == ".") mostGeneralRuleSeen = true;
        try {
	    defs.push_back(new SegDef(elems[0], elems[1], defs.size(), ignore, es, ep, normalisation));
        } catch (boost::regex_error &e) {
            ostringstream msg;
            msg << "bad regular expresion (" << ct << "): " << line;
            throw SegmenterException(msg.str());
            //cerr << "bad regular expression (" << ct << "): " << line << endl;
        }
    }
    if (!mostGeneralRuleSeen) {
        defs.push_back(new SegDef(".", "ANY", defs.size(), false, false, false));
    }
}

Segmenter::~Segmenter() {
    for (vector<SegDef *>::iterator it = defs.begin(); it != defs.end(); ++it) {
	delete *it;
    }
}

ostream& operator<<(ostream& out, const Segmenter& s) {
    out << s.defs.size() << " segmentation rules" << endl;
    for (vector<SegDef *>::const_iterator it = s.defs.begin(); it != s.defs.end(); ++it) {
	out << **it << endl;
    }
    return out;
}

void Segmenter::segment(const wstring &text, Text &res, bool keepIgnored) {
    boost::match_flag_type flags = boost::match_continuous;
    std::wstring::const_iterator start = text.begin();
    std::wstring::const_iterator end = text.end();

    boost::match_results<std::wstring::const_iterator> what;

    Paragraph curPar;
    Sentence curSent;

    unsigned offset = 0;
    //    #define WITHLONGEST

    bool rtc;
    while (start < end) {
        bool segmentFound = false;

	vector<SegDef *>::iterator it;
#ifdef WITHLONGEST
	//if (getLongestMatch) {
	unsigned int maxlen = 0;
	it = defs.end();
	for(vector<SegDef *>::iterator its = defs.begin();  its != defs.end(); ++its) {
	    rtc = boost::regex_search(start, end, what, *(*its)->re, flags);

	    if (rtc) {
		wstring seg = what[0];
		if (seg.size() > maxlen) {
		    //cerr << "NEW SEG " << (*its)->name << " "
		    // << maxlen << " --> " << seg.size()
		    // << endl;
		    maxlen = seg.size();
		    it = its;
		}
	    }
	}

	if (it != defs.end() && maxlen != 0) {
	    rtc = true;
	} else {
	    rtc = false;
	}

#endif

#ifndef WITHLONGEST
	for(vector<SegDef *>::iterator it = defs.begin();
	    it != defs.end(); ++it) {

	    // cerr << "Test " << **it << endl;
#endif
	    rtc = boost::regex_search(start, end, what, *(*it)->re, flags);
	    if (rtc) {
		wstring seg = what[0];
		if (!seg.empty()) {
		    //cerr << "NAME " << (*it)->name
		    //<< " '" << what[0] << "'"
		    // << endl;
		    start = what[0].second;

		    wstring orig = what[0];
		    unsigned int length = seg.size();
		    //	cerr << "lll: " << length << endl;
		    //char *utf8 = toUTF8(se)
		    if (keepIgnored || !(*it)->ignore) {
			//std::cerr <<  " " << offset << endl;
			if ((*it)->normalisation != 0) {
			    wchar_t *norm = ((*it)->normalisation)(orig.c_str());
			    seg = norm;
			    delete [] norm;

			    //cerr << "normalise: " << endl;
			    //wcerr << L"<" << orig << L"> ==> "
			    //      << L"<" << seg << L">" << endl;
			}

			curSent.push_back(Segment(seg, (*it)->name, (*it)->id,
						  offset, length,
						  (*it)->ignore,
						  orig
						  ));
#ifdef TILTMETHODS
			if (recodeur != 0) {
			    char *text8bit = recodeur->recoder(curSent.back().getUTF8Text().c_str());
			    if (strlen(text8bit) != 0) {
				curSent.back().add8BitText(text8bit);
			    }
			    free(text8bit);
			}
#endif
		    }
		    offset += length;

		    if ((*it)->endSent) {
			if (!curSent.empty()) curPar.push_back(curSent);
			curSent.clear();
		    }
		    if ((*it)->endPara) {
			if (!curSent.empty()) curPar.push_back(curSent);
			curSent.clear();
			if (!curPar.empty()) res.push_back(curPar);
			curPar.clear();
		    }
		    // rule matched
		    segmentFound = true;
#ifndef WITHLONGEST
		    break;
#endif
		}
	    }
#ifndef WITHLONGEST
	}
#endif
        if (!segmentFound || offset == 0) {
	    // TODO Exception
            cerr << "Jammed at " << offset << endl;
            break;
        }
	if (start >= end) {
	    break;
	}

    }
    if (!curSent.empty()) {
	curPar.push_back(curSent);
    }
    if (!curPar.empty()) {
        res.push_back(curPar);
    }
}

void Segmenter::listeDesTypesDeSegments(vector<string>& lts) const {
    lts.clear();
    for (vector<SegDef *>::const_iterator it = defs.begin(); it != defs.end(); ++it) {
	lts.push_back((*it)->name);
    }
}

void Segmenter::tableDesNumTypes(map<string,int>& tnt) const {
    tnt.clear();
    for (vector<SegDef *>::const_iterator it = defs.begin(); it != defs.end(); ++it) {
	    tnt[(*it)->name] = (*it)->id;
    }
}

ext: cc

max_stars_repo_name: Orange-OpenSource/Text-tokenizer

max_issues_repo_name: Orange-OpenSource/Text-tokenizer

max_forks_repo_name: Orange-OpenSource/Text-tokenizer


text:
#include <evt/testing/tester.hpp>

#include <boost/algorithm/string/predicate.hpp>
#include <boost/test/unit_test.hpp>

#include <evt/chain/token_database.hpp>

using namespace evt::chain::contracts;

namespace evt { namespace testing {

bool
expect_assert_message(const fc::exception& ex, string expected) {
    BOOST_TEST_MESSAGE("LOG : "
                       << "expected: " << expected << ", actual: " << ex.get_log().at(0).get_message());
    return (ex.get_log().at(0).get_message().find(expected) != std::string::npos);
}

fc::variant_object
filter_fields(const fc::variant_object& filter, const fc::variant_object& value) {
    fc::mutable_variant_object res;
    for(auto& entry : filter) {
        auto it = value.find(entry.key());
        res(it->key(), it->value());
    }
    return res;
}

base_tester::base_tester() {}

bool
base_tester::is_same_chain(base_tester& other) {
    return control->head_block_id() == other.control->head_block_id();
}

void
base_tester::init(bool push_genesis) {
    cfg.blocks_dir             = tempdir.path() / config::default_blocks_dir_name;
    cfg.state_dir              = tempdir.path() / config::default_state_dir_name;
    cfg.db_config.db_path      = tempdir.path() / config::default_token_database_dir_name;
    cfg.contracts_console      = true;
    cfg.loadtest_mode          = false;
    cfg.charge_free_mode       = false;
    cfg.max_serialization_time = std::chrono::hours(1);

    cfg.genesis.initial_timestamp = fc::time_point::from_iso_string("2020-01-01T00:00:00.000");
    cfg.genesis.initial_key       = get_public_key("evt");

    open(nullptr);

    if(push_genesis) {
        push_genesis_block();
    }
}

void
base_tester::init(controller::config config, const snapshot_reader_ptr& snapshot) {
    cfg = config;
    open(snapshot);
}

void
base_tester::close() {
    control.reset();
    chain_transactions.clear();
}

void
base_tester::open(const snapshot_reader_ptr& snapshot) {
    control.reset(new controller(cfg));
    control->add_indices();
    control->startup(snapshot);
    chain_transactions.clear();
    control->accepted_block.connect([this](const block_state_ptr& block_state) {
        FC_ASSERT(block_state->block);
        for(const auto& receipt : block_state->block->transactions) {
            chain_transactions[receipt.trx.id()] = transaction_receipt(receipt.trx);
        }
    });
}

signed_block_ptr
base_tester::push_block(signed_block_ptr b) {
    control->abort_block();
    control->push_block(b);

    auto itr = last_produced_block.find(b->producer);
    if(itr == last_produced_block.end() || block_header::num_from_id(b->id()) > block_header::num_from_id(itr->second)) {
        last_produced_block[b->producer] = b->id();
    }

    return b;
}

signed_block_ptr
base_tester::_produce_block(fc::microseconds skip_time, bool skip_pending_trxs, uint32_t skip_flag) {
    auto head      = control->head_block_state();
    auto head_time = control->head_block_time();
    auto next_time = head_time + skip_time;

    if(!control->pending_block_state() || control->pending_block_state()->header.timestamp != next_time) {
        _start_block(next_time);
    }

    auto producer = control->head_block_state()->get_scheduled_producer(next_time);
    auto priv_key = private_key_type();
    // Check if signing private key exist in the list
    auto private_key_itr = block_signing_private_keys.find(producer.block_signing_key);
    if(private_key_itr == block_signing_private_keys.end()) {
        // If it's not found, default to active k1 key
        priv_key = get_private_key((std::string)producer.producer_name);
    }
    else {
        priv_key = private_key_itr->second;
    }

    if(!skip_pending_trxs) {
        auto unapplied_trxs = control->get_unapplied_transactions();
        for(const auto& trx : unapplied_trxs) {
            auto trace = control->push_transaction(trx.second, fc::time_point::maximum());
            if(trace->except) {
                trace->except->dynamic_rethrow_exception();
            }
        }
    }

    control->finalize_block();
    control->sign_block([&](digest_type d) {
        return priv_key.sign(d);
    });

    control->commit_block();
    last_produced_block[control->head_block_state()->header.producer] = control->head_block_state()->id;

    _start_block(next_time + fc::microseconds(config::block_interval_us));
    return control->head_block_state()->block;
}

void
base_tester::_start_block(fc::time_point block_time) {
    auto head_block_number = control->head_block_num();
    auto producer          = control->head_block_state()->get_scheduled_producer(block_time);

    auto last_produced_block_num = control->last_irreversible_block_num();
    auto itr                     = last_produced_block.find(producer.producer_name);
    if(itr != last_produced_block.end()) {
        last_produced_block_num = std::max(control->last_irreversible_block_num(), block_header::num_from_id(itr->second));
    }

    control->abort_block();
    control->start_block(block_time, head_block_number - last_produced_block_num);
}

void
base_tester::produce_blocks(uint32_t n, bool empty) {
    if(empty) {
        for(uint32_t i = 0; i < n; ++i)
            produce_empty_block();
    }
    else {
        for(uint32_t i = 0; i < n; ++i)
            produce_block();
    }
}

void
base_tester::produce_blocks_until_end_of_round() {
    uint64_t blocks_per_round;
    while(true) {
        blocks_per_round = control->active_producers().producers.size() * config::producer_repetitions;
        produce_block();
        if(control->head_block_num() % blocks_per_round == (blocks_per_round - 1))
            break;
    }
}

void
base_tester::produce_blocks_for_n_rounds(const uint32_t num_of_rounds) {
    for(uint32_t i = 0; i < num_of_rounds; i++) {
        produce_blocks_until_end_of_round();
    }
}

void
base_tester::produce_min_num_of_blocks_to_spend_time_wo_inactive_prod(const fc::microseconds target_elapsed_time) {
    fc::microseconds elapsed_time;
    while(elapsed_time < target_elapsed_time) {
        for(uint32_t i = 0; i < control->head_block_state()->active_schedule.producers.size(); i++) {
            const auto time_to_skip = fc::milliseconds(config::producer_repetitions * config::block_interval_ms);
            produce_block(time_to_skip);
            elapsed_time += time_to_skip;
        }
        // if it is more than 24 hours, producer will be marked as inactive
        const auto time_to_skip = fc::seconds(23 * 60 * 60);
        produce_block(time_to_skip);
        elapsed_time += time_to_skip;
    }
}

void
base_tester::set_transaction_headers(signed_transaction& trx, const address& payer, uint32_t max_charge, uint32_t expiration) const {
    trx.expiration = control->head_block_time() + fc::seconds(expiration);
    trx.payer      = payer;
    trx.max_charge = max_charge;
    trx.set_reference_block(control->head_block_id());
}

transaction_trace_ptr
base_tester::push_transaction(packed_transaction& trx,
                              fc::time_point      deadline) {
    try {
        if(!control->pending_block_state()) {
            _start_block(control->head_block_time() + fc::microseconds(config::block_interval_us));
        }
        auto r = control->push_transaction(std::make_shared<transaction_metadata>(std::make_shared<packed_transaction>(trx)), deadline);

        if(r->except_ptr) {
            std::rethrow_exception(r->except_ptr);
        }
        if(r->except) {
            throw *r->except;
        }
        return r;
    }
    FC_CAPTURE_AND_RETHROW((transaction_header(trx.get_transaction())))
}

transaction_trace_ptr
base_tester::push_transaction(signed_transaction& trx,
                              fc::time_point      deadline) {
    try {
        if(!control->pending_block_state()) {
            _start_block(control->head_block_time() + fc::microseconds(config::block_interval_us));
        }
        auto c = packed_transaction::none;

        if(fc::raw::pack_size(trx) > 1000) {
            c = packed_transaction::zlib;
        }

        auto r = control->push_transaction(std::make_shared<transaction_metadata>(trx, c), deadline);

        if(r->except_ptr) {
            std::rethrow_exception(r->except_ptr);
        }
        if(r->except) {
            throw *r->except;
        }
        return r;
    }
    FC_CAPTURE_AND_RETHROW((transaction_header(trx)))
}

transaction_trace_ptr
base_tester::push_action(action&& act, std::vector<name>& auths, const address& payer, uint32_t max_charge) {
    try {
        signed_transaction trx;
        trx.actions.emplace_back(std::move(act));
        set_transaction_headers(trx, payer, max_charge);
        if(!auths.empty()) {
            for(auto& au : auths) {
                trx.sign(get_private_key(au), control->get_chain_id());
            }
        }
        return push_transaction(trx);
    }
    FC_CAPTURE_AND_RETHROW((act)(auths)(payer)(max_charge))
}

transaction_trace_ptr
base_tester::push_action(const action_name&       acttype,
                         const domain_name&       domain,
                         const domain_key&        key,
                         const variant_object&    data,
                         const std::vector<name>& auths,
                         const address&           payer,
                         uint32_t                 max_charge,
                         uint32_t                 expiration)
{
    try {
        auto trx = signed_transaction();
        trx.actions.emplace_back(get_action(acttype, domain, key, data));
        set_transaction_headers(trx, payer, max_charge, expiration);
        for(const auto& auth : auths) {
            trx.sign(get_private_key(auth), control->get_chain_id());
        }

        return push_transaction(trx);
    }
    FC_CAPTURE_AND_RETHROW((acttype)(domain)(key)(data)(auths)(payer)(max_charge)(expiration))
}

action
base_tester::get_action(action_name acttype, const domain_name& domain, const domain_key& key, const variant_object& data) const {
    try {
        auto& abi      = control->get_abi_serializer();
        auto& exec_ctx = control->get_execution_context();
        auto  type     = exec_ctx.get_acttype_name(acttype);
        FC_ASSERT(!type.empty(), "unknown action type ${a}", ("a", acttype));

        action act;
        act.name   = acttype;
        act.domain = domain;
        act.key    = key;
        act.data   = abi.variant_to_binary(type, data, exec_ctx);

        return act;
    }
    FC_CAPTURE_AND_RETHROW()
}

bool
base_tester::chain_has_transaction(const transaction_id_type& txid) const {
    return chain_transactions.count(txid) != 0;
}

const transaction_receipt&
base_tester::get_transaction_receipt(const transaction_id_type& txid) const {
    return chain_transactions.at(txid);
}

vector<uint8_t>
base_tester::to_uint8_vector(const string& s) {
    vector<uint8_t> v(s.size());
    copy(s.begin(), s.end(), v.begin());
    return v;
};

vector<uint8_t>
base_tester::to_uint8_vector(uint64_t x) {
    vector<uint8_t> v(sizeof(x));
    *reinterpret_cast<uint64_t*>(v.data()) = x;
    return v;
};

uint64_t
base_tester::to_uint64(fc::variant x) {
    vector<uint8_t> blob;
    fc::from_variant<uint8_t>(x, blob);
    FC_ASSERT(8 == blob.size());
    return *reinterpret_cast<uint64_t*>(blob.data());
}

string
base_tester::to_string(fc::variant x) {
    vector<uint8_t> v;
    fc::from_variant<uint8_t>(x, v);
    string s(v.size(), 0);
    copy(v.begin(), v.end(), s.begin());
    return s;
}

void
base_tester::sync_with(base_tester& other) {
    // Already in sync?
    if(control->head_block_id() == other.control->head_block_id())
        return;
    // If other has a longer chain than we do, sync it to us first
    if(control->head_block_num() < other.control->head_block_num())
        return other.sync_with(*this);

    auto sync_dbs = [](base_tester& a, base_tester& b) {
        for(auto i = 1u; i <= a.control->head_block_num(); ++i) {
            auto block = a.control->fetch_block_by_number(i);
            if(block) {  //&& !b.control->is_known_block(block->id()) ) {
                b.control->abort_block();
                b.control->push_block(block);  //, evt::chain::validation_steps::created_block);
            }
        }
    };

    sync_dbs(*this, other);
    sync_dbs(other, *this);
}

void
base_tester::push_genesis_block() {
    //produce_block();
}

void
base_tester::add_money(const address& addr, const asset& number) {
    auto& tokendb = control->token_db();

    auto s = tokendb.new_savepoint_session();

    auto str  = std::string();
    auto prop = property();
    prop.sym  = number.sym();

    if(tokendb.read_asset(addr, number.symbol_id(), str, true)) {
        extract_db_value(str, prop);
    }

    prop.amount += number.amount();

    auto dv = make_db_value(prop);
    tokendb.put_asset(addr, number.symbol_id(), dv.as_string_view());

    s.accept();
    tokendb.pop_back_savepoint();
}

bool
fc_exception_message_is::operator()(const fc::exception& ex) {
    auto message = ex.get_log().at(0).get_message();
    bool match   = (message == expected);
    if(!match) {
        BOOST_TEST_MESSAGE("LOG: expected: " << expected << ", actual: " << message);
    }
    return match;
}

bool
fc_exception_message_starts_with::operator()(const fc::exception& ex) {
    auto message = ex.get_log().at(0).get_message();
    bool match   = boost::algorithm::starts_with(message, expected);
    if(!match) {
        BOOST_TEST_MESSAGE("LOG: expected: " << expected << ", actual: " << message);
    }
    return match;
}

bool
fc_assert_exception_message_is::operator()(const fc::assert_exception& ex) {
    auto message = ex.get_log().at(0).get_message();
    bool match   = false;
    auto pos     = message.find(": ");
    if(pos != std::string::npos) {
        message = message.substr(pos + 2);
        match   = (message == expected);
    }
    if(!match) {
        BOOST_TEST_MESSAGE("LOG: expected: " << expected << ", actual: " << message);
    }
    return match;
}

bool
fc_assert_exception_message_starts_with::operator()(const fc::assert_exception& ex) {
    auto message = ex.get_log().at(0).get_message();
    bool match   = false;
    auto pos     = message.find(": ");
    if(pos != std::string::npos) {
        message = message.substr(pos + 2);
        match   = boost::algorithm::starts_with(message, expected);
    }
    if(!match) {
        BOOST_TEST_MESSAGE("LOG: expected: " << expected << ", actual: " << message);
    }
    return match;
}

}}  // namespace evt::testing

std::ostream&
operator<<(std::ostream& osm, const fc::variant& v) {
    //fc::json::to_stream( osm, v );
    osm << fc::json::to_pretty_string(v);
    return osm;
}

std::ostream&
operator<<(std::ostream& osm, const fc::variant_object& v) {
    osm << fc::variant(v);
    return osm;
}

std::ostream&
operator<<(std::ostream& osm, const fc::variant_object::entry& e) {
    osm << "{ " << e.key() << ": " << e.value() << " }";
    return osm;
}

ext: cpp

max_stars_repo_name: zhao-db/evt

max_issues_repo_name: zhao-db/evt

max_forks_repo_name: zhao-db/evt


text:
/*
   Copyright [2017-2022] [IBM Corporation]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

#include "make_kvstore.h"
#include "kv_lock.h"
#include "pool_instance.h"

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Weffc++"
#pragma GCC diagnostic ignored "-Wconversion"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop

#include <common/json.h>
#include <common/logging.h> /* format */
#include <common/str_utils.h>
#include <common/utils.h>
#include <api/kvstore_itf.h>
#include <boost/program_options.hpp>
#include <gsl/pointers>
#include <nupm/region_descriptor.h>
#include <cstddef> /* size_t */
#include <iostream> /* cerr, cout */
#include <random>
#include <string>

#define ASSERT_OK(X) ASSERT_EQ(S_OK, (X))

static std::string store;
static custom_store *cs;
static component::IKVStore_factory::map_create mc;
static std::size_t many_count_proposed;

/*
 * Ideally, these error values should be expected wherever they might occur:
 *    E_KEY_EXISTS     = E_ERROR_BASE - 1 (put, put_direct)
 *    E_KEY_NOT_FOUND  = E_ERROR_BASE - 2, (resive_value, get, get_direct, get_attributes, swap_keys, lock)
 *    E_POOL_NOT_FOUND = E_ERROR_BASE - 3, (close_pool, get_[pool_]regions, grow_pool, put, put_direct, get, get_direct, get_attribute, swap_keys, set_attribute, map, map_keys, allocate_pool_memory, free_pool_memory, flush_pool_memory)
 *    E_BAD_ALIGNMENT  = E_ERROR_BASE - 4, (resize_value)
 *    E_TOO_LARGE      = E_ERROR_BASE - 5, (resize_value, lock)
 *    E_ALREADY_OPEN   = E_ERROR_BASE - 6, (close_pool, delete_pool)
*/

std::string pool_name()
{
	return "pool/test-" + store;
}

namespace
{
	/* A pool instance whose size is adapted depending in available memory.
	 * The variable size portion is recorded in "variable."
	 */
	struct pool_instance_sized
		: pool_instance
	{
	private:
		std::size_t _variable;
	public:
		template <typename ... Args>
			pool_instance_sized(
				gsl::not_null<std::shared_ptr<IKVStore>> kvstore_
				, common::string_view name_
				, std::size_t fixed_
				, std::size_t variable_
				, Args && ... args_
			)
				: pool_instance(kvstore_, name_, fixed_ + variable_, std::forward<Args>(args_) ...)
				, _variable(variable_)
			{}
		std::size_t variable() const { return _variable; }
	};

	/* a view into a vector, limited to the first limit_ elements */
	template <typename T>
		struct limited_size_view
		{
		private:
			const T &_t;
			typename T::size_type _size;
		public:
			limited_size_view(const T &t_, typename T::size_type limit_)
				: _t(t_)
				, _size(std::min(limit_, _t.size()))
			{}
			auto begin() const { return _t.begin(); }
			auto end() const { return begin() + _size; }
			auto cbegin() const { return _t.begin(); }
			auto cend() const { return begin() + _size; }
			auto rbegin() const { return rend() - _size; }
			auto rend() const { return _t.rend(); }
			auto crbegin() const { return crend() - _size; }
			auto crend() const { return _t.crend(); }
			auto size() const { return _size; }
		};

// The fixture for testing class Foo.
class KVStore_test
	: public ::testing::Test
{
	static bool numa_notified;
public:
	using IKVStore = component::IKVStore;
	using IKVStore_factory = component::IKVStore_factory;
	using string_view = common::string_view;
	using kv_t = std::tuple<std::string, std::string>;
protected:
	static const std::string test1_pool;
	static const std::string iterator_test_pool;
	static const std::string timestamp_test_pool;
	static const std::string single_key;
	static const std::string missing_key;
	static constexpr std::size_t single_value_size = MiB(8);
	static const std::string single_value_updated_different_size;
	static const std::string single_value_updated3;

	static constexpr std::size_t single_count = 1U;
	static constexpr unsigned get_expand = 2;
	/* AVL allocator expects 128MiB region for each slab */
	static constexpr std::size_t avl_minimum = MiB(320);

	/* More testing of table splits, at a performance cost */
	static constexpr std::size_t estimated_object_count_small = 1;
	static constexpr char hello[] = "Hello world!";

	// Objects declared here can be used by all tests in the test case
	gsl::not_null<std::shared_ptr<IKVStore>> _kvstore;
	/*
	 * The number of elements we would like for "many" operations".
	 * May be reduced to what the pool can handle
	 */
	std::size_t many_count_target_proposed;
	const std::vector<kv_t> kvv;

	static constexpr std::size_t many_key_length = 8;
	static constexpr std::size_t many_value_length = 16;

	const std::string single_value;
	std::string single_value_updated_same_size;

	KVStore_test()
		: _kvstore(make_kvstore(store, cs, mc))
		, many_count_target_proposed(many_count_proposed)
		, kvv(populate_many(many_count_target_proposed, many_key_length, many_value_length))
		, single_value(hello + std::string(single_value_size - (strlen(hello)), '\0'))
		, single_value_updated_same_size("Jello world!")
	{
		assert(single_value.size() == single_value_size);
		if ( ! mc.count(+IKVStore_factory::k_numa_nodes) == 0 && ! numa_notified )
		{
			std::cerr << "If using mmap memory. Run with --numa-nodes==<numa_nodes> to allocate from numa nodes\n";
			numa_notified = true;
		}
	}

	template <typename ... Args>
		auto open_pool(string_view pool_name_, Args && ... args_) -> pool_opened
		{
			return pool_opened(_kvstore, _kvstore->open_pool(std::string(pool_name_), std::forward<Args>(args_) ...));
		}

	auto create_pool(common::string_view name_)
	{
		return pool_instance(_kvstore, name_);
	}

	template <typename ... Args>
		auto create_pool(common::string_view name_, std::size_t size_, Args && ... args_)
		{
			return pool_instance(_kvstore, name_, std::max(size_, avl_minimum), std::forward<Args>(args_) ...);
		}

	template <typename ... Args>
		auto create_pool_sized(
			common::string_view name_
			, std::size_t fixed
			, std::size_t variable
			, Args & ... args_
		) -> pool_instance_sized
		{
			fixed = std::max(fixed, avl_minimum);
			auto p = pool_instance_sized(_kvstore, name_, fixed, variable, args_ ...);
			while ( p.handle() == IKVStore::POOL_ERROR && 0 != variable )
			{
				variable = variable / 2;
				p = std::move(pool_instance_sized(_kvstore, name_, fixed, variable, args_ ...));
			}
			return p;
		}
	/*
	 * Make a pool when the pool may
	 * need to share itself, as with a lock
	 */
	template <typename ... Args>
		auto create_pool_shared(common::string_view name_, std::size_t size_, Args && ... args_)
		{
			return std::make_shared<pool_instance>(_kvstore, name_, std::max(size_, avl_minimum), std::forward<Args>(args_) ...);
		}

	template <typename ... Args>
		auto create_pool_sized_shared(
			common::string_view name_
			, std::size_t fixed
			, std::size_t variable
			, Args & ... args
		)
		{
			fixed = std::max(fixed, avl_minimum);
			auto sp = std::make_shared<pool_instance_sized>(_kvstore, name_, fixed, variable, args ...);
			while ( sp->handle() == IKVStore::POOL_ERROR && 0 != variable )
			{
				variable = variable / 2;
				sp = std::make_shared<pool_instance_sized>(_kvstore, name_, fixed, variable, args ...);
			}
			return sp;
		}

		constexpr auto mc_count_to_size(std::size_t count_)
		{
			/* count of elements multiplied
			 *  - by 64U for bucket size,
			 *  - by 3U to account for 40% table density at expansion (typical),
			 *  - by 2U to account for worst-case due to doubling strategy for increasing bucket array size
			 * required size multiplied
			 *  - by 8U to account for current AVL_LB allocator alignment requirements
			 */
			return count_ * 64U * 3U * 2U * 8U;
		}

		constexpr auto mc_size_to_count(std::size_t size_)
		{
			return size_ / (64U * 3U * 2U * 8U);
		}

		constexpr auto sv_count_to_size(std::size_t count_)
		{
			return 4U * count_ * 8U;
		}
private:

	static std::vector<kv_t> populate_many(std::size_t mct, std::size_t mkl, std::size_t mvl)
	{
		std::vector<kv_t> r;
		std::mt19937_64 r0{};
		for ( auto i = std::size_t(); i != mct; ++i )
		{
			auto key = common::format("{:x}", r0());
			key.resize(mkl, '.');
			auto value = std::to_string(i);
			value.resize(mvl, '.');
			r.emplace_back(key, value);
		}
		return r;
	}
};

constexpr std::size_t KVStore_test::estimated_object_count_small;
constexpr std::size_t KVStore_test::many_key_length;
constexpr std::size_t KVStore_test::many_value_length;
constexpr std::size_t KVStore_test::single_count;
constexpr unsigned KVStore_test::get_expand;
constexpr std::size_t KVStore_test::avl_minimum;
constexpr std::size_t KVStore_test::single_value_size;

bool KVStore_test::numa_notified = false;
const std::string KVStore_test::test1_pool = "test1.pool";
const std::string KVStore_test::timestamp_test_pool = "timestamp-test.pool";
const std::string KVStore_test::iterator_test_pool = "iterator-test.pool";

/* Keys 23-byte or fewer are stored inline. Provide one longer to force allocation */
const std::string KVStore_test::single_key = "MySingleKeyLongEnoughToForceAllocation";
const std::string KVStore_test::missing_key = "KeyNeverInserted";
const std::string KVStore_test::single_value_updated_different_size = "Hello world!";
const std::string KVStore_test::single_value_updated3 = "WeXYZ world!";

void put2(pool_opened & pool)
{
	std::string key = "MyKey";
	std::string key2 = "MyKey2";
	std::string value = "Hello world!";
	//  value.resize(value.length()+1); /* append \0 */
	value.resize(KB(8));

	pool.put(key, value.c_str(), value.length());
	pool.put(key2, value.c_str(), value.length());
}

void show_percent_used(pool_opened & pool)
{
	using IKVStore = component::IKVStore;
	std::vector<uint64_t> attr;
	auto r = pool.get_attribute(IKVStore::PERCENT_USED, attr, nullptr);
	EXPECT_EQ(cs->rc_percent_used(), r);
	if ( S_OK == r )
	{
		EXPECT_GE(100, attr[0]);
		PINF("Percent used %zu", attr[0]);
	}
}

/*
 * Following tests were copied from test1 in mapstore unit_test
 */

TEST_F(KVStore_test, OpenPool)
{
	auto pool = create_pool(test1_pool);
	ASSERT_LT(component::IKVStore::POOL_ERROR, pool.handle());
}

TEST_F(KVStore_test, BasicPut)
{
	auto pool = create_pool(test1_pool);
	ASSERT_LT(component::IKVStore::POOL_ERROR, pool.handle());
	put2(pool);
}

TEST_F(KVStore_test, BasicGet)
{
	auto pool = create_pool(test1_pool);
	ASSERT_LT(component::IKVStore::POOL_ERROR, pool.handle());
	put2(pool);
	std::string key = "MyKey";

	void * value = nullptr;
	size_t value_len = 0;
	pool.get(key, value, value_len);
	PINF("Value=(%.50s) %lu", static_cast<const char *>(value), value_len);

	ASSERT_NE(nullptr, value);
	ASSERT_EQ(KB(8), value_len);
	pool.free_memory(value);

	value = nullptr;
	value_len = 0;
	pool.get(key, value, value_len);
	PINF("Repeat Value=(%.50s) %lu", static_cast<const char *>(value), value_len);
	auto count = pool.count();
	PINF("Count = %ld", count);
	ASSERT_EQ(2, count);
	ASSERT_NE(nullptr, value);
	ASSERT_EQ(KB(8), value_len);
	pool.free_memory(value);
}

TEST_F(KVStore_test, BasicMap1)
{
	auto pool = create_pool(test1_pool);
	put2(pool);
	pool.map(
		[](const void * key,
	                 const size_t key_len,
	                 const void * value,
	                 const size_t value_len) -> int
		{
			FINF("key:({}) value({})"
				, string_view(static_cast<const char *>(key), key_len)
				, string_view(static_cast<const char *>(value), value_len)
			);
			return 0;
		}
	);
}

TEST_F(KVStore_test, ValueResize)
{
	auto pool = create_pool(test1_pool);
	put2(pool);
	pool.map(
		[](
			const void * key,
			size_t key_len,
			const void * value,
			size_t value_len) -> int
		{
			FINF("key:({}) value({}-{})"
				, string_view(static_cast<const char *>(key), key_len)
				, string_view(static_cast<const char *>(value), value_len)
				, value_len
			);
			return 0;
		}
	);

	ASSERT_OK(pool.resize_value("MyKey", KB(16), 8));

	pool.map(
		[](
			const void * key,
			size_t key_len,
			const void * value,
			size_t value_len) -> int
		{
			FINF("key:({}) value({}-{})"
				, string_view(static_cast<const char *>(key), key_len)
				, string_view(static_cast<const char *>(value), value_len)
				, value_len
			);
	                return 0;
		}
	);

}

TEST_F(KVStore_test, BasicRemove)
{
	auto pool = create_pool(test1_pool);
	put2(pool);
	pool.erase("MyKey");
}

TEST_F(KVStore_test, ClosePool1)
{
	auto pool = create_pool(test1_pool);
	EXPECT_EQ(S_OK, pool.close());
}

TEST_F(KVStore_test, ReopenPool)
{
	auto pool = create_pool(test1_pool);
	FLOG("re-opened pool: {}", reinterpret_cast<const void *>(pool.handle()));

	pool.close();
}

TEST_F(KVStore_test, Timestamps)
{
	auto pool = create_pool(timestamp_test_pool, MB(32));
	/* if timestamping is enabled */
	if(_kvstore->get_capability(IKVStore::Capability::WRITE_TIMESTAMPS)) {

		auto now = common::epoch_now();

		for(unsigned i=0;i<10;i++) {
			auto value = common::random_string(16);
			auto key = common::random_string(8);
			FLOG("adding key-value pair ({})", key);
			pool.put(key, value.c_str(), value.size());
			sleep(2);
		}

		pool.map(
			[](const void* key,
	                         const size_t key_len,
	                         const void* value,
	                         const size_t value_len,
	                         const common::tsc_time_t timestamp) -> bool
			{
				(void)value; // unused
				(void)value_len; // unused
				FLOG("Timestamped record: {} @ {}"
					, string_view(static_cast<const char *>(key), key_len)
					, timestamp.raw()
				);
				return true;
			}
			, 0, 0
		);

		PLOG("After 5 seconds");
		pool.map(
			[](const void* key,
	                         size_t key_len,
	                         const void* value,
	                         size_t value_len,
	                         common::tsc_time_t timestamp) -> bool
			{
				(void)value; // unused
				(void)value_len; // unused
				FLOG("After 5 Timestamped record: {} @ {}"
					, string_view(static_cast<const char *>(key), key_len)
					, timestamp.raw()
				);
				return true;
			}
			, now.add_seconds(5)
			, common::epoch_time_t{0,0}
		);
	}

	PLOG("Closing pool.");
	EXPECT_NE(IKVStore::POOL_ERROR, pool.handle());
	EXPECT_EQ(S_OK, pool.close());
}

/* original copied from mapstore */
TEST_F(KVStore_test, Timestamps2)
{
	auto pool = create_pool("timestamp-test.pool", MiB(32), IKVStore::FLAGS_CREATE_ONLY);

	/* if timestamping is enabled */
	if ( _kvstore->get_capability(IKVStore::Capability::WRITE_TIMESTAMPS) )
	{
		auto t0 = common::epoch_now();
		unsigned delay = 2;

		std::vector<std::string> keys;
		for ( unsigned i=0; i<10; ++i )
		{
			keys.push_back(common::random_string(8));
		}

		for ( unsigned i=0; i != keys.size(); ++i )
		{
			auto value = common::random_string(16);
			PLOG("adding key-value pair (%s)", keys[i].c_str());
			pool.put(keys[i], value.c_str(), value.size());
			sleep(delay);
		}

		std::size_t key_count = 0;

		/* All keys should be listed */
		pool.map(
			[&key_count](
				const void* key
				, const size_t key_len
				, const void* // value
				, const size_t // value_len
				, const common::tsc_time_t timestamp
			) -> bool
			{
				++key_count;
				PLOG("Timestamped record: %.*s @ %lu", int(key_len), static_cast<const char *>(key), timestamp.raw());
				return true;
			}
			, 0
			, 0
		);
		EXPECT_EQ(keys.size(), key_count);

		unsigned wait = 5;
		auto t0_plus_wait = t0;
		t0_plus_wait.add_seconds(wait);

		/* First two keys should not be listed */
		PLOG("After %u seconds", wait);
		key_count = 0;
		pool.map(
			[&key_count, wait](
				const void* key
				, const size_t key_len
				, const void* // value
				, const size_t // value_len
				, const common::tsc_time_t timestamp
			) -> bool
			{
				++key_count;
				std::ostringstream ts;
				ts << timestamp;
				PLOG("After %u Timestamped record: %.*s @ %s", wait
					, int(key_len), static_cast<const char *>(key), ts.str().c_str()
				);
				return true;
			}
			/* A time_point, expressed as seconds past epoch */
			, t0_plus_wait
			, 0
		);
		EXPECT_EQ(keys.size() - unsigned(std::ceil(double(wait)/double(delay))), key_count);

		auto t_before_swap = common::epoch_now();

		for ( unsigned i=0; i+1 < keys.size(); ++i )
		{
			auto value = common::random_string(16);
			auto key = common::random_string(8);
			PLOG("swapping keys (%s, %s)", keys[i].c_str(), keys[i+1].c_str());
			pool.swap_keys(keys[i], keys[i+1]);
			sleep(delay);
		}

		auto t_after_swap = common::epoch_now();
		t_after_swap.add_seconds(1); /* adjust for coarse granularity */

		PLOG(
			"Timestamps remaining from before swap [%lu .. %lu]"
			, t_before_swap.seconds()
			, t_after_swap.seconds()
		);
		key_count = 0;
		pool.map(
			[&key_count](
				const void* key
				, const size_t key_len
				, const void* // value
				, const size_t // value_len
				, const common::tsc_time_t timestamp
			) -> bool
			{
				++key_count;
				std::ostringstream ts;
				ts << timestamp;
				PLOG("Before swaps record: %.*s @ %s"
					, int(key_len), static_cast<const char *>(key), ts.str().c_str()
				);
				return true;
			}
			, 0
			, t_before_swap.sub_seconds(1) /* adjust for closed interval requirement */
		);
		EXPECT_EQ(cs->swap_updates_timestamp() ? 0 : keys.size(), key_count);

		key_count = 0;
		PLOG("Timestamps during swaps");
		pool.map(
			[&key_count](
				const void* key
				, const size_t key_len
				, const void* // value
				, const size_t // value_len
				, const common::tsc_time_t timestamp
			) -> bool
			{
				++key_count;
				std::ostringstream ts;
				ts << timestamp;
				PLOG("During swaps record: %.*s @ %s"
					, int(key_len), static_cast<const char *>(key), ts.str().c_str()
				);
				return true;
			}
			, t_before_swap
			, t_after_swap
		);
		EXPECT_EQ(cs->swap_updates_timestamp() ? keys.size() : 0, key_count);

		key_count = 0;
		PLOG("Timestamps after swaps");
		pool.map(
			[&key_count](
				const void* key
				, const size_t key_len
				, const void* // value
				, const size_t // value_len
				, const common::tsc_time_t timestamp
			) -> bool
			{
				++key_count;
				std::ostringstream ts;
				ts << timestamp;
				PLOG("After swaps record: %.*s @ %s"
					, int(key_len), static_cast<const char *>(key), ts.str().c_str()
				);
				return true;
			}
			, t_after_swap
			, 0
		);
		EXPECT_EQ(0, key_count);
	}

	PLOG("Closing pool.");
	EXPECT_EQ(S_OK, pool.close());
}

TEST_F(KVStore_test, Iterator)
{
	auto pool = create_pool(iterator_test_pool, MB(32));

	common::epoch_time_t now = 0;

	for(unsigned i=0;i<10;i++) {
	  auto value = common::random_string(16);
	  auto key = common::random_string(8);

	  if(i==5) { sleep(2); now = common::epoch_now(); }

	  FLOG("({}) adding key-value pair key({}) value({})", i, key, value);
	  pool.put(key, value.c_str(), value.size());
	}

	pool.map(
	              [](const void * key,
	                 const size_t key_len,
	                 const void * value,
	                 const size_t value_len) -> int
	              {
	                FINF("key:({} {}) value({})"
						, key
						, string_view(static_cast<const char *>(key), key_len)
						, string_view(static_cast<const char *>(value), value_len)
	                    , static_cast<const char *>(value));
	                return 0;
	              }
	              );

	PLOG("Iterating...");
	status_t rc;
	IKVStore::pool_reference_t ref;
	bool time_match;

	{
		auto iter = pool.open_iterator();
		while((rc = iter.deref(0, 0, ref, time_match, true)) == S_OK) {
			FLOG("iterator: key({}) value({}) {}"
				, string_view(static_cast<const char *>(ref.key), ref.key_len)
				, string_view(static_cast<const char *>(ref.value), ref.value_len)
				, ref.timestamp.seconds()
			);
		}
	}
	EXPECT_EQ(E_OUT_OF_BOUNDS, rc);

	{
		auto iter = pool.open_iterator();
		EXPECT_LT(0, now.seconds());
		while((rc = iter.deref(0, now, ref, time_match, true)) == S_OK) {
			FLOG("(time-constrained) iterator: key({}) value({}) {} (match={})"
				, string_view(static_cast<const char *>(ref.key), ref.key_len)
				, string_view(static_cast<const char *>(ref.value), ref.value_len)
				, ref.timestamp.seconds()
				, time_match ? "y":"n"
			);
		}

	}
	EXPECT_EQ(E_OUT_OF_BOUNDS, rc);

	PLOG("Disturbed iteration...");
	unsigned i=0;
	{
		auto iter = pool.open_iterator();
			while((rc = iter.deref(0, 0, ref, time_match, true)) == S_OK) {
				FLOG("iterator: key({}) value({}) {}"
					, string_view(static_cast<const char *>(ref.key), ref.key_len)
					, string_view(static_cast<const char *>(ref.value), ref.value_len)
					, ref.timestamp.seconds()
				);
				i++;
				if(i == 5) {
				/* disturb iteration */
				auto value = common::random_string(16);
				auto key = common::random_string(8);
				FLOG("adding key-value pair key({}) value({})", key, value);
				pool.put(key, value.c_str(), value.size());
			}
		}
		EXPECT_EQ(E_ITERATOR_DISTURBED, rc);
	}

	PLOG("Closing pool.");
	EXPECT_NE(IKVStore::POOL_ERROR, pool.handle());
	EXPECT_EQ(S_OK, pool.close());
}

TEST_F(KVStore_test, KeySwap)
{
	auto pool = create_pool("keyswap", MB(32));

	std::string left_key = "LeftKey";
	std::string right_key = "RightKey";
	std::string left_value = "This is left";
	std::string right_value = "This is right";

	ASSERT_OK(pool.put(left_key, left_value.c_str(), left_value.length()));
	ASSERT_OK(pool.put(right_key, right_value.c_str(), right_value.length()));

	ASSERT_OK(pool.swap_keys(left_key, right_key));

	iovec new_left{}, new_right{};
	pool.get(left_key, new_left.iov_base, new_left.iov_len);
	pool.get(right_key, new_right.iov_base, new_right.iov_len);

	FLOG("left: {}", string_view(static_cast<const char *>(new_left.iov_base), new_left.iov_len));
	FLOG("right: {}", string_view(static_cast<const char *>(new_right.iov_base), new_right.iov_len));
	EXPECT_EQ(0, strncmp(static_cast<const char *>(new_left.iov_base), right_value.c_str(), new_left.iov_len));
	EXPECT_EQ(0, strncmp(static_cast<const char *>(new_right.iov_base), left_value.c_str(), new_right.iov_len));
	pool.free_memory(new_right.iov_base);
	pool.free_memory(new_left.iov_base);

	EXPECT_EQ(S_OK, pool.close());
}

TEST_F(KVStore_test, AlignedLock)
{
	auto pool = create_pool_shared("alignedlock", MB(32));

	std::vector<size_t> alignments = {8,16,32,128,512,2048,4096};

	for(size_t alignment: alignments)
	{
		void * addr = nullptr;
		size_t value_len = 4096;

		{
			exclusive_lock l1(pool, "key", IKVStore::STORE_LOCK_WRITE, addr, value_len, alignment);
			EXPECT_EQ(S_OK_CREATED, l1.rc());
			EXPECT_EQ(true, check_aligned(addr, alignment));
			exclusive_lock l2(pool, "key", IKVStore::STORE_LOCK_WRITE, addr, value_len, alignment);
			EXPECT_EQ(E_LOCKED, l2.rc());
		}
		EXPECT_EQ(S_OK,pool->erase("key"));
	}

	EXPECT_EQ(S_OK, pool->close());
}

/*
 * Following tests were copied from test1 in hstore unit_test
 */

TEST_F(KVStore_test, CreatePool)
{
	auto pool_foo_name = "foo/" + pool_name();
	auto pool_foo = create_pool(pool_foo_name, 42, IKVStore::FLAGS_CREATE_ONLY, 42);
	{
		auto rc = pool_foo.close();
		EXPECT_EQ(S_OK, rc);
	}

	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size), IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);
	auto pool2 = open_pool(pool_name());
	{
		void *v0;
		size_t s = 4096;
		auto r0 = pool.allocate_memory(8, s, v0);
		EXPECT_EQ(S_OK, r0);
		void *v1;
		auto r1 = pool.allocate_memory(8, s, v1);
		EXPECT_EQ(S_OK, r1);
		EXPECT_NE(v0, v1);
	}

	EXPECT_EQ(S_OK, pool2.close());
	EXPECT_EQ(S_OK, pool.close());

	/* All open handles closed */
	EXPECT_EQ(IKVStore::E_POOL_NOT_FOUND, pool2.close());

	auto pool_copy = open_pool(pool_name());
	ASSERT_LT(IKVStore::POOL_ERROR, pool_copy.handle());
	{
		std::list<std::string> nl;
		auto rc = _kvstore->get_pool_names(nl);
		std::cerr << "pool_names {";
		for ( auto n : nl )
		{
			std::cerr << n << ",";
		}
		std::cerr << "}\n";
		EXPECT_EQ(S_OK, rc);
	}
	{
		EXPECT_EQ(S_OK, _kvstore->delete_pool(pool_foo_name));
	}
}

TEST_F(KVStore_test, BasicGet0)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size), IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);
	show_percent_used(pool);

	void * value = nullptr;
	size_t value_len = 0;

	EXPECT_EQ(IKVStore::E_KEY_NOT_FOUND, pool.get(single_key, value, value_len));
	pool.free_memory(value);
}

TEST_F(KVStore_test, BasicPutGetBig)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size)
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value.data(), single_value.length()));

	void * value = nullptr;
	size_t value_len = 0;
	{
		auto r = pool.get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);

		if ( S_OK == r )
		{
			FINF("Value=({}) {:x}", string_view(static_cast<char *>(value), value_len), value_len);
			EXPECT_EQ(single_value.size(), value_len);
			EXPECT_EQ(0, memcmp(single_value.data(), value, single_value.size()));
			pool.free_memory(value);
		}
	}
}

TEST_F(KVStore_test, BasicPutLocked)
{
	/* put a value.
	 * read-lock it
	 * try to replace (expecte LOCKED)
	 * try to resize (expecte LOCKED)
	 * get (expect value is unchanged)
	 */
	auto pool = create_pool_shared(
		pool_name()
		, sv_count_to_size(single_value_size) // fixed
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool->put(single_key, single_value.data(), single_value.length()));

	void *value0 = nullptr;
	std::size_t value0_len = 0;

	{
		size_t alignment = 0;
		shared_lock lk(pool, single_key, IKVStore::STORE_LOCK_READ, value0, value0_len, alignment);
		EXPECT_EQ(S_OK, lk.rc());
		if ( lk.rc() == S_OK )
		{
			EXPECT_EQ(0, memcmp(single_value.data(), value0, value0_len));
		}

		EXPECT_EQ(E_LOCKED, pool->put(single_key, single_value.data(), single_value.size()));

		EXPECT_EQ(cs->rc_resize_locked(), pool->resize_value(single_key, single_value.size(), 64)); /* mapstore: E_INVAL */

		{
			void * value = nullptr;
			size_t value_len = 0;
			auto r = pool->get(single_key, value, value_len);
			EXPECT_EQ(S_OK, r);
			if ( r == S_OK )
			{
				EXPECT_EQ(single_value.size(), value_len);
				EXPECT_EQ(0, memcmp(single_value.data(), value, value_len));
				pool->free_memory(value);
			}
		}

		EXPECT_EQ(S_OK, lk.unlock());
	}

	std::size_t small_size = 16; /* Small, but large enough to preserve all characters in "Hello world!" */

	/*
	 * resize value
	 * lock value
	 * get value (expect new size)
	 */

	/* resize to an inline size. */
	{
		auto r = pool->resize_value(single_key, small_size, 16);
		EXPECT_EQ(S_OK, r);
	}
	{
		void *v = nullptr;
		std::size_t v_len = 0;
		size_t alignment = 0;
		shared_lock lk(pool, single_key, IKVStore::STORE_LOCK_READ, v, v_len, alignment);
		EXPECT_EQ(S_OK, lk.rc());
		if ( lk.rc() == S_OK )
		{
			EXPECT_NE(nullptr, v);
			EXPECT_EQ(16, v_len);

			EXPECT_EQ(0, memcmp(single_value.data(), v, v_len));
#if 0
			/* alignment is just a suggestion, so we cannot insist on it, especiall for small-sized values */
			EXPECT_EQ(0, reinterpret_cast<std::size_t>(v) % 16);
#endif
			/* We cannot insist that v be unaligned to 256, but if it is aligned then the following test will verify nothing */
			EXPECT_NE(0, reinterpret_cast<std::size_t>(v) % 256);

			auto r = lk.unlock();
			EXPECT_EQ(S_OK, r);
		}
	}

	{
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool->get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( r == S_OK )
		{
			EXPECT_EQ(small_size, value_len);
			EXPECT_EQ(0, memcmp(single_value.data(), value, value_len));
			pool->free_memory(value);
		}
	}

	/*
	 * resize value go single_value_size/2
	 * lock value
	 * get value (expect new size)
	 */
	{
		auto r = pool->resize_value(single_key, single_value.size() / 2, 256);
		EXPECT_EQ(S_OK, r);
	}
	{
		void *v = nullptr;
		std::size_t v_len = 0;
		size_t alignment = 0;
		shared_lock lk(pool, single_key, IKVStore::STORE_LOCK_READ, v, v_len, alignment);
		EXPECT_EQ(S_OK, lk.rc());
		if ( lk.rc() == S_OK )
		{
			EXPECT_NE(nullptr, v);
			EXPECT_EQ(single_value.size() / 2, v_len);
			EXPECT_EQ(0, memcmp(single_value.data(), v, v_len));
			EXPECT_EQ(0, reinterpret_cast<std::size_t>(v) % 256);
#if 0
			/* We cannot insist that v be unaligned to 1024, but if it is aligned then the following 1024 test will verify nothing */
			EXPECT_NE(0, reinterpret_cast<std::size_t>(v) % 1024);
#endif
			auto r = lk.unlock();
			EXPECT_EQ(S_OK, r);
		}
	}

	{
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool->get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( r == S_OK )
		{
			EXPECT_EQ(single_value.size() / 2, value_len);
			EXPECT_EQ(0, memcmp(single_value.data(), value, value_len));
			pool->free_memory(value);
		}
	}

	/*
	 * resize value to single_value_size
	 * lock value
	 * get value (expect new size)
	 */
	{
		auto r = pool->resize_value(single_key, single_value.size(), 1024);
		EXPECT_EQ(S_OK, r);
	}
	{
		void *v = nullptr;
		std::size_t v_len = 0;
		size_t alignment = 0;
		shared_lock lk(pool, single_key, IKVStore::STORE_LOCK_READ, v, v_len, alignment);
		EXPECT_EQ(S_OK, lk.rc());
		if ( lk.rc() == S_OK )
		{
			EXPECT_NE(nullptr, v);
			EXPECT_EQ(single_value.size(), v_len);
			EXPECT_EQ(0, memcmp(single_value.data(), v, v_len));
			EXPECT_EQ(0, reinterpret_cast<std::size_t>(v) % 1024);
			auto r = lk.unlock();
			EXPECT_EQ(S_OK, r);
		}
	}

	{
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool->get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( r == S_OK )
		{
			EXPECT_EQ(single_value.size(), value_len);
			EXPECT_EQ(0, memcmp(single_value.data(), value, single_value.size()));
			pool->free_memory(value);
		}
	}

	/* Reqeusted change in behavior: length 0 is now a magic value which means
	 * "do not try to create an element for the key, if not found"
	 */
	{
		void *v = nullptr;
		size_t value_len = 0;
		size_t alignment = 0;
		shared_lock lk(pool, missing_key, IKVStore::STORE_LOCK_READ, v, value_len, alignment);
		EXPECT_EQ(IKVStore::E_KEY_NOT_FOUND, lk.rc());
	}

}

TEST_F(KVStore_test, BasicGet2)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size )
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value.data(), single_value.length()));

	void * value = nullptr;
	size_t value_len = 0;
	auto r = pool.get(single_key, value, value_len);
	EXPECT_EQ(S_OK, r);
	if ( S_OK == r )
	{
		FINF("Value=({}) {:x}", string_view(static_cast<char *>(value), value_len), value_len);
		EXPECT_EQ(single_value.size(), value_len);
		EXPECT_EQ(0, memcmp(single_value.data(), value, single_value.size()));
		pool.free_memory(value);
	}
}

/* hstore issue 41 specifies different implementations for same-size replace vs different-size replace. */
TEST_F(KVStore_test, BasicReplaceSameSize)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size)
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value.data(), single_value.length()));

	{
		single_value_updated_same_size.resize(single_value_size);
		EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_same_size.data(), single_value_updated_same_size.length()));
	}
	void * value = nullptr;
	size_t value_len = 0;
	auto r = pool.get(single_key, value, value_len);
	EXPECT_EQ(S_OK, r);
	if ( S_OK == r )
	{
		PINF("Value=(%.*s) %zu", static_cast<int>(value_len), static_cast<char *>(value), value_len);
		EXPECT_EQ(0, memcmp(single_value_updated_same_size.data(), value, single_value_updated_same_size.size()));
		pool.free_memory(value);
	}
}

TEST_F(KVStore_test, BasicReplaceDifferentSize)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size), IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value.data(), single_value.length()));

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	void * value = nullptr;
	size_t value_len = 0;
	auto r = pool.get(single_key, value, value_len);
	EXPECT_EQ(S_OK, r);
	if ( S_OK == r )
	{
		PINF("Value=(%.*s) %zu", static_cast<int>(value_len), static_cast<char *>(value), value_len);
		EXPECT_EQ(0, memcmp(single_value_updated_different_size.data(), value, single_value_updated_different_size.size()));
		pool.free_memory(value);
	}
}

std::size_t put_many(pool_instance &pool, const limited_size_view<std::vector<KVStore_test::kv_t>> &kvv)
{
	std::size_t many_count_actual = 0;

	for ( auto &kv : kvv )
	{
		const auto &key = std::get<0>(kv);
		const auto &value = std::get<1>(kv);
		void * old_value = nullptr;
		size_t old_value_len = 0;
		if ( S_OK == pool.get(key, old_value, old_value_len) )
		{
			pool.free_memory(old_value);
		}
		else
		{
			auto r = pool.put(key, value.data(), value.length());
			EXPECT_EQ(S_OK, r);
			if ( r == S_OK )
			{
				++many_count_actual;
			}
		}
	}
	return many_count_actual;
}

TEST_F(KVStore_test, PutMany)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value.size()) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());
	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));
	EXPECT_LE(many_count_actual, many_count_target);
	EXPECT_LE(many_count_target * 99 / 100, many_count_actual);
}

TEST_F(KVStore_test, BasicMap2)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());
	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	auto value_len_sum = std::size_t();
	pool.map(
		[&value_len_sum] (
				const void * // key
				, const size_t // key_len
				, const void * // value
				, const size_t value_len
			) -> int
			{
				value_len_sum += value_len;
				return 0;
			}
		);
	EXPECT_EQ(single_value_updated_different_size.length() + many_count_actual * many_value_length, value_len_sum);
}

TEST_F(KVStore_test, BasicMapKeys)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value.data(), single_value.length()));
	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	auto key_len_sum = std::size_t();
	pool.map_keys(
		[&key_len_sum](
				const std::string &key
		) -> int
		{
			key_len_sum += key.size();
			return 0;
		}
	);
	EXPECT_EQ(single_key.size() + many_count_actual * many_key_length, key_len_sum);
}

TEST_F(KVStore_test, Count1)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());
	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	/* count should reflect Put, PutMany */
	EXPECT_EQ(many_count_actual, pool.count());
}

TEST_F(KVStore_test, CountByBucket)
{
/* buckets implemented, but not available at the kvstore interface */
#if 0
	std::uint64_t count = 0;
	pool->debug(2 /* COUNT_BY_BUCKET */, reinterpret_cast<std::uint64_t>(&count));
	/* should reflect Put, PutMany */
	EXPECT_EQ(single_count + many_count_actual, count);
#endif
}

TEST_F(KVStore_test, ClosePool2)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());
	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));
	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	show_percent_used(pool);

	pool.close();

	auto pool2 = open_pool(pool_name(), 0);
	ASSERT_LT(IKVStore::POOL_ERROR, pool2.handle());

	show_percent_used(pool2);

	{
		auto count = pool2.count();
		/* count should reflect Put, PutMany */
		EXPECT_EQ(single_count + many_count_actual, count);
	}

	{
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool2.get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( S_OK == r )
		{
			PINF("Value=(%.*s) %zu", static_cast<int>(value_len), static_cast<char *>(value), value_len);
			pool2.free_memory(value);
		}
	}
}

TEST_F(KVStore_test, BasicGetAttribute)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size)
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	{
		std::vector<uint64_t> attr;
		auto r = pool.get_attribute(IKVStore::VALUE_LEN, attr, &single_key);
		EXPECT_EQ(S_OK, r);
		if ( S_OK == r )
		{
			EXPECT_EQ(1, attr.size());
			if ( 1 == attr.size() )
			{
				EXPECT_EQ(attr[0], single_value_updated_different_size.length());
			}
		}
		r = pool.get_attribute(IKVStore::Attribute(0), attr, &single_key);
		EXPECT_EQ(cs->rc_unknown_attribute(), r);
		r = pool.get_attribute(IKVStore::VALUE_LEN, attr, nullptr);
		EXPECT_EQ(cs->rc_attribute_key_null_ptr(), r);
		r = pool.get_attribute(IKVStore::VALUE_LEN, attr, &missing_key);
		EXPECT_EQ(IKVStore::E_KEY_NOT_FOUND, r);
	}

	/* verify that item still exists */
	{ // 511
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool.get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( S_OK == r )
		{
			PINF("Value=(%.*s) %zu", static_cast<int>(value_len), static_cast<char *>(value), value_len);
			pool.free_memory(value);
		}
	}
}

TEST_F(KVStore_test, ResizeAttribute)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size)
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	{
		std::vector<uint64_t> attr0;
		auto r0 = pool.get_attribute(IKVStore::AUTO_HASHTABLE_EXPANSION, attr0, nullptr);
		EXPECT_EQ(cs->rc_attribute_hashtable_expansion(), r0);
		if ( r0 == S_OK )
		{
			ASSERT_EQ(1, attr0.size());
			EXPECT_EQ(1, attr0[0]);

			{
				std::vector<uint64_t> attr{0};
				EXPECT_EQ(S_OK, pool.set_attribute(IKVStore::AUTO_HASHTABLE_EXPANSION, attr, nullptr));
				EXPECT_EQ(1, attr.size());
			}

			{
				std::vector<uint64_t> attr;
				EXPECT_EQ(S_OK, pool.get_attribute(IKVStore::AUTO_HASHTABLE_EXPANSION, attr, nullptr));
				EXPECT_EQ(1, attr.size());
				if ( 1 == attr.size() )
				{
					EXPECT_EQ(0, attr[0]);
				}
			}

			{
				std::vector<uint64_t> attr{34};
				EXPECT_EQ(S_OK, pool.set_attribute(IKVStore::AUTO_HASHTABLE_EXPANSION, attr, nullptr));
				EXPECT_EQ(1, attr.size());
			}

			{
				std::vector<uint64_t> attr;
				EXPECT_EQ(S_OK, pool.get_attribute(IKVStore::AUTO_HASHTABLE_EXPANSION, attr, nullptr));
				EXPECT_EQ(1, attr.size());
				if ( 1 == attr.size() )
				{
					EXPECT_EQ(1, attr[0]);
				}
			}
		}
	}
}

TEST_F(KVStore_test, GetMany)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const limited_size_view lsv(kvv, many_count_target);
	const auto many_count_actual = put_many(pool, lsv);

	auto count = pool.count();
	/* count should reflect PutMany */
	EXPECT_EQ(single_count + many_count_actual, count);

	show_percent_used(pool);

	for ( auto i = 0; i != get_expand; ++i )
	{
		std::size_t mismatch_count = 0;
		for ( auto &kv : lsv )
		{
			const auto &key = std::get<0>(kv);
			const auto &ev = std::get<1>(kv);
			char value[many_value_length * 2];
			std::size_t value_len = many_value_length * 2;
			void *vp = value;
			auto r = pool.get(key, vp, value_len);
			EXPECT_EQ(S_OK, r);

			if ( r != S_OK ) break;

			EXPECT_EQ(vp, static_cast<void *>(value));
			EXPECT_EQ(ev.size(), value_len);
			mismatch_count += ( ev.size() != value_len || 0 != memcmp(ev.data(), value, ev.size()) );
		}
		EXPECT_EQ(lsv.size(), many_count_actual + mismatch_count);
	}
}

TEST_F(KVStore_test, GetManyPreallocated)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, many_count_target_proposed
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const limited_size_view lsv(kvv, many_count_target);
	const auto many_count_actual = put_many(pool, lsv);

	auto count = pool.count();
	/* count should reflect PutMany */
	EXPECT_EQ(single_count + many_count_actual, count);

	show_percent_used(pool);

	for ( auto i = 0; i != get_expand; ++i )
	{
		std::size_t mismatch_count = 0;
		for ( auto &kv : lsv )
		{
			const auto &key = std::get<0>(kv);
			const auto &ev = std::get<1>(kv);
			char value[many_value_length * 2];
			std::size_t value_len = many_value_length * 2;
			void *vp = value;
			auto r = pool.get(key, vp, value_len);
			EXPECT_EQ(S_OK, r);

			if ( r != S_OK ) break;

			EXPECT_EQ(vp, static_cast<void *>(value));
			EXPECT_EQ(ev.size(), value_len);
			mismatch_count += ( ev.size() != value_len || 0 != memcmp(ev.data(), value, ev.size()) );
		}
		EXPECT_EQ(lsv.size(), many_count_actual + mismatch_count);
	}
}

TEST_F(KVStore_test, GetManyAllocating)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const limited_size_view lsv(kvv, many_count_target);
	const auto many_count_actual = put_many(pool, lsv);

	for ( auto i = 0; i != get_expand; ++i )
	{
		std::size_t mismatch_count = 0;
		for ( auto &kv : lsv )
		{
			const auto &key = std::get<0>(kv);
			const auto &ev = std::get<1>(kv);
			void * value = nullptr;
			std::size_t value_len = 0;
			auto r = pool.get(key, value, value_len);
			EXPECT_EQ(S_OK, r);

			if ( r != S_OK ) break;

			EXPECT_EQ(ev.size(), value_len);
			mismatch_count += ( ev.size() != value_len || 0 != memcmp(ev.data(), value, ev.size()) );
			pool.free_memory(value);
		}
		EXPECT_EQ(lsv.size(), many_count_actual + mismatch_count);
	}
}

TEST_F(KVStore_test, GetDirectMany)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const limited_size_view lsv(kvv, many_count_target);
	const auto many_count_actual = put_many(pool, lsv);

	for ( auto i = 0; i != get_expand; ++i )
	{
		std::size_t mismatch_count = 0;
		for ( auto &kv : lsv )
		{
			const auto &key = std::get<0>(kv);
			const auto &ev = std::get<1>(kv);
			char value[many_value_length * 2];
			size_t value_len = many_value_length * 2;
			auto r = pool.get_direct(key, value, value_len);
			EXPECT_EQ(S_OK, r);

			if ( r != S_OK ) break;

			EXPECT_EQ(ev.size(), value_len);
			mismatch_count += ( ev.size() != value_len || 0 != memcmp(ev.data(), value, ev.size()) );
		}
		EXPECT_EQ(lsv.size(), many_count_actual + mismatch_count);
	}
}

TEST_F(KVStore_test, GetRegions)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);

	const auto many_count_target = mc_size_to_count(pool.variable());

	nupm::region_descriptor v;
	auto r = pool.get_regions(v);
	EXPECT_EQ(S_OK, r);
	if ( S_OK == r )
	{
		EXPECT_EQ(1, v.address_map().size());
		if ( 1 == v.address_map().size() )
		{
			PMAJOR("Pool region at %p len %zu", ::base(v.address_map().front()), ::size(v.address_map().front()));
			auto iov_base = reinterpret_cast<std::uintptr_t>(::base(v.address_map().front()));
			/* region no longer needs to be well-aligned, but heap_cc still aligns to a
			 * page boundary.
			 */
			EXPECT_EQ(iov_base & 0xfff, 0);
			EXPECT_GT(::size(v.address_map().front()), mc_count_to_size(many_count_target)/8U);
			EXPECT_LT(::size(v.address_map().front()), GiB(512));
		}
	}
}

TEST_F(KVStore_test, LockMany)
{
	auto pool = create_pool_sized_shared(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool->handle());

	const auto many_count_target = mc_size_to_count(pool->variable());
	const auto many_count_actual = put_many(*pool, limited_size_view(kvv, many_count_target));

	const std::size_t lock_count = std::min(many_count_actual, std::size_t(60));

	/* Lock for read (should succeed)
	 * Lock again for read (should succeed).
	 * Lock for write (should fail).
	 * Lock a non-existent key for write, creating the key).
	 *
	 * Undo the three successful locks.
	 */
	unsigned ct = 0;
	for ( auto &kv : kvv )
	{
		if ( ct == lock_count ) { break; }
		const auto &key = std::get<0>(kv);
		const auto &ev = std::get<1>(kv);
		const auto key_new = std::get<0>(kv) + "x";
		void *value0 = nullptr;
		std::size_t value0_len = 0;

		size_t alignment = 0;
		shared_lock r0(pool, key, IKVStore::STORE_LOCK_READ, value0, value0_len, alignment);
		EXPECT_EQ(S_OK, r0.rc());
		if ( S_OK == r0.rc() )
		{
			EXPECT_EQ(many_value_length, value0_len);
			EXPECT_EQ(0, memcmp(ev.data(), value0, ev.size()));
			EXPECT_EQ(0, memcmp(key.data(), r0.key(), key.size()));
		}
		void * value1 = nullptr;
		std::size_t value1_len = 0;
		alignment = 0;
		shared_lock r1(pool, key, IKVStore::STORE_LOCK_READ, value1, value1_len, alignment);
		EXPECT_EQ(S_OK, r1.rc());
		if ( S_OK == r1.rc() )
		{
			EXPECT_EQ(many_value_length, value1_len);
			EXPECT_EQ(0, memcmp(ev.data(), value1, ev.size()));
			EXPECT_EQ(0, memcmp(key.data(), r1.key(), key.size()));
		}
		/* Exclusive locking test. */

		void * value2 = nullptr;
		std::size_t value2_len = 0;
		alignment = 0;
		exclusive_lock r2(pool, key, IKVStore::STORE_LOCK_WRITE, value2, value2_len, alignment);
		/* Undocumented behavior: lock conflict returns E_LOCKED */
		EXPECT_EQ(E_LOCKED, r2.rc());

		void * value3 = nullptr;
		std::size_t value3_len = many_value_length;
		try
		{
			alignment = 0;
			exclusive_lock r3(pool, key_new, IKVStore::STORE_LOCK_WRITE, value3, value3_len, alignment);
			/* Used to return S_OK; no longer does so */
			EXPECT_EQ(S_OK_CREATED, r3.rc());
			if ( S_OK == r3.rc() )
			{
				EXPECT_EQ(many_value_length, value3_len);
				EXPECT_NE(nullptr, value3);
				EXPECT_EQ(0, memcmp(key_new.data(), r3.key(), key_new.size()));
			}
			++ct;
		}
		catch (const std::system_error &)
		{
			/* Lock a non-existent key will fail if the key locked for read needs to be moved. */
		}
	}
}

TEST_F(KVStore_test, Size2c)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	/* verify that item still exists */
	{ // 770
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool.get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( S_OK == r )
		{
			PINF("line %d Value=(%.*s) %zu", __LINE__, static_cast<int>(value_len), static_cast<char *>(value), value_len);
			pool.free_memory(value);
		}
	}
	auto count = pool.count();
	/* count should reflect put(single_key ...), put_many */
	EXPECT_EQ(single_count + many_count_actual, count);
}

/* Missing:
 *  - test of invalid parameters
 *    - offsets greater than sizeof data
 *    - non-existent key
 *    - invalid operations
 *  - test of crash recovery
 */
TEST_F(KVStore_test, BasicUpdate)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	/* verify that item still exists */
	{ // 796
		void * value = nullptr;
		size_t value_len = 0;
		auto r = pool.get(single_key, value, value_len);
		EXPECT_EQ(S_OK, r);
		if ( S_OK == r )
		{
			PINF("line %d Value=(%.*s) %zu", __LINE__, static_cast<int>(value_len), static_cast<char *>(value), value_len);
			pool.free_memory(value);
		}
	}
	{
		std::vector<std::unique_ptr<IKVStore::Operation>> v;
		v.emplace_back(std::make_unique<IKVStore::Operation_write>(0, 1, "W"));
		v.emplace_back(std::make_unique<IKVStore::Operation_write>(2, 3, "XYZ"));
		std::vector<IKVStore::Operation *> v2;
		std::transform(v.begin(), v.end(), std::back_inserter(v2), [] (const auto &i) { return i.get(); });
		auto r0 =
			pool.atomic_update(
				single_key
				, v2
			);
		EXPECT_EQ(cs->rc_atomic_update(), r0);
		if ( r0 == S_OK )
		{
			void * value = nullptr;
			size_t value_len = 0;
			auto r = pool.get(single_key, value, value_len);
			EXPECT_EQ(S_OK, r);
			if ( S_OK == r )
			{
				PINF("Value=(%.*s) %zu", static_cast<int>(value_len), static_cast<char *>(value), value_len);
				EXPECT_EQ(single_value_updated_different_size.size(), value_len);
				EXPECT_EQ(0, memcmp(single_value_updated3.data(), value, single_value_updated3.size()));
				pool.free_memory(value);
			}
		}
	}

	auto count = pool.count();
	EXPECT_EQ(single_count + many_count_actual, count);
}

TEST_F(KVStore_test, BasicErase)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	EXPECT_EQ(S_OK, pool.put(single_key, single_value_updated_different_size.data(), single_value_updated_different_size.length()));

	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	show_percent_used(pool);

	EXPECT_EQ(S_OK, pool.erase(single_key));

	EXPECT_EQ(many_count_actual, pool.count());
}

TEST_F(KVStore_test, EraseMany)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	const auto many_count_target = mc_size_to_count(pool.variable());
	const auto many_count_actual = put_many(pool, limited_size_view(kvv, many_count_target));

	auto erase_count = 0;
	for ( auto &kv : kvv )
	{
		const auto &key = std::get<0>(kv);
		auto r = pool.erase(key);
		if ( r == S_OK )
		{
			++erase_count;
		}
	}
	EXPECT_LE(many_count_actual, erase_count);
	EXPECT_EQ(0, pool.count());
}

TEST_F(KVStore_test, AllocDealloc4K)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	show_percent_used(pool);

	std::set<void *> allocations{};

	/* Many 4K-aligned allocations */
	for ( auto i = 0; i != 1000; ++i )
	{
		void *v = nullptr;
		size_t s = 4096;
		auto r = pool.allocate_memory(8, s, v);
		EXPECT_EQ(S_OK, r);
		EXPECT_EQ(0, allocations.count(v));
		allocations.insert(v);
		EXPECT_EQ(1, allocations.count(v));
	}

	/* close and reopen the pool */
	{
		auto rc = pool.close();
		EXPECT_EQ(S_OK, rc);

		auto pool2 = open_pool(pool_name());
		ASSERT_LT(0, int64_t(pool2.handle()));

		/* Many 4K-aligned frees */
		for ( auto v : allocations )
		{
			auto r = pool2.free_memory(v, 8);
			EXPECT_EQ(S_OK, r);
		}
	}
}

TEST_F(KVStore_test, AllocDealloc)
{
	auto pool = create_pool_sized(
		pool_name()
		, sv_count_to_size(single_value_size) /* fixed */
		, mc_count_to_size(many_count_target_proposed) /* variable */
		, IKVStore::FLAGS_CREATE_ONLY
		, estimated_object_count_small
	);
	ASSERT_LT(IKVStore::POOL_ERROR, pool.handle());

	show_percent_used(pool);

	status_t r = S_OK;
#if 0
	/* alignment is now a "hint", so 0 alignment is no longer an error */
	void *v = nullptr;
	/* In this function alignment is now a "hint", so 0 alignment is no longer an error */
	r = pool->allocate_memory(100, 0, v);
	/* exceedingly small alignment is also no longer an error */
	auto r = pool->allocate_memory(100, 1, v);
	EXPECT_EQ(IKVStore::E_BAD_ALIGNMENT, r); /* alignment less than sizeof(void *) */
#endif

	/* allocate various sizes */
	void *v128 = nullptr;
	void *v160 = nullptr;
	void *v704 = nullptr;
	void *v2048 = nullptr;
	void *v0 = nullptr;
	r = pool.allocate_memory(128, 32, v128);
	EXPECT_EQ(S_OK, r);
	EXPECT_NE(v128, nullptr);
	r = pool.allocate_memory(160, 8, v160);
	EXPECT_EQ(S_OK, r);
	EXPECT_NE(v160, nullptr);
	r = pool.allocate_memory(704, 64, v704);
	EXPECT_EQ(S_OK, r);
	EXPECT_NE(v704, nullptr);
	r = pool.allocate_memory(2048, 2048, v2048);
	EXPECT_EQ(S_OK, r);
	EXPECT_NE(v2048, nullptr);
	auto r0 = pool.allocate_memory(0, 8, v0);
	EXPECT_EQ(cs->rc_allocate_pool_memory_size_0(), r0); /* mapstore fails */
	if ( r0 == S_OK )
	{
		EXPECT_NE(v0, nullptr);
	}
	else
	{
		EXPECT_EQ(v0, nullptr);
	}

	/* close and reopen the pool */
	{
		auto rc = pool.close();
		EXPECT_EQ(S_OK, rc);

		auto pool2 = open_pool(pool_name());
		ASSERT_LT(IKVStore::POOL_ERROR, pool2.handle());

		if ( r0 == S_OK && v0 ) { EXPECT_EQ(S_OK, pool2.free_memory(v0, 0)); }
		if ( v2048 ) { EXPECT_EQ(S_OK, pool2.free_memory(v2048, 2048)); }
		if ( v704 ) { EXPECT_EQ(S_OK, pool2.free_memory(v704, 704)); }
		if ( v160 ) { EXPECT_EQ(S_OK, pool2.free_memory(v160, 160)); }
		if ( v128 ) { EXPECT_EQ(S_OK, pool2.free_memory(v128, 128)); }
#if 0
		/* Removed: unless the free logic validates arguments, might
		 * corrupt memory rather than failing.
		 */
		if ( v128 ) { EXPECT_EQ(E_INVALID_ARG, pool2.free_memory(v128, 128)); } /* not found */
#endif
	}
}

TEST_F(KVStore_test, DeletePool)
{
	auto pool = create_pool(
		pool_name()
		, sv_count_to_size(single_value_size)
		, IKVStore::FLAGS_CREATE_ONLY, estimated_object_count_small
	);

	show_percent_used(pool);

	pool.close();
	_kvstore->delete_pool(pool_name());
}

TEST_F(KVStore_test, OutOfMemory)
{
	auto pool = create_pool("oom", MiB(32), IKVStore::FLAGS_CREATE_ONLY);

	{
		status_t rc = S_OK;
		while ( rc == S_OK )
		{
			void *p = 0;
			rc = pool.allocate_memory(MiB(1), 8, p);
			(void) p;
		}
		EXPECT_EQ(cs->rc_out_of_memory(), rc); /* IMVStore::E_TOO_LARGE or E_INVAL */
	}
	ASSERT_EQ(S_OK, pool.close());
}

TEST_F(KVStore_test, NumaMask)
{
	auto pool = create_pool("numaMask", MiB(1));

	auto it = mc.find(+IKVStore_factory::k_numa_nodes);
	if ( it != mc.end() && it->second.size() != 0 )
	{
		std::vector<uint64_t> numa_mask_result;
		auto r = pool.get_attribute(component::IKVStore::NUMA_MASK, numa_mask_result);
		if ( r == S_OK )
		{
			EXPECT_EQ(1, numa_mask_result.size());
			if ( 1 == numa_mask_result.size() )
			{
				auto numa_mask = numa_mask_result[0];
				/* a trivial mask (1 node, less than 3 chars) should yield exactly one 1 bit */
				if ( it->second.size() < 3 )
		        {
					EXPECT_NE(0, numa_mask);
					EXPECT_EQ(0, (numa_mask & (numa_mask-1)));
				}
				std::cout << "Numa node mask is " << std::hex << numa_mask << "\n";
			}
		}
	}
}

} // namespace

namespace c_json = common::json;
using json = c_json::serializer<c_json::dummy_writer>;
json::array devdax_location
	( json::object
		( json::member("path", "/dev/dax0.0")
		, json::member("addr", 0x9000000000)
		)
	);

auto fsdax_location =
	json::array
	( json::object
		( json::member("path", "/mnt/pmem1")
		, json::member("addr", 0x9000000000)
		)
	);

int main(int argc, char **argv)
{
	::testing::InitGoogleTest(&argc, argv);

	namespace po = boost::program_options;

	po::options_description            desc("Options");
	po::positional_options_description g_pos; /* no positional options */

	desc.add_options()
		("help,h", "Show help")
		("numa-nodes", po::value<std::string>(), "Numa node specification (mapstore only)")
		("store", po::value<std::string>()->default_value("mapstore"), "Store type to test: e.g., hstore, hstore-cc, mapstore")
		("daxconfig"
			, po::value<std::string>()->default_value(
				json::array(
					json::object(
						json::member("path", "/dev/dax0.0")
						, json::member("addr", 0x9000000000)
					)
		        ).str()
			)
			, "dax configuration (hstore* only), in JSON. default [{\"path\": \"/dev/dax0.0\", \"addr\": \"0x9000000000\"}]"
		)
		("mm-plugin-path", po::value<std::string>(), "path to memory manager plugin")
		("debug", po::value<std::string>(), "store debug level")
		("many-count", po::value<std::size_t>()->default_value(2000000), "number of KV pairs to test for \"small KV pair, large quantitiy\" tests")
		;

	po::variables_map vm;
	po::store(po::command_line_parser(argc, argv).options(desc).positional(g_pos).run(), vm);

	if (vm.count("help") > 0)
	{
	    std::cout << desc;
	  	return -1;
	}

	store = vm["store"].as<std::string>();
	using IKVStore_factory = component::IKVStore_factory;
	mc = IKVStore_factory::map_create
		{
			{+IKVStore_factory::k_name, "numa0"},
			{+IKVStore_factory::k_dax_config, vm["daxconfig"].as<std::string>()}
		};
	if ( vm.count("numa-nodes") )
	{
		mc.insert( {+IKVStore_factory::k_numa_nodes, vm["numa-nodes"].as<std::string>()} );
	}
	if ( vm.count("mm-plugin-path") )
	{
        mc.insert( {+IKVStore_factory::k_mm_plugin_path, vm["mm-plugin-path"].as<std::string>()} );
	}
	if ( vm.count("debug") )
	{
        mc.insert( {+IKVStore_factory::k_debug, vm["debug"].as<std::string>()} );
	}
	many_count_proposed = vm["many-count"].as<std::size_t>();

	cs = locate_custom_store(store);
	auto r = RUN_ALL_TESTS();

	return r;
}

max_stars_repo_name: IBM/artemis

max_issues_repo_name: IBM/artemis

max_forks_repo_name: IBM/artemis


text:
#include "src/libs/include/helpers.h"

#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <QDebug>
#include <QApplication>
#include <QRegularExpression>
#include <boost/algorithm/string.hpp>

Helpers::Helpers() {

}

std::string Helpers::buildTitle(const std::string &input) {
    std::string title = Helpers::split(input, '\n').at(0);
    boost::trim(title);
    if (input.empty() || title.empty() || (title.length() == 1 && title.at(0) == ' '))
        return "untitled";

    if (title.at(0) != '#') {
        return title;
    } else {
        title = title.substr(1, title.length());
        boost::trim(title);
        return title;
    }
}

std::vector<std::string> Helpers::split(const std::string &str, char delimiter) {
    std::vector<std::string> tokens;
    std::string token;
    std::istringstream tokenStream(str);

    while (std::getline(tokenStream, token, delimiter))
       tokens.push_back(token);

    return tokens;
}

QString Helpers::getWordCount(QString &content) {
    int wordCount = content.split(QRegExp("(\\s|\\n|\\r)+"), QString::SkipEmptyParts).count();
    return QString::number(wordCount);
}

std::string Helpers::getFileContent(std::string path) {
    try {
        std::ifstream ifs(path);
        std::string output((std::istreambuf_iterator<char>(ifs)),
                            (std::istreambuf_iterator<char>()));
        ifs.close();
        return output;
    } catch(const std::exception& e) { }

    return "";
}

std::vector<std::string> Helpers::split(std::string &str, char delimiter) {
    std::vector<std::string> tokens;
    std::string token;
    std::istringstream tokenStream(str);

    while (std::getline(tokenStream, token, delimiter))
       tokens.push_back(token);

    return tokens;
}

bool Helpers::checkListItem(QString &line) {
    QRegularExpression regex1("^\\s*\\*\\s");
    QRegularExpression regex2("^\\s*\\d*\\.\\s");
    return (regex1.match(line).hasMatch() || regex2.match(line).hasMatch());
}

bool Helpers::checkUnorderedListItem(QString &line) {
    QRegularExpression regex("^\\s*\\*\\s");
    return (regex.match(line).hasMatch());
}

bool Helpers::checkEmptyListItem(QString &line) {
    QRegularExpression regex1("^\\s*\\*([[:blank:]]){1,}$");
    QRegularExpression regex2("^\\s*\\d*\\.([[:blank:]]){1,}$");
    return (regex1.match(line).hasMatch() || regex2.match(line).hasMatch());
}

int Helpers::getSpaceCount(QString &line) {
    int count = 0;
    for (auto& c : line) {
        if (c == ' ') {
            count += 1;
        } else {
            break;
        }
    }
    return count;
}

QString Helpers::getNextNumber(QString &line) {
    QString number = "";
    QString trimmedLine = line.trimmed();
    for (auto& c : trimmedLine) {
        if (!c.isDigit())
            break;
        number += c;
    }
    if (number != "") {
        try {
            int newNumber = number.toInt() + 1;
            return QString::number(newNumber);
        } catch (std::exception &ex) {
            return "";
        }
    } else {
        return "";
    }
}

void Helpers::exportHTML(std::string &htmlSavePath, std::string &content) {
    std::ofstream out(htmlSavePath);
    std::string headerPath = "html/header.html";
    std::string footerPath = "html/footer.html";

#ifdef Q_OS_DARWIN
    headerPath = QString(QApplication::applicationDirPath() + "/../Resources/header.html").toStdString();
    footerPath = QString(QApplication::applicationDirPath() + "/../Resources/footer.html").toStdString();
#endif

    std::string output = Helpers::getFileContent(headerPath);
    output += content;
    output += getFileContent(footerPath);

    out << output;
    out.close();
}

std::string Helpers::makeList(std::string type, std::string &selection) {
    std::vector<std::string> lines = Helpers::split(selection, '\n');

    if (type == "unordered") {
        for (auto &line : lines) {
            if (!line.empty())
                line = "* " + line;
        }
    } else if (type == "ordered") {
        for (size_t i = 0; i < lines.size(); i++)
            lines.at(i) = std::to_string(i + 1) + ". " + lines.at(i);
    }

    std::string text = "";

    for (auto &line : lines)
        text += line + "\n";

    return text;
}

bool Helpers::isAutocompletionChar(QChar &c) {
    return (c == '{' || c == '[' ||  c == '(' ||  c == '\'' ||  c == '"' ||  c == '*'
            ||  c == '`' ||  c == '~' || c == '}' || c == ']' ||  c == ')');
}


text:
#ifndef  BOOST_SERIALIZATION_COLLECTIONS_LOAD_IMP_HPP
#define BOOST_SERIALIZATION_COLLECTIONS_LOAD_IMP_HPP

// MS compatible compilers support #pragma once
#if defined(_MSC_VER)
# pragma once
#endif

#if defined(_MSC_VER) && (_MSC_VER <= 1020)
#  pragma warning (disable : 4786) // too long name, harmless warning
#endif

/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
// collections_load_imp.hpp: serialization for loading stl collections

// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com .
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

//  See http://www.boost.org for updates, documentation, and revision history.

// helper function templates for serialization of collections

#include <boost/assert.hpp>
#include <cstddef> // size_t
#include <boost/config.hpp> // msvc 6.0 needs this for warning suppression
#if defined(BOOST_NO_STDC_NAMESPACE)
namespace std{
    using ::size_t;
} // namespace std
#endif
#include <boost/detail/workaround.hpp>

#include <boost/archive/detail/basic_iarchive.hpp>
#include <boost/serialization/access.hpp>
#include <boost/serialization/nvp.hpp>
#include <boost/serialization/detail/stack_constructor.hpp>
#include <boost/serialization/collection_size_type.hpp>
#include <boost/serialization/item_version_type.hpp>
#include <boost/serialization/detail/is_default_constructible.hpp>
#include <boost/utility/enable_if.hpp>
#include <boost/move/utility_core.hpp>

namespace boost{
namespace serialization {
namespace stl {

//////////////////////////////////////////////////////////////////////
// implementation of serialization for STL containers
//

template<
    class Archive,
    class T
>
typename boost::enable_if<
    typename detail::is_default_constructible<
        typename T::value_type
    >,
    void
>::type
collection_load_impl(
    Archive & ar,
    T & t,
    collection_size_type count,
    item_version_type /*item_version*/
){
    t.resize(count);
    typename T::iterator hint;
    hint = t.begin();
    while(count-- > 0){
        ar >> boost::serialization::make_nvp("item", *hint++);
    }
}

template<
    class Archive,
    class T
>
typename boost::disable_if<
    typename detail::is_default_constructible<
        typename T::value_type
    >,
    void
>::type
collection_load_impl(
    Archive & ar,
    T & t,
    collection_size_type count,
    item_version_type item_version
){
    t.clear();
    while(count-- > 0){
        detail::stack_construct<Archive, typename T::value_type> u(ar, item_version);
        ar >> boost::serialization::make_nvp("item", u.reference());
        t.push_back(boost::move(u.reference()));
        ar.reset_object_address(& t.back() , & u.reference());
     }
}

} // namespace stl
} // namespace serialization
} // namespace boost

#endif //BOOST_SERIALIZATION_COLLECTIONS_LOAD_IMP_HPP

ext: hpp