text:
//------------------------------------------------------------------------------
/*
    This file is part of rippled: https://github.com/ripple/rippled
    Copyright (c) 2020 Ripple Labs Inc.

    Permission to use, copy, modify, and/or distribute this software for any
    purpose  with  or without fee is hereby granted, provided that the above
    copyright notice and this permission notice appear in all copies.

    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================

#include <ripple/app/ledger/LedgerMaster.h>
#include <ripple/app/ledger/LedgerToJson.h>
#include <ripple/beast/hash/hash_append.h>
#include <ripple/beast/utility/temp_dir.h>
#include <ripple/core/ConfigSections.h>
#include <ripple/nodestore/DatabaseShard.h>
#include <ripple/nodestore/DummyScheduler.h>
#include <ripple/nodestore/Manager.h>
#include <ripple/nodestore/impl/DecodedBlob.h>
#include <ripple/nodestore/impl/EncodedBlob.h>
#include <ripple/nodestore/impl/Shard.h>
#include <ripple/protocol/digest.h>
#include <boost/algorithm/hex.hpp>
#include <chrono>
#include <fstream>
#include <iostream>
#include <openssl/ripemd.h>
#include <test/jtx.h>
#include <test/nodestore/TestBase.h>

namespace ripple {
namespace NodeStore {

/** std::uniform_int_distribution is platform dependent.
 *  Unit test for deterministic shards is the following: it generates
 *  predictable accounts and transactions, packs them into ledgers
 *  and makes the shard. The hash of this shard should be equal to the
 *  given value. On different platforms (precisely, Linux and Mac)
 *  hashes of the resulting shard was different. It was unvestigated
 *  that the problem is in the class std::uniform_int_distribution
 *  which generates different pseudorandom sequences on different
 *  platforms, but we need predictable sequence.
 */
template <class IntType = int>
struct uniformIntDistribution
{
    using resultType = IntType;

    const resultType A, B;

    struct paramType
    {
        const resultType A, B;

        paramType(resultType aa, resultType bb) : A(aa), B(bb)
        {
        }
    };

    explicit uniformIntDistribution(
        const resultType a = 0,
        const resultType b = std::numeric_limits<resultType>::max())
        : A(a), B(b)
    {
    }

    explicit uniformIntDistribution(const paramType& params)
        : A(params.A), B(params.B)
    {
    }

    template <class Generator>
    resultType
    operator()(Generator& g) const
    {
        return rnd(g, A, B);
    }

    template <class Generator>
    resultType
    operator()(Generator& g, const paramType& params) const
    {
        return rnd(g, params.A, params.B);
    }

    resultType
    a() const
    {
        return A;
    }

    resultType
    b() const
    {
        return B;
    }

    resultType
    min() const
    {
        return A;
    }

    resultType
    max() const
    {
        return B;
    }

private:
    template <class Generator>
    resultType
    rnd(Generator& g, const resultType a, const resultType b) const
    {
        static_assert(
            std::is_convertible<typename Generator::result_type, resultType>::
                value,
            "Ups...");
        static_assert(
            Generator::min() == 0, "If non-zero we have handle the offset");
        const resultType range = b - a + 1;
        assert(Generator::max() >= range);  // Just for safety
        const resultType rejectLim = g.max() % range;
        resultType n;
        do
            n = g();
        while (n <= rejectLim);
        return (n % range) + a;
    }
};

template <class Engine, class Integral>
Integral
randInt(Engine& engine, Integral min, Integral max)
{
    assert(max > min);

    // This should have no state and constructing it should
    // be very cheap. If that turns out not to be the case
    // it could be hand-optimized.
    return uniformIntDistribution<Integral>(min, max)(engine);
}

template <class Engine, class Integral>
Integral
randInt(Engine& engine, Integral max)
{
    return randInt(engine, Integral(0), max);
}

// Tests DatabaseShard class
//
class DatabaseShard_test : public TestBase
{
    static constexpr std::uint32_t maxSizeGb = 10;
    static constexpr std::uint32_t ledgersPerShard = 256;
    static constexpr std::uint32_t earliestSeq = ledgersPerShard + 1;
    static constexpr std::uint32_t dataSizeMax = 4;
    static constexpr std::uint32_t iniAmount = 1000000;
    static constexpr std::uint32_t nTestShards = 4;
    static constexpr std::chrono::seconds shardStoreTimeout =
        std::chrono::seconds(60);
    test::SuiteJournal journal_;
    beast::temp_dir defNodeDir;

    struct TestData
    {
        /* ring used to generate pseudo-random sequence */
        beast::xor_shift_engine rng_;
        /* number of shards to generate */
        int nShards_;
        /* vector of accounts used to send test transactions */
        std::vector<test::jtx::Account> accounts_;
        /* nAccounts_[i] is the number of these accounts existed before i-th
         * ledger */
        std::vector<int> nAccounts_;
        /* payAccounts_[i][j] = {from, to} is the pair which consists of two
         * number of acoounts: source and destinations, which participate in
         * j-th payment on i-th ledger */
        std::vector<std::vector<std::pair<int, int>>> payAccounts_;
        /* xrpAmount_[i] is the amount for all payments on i-th ledger */
        std::vector<int> xrpAmount_;
        /* ledgers_[i] is the i-th ledger which contains the above described
         * accounts and payments */
        std::vector<std::shared_ptr<const Ledger>> ledgers_;

        TestData(
            std::uint64_t const seedValue,
            int dataSize = dataSizeMax,
            int nShards = 1)
            : rng_(seedValue), nShards_(nShards)
        {
            std::uint32_t n = 0;
            std::uint32_t nLedgers = ledgersPerShard * nShards;

            nAccounts_.reserve(nLedgers);
            payAccounts_.reserve(nLedgers);
            xrpAmount_.reserve(nLedgers);

            for (std::uint32_t i = 0; i < nLedgers; ++i)
            {
                int p;
                if (n >= 2)
                    p = randInt(rng_, 2 * dataSize);
                else
                    p = 0;

                std::vector<std::pair<int, int>> pay;
                pay.reserve(p);

                for (int j = 0; j < p; ++j)
                {
                    int from, to;
                    do
                    {
                        from = randInt(rng_, n - 1);
                        to = randInt(rng_, n - 1);
                    } while (from == to);

                    pay.push_back(std::make_pair(from, to));
                }

                n += !randInt(rng_, nLedgers / dataSize);

                if (n > accounts_.size())
                {
                    char str[9];
                    for (int j = 0; j < 8; ++j)
                        str[j] = 'a' + randInt(rng_, 'z' - 'a');
                    str[8] = 0;
                    accounts_.emplace_back(str);
                }

                nAccounts_.push_back(n);
                payAccounts_.push_back(std::move(pay));
                xrpAmount_.push_back(randInt(rng_, 90) + 10);
            }
        }

        bool
        isNewAccounts(int seq)
        {
            return nAccounts_[seq] > (seq ? nAccounts_[seq - 1] : 0);
        }

        void
        makeLedgerData(test::jtx::Env& env_, std::uint32_t seq)
        {
            using namespace test::jtx;

            if (isNewAccounts(seq))
                env_.fund(XRP(iniAmount), accounts_[nAccounts_[seq] - 1]);

            for (std::uint32_t i = 0; i < payAccounts_[seq].size(); ++i)
            {
                env_(
                    pay(accounts_[payAccounts_[seq][i].first],
                        accounts_[payAccounts_[seq][i].second],
                        XRP(xrpAmount_[seq])));
            }
        }

        bool
        makeLedgers(test::jtx::Env& env_)
        {
            for (std::uint32_t i = 3; i <= ledgersPerShard; ++i)
            {
                if (!env_.close())
                    return false;
                std::shared_ptr<const Ledger> ledger =
                    env_.app().getLedgerMaster().getClosedLedger();
                if (ledger->info().seq != i)
                    return false;
            }

            for (std::uint32_t i = 0; i < ledgersPerShard * nShards_; ++i)
            {
                makeLedgerData(env_, i);
                if (!env_.close())
                    return false;
                std::shared_ptr<const Ledger> ledger =
                    env_.app().getLedgerMaster().getClosedLedger();
                if (ledger->info().seq != i + ledgersPerShard + 1)
                    return false;
                ledgers_.push_back(ledger);
            }

            return true;
        }
    };

    void
    testLedgerData(
        TestData& data,
        std::shared_ptr<Ledger> ledger,
        std::uint32_t seq)
    {
        using namespace test::jtx;

        auto rootCount{0};
        auto accCount{0};
        auto sothCount{0};
        for (auto const& sles : ledger->sles)
        {
            if (sles->getType() == ltACCOUNT_ROOT)
            {
                int sq = sles->getFieldU32(sfSequence);
                int reqsq = -1;
                const auto id = sles->getAccountID(sfAccount);

                for (int i = 0; i < data.accounts_.size(); ++i)
                {
                    if (id == data.accounts_[i].id())
                    {
                        reqsq = ledgersPerShard + 1;
                        for (int j = 0; j <= seq; ++j)
                            if (data.nAccounts_[j] > i + 1 ||
                                (data.nAccounts_[j] == i + 1 &&
                                 !data.isNewAccounts(j)))
                            {
                                for (int k = 0; k < data.payAccounts_[j].size();
                                     ++k)
                                    if (data.payAccounts_[j][k].first == i)
                                        reqsq++;
                            }
                            else
                                reqsq++;
                        ++accCount;
                        break;
                    }
                }
                if (reqsq == -1)
                {
                    reqsq = data.nAccounts_[seq] + 1;
                    ++rootCount;
                }
                BEAST_EXPECT(sq == reqsq);
            }
            else
                ++sothCount;
        }
        BEAST_EXPECT(rootCount == 1);
        BEAST_EXPECT(accCount == data.nAccounts_[seq]);
        BEAST_EXPECT(sothCount == 3);

        auto iniCount{0};
        auto setCount{0};
        auto payCount{0};
        auto tothCount{0};
        for (auto const& tx : ledger->txs)
        {
            if (tx.first->getTxnType() == ttPAYMENT)
            {
                std::int64_t xrpAmount =
                    tx.first->getFieldAmount(sfAmount).xrp().decimalXRP();
                if (xrpAmount == iniAmount)
                    ++iniCount;
                else
                {
                    ++payCount;
                    BEAST_EXPECT(xrpAmount == data.xrpAmount_[seq]);
                }
            }
            else if (tx.first->getTxnType() == ttACCOUNT_SET)
                ++setCount;
            else
                ++tothCount;
        }
        int newacc = data.isNewAccounts(seq) ? 1 : 0;
        BEAST_EXPECT(iniCount == newacc);
        BEAST_EXPECT(setCount == newacc);
        BEAST_EXPECT(payCount == data.payAccounts_[seq].size());
        BEAST_EXPECT(tothCount == !seq);
    }

    bool
    saveLedger(
        Database& db,
        Ledger const& ledger,
        std::shared_ptr<Ledger const> const& next = {})
    {
        // Store header
        {
            Serializer s(128);
            s.add32(HashPrefix::ledgerMaster);
            addRaw(ledger.info(), s);
            db.store(
                hotLEDGER,
                std::move(s.modData()),
                ledger.info().hash,
                ledger.info().seq);
        }

        // Store the state map
        auto visitAcc = [&](SHAMapAbstractNode& node) {
            Serializer s;
            node.addRaw(s, snfPREFIX);
            db.store(
                node.getType() == SHAMapAbstractNode::TNType::tnINNER
                    ? hotUNKNOWN
                    : hotACCOUNT_NODE,
                std::move(s.modData()),
                node.getNodeHash().as_uint256(),
                ledger.info().seq);
            return true;
        };

        if (ledger.stateMap().getHash().isNonZero())
        {
            if (!ledger.stateMap().isValid())
                return false;
            if (next && next->info().parentHash == ledger.info().hash)
            {
                auto have = next->stateMap().snapShot(false);
                ledger.stateMap().snapShot(false)->visitDifferences(
                    &(*have), visitAcc);
            }
            else
                ledger.stateMap().snapShot(false)->visitNodes(visitAcc);
        }

        // Store the transaction map
        auto visitTx = [&](SHAMapAbstractNode& node) {
            Serializer s;
            node.addRaw(s, snfPREFIX);
            db.store(
                node.getType() == SHAMapAbstractNode::TNType::tnINNER
                    ? hotUNKNOWN
                    : hotTRANSACTION_NODE,
                std::move(s.modData()),
                node.getNodeHash().as_uint256(),
                ledger.info().seq);
            return true;
        };

        if (ledger.info().txHash.isNonZero())
        {
            if (!ledger.txMap().isValid())
                return false;
            ledger.txMap().snapShot(false)->visitNodes(visitTx);
        }

        return true;
    }

    void
    checkLedger(TestData& data, DatabaseShard& db, Ledger const& ledger)
    {
        auto fetched = db.fetchLedger(ledger.info().hash, ledger.info().seq);
        if (!BEAST_EXPECT(fetched))
            return;

        testLedgerData(data, fetched, ledger.info().seq - ledgersPerShard - 1);

        // verify the metadata/header info by serializing to json
        BEAST_EXPECT(
            getJson(
                LedgerFill{ledger, LedgerFill::full | LedgerFill::expand}) ==
            getJson(
                LedgerFill{*fetched, LedgerFill::full | LedgerFill::expand}));

        BEAST_EXPECT(
            getJson(
                LedgerFill{ledger, LedgerFill::full | LedgerFill::binary}) ==
            getJson(
                LedgerFill{*fetched, LedgerFill::full | LedgerFill::binary}));

        // walk shamap and validate each node
        auto fcompAcc = [&](SHAMapAbstractNode& node) -> bool {
            Serializer s;
            node.addRaw(s, snfPREFIX);
            auto nSrc{NodeObject::createObject(
                node.getType() == SHAMapAbstractNode::TNType::tnINNER
                    ? hotUNKNOWN
                    : hotACCOUNT_NODE,
                std::move(s.modData()),
                node.getNodeHash().as_uint256())};
            if (!BEAST_EXPECT(nSrc))
                return false;

            auto nDst =
                db.fetch(node.getNodeHash().as_uint256(), ledger.info().seq);
            if (!BEAST_EXPECT(nDst))
                return false;

            BEAST_EXPECT(isSame(nSrc, nDst));

            return true;
        };
        if (ledger.stateMap().getHash().isNonZero())
            ledger.stateMap().snapShot(false)->visitNodes(fcompAcc);

        auto fcompTx = [&](SHAMapAbstractNode& node) -> bool {
            Serializer s;
            node.addRaw(s, snfPREFIX);
            auto nSrc{NodeObject::createObject(
                node.getType() == SHAMapAbstractNode::TNType::tnINNER
                    ? hotUNKNOWN
                    : hotTRANSACTION_NODE,
                std::move(s.modData()),
                node.getNodeHash().as_uint256())};
            if (!BEAST_EXPECT(nSrc))
                return false;

            auto nDst =
                db.fetch(node.getNodeHash().as_uint256(), ledger.info().seq);
            if (!BEAST_EXPECT(nDst))
                return false;

            BEAST_EXPECT(isSame(nSrc, nDst));

            return true;
        };
        if (ledger.info().txHash.isNonZero())
            ledger.txMap().snapShot(false)->visitNodes(fcompTx);
    }

    std::string
    bitmask2Rangeset(std::uint64_t bitmask)
    {
        std::string set;
        if (!bitmask)
            return set;
        bool empty = true;

        for (std::uint32_t i = 0; i < 64 && bitmask; i++)
        {
            if (bitmask & (1ll << i))
            {
                if (!empty)
                    set += ",";
                set += std::to_string(i);
                empty = false;
            }
        }

        RangeSet<std::uint32_t> rs;
        from_string(rs, set);
        return to_string(rs);
    }

    std::unique_ptr<Config>
    testConfig(
        std::string const& testName,
        std::string const& backendType,
        std::string const& shardDir,
        std::string const& nodeDir = std::string())
    {
        using namespace test::jtx;

        if (testName != "")
        {
            std::string caseName =
                "DatabaseShard " + testName + " with backend " + backendType;
            testcase(caseName);
        }

        return envconfig([&](std::unique_ptr<Config> cfg) {
            cfg->overwrite(ConfigSection::shardDatabase(), "type", backendType);
            cfg->overwrite(ConfigSection::shardDatabase(), "path", shardDir);
            cfg->overwrite(
                ConfigSection::shardDatabase(),
                "max_size_gb",
                std::to_string(maxSizeGb));
            cfg->overwrite(
                ConfigSection::shardDatabase(),
                "ledgers_per_shard",
                std::to_string(ledgersPerShard));
            cfg->overwrite(
                ConfigSection::shardDatabase(),
                "earliest_seq",
                std::to_string(earliestSeq));
            cfg->overwrite(ConfigSection::nodeDatabase(), "type", backendType);
            cfg->overwrite(
                ConfigSection::nodeDatabase(),
                "max_size_gb",
                std::to_string(maxSizeGb));
            cfg->overwrite(
                ConfigSection::nodeDatabase(),
                "earliest_seq",
                std::to_string(earliestSeq));
            if (nodeDir.empty())
                cfg->overwrite(
                    ConfigSection::nodeDatabase(), "path", defNodeDir.path());
            else
                cfg->overwrite(ConfigSection::nodeDatabase(), "path", nodeDir);
            return cfg;
        });
    }

    std::optional<int>
    waitShard(
        DatabaseShard& db,
        int shardNumber,
        std::chrono::seconds timeout = shardStoreTimeout)
    {
        RangeSet<std::uint32_t> rs;
        auto start = std::chrono::system_clock::now();
        auto end = start + timeout;
        while (!from_string(rs, db.getCompleteShards()) ||
               !boost::icl::contains(rs, shardNumber))
        {
            if (!BEAST_EXPECT(std::chrono::system_clock::now() < end))
                return {};
            std::this_thread::yield();
        }

        return shardNumber;
    }

    std::optional<int>
    createShard(TestData& data, DatabaseShard& db, int maxShardNumber = 1)
    {
        int shardNumber = -1;

        for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
        {
            auto ind = db.prepareLedger((maxShardNumber + 1) * ledgersPerShard);
            if (!BEAST_EXPECT(ind != boost::none))
                return {};
            shardNumber = db.seqToShardIndex(*ind);
            int arrInd = *ind - ledgersPerShard - 1;
            BEAST_EXPECT(
                arrInd >= 0 && arrInd < maxShardNumber * ledgersPerShard);
            BEAST_EXPECT(saveLedger(db, *data.ledgers_[arrInd]));
            if (arrInd % ledgersPerShard == (ledgersPerShard - 1))
            {
                uint256 const finalKey_{0};
                Serializer s;
                s.add32(Shard::version);
                s.add32(db.firstLedgerSeq(shardNumber));
                s.add32(db.lastLedgerSeq(shardNumber));
                s.addRaw(data.ledgers_[arrInd]->info().hash.data(), 256 / 8);
                db.store(hotUNKNOWN, std::move(s.modData()), finalKey_, *ind);
            }
            db.setStored(data.ledgers_[arrInd]);
        }

        return waitShard(db, shardNumber);
    }

    void
    testStandalone(std::string const& backendType)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        Env env{*this, testConfig("standalone", backendType, shardDir.path())};
        DummyScheduler scheduler;
        RootStoppable parent("TestRootStoppable");

        std::unique_ptr<DatabaseShard> db =
            make_ShardStore(env.app(), parent, scheduler, 2, journal_);

        BEAST_EXPECT(db);
        BEAST_EXPECT(db->ledgersPerShard() == db->ledgersPerShardDefault);
        BEAST_EXPECT(db->init());
        BEAST_EXPECT(db->ledgersPerShard() == ledgersPerShard);
        BEAST_EXPECT(db->seqToShardIndex(ledgersPerShard + 1) == 1);
        BEAST_EXPECT(db->seqToShardIndex(2 * ledgersPerShard) == 1);
        BEAST_EXPECT(db->seqToShardIndex(2 * ledgersPerShard + 1) == 2);
        BEAST_EXPECT(
            db->earliestShardIndex() == (earliestSeq - 1) / ledgersPerShard);
        BEAST_EXPECT(db->firstLedgerSeq(1) == ledgersPerShard + 1);
        BEAST_EXPECT(db->lastLedgerSeq(1) == 2 * ledgersPerShard);
        BEAST_EXPECT(db->getRootDir().string() == shardDir.path());
    }

    void
    testCreateShard(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        Env env{*this, testConfig("createShard", backendType, shardDir.path())};
        DatabaseShard* db = env.app().getShardStore();
        BEAST_EXPECT(db);

        TestData data(seedValue);
        if (!BEAST_EXPECT(data.makeLedgers(env)))
            return;

        if (!createShard(data, *db, 1))
            return;

        for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
            checkLedger(data, *db, *data.ledgers_[i]);
    }

    void
    testReopenDatabase(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        {
            Env env{
                *this,
                testConfig("reopenDatabase", backendType, shardDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            TestData data(seedValue, 4, 2);
            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            for (std::uint32_t i = 0; i < 2; ++i)
                if (!createShard(data, *db, 2))
                    return;
        }
        {
            Env env{*this, testConfig("", backendType, shardDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            TestData data(seedValue, 4, 2);
            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            for (std::uint32_t i = 1; i <= 2; ++i)
                waitShard(*db, i);

            for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
                checkLedger(data, *db, *data.ledgers_[i]);
        }
    }

    void
    testGetCompleteShards(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        Env env{
            *this,
            testConfig("getCompleteShards", backendType, shardDir.path())};
        DatabaseShard* db = env.app().getShardStore();
        BEAST_EXPECT(db);

        TestData data(seedValue, 2, nTestShards);
        if (!BEAST_EXPECT(data.makeLedgers(env)))
            return;

        BEAST_EXPECT(db->getCompleteShards() == "");

        std::uint64_t bitMask = 0;

        for (std::uint32_t i = 0; i < nTestShards; ++i)
        {
            auto n = createShard(data, *db, nTestShards);
            if (!BEAST_EXPECT(n && *n >= 1 && *n <= nTestShards))
                return;
            bitMask |= 1ll << *n;
            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(bitMask));
        }
    }

    void
    testPrepareShard(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        Env env{
            *this, testConfig("prepareShard", backendType, shardDir.path())};
        DatabaseShard* db = env.app().getShardStore();
        BEAST_EXPECT(db);

        TestData data(seedValue, 1, nTestShards);
        if (!BEAST_EXPECT(data.makeLedgers(env)))
            return;

        std::uint64_t bitMask = 0;
        BEAST_EXPECT(db->getPreShards() == "");

        for (std::uint32_t i = 0; i < nTestShards * 2; ++i)
        {
            std::uint32_t n = randInt(data.rng_, nTestShards - 1) + 1;
            if (bitMask & (1ll << n))
            {
                db->removePreShard(n);
                bitMask &= ~(1ll << n);
            }
            else
            {
                db->prepareShard(n);
                bitMask |= 1ll << n;
            }
            BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
        }

        // test illegal cases
        // adding shards with too large number
        db->prepareShard(0);
        BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
        db->prepareShard(nTestShards + 1);
        BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
        db->prepareShard(nTestShards + 2);
        BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));

        // create shards which are not prepared for import
        BEAST_EXPECT(db->getCompleteShards() == "");

        std::uint64_t bitMask2 = 0;

        for (std::uint32_t i = 0; i < nTestShards; ++i)
        {
            auto n = createShard(data, *db, nTestShards);
            if (!BEAST_EXPECT(n && *n >= 1 && *n <= nTestShards))
                return;
            bitMask2 |= 1ll << *n;
            BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask));
            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(bitMask2));
            BEAST_EXPECT((bitMask & bitMask2) == 0);
            if ((bitMask | bitMask2) == ((1ll << nTestShards) - 1) << 1)
                break;
        }

        // try to create another shard
        BEAST_EXPECT(
            db->prepareLedger((nTestShards + 1) * ledgersPerShard) ==
            boost::none);
    }

    void
    testImportShard(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir importDir;
        TestData data(seedValue, 2);

        {
            Env env{
                *this,
                testConfig("importShard", backendType, importDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            if (!createShard(data, *db, 1))
                return;

            for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
                checkLedger(data, *db, *data.ledgers_[i]);

            data.ledgers_.clear();
        }

        boost::filesystem::path importPath(importDir.path());
        importPath /= "1";

        {
            beast::temp_dir shardDir;
            Env env{*this, testConfig("", backendType, shardDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            db->prepareShard(1);
            BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(2));
            if (!BEAST_EXPECT(db->importShard(1, importPath)))
                return;
            BEAST_EXPECT(db->getPreShards() == "");

            auto n = waitShard(*db, 1);
            if (!BEAST_EXPECT(n && *n == 1))
                return;

            for (std::uint32_t i = 0; i < ledgersPerShard; ++i)
                checkLedger(data, *db, *data.ledgers_[i]);
        }
    }

    void
    testCorruptedDatabase(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        {
            TestData data(seedValue, 4, 2);
            {
                Env env{
                    *this,
                    testConfig(
                        "corruptedDatabase", backendType, shardDir.path())};
                DatabaseShard* db = env.app().getShardStore();
                BEAST_EXPECT(db);

                if (!BEAST_EXPECT(data.makeLedgers(env)))
                    return;

                for (std::uint32_t i = 0; i < 2; ++i)
                    if (!BEAST_EXPECT(createShard(data, *db, 2)))
                        return;
            }

            boost::filesystem::path path = shardDir.path();
            path /= std::string("2");
            path /= backendType + ".dat";

            FILE* f = fopen(path.string().c_str(), "r+b");
            if (!BEAST_EXPECT(f))
                return;
            char buf[256];
            beast::rngfill(buf, sizeof(buf), data.rng_);
            BEAST_EXPECT(fwrite(buf, 1, 256, f) == 256);
            fclose(f);
        }
        {
            Env env{*this, testConfig("", backendType, shardDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            TestData data(seedValue, 4, 2);
            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            for (std::uint32_t i = 1; i <= 1; ++i)
                waitShard(*db, i);

            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x2));

            for (std::uint32_t i = 0; i < 1 * ledgersPerShard; ++i)
                checkLedger(data, *db, *data.ledgers_[i]);
        }
    }

    void
    testIllegalFinalKey(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        for (int i = 0; i < 5; ++i)
        {
            beast::temp_dir shardDir;
            {
                Env env{
                    *this,
                    testConfig(
                        (i == 0 ? "illegalFinalKey" : ""),
                        backendType,
                        shardDir.path())};
                DatabaseShard* db = env.app().getShardStore();
                BEAST_EXPECT(db);

                TestData data(seedValue + i, 2);
                if (!BEAST_EXPECT(data.makeLedgers(env)))
                    return;

                int shardNumber = -1;
                for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
                {
                    auto ind = db->prepareLedger(2 * ledgersPerShard);
                    if (!BEAST_EXPECT(ind != boost::none))
                        return;
                    shardNumber = db->seqToShardIndex(*ind);
                    int arrInd = *ind - ledgersPerShard - 1;
                    BEAST_EXPECT(arrInd >= 0 && arrInd < ledgersPerShard);
                    BEAST_EXPECT(saveLedger(*db, *data.ledgers_[arrInd]));
                    if (arrInd % ledgersPerShard == (ledgersPerShard - 1))
                    {
                        uint256 const finalKey_{0};
                        Serializer s;
                        s.add32(Shard::version + (i == 0));
                        s.add32(db->firstLedgerSeq(shardNumber) + (i == 1));
                        s.add32(db->lastLedgerSeq(shardNumber) - (i == 3));
                        s.addRaw(
                            data.ledgers_[arrInd - (i == 4)]
                                ->info()
                                .hash.data(),
                            256 / 8);
                        db->store(
                            hotUNKNOWN,
                            std::move(s.modData()),
                            finalKey_,
                            *ind);
                    }
                    db->setStored(data.ledgers_[arrInd]);
                }

                if (i == 2)
                    waitShard(*db, shardNumber);
                else
                {
                    boost::filesystem::path path(shardDir.path());
                    path /= "1";
                    boost::system::error_code ec;
                    auto start = std::chrono::system_clock::now();
                    auto end = start + shardStoreTimeout;
                    while (std::chrono::system_clock::now() < end &&
                           boost::filesystem::exists(path, ec))
                    {
                        std::this_thread::yield();
                    }
                }

                BEAST_EXPECT(
                    db->getCompleteShards() ==
                    bitmask2Rangeset(i == 2 ? 2 : 0));
            }

            {
                Env env{*this, testConfig("", backendType, shardDir.path())};
                DatabaseShard* db = env.app().getShardStore();
                BEAST_EXPECT(db);

                TestData data(seedValue + i, 2);
                if (!BEAST_EXPECT(data.makeLedgers(env)))
                    return;

                if (i == 2)
                    waitShard(*db, 1);

                BEAST_EXPECT(
                    db->getCompleteShards() ==
                    bitmask2Rangeset(i == 2 ? 2 : 0));

                if (i == 2)
                {
                    for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
                        checkLedger(data, *db, *data.ledgers_[j]);
                }
            }
        }
    }

    void
    testImport(std::string const& backendType, std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        beast::temp_dir shardDir;
        {
            beast::temp_dir nodeDir;
            Env env{
                *this,
                testConfig(
                    "import", backendType, shardDir.path(), nodeDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            Database& ndb = env.app().getNodeStore();
            BEAST_EXPECT(db);

            TestData data(seedValue, 4, 2);
            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
                BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i]));

            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0));
            db->import(ndb);
            for (std::uint32_t i = 1; i <= 2; ++i)
                waitShard(*db, i);
            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x6));
        }
        {
            Env env{*this, testConfig("", backendType, shardDir.path())};
            DatabaseShard* db = env.app().getShardStore();
            BEAST_EXPECT(db);

            TestData data(seedValue, 4, 2);
            if (!BEAST_EXPECT(data.makeLedgers(env)))
                return;

            for (std::uint32_t i = 1; i <= 2; ++i)
                waitShard(*db, i);

            BEAST_EXPECT(db->getCompleteShards() == bitmask2Rangeset(0x6));

            for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i)
                checkLedger(data, *db, *data.ledgers_[i]);
        }
    }

    std::string
    ripemd160File(std::string filename)
    {
        using beast::hash_append;
        std::ifstream input(filename, std::ios::in | std::ios::binary);
        char buf[4096];
        ripemd160_hasher h;

        while (input.read(buf, 4096), input.gcount() > 0)
            hash_append(h, buf, input.gcount());

        auto const binResult = static_cast<ripemd160_hasher::result_type>(h);
        const auto charDigest = binResult.data();
        std::string result;
        boost::algorithm::hex(
            charDigest,
            charDigest + sizeof(binResult),
            std::back_inserter(result));

        return result;
    }

    void
    testDeterministicShard(
        std::string const& backendType,
        std::uint64_t const seedValue)
    {
        using namespace test::jtx;

        std::string ripemd160Key("4CFA8985836B549EC99D2E9705707F488DC91E4E"),
            ripemd160Dat("8CC61F503C36339803F8C2FC652C1102DDB889F1");

        for (int i = 0; i < 2; i++)
        {
            beast::temp_dir shardDir;
            {
                Env env{
                    *this,
                    testConfig(
                        (i ? "" : "deterministicShard"),
                        backendType,
                        shardDir.path())};
                DatabaseShard* db = env.app().getShardStore();
                BEAST_EXPECT(db);

                TestData data(seedValue, 4);
                if (!BEAST_EXPECT(data.makeLedgers(env)))
                    return;

                if (createShard(data, *db) < 0)
                    return;
            }
            {
                Env env{*this, testConfig("", backendType, shardDir.path())};
                DatabaseShard* db = env.app().getShardStore();
                BEAST_EXPECT(db);

                TestData data(seedValue, 4);
                if (!BEAST_EXPECT(data.makeLedgers(env)))
                    return;

                waitShard(*db, 1);

                for (std::uint32_t j = 0; j < ledgersPerShard; ++j)
                    checkLedger(data, *db, *data.ledgers_[j]);
            }

            boost::filesystem::path path(shardDir.path());
            path /= "1";
            boost::filesystem::path keypath = path / (backendType + ".key");
            std::string key = ripemd160File(keypath.string());
            boost::filesystem::path datpath = path / (backendType + ".dat");
            std::string dat = ripemd160File(datpath.string());

            std::cerr << "Iteration " << i << ": RIPEMD160[" << backendType
                      << ".key] = " << key << std::endl;
            std::cerr << "Iteration " << i << ": RIPEMD160[" << backendType
                      << ".dat] = " << dat << std::endl;

            BEAST_EXPECT(key == ripemd160Key);
            BEAST_EXPECT(dat == ripemd160Dat);
        }
    }

    void
    testAll(std::string const& backendType)
    {
        std::uint64_t const seedValue = 51;
        testStandalone(backendType);
        testCreateShard(backendType, seedValue);
        testReopenDatabase(backendType, seedValue + 5);
        testGetCompleteShards(backendType, seedValue + 10);
        testPrepareShard(backendType, seedValue + 20);
        testImportShard(backendType, seedValue + 30);
        testCorruptedDatabase(backendType, seedValue + 40);
        testIllegalFinalKey(backendType, seedValue + 50);
        testImport(backendType, seedValue + 60);
        testDeterministicShard(backendType, seedValue + 70);
    }

public:
    DatabaseShard_test() : journal_("DatabaseShard_test", *this)
    {
    }

    void
    run() override
    {
        testAll("nudb");

#if RIPPLE_ROCKSDB_AVAILABLE
//      testAll ("rocksdb");
#endif

#if RIPPLE_ENABLE_SQLITE_BACKEND_TESTS
        testAll("sqlite");
#endif
    }
};

BEAST_DEFINE_TESTSUITE(DatabaseShard, NodeStore, ripple);

}  // namespace NodeStore
}  // namespace ripple

max_stars_repo_name: mayurbhandary/rippled

max_issues_repo_name: mayurbhandary/rippled

max_forks_repo_name: mayurbhandary/rippled


text:
//------------------------------------------------------------------------------
/// \file Invoke_tests.cpp
/// \author Ernest Yeung
/// \email  ernestyalumni@gmail.com
/// \ref https://en.cppreference.com/w/cpp/utility/functional/invoke
/// https://en.cppreference.com/w/cpp/types/is_invocable
//------------------------------------------------------------------------------

#include <boost/test/unit_test.hpp>
#include <type_traits>

using std::is_invocable;
using std::is_invocable_r;

BOOST_AUTO_TEST_SUITE(Cpp)
BOOST_AUTO_TEST_SUITE(Utilities)
BOOST_AUTO_TEST_SUITE(FunctionObjects)
BOOST_AUTO_TEST_SUITE(Invoke_tests)

BOOST_AUTO_TEST_SUITE_END() // Invoke_tests

BOOST_AUTO_TEST_SUITE(Invocable_tests)

auto func2(char) -> int (*)()
{
  return nullptr;
}

//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
BOOST_AUTO_TEST_CASE(IsInvocableDeterminesIfFunctionTypeCanBeInvoked)
{
  BOOST_TEST(is_invocable<int()>::value);
}

//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
BOOST_AUTO_TEST_CASE(
  IsInvocableDeterminesIfFunctionTypeCanBeInvokedToYieldReturnType)
{
  BOOST_TEST((is_invocable_r<int, int()>::value));
}

//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
BOOST_AUTO_TEST_CASE(
  IsInvocableDeterminesIfFunctionTypeCanBeInvokedWithArgumentTypes)
{
  BOOST_TEST((is_invocable_r<void, void(int), int>::value));
  BOOST_TEST((is_invocable_r<int(*)(), decltype(func2), char>::value));
}

BOOST_AUTO_TEST_SUITE_END() // Invocable_tests

BOOST_AUTO_TEST_SUITE_END() // FunctionObjects
BOOST_AUTO_TEST_SUITE_END() // Utilities
BOOST_AUTO_TEST_SUITE_END() // Cpp

ext: cpp

max_stars_repo_name: ernestyalumni/HrdwCCppCUDA

max_issues_repo_name: ernestyalumni/HrdwCCppCUDA

max_forks_repo_name: ernestyalumni/HrdwCCppCUDA


text:
/**
 * Non-metric Space Library
 *
 * Authors: Bilegsaikhan Naidan (https://github.com/bileg), Leonid Boytsov (http://boytsov.info).
 * With contributions from Lawrence Cayton (http://lcayton.com/) and others.
 *
 * For the complete list of contributors and further details see:
 * https://github.com/searchivarius/NonMetricSpaceLib
 *
 * Copyright (c) 2015
 *
 * This code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <cmath>
#include <memory>
#include <limits>
#include <string>
#include <sstream>
#include <vector>
#include <fstream>
#include <map>

#include <boost/program_options.hpp>
#include <cluster_util.h>

#include "init.h"
#include "global.h"
#include "utils.h"
#include "ztimer.h"
#include "experiments.h"
#include "tune.h"
#include "method/vptree.h"
#include "method/proj_vptree.h"
#include "method/perm_bin_vptree.h"
#include "logging.h"
#include "spacefactory.h"
#include "cluster_util.h"
#include "params_def.h"

#include "meta_analysis.h"
#include "params.h"

using namespace similarity;

#define CLUST_TYPE_PARAM_OPT "clustType,t"
const string CLUST_TYPE_PARAM_MSG="A type of cluster: " + CLUST_TYPE_CLARAN + "," + CLUST_TYPE_FIRMAL;
#define CLUST_QTY_PARAM_OPT "clustQty,c"
#define CLUST_QTY_PARAM_MSG "A # of clusters"
const IdTypeUnsign CLUST_QTY_PARAM_DEFAULT = 100;
#define IN_CLUST_SWAP_ATT_PARAM_OPT "swapAtt,W"
#define IN_CLUST_SWAP_ATT_PARAM_MSG "The number of in claster swap attempts (in order to find a better center)"
const IdTypeUnsign IN_CLUST_SWAP_ATT_PARAM_DEFAULT = CLARANS_SWAP_ATTEMPTS;
#define IN_CLUST_SAMPLE_QTY_PARAM_OPT "clustSampleQty,Q"
#define IN_CLUST_SAMPLE_QTY_PARAM_MSG "The number of sampled points inside the cluster to compute a cluster configuration cost"
const IdTypeUnsign IN_CLUST_SAMPLE_QTY_PARAM_DEFAULT = CLARANS_SAMPLE_QTY;
#define RAND_REST_QTY_PARAM_OPT "randRestartQty,R"
#define RAND_REST_QTY_PARAM_MSG "The number of random restarts"
const size_t RAND_REST_QTY_PARAM_DEFAULT = 5;
#define SEARCH_CLOSE_ITER_QTY_PARAM_OPT "searchCloseIterQty,I"
#define SEARCH_CLOSE_ITER_QTY_PARAM_MSG "A number of search iterations to find a point that is close to already selected centers"
const IdTypeUnsign SEARCH_CLOSE_ITER_QTY_PARAM_DEFAULT = 200;
#define DIST_SAMPLE_QTY_PARAM_OPT "distSampleQty,S"
#define DIST_SAMPLE_QTY_PARAM_MSG "A number of samples to determine the distribution of distances"
const IdTypeUnsign DIST_SAMPLE_QTY_PARAM_DEFAULT = SAMPLE_LIST_CLUST_DEFAULT_SAMPLE_QTY;
#define MAX_META_ITER_QTY_PARAM_OPT "maxMetaIterQty,M"
#define MAX_META_ITER_QTY_PARAM_MSG "A maximum number of meta iterations"
const IdTypeUnsign MAX_META_ITER_QTY_PARAM_DEFAULT = 10;
#define KEEP_FRAC_QTY_PARAM_OPT "keepFrac,F"
#define KEEP_FRAC_QTY_PARAM_MSG "Percentage of assigned points kept after a meta-iteration is finished"
const float KEEP_FRAC_QTY_PARAM_DEFAULT = 0.2;

using std::vector;
using std::map;
using std::make_pair;
using std::string;
using std::stringstream;

namespace po = boost::program_options;

static void Usage(const char *prog,
                  const po::options_description& desc) {
    std::cout << prog << std::endl
              << desc << std::endl;
}

template <typename dist_t>
void RunExper(
    bool                   PrintProgress,
    const string&          DistType,
    const string&          SpaceType,
    shared_ptr<AnyParams>  SpaceParams,
    const string&          DataFile,
    IdTypeUnsign           MaxNumData,
    const string&          ClustType,
    IdTypeUnsign           ClustQty,
    // reductive CLARANS parameters
    IdTypeUnsign           maxMetaIterQty,
    float                  keepFrac,
    // CLARANS & reductive CLARANS parameters
    IdTypeUnsign           inClusterSwapAttempts,
    IdTypeUnsign           inClusterSampleQty, // Number of random points to estimate if the swap was beneficial
    size_t                 randRestQty,
    // FIRMAL parameters
    size_t                 searchCloseIterQty, // A # of search iterations to find a point that is close to already selected centers
                                          // For good performance it should be in the order of sqrt(data.size())
    size_t                 sampleDistQty      // A # of sample to determine the distribution of distances)
) {

  unique_ptr<Space<dist_t>> space (SpaceFactoryRegistry<dist_t>::
                                   Instance().CreateSpace(SpaceType, *SpaceParams));

  ObjectVector              data;
  vector<string>            tmp;

  unique_ptr<DataFileInputState> inpState(space->ReadDataset(data, tmp, DataFile, MaxNumData));
  space->UpdateParamsFromFile(*inpState);

  LOG(LIB_INFO) << "Read: " << data.size() << " entries.";

  ObjectVector                                      vCenters;
  ObjectVector                                      vUnassigned;
  vector<shared_ptr<DistObjectPairVector<dist_t>>>  vClusterAssign;

  if (ClustType == CLUST_TYPE_CLARAN) {
    ClusterUtils<dist_t>::doCLARANS(PrintProgress, *space, data, ClustQty, vCenters, vClusterAssign,
                                    inClusterSwapAttempts, inClusterSampleQty, randRestQty);
  } else if (ClustType == CLUST_TYPE_REDUCT_CLARAN) {
    ClusterUtils<dist_t>::doReductiveCLARANS(PrintProgress, *space, data,
                                             maxMetaIterQty, keepFrac,
                                             ClustQty, vCenters, vClusterAssign, vUnassigned,
                                             inClusterSwapAttempts, inClusterSampleQty);
  } else if (ClustType == CLUST_TYPE_FIRMAL) {
    ClusterUtils<dist_t>::doFIRMAL(PrintProgress, *space, data, ClustQty, vCenters, vClusterAssign, vUnassigned,
                                   searchCloseIterQty, sampleDistQty);
  } else {
    throw runtime_error("Unsupported clustering type: " + ClustType);
  }

  LOG(LIB_INFO) << "The number of unassigned points: " << vUnassigned.size();

  ClusterUtils<dist_t>::printAndVerifyClusterStat(*space, vCenters, vClusterAssign, 1000);
}

void ParseCommandLineForClustering(int argc, char*argv[],
   bool&                   PrintProgress,
   string&                 LogFile,
   string&                 DistType,
   string&                 SpaceType,
   shared_ptr<AnyParams>&  SpaceParams,
   string&                 DataFile,
   unsigned&               MaxNumData,
   string&                 ClustType,
   IdTypeUnsign&           ClustQty,
// reductive CLARANS parameters
   IdTypeUnsign&           maxMetaIterQty,
   float&                  keepFrac,
// CLARANS parameters
   IdTypeUnsign&           inClusterSwapAttempts,
   IdTypeUnsign&           inClusterSampleQty, // Number of random points to estimate if the swap was beneficial
   IdTypeUnsign&           RandRestQty,
// FIRMAL parameters
   IdTypeUnsign&           SearchCloseIterQty,// A # of search iterations to find a point that is close to already selected centers
// For good performance it should be in the order of sqrt(data.size())
   IdTypeUnsign&           SampleDistQty      // A # of sample to determine the distribution of distances)

) {
  bool NoPrintProgress;

  po::options_description ProgOptDesc("Allowed options");
  ProgOptDesc.add_options()
      (HELP_PARAM_OPT,          HELP_PARAM_MSG)
      (NO_PROGRESS_PARAM_OPT,   po::bool_switch(&NoPrintProgress), NO_PROGRESS_PARAM_MSG)
      (SPACE_TYPE_PARAM_OPT,    po::value<string>(&SpaceType)->required(),                    SPACE_TYPE_PARAM_MSG)
      (DIST_TYPE_PARAM_OPT,     po::value<string>(&DistType)->default_value(DIST_TYPE_FLOAT), DIST_TYPE_PARAM_MSG)
      (DATA_FILE_PARAM_OPT,     po::value<string>(&DataFile)->required(),                     DATA_FILE_PARAM_MSG)
      (MAX_NUM_DATA_PARAM_OPT,  po::value<unsigned>(&MaxNumData)->default_value(MAX_NUM_DATA_PARAM_DEFAULT), MAX_NUM_DATA_PARAM_MSG)
      (LOG_FILE_PARAM_OPT,      po::value<string>(&LogFile)->default_value(LOG_FILE_PARAM_DEFAULT),          LOG_FILE_PARAM_MSG)
      (CLUST_TYPE_PARAM_OPT,    po::value<string>(&ClustType)->required(),      CLUST_TYPE_PARAM_MSG.c_str())
      (CLUST_QTY_PARAM_OPT,    po::value<IdTypeUnsign>(&ClustQty)->default_value(CLUST_QTY_PARAM_DEFAULT), CLUST_QTY_PARAM_MSG)
      (IN_CLUST_SWAP_ATT_PARAM_OPT, po::value<IdTypeUnsign>(&inClusterSwapAttempts)->default_value(IN_CLUST_SWAP_ATT_PARAM_DEFAULT), IN_CLUST_SWAP_ATT_PARAM_MSG)
      (IN_CLUST_SAMPLE_QTY_PARAM_OPT, po::value<IdTypeUnsign>(&inClusterSampleQty)->default_value(IN_CLUST_SAMPLE_QTY_PARAM_DEFAULT), IN_CLUST_SAMPLE_QTY_PARAM_MSG)
      (RAND_REST_QTY_PARAM_OPT, po::value<IdTypeUnsign>(&RandRestQty)->default_value(RAND_REST_QTY_PARAM_DEFAULT), RAND_REST_QTY_PARAM_MSG)
      (SEARCH_CLOSE_ITER_QTY_PARAM_OPT,po::value<IdTypeUnsign>(&SearchCloseIterQty)->default_value(SEARCH_CLOSE_ITER_QTY_PARAM_DEFAULT), SEARCH_CLOSE_ITER_QTY_PARAM_MSG)
      (DIST_SAMPLE_QTY_PARAM_OPT, po::value<IdTypeUnsign>(&SampleDistQty)->default_value(DIST_SAMPLE_QTY_PARAM_DEFAULT), DIST_SAMPLE_QTY_PARAM_MSG)
      (MAX_META_ITER_QTY_PARAM_OPT, po::value<IdTypeUnsign>(&maxMetaIterQty)->default_value(MAX_META_ITER_QTY_PARAM_DEFAULT), MAX_META_ITER_QTY_PARAM_MSG)
      (KEEP_FRAC_QTY_PARAM_OPT, po::value<float>(&keepFrac)->default_value(KEEP_FRAC_QTY_PARAM_DEFAULT), KEEP_FRAC_QTY_PARAM_MSG)
      ;

  PrintProgress = !NoPrintProgress;

  po::variables_map vm;
  try {
    po::store(po::parse_command_line(argc, argv, ProgOptDesc), vm);
    po::notify(vm);
  } catch (const exception& e) {
    Usage(argv[0], ProgOptDesc);
    LOG(LIB_FATAL) << e.what();
  }

  if (vm.count("help")  ) {
    Usage(argv[0], ProgOptDesc);
    exit(0);
  }

  ToLower(ClustType);
  ToLower(SpaceType);

  try {
    {
      vector<string> SpaceDesc;
      string str = SpaceType;
      ParseSpaceArg(str, SpaceType, SpaceDesc);
      SpaceParams = shared_ptr<AnyParams>(new AnyParams(SpaceDesc));
    }

    if (DataFile.empty()) {
      LOG(LIB_FATAL) << "data file is not specified!";
    }

    if (!DoesFileExist(DataFile)) {
      LOG(LIB_FATAL) << "data file " << DataFile << " doesn't exist";
    }
    CHECK_MSG(MaxNumData < MAX_DATASET_QTY, "The maximum number of points should not exceed" + ConvertToString(MAX_DATASET_QTY));
  } catch (const exception& e) {
    LOG(LIB_FATAL) << "Exception: " << e.what();
  }
}

int main(int argc, char* argv[]) {
  WallClockTimer timer;
  timer.reset();

  bool                    PrintProgress;
  string                  LogFile;
  string                  DistType;
  string                  SpaceType;
  shared_ptr<AnyParams>   SpaceParams;
  string                  ClustType;
  IdTypeUnsign            ClustQty;
  string                  DataFile;
  IdTypeUnsign            MaxNumData;

// reductive CLARANS parameters
  IdTypeUnsign            maxMetaIterQty;
  float                   keepFrac;
  // CLARANS parameters
  IdTypeUnsign           inClusterSwapAttempts;
  IdTypeUnsign           inClusterSampleQty; // Number of random points to estimate if the swap was beneficial
  IdTypeUnsign           RandRestQty;
  // FIRMAL parameters
  IdTypeUnsign           SearchCloseIterQty;// A # of search iterations to find a point that is close to already selected centers
  // For good performance it should be in the order of sqrt(data.size())
  IdTypeUnsign           SampleDistQty;      // A # of sample to determine the distribution of distances)

  ParseCommandLineForClustering(argc, argv,
                                PrintProgress,
                                LogFile,
                                DistType,
                                SpaceType,
                                SpaceParams,
                                DataFile,
                                MaxNumData,
                                ClustType,
                                ClustQty,
      // Reductive CLARANS parameters
                                maxMetaIterQty,
                                keepFrac,
      // CLARANS parameters
                                inClusterSwapAttempts,
                                inClusterSampleQty,
                                RandRestQty,
      // FIRMAL paramaters
                                SearchCloseIterQty,
                                SampleDistQty
  );

  initLibrary(LogFile.empty() ? LIB_LOGSTDERR:LIB_LOGFILE, LogFile.c_str());

  ToLower(DistType);
  ToLower(ClustType);

  if (!SpaceParams) {
    LOG(LIB_FATAL) << "Failed to initialized space parameters!";
  }

  if (DIST_TYPE_INT == DistType) {
    RunExper<int>(PrintProgress,
                  DistType,
                  SpaceType,
                  SpaceParams,
                  DataFile,
                  MaxNumData,
                  ClustType,
                  ClustQty,
                  maxMetaIterQty,
                  keepFrac,
                  inClusterSwapAttempts,
                  inClusterSampleQty,
                  RandRestQty,
                  SearchCloseIterQty,
                  SampleDistQty
    );
  } else if (DIST_TYPE_FLOAT == DistType) {
    RunExper<float>(PrintProgress,
                    DistType,
                    SpaceType,
                    SpaceParams,
                    DataFile,
                    MaxNumData,
                    ClustType,
                    ClustQty,
                    maxMetaIterQty,
                    keepFrac,
                    inClusterSwapAttempts,
                    inClusterSampleQty,
                    RandRestQty,
                    SearchCloseIterQty,
                    SampleDistQty);
  } else if (DIST_TYPE_DOUBLE == DistType) {
    RunExper<double>(PrintProgress,
                     DistType,
                     SpaceType,
                     SpaceParams,
                     DataFile,
                     MaxNumData,
                     ClustType,
                     ClustQty,
                     maxMetaIterQty,
                     keepFrac,
                     inClusterSwapAttempts,
                     inClusterSampleQty,
                     RandRestQty,
                     SearchCloseIterQty,
                     SampleDistQty);
  } else {
    LOG(LIB_FATAL) << "Unknown distance value type: " << DistType;
  }

  timer.split();
  LOG(LIB_INFO) << "Time elapsed = " << timer.elapsed() / 1e6;
  LOG(LIB_INFO) << "Finished at " << LibGetCurrentTime();

  return 0;
}

ext: cc