text:
// やめるのだフェネックで学ぶC++の実証コード(インラインアセンブリ)
#include <climits>
#include <cstring>
#include <algorithm>
#include <functional>
#include <iomanip>
#include <iostream>
#include <limits>
#include <string>
#include <unordered_map>
#include <vector>
#include <boost/math/constants/constants.hpp>
#include <boost/multiprecision/cpp_dec_float.hpp>
#include <gtest/gtest.h>
#include "cFriendsCommon.h"
#include "cppFriends.hpp"
#include "cppFriendsClang.hpp"

class TestSaturationArithmetic : public ::testing::Test{};

TEST_F(TestSaturationArithmetic, Add) {
    constexpr size_t XmmRegisterSizeInByte = 16;
    constexpr size_t NumberOfRegister = 2;
    std::aligned_storage<XmmRegisterSizeInByte, XmmRegisterSizeInByte>::type xmmRegisters[NumberOfRegister];
    const uint8_t presetValue[XmmRegisterSizeInByte * NumberOfRegister] = {
        0,   0, 1, 1,   1,   1, 2, 2,   2,   2,   2, 254, 254, 254, 255, 255,   // 足される数
        0, 255, 0, 1, 254, 255, 0, 1, 253, 254, 255,   0,   1,   2,   0,   1    // 足す数
    };

    static_assert(sizeof(xmmRegisters) >= sizeof(presetValue), "Too large");
    ::memmove(xmmRegisters, presetValue, sizeof(presetValue));

    const uint8_t expected[XmmRegisterSizeInByte] = {
        0, 255, 1, 2, 255, 255, 2, 3, 255, 255, 255, 254, 255, 255, 255, 255};  // 和

    // Windows/Linuxの両方で使えるscratch registers
    asm volatile (
        "movdqa  (%0),   %%xmm4 \n\t"
        "movdqa  16(%0), %%xmm5 \n\t"
        "paddusb %%xmm5, %%xmm4 \n\t"
        "movdqa  %%xmm4, (%0)    \n\t"
        ::"r"(xmmRegisters):"xmm4", "xmm5", "memory");

    static_assert(sizeof(expected) <= sizeof(xmmRegisters), "Too large");
    EXPECT_EQ(0, ::memcmp(expected, xmmRegisters, sizeof(expected)));
}

TEST_F(TestSaturationArithmetic, Sub) {
    constexpr size_t XmmRegisterSizeInByte = 16;
    constexpr size_t NumberOfRegister = 2;
    std::aligned_storage<XmmRegisterSizeInByte, XmmRegisterSizeInByte>::type xmmRegisters[NumberOfRegister];
    const uint8_t presetValue[XmmRegisterSizeInByte * NumberOfRegister] = {
        0,   0, 0,   0,   0, 1, 1, 1, 254, 254, 254, 254, 255, 255, 255, 255,  // 引かれる数
        0,   1, 2, 254, 255, 0, 1, 2,   1, 253, 254, 255,   0,   1, 254, 255   // 引く数
    };
    static_assert(sizeof(xmmRegisters) >= sizeof(presetValue), "Too large");
    ::memmove(xmmRegisters, presetValue, sizeof(presetValue));

    const uint8_t expected[XmmRegisterSizeInByte] = {
        0,   0, 0,   0,   0, 1, 0, 0, 253,   1,   0,   0, 255, 254,   1,   0}; // 差

    // Windows/Linuxの両方で使えるscratch registers
    asm volatile (
        "movdqa  (%0),   %%xmm4 \n\t"
        "movdqa  16(%0), %%xmm5 \n\t"
        "psubusb %%xmm5, %%xmm4 \n\t"
        "movdqa  %%xmm4, (%0)    \n\t"
        ::"r"(xmmRegisters):"xmm4", "xmm5", "memory");

    static_assert(sizeof(expected) <= sizeof(xmmRegisters), "Too large");
    EXPECT_EQ(0, ::memcmp(expected, xmmRegisters, sizeof(expected)));
}

TEST_F(TestSaturationArithmetic, ShiftLeft) {
    using Reg32 = uint32_t;
    Reg32 expected = 1;

    for(Reg32 width = 0; width < 35; ++width) {
        Reg32 src = 1;
        Reg32 temp = 0;
        Reg32 actual = 3;

        asm volatile (
            "xor   %%ebx, %%ebx \n\t"
            "cmp   $0x1f, %%ecx \n\t"
            "shlx  %%ecx, %%edx, %%eax \n\t"
            "cmova %%ebx, %%eax  \n\t"
            :"=a"(actual),"=b"(temp):"d"(src),"c"(width):);

        EXPECT_EQ(expected, actual);
        expected <<= 1;
    }
}

TEST_F(TestSaturationArithmetic, ShiftRight) {
    using Reg32 = uint32_t;
    Reg32 expected = 0x80000000;
    for(Reg32 width = 0; width < 35; ++width) {
        Reg32 src = 0x80000000;
        Reg32 temp = 0;
        Reg32 actual = 1;

        asm volatile (
            "mov   %%edx, %%ebx \n\t"
            "sar   $0x1f, %%ebx \n\t"
            "cmp   $0x1f, %%ecx \n\t"
            "sarx  %%ecx, %%edx, %%eax \n\t"
            "cmova %%ebx, %%eax  \n\t"
            :"=a"(actual),"=b"(temp):"d"(src),"c"(width):);

        EXPECT_EQ(expected, actual);
        expected = expected | (expected >> 1);
    }
}

namespace {
    using ProcessorClock = uint64_t;
    using ProcessorClockSet = std::vector<ProcessorClock>;
    using ProcessorClockMap = std::unordered_map<ProcessorClock, size_t>;
    std::string g_expString;

    ProcessorClock getProcessorClock() {
        ProcessorClock clock = 0;
        auto pClock = &clock;

        // クロックだけ測る
        asm volatile (
            "rdtsc \n\t"
            "mov %%eax, (%0)  \n\t"
            "mov %%edx, 4(%0) \n\t"
            ::"r"(pClock):"eax", "edx", "memory");
        return clock;
    }

    ProcessorClock getProcessorClockWithLoad() {
        // 負荷を掛ける
        using LongFloat = boost::multiprecision::number<boost::multiprecision::cpp_dec_float<50>>;
        g_expString += boost::math::constants::e<LongFloat>().str().at(0);
        return getProcessorClock();
    }

    using FuncGetClock = std::function<ProcessorClock()>;
    void checkProcessorClock(FuncGetClock& f) {
        ProcessorClockSet clockSet;
        for(size_t i = 0; i < 120000; ++i) {
            clockSet.push_back(f());
        }

        // 周回しないものと仮定している
        ProcessorClockMap clockMap;
        constexpr ProcessorClock mod = 12;

        for(ProcessorClock i=0; i<mod; ++i) {
            clockMap[i] = 0;
        }

        decltype(clockSet)::value_type prevTimestamp = clockSet.at(0);
        for(auto timestamp : clockSet) {
            ASSERT_LE(prevTimestamp, timestamp);
            clockMap[timestamp % mod] += 1;
            prevTimestamp = timestamp;
        }

        // 出現確率が二項分布なら、標準偏差は sqrt(n*p*(1-p))になる
        // p=1/12なら、1σ = sqrt(n) * 0.28, 6σ = sqrt(n) * 1.66
        // n=120000なら、平均=10000, 6σ=575なので、9000回は出るはず
        // 9000回に満たないのはおかしい
        bool foundBias = false;
        for(ProcessorClock i=0; i<mod; ++i) {
            auto count = clockMap[i];
            std::cout << count << ":";
            foundBias |= (count < 9000);
        }

        EXPECT_TRUE(foundBias);
        std::cout << "\n";
        return;
    }

    using NallowClock = uint32_t;
    // コンテキストスイッチがなければ、これ以上の時刻差は観測されないはず
    constexpr NallowClock ClockThreshold = 4000;
    // Signedとみなして巨大な数なら時刻が逆転している
    constexpr NallowClock MaxDiff = 2 + std::numeric_limits<NallowClock>::max() / 2;

    // atomicにリアルタイムクロックを読まないと不適切な値が得られるという例
    inline NallowClock getProcessorClockBad(ProcessorClock& clock) {
        uint32_t mixed = 0;
        auto pMixed = &mixed;
        auto pClock = &clock;

        asm volatile (
            "rdtsc \n\t"  // 下位12bitを読む
            "andl $0xfff, %%eax \n\t"
            "movl %%eax,  %%ebx \n\t"
            "rdtsc \n\t"  // 上位bitを読む
            "movl %%eax, (%1)  \n\t"
            "movl %%edx, 4(%1) \n\t"
            "andl $0xfffff000, %%eax \n\t"
            "orl  %%ebx, %%eax \n\t"
            "movl %%eax, (%0)  \n\t"
            ::"r"(pMixed),"r"(pClock):"eax","ebx","edx","memory");

        return static_cast<NallowClock>(mixed);
    }

    NallowClock getProcessorClockGood(ProcessorClock& clock) {
        uint32_t mixed = 0;
        auto pMixed = &mixed;
        auto pClock = &clock;

        asm volatile (
            "1: \n\t"
            "rdtsc \n\t"  // 上位bitを読む
            "andl $0xfffff000, %%eax \n\t"
            "movl %%eax,  %%ecx \n\t"
            "rdtsc \n\t"  // 下位12bitを読む
            "andl $0xfff, %%eax \n\t"
            "movl %%eax,  %%edi \n\t"
            "rdtsc \n\t"  // もう一度上位bitを読む
            "movl %%eax, (%1)  \n\t"
            "movl %%edx, 4(%1) \n\t"
            "andl $0xfffff000, %%eax \n\t"
            "cmpl %%ecx,  %%eax \n\t"
            "jne  1b \n\t"  // 読み直す
            "orl  %%edi, %%eax \n\t"
            "movl %%eax, (%0)  \n\t"
            ::"r"(pMixed),"r"(pClock):"eax","ecx","edx","edi","memory");

        return static_cast<NallowClock>(mixed);
    }
}

class TestProcessorClock : public ::testing::Test{
    virtual void SetUp() override {
        g_expString.clear();
    }

    virtual void TearDown() override {
        g_expString.clear();
    }
};

TEST_F(TestProcessorClock, Light) {
    FuncGetClock f(getProcessorClock);
    checkProcessorClock(f);
}

TEST_F(TestProcessorClock, Heavy) {
    FuncGetClock f(getProcessorClockWithLoad);
//  失敗することがある
//  checkProcessorClock(f);
}

TEST_F(TestProcessorClock, Bad) {
    ProcessorClock previousClock = 0;
    auto previous = getProcessorClockBad(previousClock);
    bool found = false;

    // 1秒くらい、十分たくさん行うが、失敗する可能性がないとは言えない
    for(int i=0; i<0x4000000 && !found; ++i) {
        ProcessorClock currentClock = 0;
        auto current = getProcessorClockBad(currentClock);
        // Wrap around
        auto diff = current - previous;
        ASSERT_TRUE(diff);

        // クロックが大きく飛んだ時は、コンテキストスイッチが入ったとみなす
        if (ClockThreshold > (currentClock - previousClock)) {
            // 閾値より大きい差は、getProcessorClockで弾いているはず
            found |= (MaxDiff <= diff);
        }
        previous = current;
        previousClock = currentClock;
    }

    ASSERT_TRUE(found);
}

TEST_F(TestProcessorClock, Good) {
    ProcessorClock previousClock = 0;
    auto previous = getProcessorClockGood(previousClock);

    for(int i=0; i<0x2000000; ++i) {
        ProcessorClock currentClock = 0;
        auto current = getProcessorClockGood(currentClock);
        auto diff = current - previous;
        ASSERT_TRUE(diff);

        // クロックが大きく飛んだ時は、コンテキストスイッチが入ったとみなす
        if (ClockThreshold > (currentClock - previousClock)) {
            ASSERT_GT(MaxDiff, diff);
        }
        previous = current;
        previousClock = currentClock;
    }
}

namespace {
    using Log2Arg = unsigned int;
    using Log2Result = int;
    constexpr Log2Result Log2ResultInvalid = -1;

    Log2Result log2Builtin(Log2Arg arg) {
        return (arg) ? (31 - __builtin_clz(arg)) : Log2ResultInvalid;
    }

    Log2Result log2Asm(Log2Arg arg) {
        Log2Result result = 0;
        Log2Result invalid = Log2ResultInvalid;

        asm volatile (
            "bsr    %1, %0 \n\t"
            "cmovz  %2, %0 \n\t"
            :"=r"(result):"r"(arg),"r"(invalid):);

        return result;
    }

    struct Log2TestCase {
        Log2Arg arg;
        Log2Result expected;
    };

    const Log2TestCase g_log2TestCaseSet [] = {
        {0, Log2ResultInvalid},
        {1,0}, {2,1}, {3,1}, {4,2}, {5,2}, {6,2}, {7,2},
        {8,3}, {9,3}, {14,3}, {15,3}, {16,4}, {17,4}, {31,4},
        {0x7fffffffu, 30}, {0x80000000u, 31}, {0xffffffffu, 31}
    };
}

class TestLog2 : public ::testing::Test{};

TEST_F(TestLog2, Builtin) {
    for(auto& testcase : g_log2TestCaseSet) {
        EXPECT_EQ(testcase.expected, log2Builtin(testcase.arg));
    }
}

TEST_F(TestLog2, Asm) {
    for(auto& testcase : g_log2TestCaseSet) {
        EXPECT_EQ(testcase.expected, log2Asm(testcase.arg));
    }
}

TEST_F(TestLog2, LongLong) {
    int count = 0;
    unsigned long long big = 5000000000000000ull;  // 5000兆

    // unsignedなのでいつかは全ビット0になって終わる
    while(big) {
        ++count;
        big /= 2;
    }

    EXPECT_EQ(53, count);
}

class TestDivideBy2 : public ::testing::Test{};

/*
負の数に対する除算に対して、コンパイラがどんなコードを出力するか確認する
Cで書くとこうなる
  int divide_by_2(int src) {
      return src / 2;
  }

以下はGCCの出力である
  mov  eax, ecx
  shr  eax, 31
  add  eax, ecx
  sar  eax

以下はclang-LLVMの出力である
  mov  eax, ecx
  shr  eax, 31
  lea  eax, [rax + rcx]
  sar  eax

http://www.microhowto.info/howto/round_towards_minus_infinity_when_dividing_integers_in_c_or_c++.html
によると、割る数または割られる数の少なくとも一方が負の場合の丸めは実装依存である。
*/

TEST_F(TestDivideBy2, RoundToZero) {
    // 負の数を2で割るときに、単にsar 1回ではうまくいかない
    // -1を2で割っても-1になってしまう
    // 負の数で小数点以下切り捨て(round to 0)なら1足してからシフトする必要がある
    // -8 : 1000b <+1> 1001b <sar> 1100b (-4)
    // -7 : 1001b <+1> 1010b <sar> 1101b (-3)
    // -2 : 1110b <+1> 1111b <sar> 1111b (-1)
    // -1 : 1111b <+1> 0 <sar> 0
    // 正の数には1足さないので、単にシフトしてLSBを捨てるだけである

    struct TestCase {
        int32_t arg;
        int32_t expected;
        int32_t expectedShifted;
    };

    const TestCase testCaseSet[] = {
        {0, 0, 0},
        {1, 0, 0},
        {2, 1, 1},
        {3, 1, 1},
        {4, 2, 2},
        {7, 3, 3},
        {8, 4, 4},
        {9, 4, 4},
        { 2147483645,  1073741822,  1073741822},
        { 2147483646,  1073741823,  1073741823},
        { 2147483647,  1073741823,  1073741823},
        {-2147483648, -1073741824, -1073741824},
        {-2147483647, -1073741823, -1073741824},
        {-2147483646, -1073741823, -1073741823},
        {-9, -4, -5},
        {-8, -4, -4},
        {-7, -3, -4},
        {-4, -2, -2},
        {-3, -1, -2},
        {-2, -1, -1},
        {-1,  0, -1}
    };

    static_assert(std::numeric_limits<int32_t>::min() == -2147483648, "");
    static_assert(std::numeric_limits<int32_t>::max() == 2147483647, "");

    for(const auto& testCase : testCaseSet) {
        auto arg = testCase.arg;
        decltype(arg) actual = 0;

        // 最近のCPUなら、3オペレータシフト命令を使える
        asm volatile (
            "movl  %%ecx, %%eax \n\t"  // 第一引数 = ecx
            "shrl  $31,   %%eax \n\t"  // 第一引数が非負なら0, 負なら1
            "addl  %%ecx, %%eax \n\t"  // 第一引数が非負ならそのまま, 負なら1足したものを
            "sarl  %%eax        \n\t"  // 符号を保ったままシフトする
            :"=a"(actual):"c"(arg):);

        EXPECT_EQ(testCase.expected, actual);
        EXPECT_EQ(testCase.expected, testCase.arg / 2);

        // 負の数では、/2 を負の無限大方向に切り上げるのと同じになる
        // それはそれで正しいのかもしれないが、-1を2で割って-1になるのは直観に反する
        auto shifted = testCase.arg;
        shifted >>= 1;
        EXPECT_EQ(testCase.expectedShifted, shifted);
    }
}

namespace {
    int32_t divideByZeroUnsafe(int32_t dividend, int32_t divisor, int32_t special) {
        int32_t result = dividend;
        asm volatile (
            "cdq    \n\t"
            "idiv   %%ebx \n\t"
            "or     %%ebx, %%ebx \n\t"
            "cmovz  %%esi, %%eax \n\t"
            :"+a"(result):"b"(divisor),"S"(special):"edx");

        return result;
    }

    int32_t divideByZeroMuchSafe(int32_t dividend, int32_t divisor, int32_t special) {
        // clangの出力を元に作る
        int32_t result = dividend;
        asm volatile (
            "or    %%ebx, %%ebx \n\t"
            "cmovz %%esi, %%eax \n\t"
            "jz    1f \n\t"
            "cdq   \n\t"
            "idiv  %%ebx \n\t"
            "1:"
            :"+a"(result):"b"(divisor),"S"(special):"edx");

        return result;
    }

    int32_t divideByZero(int32_t dividend, int32_t divisor, int32_t special) {
        return (divisor) ? (dividend / divisor) : special;
    }
}

class TestDivideByZero : public ::testing::Test{};

TEST_F(TestDivideByZero, Unsafe) {
    EXPECT_EQ(3,  divideByZeroUnsafe(15, 4, -1));
}

TEST_F(TestDivideByZero, MuchSafe) {
    EXPECT_EQ(3,  divideByZeroMuchSafe(15, 4, -1));
    EXPECT_EQ(-1, divideByZeroMuchSafe(15, 0, -1));
}

TEST_F(TestDivideByZero, Cpp) {
    EXPECT_EQ(3,  divideByZero(15, 4, -1));
    EXPECT_EQ(-1, divideByZero(15, 0, -1));
#if 0
    // int32_tの範囲で解を表現できないのでCPU例外が発生するはずだが
    // 定数式としてコンパイラが値を求めた場合は実行時例外にならない
    constexpr int32_t intMin = std::numeric_limits<int32_t>::min();
    EXPECT_EQ(intMin, divideByZero(intMin, -1, 0));
#endif
}

class TestBitManipulation : public ::testing::Test{
protected:
    using IntType = uint32_t;
    static constexpr IntType bitCount = static_cast<IntType>(sizeof(IntType) * 8);
    static const IntType TestSet[2];
};

const TestBitManipulation::IntType TestBitManipulation::TestSet[2] = {0xffffffffu, 0xaaaaaaaau};

#ifdef CPPFRIENDS_AVX2
// iビット目以上を0にする
TEST_F(TestBitManipulation, BitMask) {
    for(auto src : TestSet) {
        IntType expected = 0;

        for(IntType i = 0; i <= bitCount*3; ++i) {
            IntType actualCpp = src;
            IntType mask = 1;
            if (i < bitCount) {
                mask <<= i;  // 多すぎるシフトに対する動作は未定義
            } else {
                mask = 0;
            }
            --mask;
            actualCpp = src & mask;
            EXPECT_EQ(expected, actualCpp);

            // ビット位置はmod 32/64ではなくsaturated
            IntType actualAsm = src;
            asm volatile (
                "bzhi  %2, %1, %0 \n\t"
                :"=r"(actualAsm):"r"(src),"r"(i):);
            EXPECT_EQ(expected, actualAsm);

            if (src & 1) {
                expected <<= 1;
                ++expected;
            } else {
                if (i & 1) {
                    expected <<= 2;
                    expected |= 2;
                }
            }
        }
    }
}

// 整数を2^n * 奇数で表現できるときの2^nを求める
TEST_F(TestBitManipulation, PowerOf2) {
    for(auto original : TestSet) {
        IntType src = original;
        IntType expected = (original & 1) ? 1 : 2;

        for(IntType index = 0; index <= bitCount; ++index) {
            IntType actualCpp = src & -src;
            EXPECT_EQ(expected, actualCpp);

            IntType actualAsm = src;
            // 0には0を返す
            asm volatile (
                "blsi  %1, %0 \n\t"
                :"=r"(actualAsm):"r"(src):);
            EXPECT_EQ(expected, actualAsm);

            if (original & 1) {
                expected <<= 1;
            } else {
                if (index & 1) {
                    expected <<= 2;
                }
            }

            IntType mask = 1;
            mask <<= index;
            src &= ~mask;
        }
    }
}
#endif

class TestProcessorExceptionDeathTest : public ::testing::Test{};

TEST_F(TestProcessorExceptionDeathTest, Unsafe) {
    // 実行すると0除算エラーになる
    ASSERT_DEATH(divideByZeroUnsafe(15, 0, -1), "");
}

TEST_F(TestProcessorExceptionDeathTest, Abs) {
    EXPECT_EQ(0,  ProcessorException::abs_int(0));
    EXPECT_EQ(1,  ProcessorException::abs_int(1));
    EXPECT_EQ(1,  ProcessorException::abs_int(-1));

    constexpr int intMax = std::numeric_limits<int>::max();
    constexpr int intMin = std::numeric_limits<int>::min();
    EXPECT_EQ(intMax, ProcessorException::abs_int(intMax));
    EXPECT_EQ(intMin, ProcessorException::abs_int(intMin));
}

#if !defined(__MINGW32__) && !defined(__MINGW64__)
// MinGWでは、FPEが発生するが処理が先に進まない
TEST_F(TestProcessorExceptionDeathTest, MuchSafe) {
    // int32_tの範囲で解を表現できないのでCPU例外が発生する
    constexpr int32_t intMin = std::numeric_limits<int32_t>::min();
    ASSERT_DEATH(divideByZeroMuchSafe(intMin, -1, 0), "");
}

TEST_F(TestProcessorExceptionDeathTest, TooLarge) {
    EXPECT_EQ(3,  ProcessorException::may_divide_by_zero(15, 4, -1));
    EXPECT_EQ(-1, ProcessorException::may_divide_by_zero(15, 0, -1));
    // int32_tの範囲で解を表現できないのでCPU例外が発生する
    constexpr int32_t intMin = std::numeric_limits<int32_t>::min();
    ASSERT_DEATH(ProcessorException::may_divide_by_zero(intMin, -1, 0), "");
}

TEST_F(TestProcessorExceptionDeathTest, IntMaxMin) {
    if ((sizeof(int32_t) == sizeof(int)) && (std::numeric_limits<int32_t>::min() == INT_MIN)) {
        EXPECT_EQ(INT_MIN, ProcessorException::abs_int(INT_MIN));
    }
    ASSERT_DEATH(ProcessorException::may_divide_by_zero(INT_MIN, -1, 0), "");
}
#endif

// 元記事は下記によるもの。64bit Cygwin向けに変更した。
// http://blog.onlinedisassembler.com/blog/?p=23
// 15 bytesもある命令。実行はできない。
void LongestInstruction(void) {
#ifdef __x86_64__
    // objdumpの出力
    // <_Z18LongestInstructionv>:
    // 64 67 f0 43 81 84 88    lock addl $0xa0a0a0a0,%fs:0x50505050(%r8d,%r9d,4)
    // 50 50 50 50 a0 a0 a0
    // a0
    asm volatile (
        "lock addl $0xa0a0a0a0, %%fs:0x50505050(%%r8d, %%r9d, 4)\n\t"
        :::);
#endif
}

namespace {
    uint8_t calculate2ndOrGreaterDigits(uint8_t num) {
        uint8_t result = 0;
        // min(255, (num + 1)) * 51 / 512
        asm volatile (
            // min(255, (num + 1))
            "add    $1, %%al \n\t"
            "sbb    $0, %%al \n\t"
            // += num
            "xor    %%bl, %%bl \n\t"
            "mov    %%al, %%cl \n\t"
            "xor    %%dl, %%dl \n\t"
            // += num * 2
            "add    %%al, %%al \n\t"
            "adc    %%bl, %%bl \n\t"
            "add    %%al, %%cl \n\t"
            "adc    %%bl, %%dl \n\t"
            // num * 48 = num * 3 << 4
            "mov    %%cl, %%al \n\t"
            "mov    %%dl, %%bl \n\t"
            "add    %%al, %%al \n\t"
            "adc    %%bl, %%bl \n\t"
            "add    %%al, %%al \n\t"
            "adc    %%bl, %%bl \n\t"
            "add    %%al, %%al \n\t"
            "adc    %%bl, %%bl \n\t"
            "add    %%al, %%al \n\t"
            "adc    %%bl, %%bl \n\t"
            // += num * 48
            "add    %%al, %%cl \n\t"
            "adc    %%bl, %%dl \n\t"
            // /= 512
            "shr    %%dl \n\t"
            :"=d"(result),"+a"(num)::"rbx", "rcx");
       return result;
    }

    uint8_t calculate1stDigit(uint8_t num, uint8_t secondOrGreaterDigits) {
        uint8_t result = 0;
        asm volatile (
            // num - (secondOrGreaterDigits * 10)
            "mov    %%al, %%dl \n\t"
            "mov    %%bl, %%cl \n\t"
            "shl    %%bl \n\t"
            "shl    %%bl \n\t"
            "add    %%cl, %%bl \n\t"
            "shl    %%bl \n\t"
            "sub    %%bl, %%dl \n\t"
            :"=d"(result),"+a"(num),"+b"(secondOrGreaterDigits)::"rcx");
       return result;
    }
}

class TestSplitDigits : public ::testing::Test{};

TEST_F(TestSplitDigits, All) {
    // Calclulate digits for uint8_t
    uint8_t i=0;
    for(;;) {
        const auto digits23 = calculate2ndOrGreaterDigits(i);
        const auto digits1 = calculate1stDigit(i, digits23);
        const auto digits3 = calculate2ndOrGreaterDigits(digits23);
        const auto digits2 = calculate1stDigit(digits23, digits3);
        const uint32_t actualDigits1 = digits1;
        const uint32_t actualDigits2 = digits2;
        const uint32_t actualDigits3 = digits3;

        uint32_t expected = i;
        const auto expectedDigits1 = expected % 10;
        expected /= 10;
        const auto expectedDigits2 = expected % 10;
        const auto expectedDigits3 = expected / 10;

        // uint32_tにすると、値が文字ではなく数値として表示される
        EXPECT_EQ(expectedDigits1, actualDigits1);
        EXPECT_EQ(expectedDigits2, actualDigits2);
        EXPECT_EQ(expectedDigits3, actualDigits3);
        if (!++i) {
            break;
        }
    }
}

class TestAsmInstructions : public ::testing::Test{};

TEST_F(TestAsmInstructions, Coffee) {
    uint32_t arg = 0x8000;
    uint32_t result = 0;
    constexpr uint32_t expected = 0xff00;

    // sar bh, 0xee = c0h, ffh, eeh
    asm volatile (
//      ".byte 0xc0, 0xff, 0xee \n\t"
        "sarb  $0xee, %%bh \n\t"
        "mov   %%ebx, %%eax \n\t"
        :"=a"(result),"+b"(arg)::);
    EXPECT_EQ(expected, arg);
    EXPECT_EQ(expected, result);
}

TEST_F(TestAsmInstructions, Div3_16bit) {
    uint16_t arg = 0;
    do {
        uint16_t expected = (arg % 3) ? 0 : 1;
        uint16_t result = 0;
        asm volatile (
            "mov   $0x5555, %%ax \n\t"
            "mul   %%bx \n\t"
            "mov   %%dx, %%ax \n\t"
            "add   %%ax, %%dx \n\t"
            "add   %%ax, %%dx \n\t"
            "mov   %%bx, %%ax \n\t"
            "sub   %%dx, %%ax \n\t"
            "mov   %%ax, %%dx \n\t"
            "shr   $1, %%ax \n\t"
            "xor   %%dx, %%ax \n\t"
            "not   %%ax \n\t"
            "and   $1, %%ax \n\t"
            :"=a"(result):"b"(arg):"rdx");
        if (expected != result) {
            std::cerr << arg;
        }
        ASSERT_EQ(expected, result);
    } while(++arg);
}

TEST_F(TestAsmInstructions, Div3_32bit) {
    uint32_t arg = 0;
    do {
        uint32_t expected = (arg % 3) ? 0 : 1;
        uint32_t result = 0;
        asm volatile (
            "mov   $0x55555555, %%eax \n\t"
            "mul   %%ebx \n\t"
            "mov   %%edx, %%eax \n\t"
            "add   %%eax, %%edx \n\t"
            "add   %%eax, %%edx \n\t"
            "mov   %%ebx, %%eax \n\t"
            "sub   %%edx, %%eax \n\t"
            "mov   %%eax, %%edx \n\t"
            "shr   $1, %%eax \n\t"
            "xor   %%edx, %%eax \n\t"
            "not   %%eax \n\t"
            "and   $1, %%eax \n\t"
            :"=a"(result):"b"(arg):"rdx");
        if (expected != result) {
            std::cerr << arg;
        }
        ASSERT_EQ(expected, result);
    } while(++arg);
}

namespace {
    using FilledBitSet = uint32_t;

    FilledBitSet SplitByMsb(FilledBitSet arg, FilledBitSet& count) {
        uint32_t result = 0;
        asm volatile (
            "lzcnt %%eax, %%ecx \n\t"
            "mov   $-1, %%ebx \n\t"
            "shrx  %%ecx, %%ebx, %%edx \n\t"
            "cmovb %%eax, %%edx \n\t"
            :"=d"(result),"=c"(count):"a"(arg):"rbx");
        return result;
    }

    FilledBitSet SetOnesForLeadingZeros(FilledBitSet arg, FilledBitSet& count) {
        uint32_t result = 0;
        asm volatile (
            "lzcnt %%eax, %%ecx \n\t"
            "shlx  %%ecx, %%eax, %%ebx \n\t"
            "sarx  %%ecx, %%ebx, %%edx \n\t"
            :"=d"(result),"=c"(count):"a"(arg):"rbx");
        return result;
    }
}

TEST_F(TestAsmInstructions, SetLowOnes) {
    FilledBitSet count = 0;
    constexpr FilledBitSet BitSetWidth = sizeof(FilledBitSet) * 8;
    static_assert(BitSetWidth == 32, "Unexpected uint size");

    for(FilledBitSet i=0; i < BitSetWidth; ++i) {
        const FilledBitSet expectedCount = 31 - i;
        FilledBitSet pattern = 1;
        pattern <<= i;
        const FilledBitSet mask = pattern - 1;
        const FilledBitSet expected = pattern | mask;

        EXPECT_EQ(expected, SplitByMsb(pattern, count));
        EXPECT_EQ(expectedCount, count);

        uint32_t lowBits = 0xaaaaaaaa & mask;
        EXPECT_EQ(expected, SplitByMsb(pattern | lowBits, count));
        EXPECT_EQ(expectedCount, count);

        lowBits = 0x55555555 & mask;
        EXPECT_EQ(expected, SplitByMsb(pattern | lowBits, count));
        EXPECT_EQ(expectedCount, count);
    }

    EXPECT_FALSE(SplitByMsb(0, count));
    EXPECT_EQ(BitSetWidth, count);

    EXPECT_EQ(0xffffffffu, SplitByMsb(0x80000000u, count));
    EXPECT_EQ(0, count);

    EXPECT_EQ(0xffffffffu, SplitByMsb(0xc0000000u, count));
    EXPECT_EQ(0, count);
}

TEST_F(TestAsmInstructions, SetHighOnes) {
    FilledBitSet count = 0;
    constexpr FilledBitSet BitSetWidth = sizeof(FilledBitSet) * 8;
    static_assert(BitSetWidth == 32, "Unexpected uint size");

    for(FilledBitSet i=0; i < BitSetWidth; ++i) {
        const FilledBitSet expectedCount = 31 - i;
        FilledBitSet diff = 1;
        diff <<= i;
        FilledBitSet mask = diff - 1;
        FilledBitSet expected = 0;
        expected -= diff;

        uint32_t arg = diff;
        EXPECT_EQ(expected, SetOnesForLeadingZeros(arg, count));
        EXPECT_EQ(expectedCount, count);

        uint32_t highBits = ~mask;
        uint32_t lowBits = 0xaaaaaaaa & mask;
        expected = highBits | lowBits;
        EXPECT_EQ(expected, SetOnesForLeadingZeros(arg | lowBits, count));
        EXPECT_EQ(expectedCount, count);

        lowBits = 0x55555555 & mask;
        expected = highBits | lowBits;
        EXPECT_EQ(expected, SetOnesForLeadingZeros(arg | lowBits, count));
        EXPECT_EQ(expectedCount, count);
    }

    EXPECT_FALSE(SetOnesForLeadingZeros(0, count));
    EXPECT_EQ(BitSetWidth, count);

    EXPECT_EQ(0x80000000u, SetOnesForLeadingZeros(0x80000000u, count));
    EXPECT_EQ(0, count);

    EXPECT_EQ(0xc0000000u, SetOnesForLeadingZeros(0xc0000000u, count));
    EXPECT_EQ(0, count);
}

TEST_F(TestAsmInstructions, BitScanReverse) {
    constexpr FilledBitSet BitSetWidth = sizeof(FilledBitSet) * 8;
    static_assert(BitSetWidth == 32, "Unexpected uint size");

    for(FilledBitSet i=0; i < BitSetWidth; ++i) {
        const FilledBitSet expectedBsr = i;
        FilledBitSet arg = 1;
        arg <<= i;

        FilledBitSet actual = 0;
        asm volatile (
            "bsr %%eax, %%edx \n\t"
            :"=d"(actual):"a"(arg):);
        EXPECT_EQ(expectedBsr, actual);

        const FilledBitSet expectedLzcnt = BitSetWidth - 1 - i;
        actual = 0;
        asm volatile (
            "lzcnt %%eax, %%edx \n\t"
            :"=d"(actual):"a"(arg):);
        EXPECT_EQ(expectedLzcnt, actual);
    }

    constexpr FilledBitSet argZero = 0;
    FilledBitSet actual = 0;
    asm volatile (
        "tzcnt %%eax, %%edx \n\t"
        :"=d"(actual):"a"(argZero):);
    EXPECT_EQ(BitSetWidth, actual);

    actual = 0;
    asm volatile (
        "lzcnt %%eax, %%edx \n\t"
        :"=d"(actual):"a"(argZero):);
    EXPECT_EQ(BitSetWidth, actual);

    constexpr uint64_t argZero64 = 0;
    uint64_t actual64 = 0;
    asm volatile (
        "tzcnt %%rax, %%rdx \n\t"
        :"=d"(actual64):"a"(argZero64):);
    EXPECT_EQ(64, actual64);

    actual64 = 0;
    asm volatile (
        "lzcnt %%rax, %%rdx \n\t"
        :"=d"(actual64):"a"(argZero64):);
    EXPECT_EQ(64, actual64);
}

namespace {
    struct XmmRegValue {
        alignas(16) unsigned char values[16] {0};
    };

    void ReLU(XmmRegValue& xmmRegValue) {
        asm volatile (
            "xorps  %%xmm4, %%xmm4 \n\t"
            "movaps (%0),   %%xmm5 \n\t"
            "maxps  %%xmm5, %%xmm4 \n\t"
            "movaps %%xmm4, (%0)   \n\t"
            ::"r"(xmmRegValue.values):"xmm4", "xmm5", "memory");
        return;
    }

    int16_t SaturatedAddInt16(int16_t x, int16_t y) {
        XmmRegValue reg;
        ::memmove(reg.values, &x, sizeof(x));
        ::memmove(reg.values + 2, &y, sizeof(y));

        asm volatile (
            "movdqa  (%0), %%xmm4 \n\t"
            "phaddsw %%xmm4, %%xmm4 \n\t"
            "movdqa  %%xmm4, (%0) \n\t"
            ::"r"(reg.values):"xmm4", "memory");

        return *(std::launder(reinterpret_cast<int16_t*>(reg.values)));
    }

    uint16_t SaturatedAddUint16(uint16_t x, uint16_t y) {
        uint16_t sum = 0;
        asm volatile (
            "mov  %%bx, %%ax \n\t"
            "add  %%cx, %%ax \n\t"
            "sbb  %%r10w, %%r10w \n\t"
            "or   %%r10w, %%ax \n\t"
            :"=a"(sum):"b"(x),"c"(y):"r10");

        return sum;
    }

    uint16_t SaturatedAddUint16cmov(uint16_t x, uint16_t y) {
        uint16_t sum = 0;
        asm volatile (
            "xor   %%r10w, %%r10w \n\t"
            "not   %%r10w \n\t"
            "mov   %%bx, %%ax \n\t"
            "add   %%cx, %%ax \n\t"
            "cmovc %%r10w, %%ax \n\t"
            :"=a"(sum):"b"(x),"c"(y):"r10");

        return sum;
    }

    void threeBytesXchg(void) {
        // 0x48 REX.W 0100b (fixed) and 1000b (64 Bit Operand Size)
        // 0x87 XCHG
        // 0xc0
        //   2-bit ModR/M.mod (11b : direct register access)
        //   3-bit a destination register operand
        //   3-bit a source register operand
        asm volatile (
            ".byte  0x48, 0x87, 0xc0 \n\t"
            :::"rax", "memory");
        return;
    }
}

TEST_F(TestAsmInstructions, MaxDoubles) {
    using FloatingType = float;
    XmmRegValue actual;
    constexpr size_t n_float = sizeof(actual.values) / sizeof(FloatingType);
    FloatingType values[n_float] {2.5, 10, -7.5, 1.0};
    const FloatingType expected[n_float] {2.5, 10, 0, 1.0};
    static_assert(sizeof(actual.values) == sizeof(values), "Must have a same size");
    static_assert(sizeof(actual.values) == sizeof(expected), "Must have a same size");

    ::memmove(actual.values, values, sizeof(values));
    ReLU(actual);
    EXPECT_EQ(0, ::memcmp(expected, &actual.values, sizeof(expected)));
}

TEST_F(TestAsmInstructions, SaturatedAddInt16) {
    using IntType = int16_t;
    using BigIntType = int64_t;
    constexpr auto minValue = std::numeric_limits<IntType>::min();
    constexpr auto maxValue = std::numeric_limits<IntType>::max();

    const std::vector<int16_t> valueSet {
        minValue, minValue + 1,
        minValue/2 - 1, minValue/2, minValue/2 + 1,
        -1, 0, 1,
        maxValue/2 - 1, maxValue/2, maxValue/2 + 1,
        maxValue - 1, maxValue
    };

    constexpr BigIntType lowerLimit = minValue;
    constexpr BigIntType upperLimit = maxValue;
    for (const auto& x : valueSet) {
        for (const auto& y : valueSet) {
            BigIntType intX = x;
            BigIntType intY = y;
            BigIntType expected = std::clamp(intX + intY, lowerLimit, upperLimit);
            BigIntType actual = SaturatedAddInt16(x, y);
            EXPECT_EQ(expected, actual);
        }
    }
}

TEST_F(TestAsmInstructions, SaturatedAddUint16) {
    using UintType = uint16_t;
    using BigUintType = uint64_t;
    constexpr auto maxValue = std::numeric_limits<UintType>::max();
    const std::vector<uint16_t> valueSet {
        0, 1, maxValue/2 - 1, maxValue/2, maxValue/2 + 1,
        maxValue - 1, maxValue
    };

    constexpr BigUintType upperLimit = maxValue;
    for (const auto& x : valueSet) {
        for (const auto& y : valueSet) {
            BigUintType uintX = x;
            BigUintType uintY = y;
            BigUintType expected = std::min(uintX + uintY, upperLimit);
            BigUintType actual = SaturatedAddUint16(x, y);
            EXPECT_EQ(expected, actual);
            actual = SaturatedAddUint16cmov(x, y);
            EXPECT_EQ(expected, actual);
        }
    }
}

#if 1
TEST_F(TestAsmInstructions, SaturatedAddInt16Full) {
    using IntType = int16_t;
    using BigIntType = int64_t;
    constexpr auto minValue = std::numeric_limits<IntType>::min();
    constexpr auto maxValue = std::numeric_limits<IntType>::max();
    constexpr BigIntType lowerLimit = minValue;
    constexpr BigIntType upperLimit = maxValue;
    for (IntType x = minValue; x < maxValue; ++x) {
        for (IntType y = minValue; y < maxValue; ++y) {
            BigIntType intX = x;
            BigIntType intY = y;
            BigIntType expected = std::clamp(intX + intY, lowerLimit, upperLimit);
            BigIntType actual = SaturatedAddInt16(x, y);
            EXPECT_EQ(expected, actual);
        }
    }
}

TEST_F(TestAsmInstructions, SaturatedAddUint16Full) {
    using UintType = uint16_t;
    using BigUintType = uint64_t;
    constexpr auto maxValue = std::numeric_limits<UintType>::max();

    constexpr BigUintType upperLimit = maxValue;
    for (UintType x = 0; x < maxValue; ++x) {
        for (UintType y = 0; y < maxValue; ++y) {
            BigUintType uintX = x;
            BigUintType uintY = y;
            BigUintType expected = std::min(uintX + uintY, upperLimit);
            BigUintType actual = SaturatedAddUint16(x, y);
            EXPECT_EQ(expected, actual);
            actual = SaturatedAddUint16cmov(x, y);
            EXPECT_EQ(expected, actual);
        }
    }
}
#endif

TEST_F(TestAsmInstructions, ThreeBytesXchg) {
    threeBytesXchg();
}

/*
  Local Variables:
  mode: c++
  coding: utf-8-dos
  tab-width: nil
  c-file-style: "stroustrup"
  End:
*/

max_stars_repo_name: zettsu-t/cPlusPlusFriend

max_issues_repo_name: zettsu-t/cPlusPlusFriend

max_forks_repo_name: zettsu-t/cPlusPlusFriend


text:
// Copyright (c) 2012-2017, The CryptoNote developers, The Bytecoin developers
//
// This file is part of Bytecoin.
//
// Bytecoin is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Bytecoin is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Bytecoin.  If not, see <http://www.gnu.org/licenses/>.

#include "CryptoNoteSerialization.h"

#include <algorithm>
#include <sstream>
#include <stdexcept>

#include <boost/variant/static_visitor.hpp>
#include <boost/variant/apply_visitor.hpp>

#include "Serialization/ISerializer.h"
#include "Serialization/SerializationOverloads.h"
#include "Serialization/BinaryInputStreamSerializer.h"
#include "Serialization/BinaryOutputStreamSerializer.h"

#include "Common/StringOutputStream.h"
#include "crypto/crypto.h"

#include "Account.h"
#include <config/CryptoNoteConfig.h>
#include "CryptoNoteFormatUtils.h"
#include "CryptoNoteTools.h"
#include "TransactionExtra.h"

using namespace Common;

namespace {

using namespace CryptoNote;
using namespace Common;

uint64_t getSignaturesCount(const TransactionInput& input) {
  struct txin_signature_size_visitor : public boost::static_visitor < uint64_t > {
    uint64_t operator()(const BaseInput& txin) const { return 0; }
    uint64_t operator()(const KeyInput& txin) const { return txin.outputIndexes.size(); }
  };

  return boost::apply_visitor(txin_signature_size_visitor(), input);
}

struct BinaryVariantTagGetter: boost::static_visitor<uint8_t> {
  uint8_t operator()(const CryptoNote::BaseInput) { return  0xff; }
  uint8_t operator()(const CryptoNote::KeyInput) { return  0x2; }
  uint8_t operator()(const CryptoNote::KeyOutput) { return  0x2; }
  uint8_t operator()(const CryptoNote::Transaction) { return  0xcc; }
  uint8_t operator()(const CryptoNote::BlockTemplate) { return  0xbb; }
};

struct VariantSerializer : boost::static_visitor<> {
  VariantSerializer(CryptoNote::ISerializer& serializer, const std::string& name) : s(serializer), name(name) {}

  template <typename T>
  void operator() (T& param) { s(param, name); }

  CryptoNote::ISerializer& s;
  std::string name;
};

void getVariantValue(CryptoNote::ISerializer& serializer, uint8_t tag, CryptoNote::TransactionInput& in) {
  switch(tag) {
  case 0xff: {
    CryptoNote::BaseInput v;
    serializer(v, "value");
    in = v;
    break;
  }
  case 0x2: {
    CryptoNote::KeyInput v;
    serializer(v, "value");
    in = v;
    break;
  }
  default:
    throw std::runtime_error("Unknown variant tag");
  }
}

void getVariantValue(CryptoNote::ISerializer& serializer, uint8_t tag, CryptoNote::TransactionOutputTarget& out) {
  switch(tag) {
  case 0x2: {
    CryptoNote::KeyOutput v;
    serializer(v, "data");
    out = v;
    break;
  }
  default:
    throw std::runtime_error("Unknown variant tag");
  }
}

template <typename T>
bool serializePod(T& v, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializer.binary(&v, sizeof(v), name);
}

bool serializeVarintVector(std::vector<uint32_t>& vector, CryptoNote::ISerializer& serializer, Common::StringView name) {
  uint64_t size = vector.size();

  if (!serializer.beginArray(size, name)) {
    vector.clear();
    return false;
  }

  vector.resize(size);

  for (uint64_t i = 0; i < size; ++i) {
    serializer(vector[i], "");
  }

  serializer.endArray();
  return true;
}

}

namespace Crypto {

bool serialize(PublicKey& pubKey, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(pubKey, name, serializer);
}

bool serialize(SecretKey& secKey, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(secKey, name, serializer);
}

bool serialize(Hash& h, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(h, name, serializer);
}

bool serialize(KeyImage& keyImage, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(keyImage, name, serializer);
}

bool serialize(chacha8_iv& chacha, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(chacha, name, serializer);
}

bool serialize(Signature& sig, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(sig, name, serializer);
}

bool serialize(EllipticCurveScalar& ecScalar, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(ecScalar, name, serializer);
}

bool serialize(EllipticCurvePoint& ecPoint, Common::StringView name, CryptoNote::ISerializer& serializer) {
  return serializePod(ecPoint, name, serializer);
}

}

namespace CryptoNote {

void serialize(TransactionPrefix& txP, ISerializer& serializer) {
  serializer(txP.version, "version");

  if (CURRENT_TRANSACTION_VERSION < txP.version && serializer.type() == ISerializer::INPUT) {
    throw std::runtime_error("Wrong transaction version");
  }

  serializer(txP.unlockTime, "unlock_time");
  serializer(txP.inputs, "vin");
  serializer(txP.outputs, "vout");
  serializeAsBinary(txP.extra, "extra", serializer);
}

void serialize(BaseTransaction& tx, ISerializer& serializer) {
  serializer(tx.version, "version");
  serializer(tx.unlockTime, "unlock_time");
  serializer(tx.inputs, "vin");
  serializer(tx.outputs, "vout");
  serializeAsBinary(tx.extra, "extra", serializer);

  if (tx.version >= TRANSACTION_VERSION_2) {
    uint64_t ignored = 0;
    serializer(ignored, "ignored");
  }
}

void serialize(Transaction& tx, ISerializer& serializer) {
  serialize(static_cast<TransactionPrefix&>(tx), serializer);

  uint64_t sigSize = tx.inputs.size();
  //TODO: make arrays without sizes
//  serializer.beginArray(sigSize, "signatures");

  // ignore base transaction
  if (serializer.type() == ISerializer::INPUT && !(sigSize == 1 && tx.inputs[0].type() == typeid(BaseInput))) {
    tx.signatures.resize(sigSize);
  }

  bool signaturesNotExpected = tx.signatures.empty();
  if (!signaturesNotExpected && tx.inputs.size() != tx.signatures.size()) {
    throw std::runtime_error("Serialization error: unexpected signatures size");
  }

  for (uint64_t i = 0; i < tx.inputs.size(); ++i) {
    uint64_t signatureSize = getSignaturesCount(tx.inputs[i]);
    if (signaturesNotExpected) {
      if (signatureSize == 0) {
        continue;
      } else {
        throw std::runtime_error("Serialization error: signatures are not expected");
      }
    }

    if (serializer.type() == ISerializer::OUTPUT) {
      if (signatureSize != tx.signatures[i].size()) {
        throw std::runtime_error("Serialization error: unexpected signatures size");
      }

      for (Crypto::Signature& sig : tx.signatures[i]) {
        serializePod(sig, "", serializer);
      }

    } else {
      std::vector<Crypto::Signature> signatures(signatureSize);
      for (Crypto::Signature& sig : signatures) {
        serializePod(sig, "", serializer);
      }

      tx.signatures[i] = std::move(signatures);
    }
  }
//  serializer.endArray();
}

void serialize(TransactionInput& in, ISerializer& serializer) {
  if (serializer.type() == ISerializer::OUTPUT) {
    BinaryVariantTagGetter tagGetter;
    uint8_t tag = boost::apply_visitor(tagGetter, in);
    serializer.binary(&tag, sizeof(tag), "type");

    VariantSerializer visitor(serializer, "value");
    boost::apply_visitor(visitor, in);
  } else {
    uint8_t tag;
    serializer.binary(&tag, sizeof(tag), "type");

    getVariantValue(serializer, tag, in);
  }
}

void serialize(BaseInput& gen, ISerializer& serializer) {
  serializer(gen.blockIndex, "height");
}

void serialize(KeyInput& key, ISerializer& serializer) {
  serializer(key.amount, "amount");
  serializeVarintVector(key.outputIndexes, serializer, "key_offsets");
  serializer(key.keyImage, "k_image");
}

void serialize(TransactionOutput& output, ISerializer& serializer) {
  serializer(output.amount, "amount");
  serializer(output.target, "target");
}

void serialize(TransactionOutputTarget& output, ISerializer& serializer) {
  if (serializer.type() == ISerializer::OUTPUT) {
    BinaryVariantTagGetter tagGetter;
    uint8_t tag = boost::apply_visitor(tagGetter, output);
    serializer.binary(&tag, sizeof(tag), "type");

    VariantSerializer visitor(serializer, "data");
    boost::apply_visitor(visitor, output);
  } else {
    uint8_t tag;
    serializer.binary(&tag, sizeof(tag), "type");

    getVariantValue(serializer, tag, output);
  }
}

void serialize(KeyOutput& key, ISerializer& serializer) {
  serializer(key.key, "key");
}

void serialize(ParentBlockSerializer& pbs, ISerializer& serializer) {
  serializer(pbs.m_parentBlock.majorVersion, "majorVersion");

  serializer(pbs.m_parentBlock.minorVersion, "minorVersion");
  serializer(pbs.m_timestamp, "timestamp");
  serializer(pbs.m_parentBlock.previousBlockHash, "prevId");
  serializer.binary(&pbs.m_nonce, sizeof(pbs.m_nonce), "nonce");

  if (pbs.m_hashingSerialization) {
    Crypto::Hash minerTxHash;
    if (!getBaseTransactionHash(pbs.m_parentBlock.baseTransaction, minerTxHash)) {
      throw std::runtime_error("Get transaction hash error");
    }

    Crypto::Hash merkleRoot;
    Crypto::tree_hash_from_branch(pbs.m_parentBlock.baseTransactionBranch.data(), pbs.m_parentBlock.baseTransactionBranch.size(), minerTxHash, 0, merkleRoot);

    serializer(merkleRoot, "merkleRoot");
  }

  uint64_t txNum = static_cast<uint64_t>(pbs.m_parentBlock.transactionCount);
  serializer(txNum, "numberOfTransactions");
  pbs.m_parentBlock.transactionCount = static_cast<uint16_t>(txNum);
  if (pbs.m_parentBlock.transactionCount < 1) {
    throw std::runtime_error("Wrong transactions number");
  }

  if (pbs.m_headerOnly) {
    return;
  }

  uint64_t branchSize = Crypto::tree_depth(pbs.m_parentBlock.transactionCount);
  if (serializer.type() == ISerializer::OUTPUT) {
    if (pbs.m_parentBlock.baseTransactionBranch.size() != branchSize) {
      throw std::runtime_error("Wrong miner transaction branch size");
    }
  } else {
    pbs.m_parentBlock.baseTransactionBranch.resize(branchSize);
  }

//  serializer(m_parentBlock.baseTransactionBranch, "baseTransactionBranch");
  //TODO: Make arrays with computable size! This code won't work with json serialization!
  for (Crypto::Hash& hash: pbs.m_parentBlock.baseTransactionBranch) {
    serializer(hash, "");
  }

  serializer(pbs.m_parentBlock.baseTransaction, "minerTx");

  TransactionExtraMergeMiningTag mmTag;
  if (!getMergeMiningTagFromExtra(pbs.m_parentBlock.baseTransaction.extra, mmTag)) {
    throw std::runtime_error("Can't get extra merge mining tag");
  }

  if (mmTag.depth > 8 * sizeof(Crypto::Hash)) {
    throw std::runtime_error("Wrong merge mining tag depth");
  }

  if (serializer.type() == ISerializer::OUTPUT) {
    if (mmTag.depth != pbs.m_parentBlock.blockchainBranch.size()) {
      throw std::runtime_error("Blockchain branch size must be equal to merge mining tag depth");
    }
  } else {
    pbs.m_parentBlock.blockchainBranch.resize(mmTag.depth);
  }

//  serializer(m_parentBlock.blockchainBranch, "blockchainBranch");
  //TODO: Make arrays with computable size! This code won't work with json serialization!
  for (Crypto::Hash& hash: pbs.m_parentBlock.blockchainBranch) {
    serializer(hash, "");
  }
}

void serializeBlockHeader(BlockHeader& header, ISerializer& serializer) {
  serializer(header.majorVersion, "major_version");
  if (header.majorVersion > BLOCK_MAJOR_VERSION_5) {
    throw std::runtime_error("Wrong major version");
  }

  serializer(header.minorVersion, "minor_version");
  if (header.majorVersion == BLOCK_MAJOR_VERSION_0) {
    serializer(header.timestamp, "timestamp");
    serializer(header.previousBlockHash, "prev_id");
    serializer.binary(&header.nonce, sizeof(header.nonce), "nonce");
  } else if (header.majorVersion >= BLOCK_MAJOR_VERSION_1) {
    serializer(header.previousBlockHash, "prev_id");
  } else {
    throw std::runtime_error("Wrong major version");
  }
}

void serialize(BlockHeader& header, ISerializer& serializer) {
  serializeBlockHeader(header, serializer);
}

void serialize(BlockTemplate& block, ISerializer& serializer) {
  serializeBlockHeader(block, serializer);

  if (block.majorVersion >= BLOCK_MAJOR_VERSION_1) {
    auto parentBlockSerializer = makeParentBlockSerializer(block, false, false);
    serializer(parentBlockSerializer, "parent_block");
  }

  serializer(block.baseTransaction, "miner_tx");
  serializer(block.transactionHashes, "tx_hashes");
}

void serialize(AccountPublicAddress& address, ISerializer& serializer) {
  serializer(address.spendPublicKey, "m_spend_public_key");
  serializer(address.viewPublicKey, "m_view_public_key");
}

void serialize(AccountKeys& keys, ISerializer& s) {
  s(keys.address, "m_account_address");
  s(keys.spendSecretKey, "m_spend_secret_key");
  s(keys.viewSecretKey, "m_view_secret_key");
}

void doSerialize(TransactionExtraMergeMiningTag& tag, ISerializer& serializer) {
  uint64_t depth = static_cast<uint64_t>(tag.depth);
  serializer(depth, "depth");
  tag.depth = static_cast<uint64_t>(depth);
  serializer(tag.merkleRoot, "merkle_root");
}

void serialize(TransactionExtraMergeMiningTag& tag, ISerializer& serializer) {
  if (serializer.type() == ISerializer::OUTPUT) {
    std::string field;
    StringOutputStream os(field);
    BinaryOutputStreamSerializer output(os);
    doSerialize(tag, output);
    serializer(field, "");
  } else {
    std::string field;
    serializer(field, "mm_tag");
    MemoryInputStream stream(field.data(), field.size());
    BinaryInputStreamSerializer input(stream);
    doSerialize(tag, input);
  }
}

void serialize(KeyPair& keyPair, ISerializer& serializer) {
  serializer(keyPair.secretKey, "secret_key");
  serializer(keyPair.publicKey, "public_key");
}

// unpack to strings to maintain protocol compatibility with older versions
void serialize(RawBlock& rawBlock, ISerializer& serializer) {
  if (serializer.type() == ISerializer::INPUT) {
    uint64_t blockSize;
    serializer(blockSize, "block_size");
    rawBlock.block.resize(static_cast<uint64_t>(blockSize));
  } else {
    uint64_t blockSize = rawBlock.block.size();
    serializer(blockSize, "block_size");
  }

  serializer.binary(rawBlock.block.data(), rawBlock.block.size(), "block");

  if (serializer.type() == ISerializer::INPUT) {
    uint64_t txCount;
    serializer(txCount, "tx_count");
    rawBlock.transactions.resize(static_cast<uint64_t>(txCount));

    for (auto& txBlob : rawBlock.transactions) {
      uint64_t txSize;
      serializer(txSize, "tx_size");
      txBlob.resize(txSize);
      serializer.binary(txBlob.data(), txBlob.size(), "transaction");
    }
  } else {
    uint64_t txCount = rawBlock.transactions.size();
    serializer(txCount, "tx_count");

    for (auto& txBlob : rawBlock.transactions) {
      uint64_t txSize = txBlob.size();
      serializer(txSize, "tx_size");
      serializer.binary(txBlob.data(), txBlob.size(), "transaction");
    }
  }
}

} //namespace CryptoNote

ext: cpp

max_stars_repo_name: plenteum/plenteum-testnet

max_issues_repo_name: plenteum/plenteum-testnet

max_forks_repo_name: plenteum/plenteum-testnet


text:
/*
 * Copyright (c) 2011-2021, The DART development contributors
 * All rights reserved.
 *
 * The list of contributors can be found at:
 *   https://github.com/dartsim/dart/blob/master/LICENSE
 *
 * This file is provided under the following "BSD-style" License:
 *   Redistribution and use in source and binary forms, with or
 *   without modification, are permitted provided that the following
 *   conditions are met:
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 *   CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 *   INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 *   MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 *   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
 *   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 *   AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *   LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *   ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *   POSSIBILITY OF SUCH DAMAGE.
 */

#include "dart/optimizer/nlopt/NloptSolver.hpp"

#include <memory>

#include <Eigen/Dense>

#include "dart/common/Console.hpp"
#include "dart/common/StlHelpers.hpp"
#include "dart/optimizer/Function.hpp"
#include "dart/optimizer/Problem.hpp"

namespace dart {
namespace optimizer {

//==============================================================================
NloptSolver::NloptSolver(
    const Solver::Properties& properties, nlopt::algorithm alg)
  : NloptSolver(properties, convertAlgorithm(alg))
{
  // Do nothing
}

//==============================================================================
NloptSolver::NloptSolver(
    const Solver::Properties& properties, NloptSolver::Algorithm alg)
  : Solver(properties), mOpt(nullptr), mAlg(convertAlgorithm(alg)), mMinF(0.0)
{
  // Do nothing
}

//==============================================================================
NloptSolver::NloptSolver(std::shared_ptr<Problem> problem, nlopt::algorithm alg)
  : NloptSolver(std::move(problem), convertAlgorithm(alg))
{
  // Do nothing
}

//==============================================================================
NloptSolver::NloptSolver(
    std::shared_ptr<Problem> problem, NloptSolver::Algorithm alg)
  : Solver(std::move(problem)),
    mOpt(nullptr),
    mAlg(convertAlgorithm(alg)),
    mMinF(0.0)
{
  // Do nothing
}

//==============================================================================
NloptSolver::~NloptSolver()
{
  // Do nothing
}

//==============================================================================
static std::vector<double> convertToStd(const Eigen::VectorXd& v)
{
  return std::vector<double>(v.data(), v.data() + v.size());
}

//==============================================================================
static Eigen::VectorXd convertToEigen(const std::vector<double>& v)
{
  Eigen::VectorXd result(v.size());
  for (std::size_t i = 0; i < v.size(); ++i)
    result[static_cast<Eigen::Index>(i)] = v[i];

  return result;
}

//==============================================================================
bool NloptSolver::solve()
{
  // Allocate a new nlopt::opt structure if needed
  std::size_t dimension = mProperties.mProblem->getDimension();
  if (nullptr == mOpt || mOpt->get_dimension() != dimension
      || mOpt->get_algorithm() != mAlg)
  {
    mOpt = std::make_unique<nlopt::opt>(mAlg, dimension);
  }
  else
  {
    mOpt->remove_equality_constraints();
    mOpt->remove_inequality_constraints();
  }

  const std::shared_ptr<Problem>& problem = mProperties.mProblem;

  mOpt->set_maxeval(static_cast<int>(mProperties.mNumMaxIterations));
  mOpt->set_xtol_rel(mProperties.mTolerance);
  mOpt->set_lower_bounds(convertToStd(problem->getLowerBounds()));
  mOpt->set_upper_bounds(convertToStd(problem->getUpperBounds()));

  // Set up the nlopt::opt
  mOpt->set_min_objective(
      NloptSolver::_nlopt_func, problem->getObjective().get());

  for (std::size_t i = 0; i < problem->getNumEqConstraints(); ++i)
  {
    FunctionPtr fn = problem->getEqConstraint(i);
    try
    {
      mOpt->add_equality_constraint(
          NloptSolver::_nlopt_func, fn.get(), mProperties.mTolerance);
    }
    catch (const std::invalid_argument& e)
    {
      dterr << "[NloptSolver::solve] Encountered exception [" << e.what()
            << "] while adding an equality constraint to an Nlopt solver. "
            << "Check whether your algorithm [" << nlopt::algorithm_name(mAlg)
            << "] (" << mAlg << ") supports equality constraints!\n";
      assert(false);
    }
    catch (const std::exception& e)
    {
      dterr << "[NloptSolver::solve] Encountered exception [" << e.what()
            << "] while adding an equality constraint to the Nlopt solver. "
            << "This might be a bug in DART; please report this!\n";
      assert(false);
    }
  }

  for (std::size_t i = 0; i < problem->getNumIneqConstraints(); ++i)
  {
    FunctionPtr fn = problem->getIneqConstraint(i);
    try
    {
      mOpt->add_inequality_constraint(
          NloptSolver::_nlopt_func, fn.get(), mProperties.mTolerance);
    }
    catch (const std::invalid_argument& e)
    {
      dterr << "[NloptSolver::solve] Encountered exception [" << e.what()
            << "] while adding an inequality constraint to an Nlopt solver. "
            << "Check whether your algorithm [" << nlopt::algorithm_name(mAlg)
            << "] (" << mAlg << ") supports inequality constraints!\n";
      assert(false);
    }
    catch (const std::exception& e)
    {
      dterr << "[NloptSolver::solve] Encountered exception [" << e.what()
            << "] while adding an inequality constraint to the Nlopt solver. "
            << "This might be a bug in DART; please report this!\n";
      assert(false);
    }
  }

  // Optimize
  mX = convertToStd(problem->getInitialGuess());
  nlopt::result result = mOpt->optimize(mX, mMinF);

  // If the result is not in this range, then it failed
  if (!(nlopt::SUCCESS <= result && result <= nlopt::XTOL_REACHED))
    return false;

  // Store optimal and optimum values
  problem->setOptimumValue(mMinF);
  problem->setOptimalSolution(convertToEigen(mX));

  return true;
}

//==============================================================================
Eigen::VectorXd NloptSolver::getLastConfiguration() const
{
  return convertToEigen(mX);
}

//==============================================================================
std::string NloptSolver::getType() const
{
  return "NloptSolver";
}

//==============================================================================
std::shared_ptr<Solver> NloptSolver::clone() const
{
  return std::make_shared<NloptSolver>(getSolverProperties(), getAlgorithm2());
}

//==============================================================================
void NloptSolver::copy(const NloptSolver& other)
{
  setProperties(other.getSolverProperties());
  setAlgorithm(other.getAlgorithm2());
}

//==============================================================================
NloptSolver& NloptSolver::operator=(const NloptSolver& other)
{
  copy(other);
  return *this;
}

//==============================================================================
void NloptSolver::setAlgorithm(nlopt::algorithm alg)
{
  setAlgorithm(convertAlgorithm(alg));
}

//==============================================================================
void NloptSolver::setAlgorithm(NloptSolver::Algorithm alg)
{
  mAlg = convertAlgorithm(alg);
}

//==============================================================================
nlopt::algorithm NloptSolver::getAlgorithm() const
{
  return convertAlgorithm(getAlgorithm2());
}

//==============================================================================
NloptSolver::Algorithm NloptSolver::getAlgorithm2() const
{
  return convertAlgorithm(mAlg);
}

//==============================================================================
#define NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(alg_name)                          \
  case alg_name:                                                               \
    return nlopt::algorithm::alg_name;

//==============================================================================
#define NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(alg_name)                          \
  case nlopt::algorithm::alg_name:                                             \
    return alg_name;

//==============================================================================
nlopt::algorithm NloptSolver::convertAlgorithm(NloptSolver::Algorithm algorithm)
{
  switch (algorithm)
  {
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT_L)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT_L_RAND)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT_NOSCAL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT_L_NOSCAL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_DIRECT_L_RAND_NOSCAL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_ORIG_DIRECT)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_ORIG_DIRECT_L)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GD_STOGO)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GD_STOGO_RAND)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_LBFGS_NOCEDAL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_LBFGS)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_PRAXIS)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_VAR1)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_VAR2)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_TNEWTON)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_TNEWTON_RESTART)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_TNEWTON_PRECOND)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_TNEWTON_PRECOND_RESTART)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_CRS2_LM)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_MLSL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GD_MLSL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GD_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_MMA)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_COBYLA)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_NEWUOA)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_NEWUOA_BOUND)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_NELDERMEAD)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_SBPLX)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_AUGLAG)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_AUGLAG)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LN_BOBYQA)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_ISRES)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(AUGLAG)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(G_MLSL)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(G_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_SLSQP)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(LD_CCSAQ)
    NLOPTSOLVER_ALGORITHM_DART_TO_NLOPT(GN_ESCH)
    default:
      dtwarn << "[NloptSolver] Attempt to convert unsupported algorithm '"
             << algorithm << "'. Use nlopt::algorithm::LN_COBYLA instead. \n";
      return nlopt::algorithm::LN_COBYLA;
  }
}

//==============================================================================
NloptSolver::Algorithm NloptSolver::convertAlgorithm(nlopt::algorithm algorithm)
{
  switch (algorithm)
  {
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT_L)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT_L_RAND)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT_NOSCAL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT_L_NOSCAL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_DIRECT_L_RAND_NOSCAL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_ORIG_DIRECT)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_ORIG_DIRECT_L)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GD_STOGO)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GD_STOGO_RAND)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_LBFGS_NOCEDAL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_LBFGS)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_PRAXIS)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_VAR1)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_VAR2)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_TNEWTON)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_TNEWTON_RESTART)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_TNEWTON_PRECOND)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_TNEWTON_PRECOND_RESTART)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_CRS2_LM)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_MLSL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GD_MLSL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GD_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_MMA)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_COBYLA)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_NEWUOA)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_NEWUOA_BOUND)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_NELDERMEAD)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_SBPLX)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_AUGLAG)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_AUGLAG)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LN_BOBYQA)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_ISRES)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(AUGLAG)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(AUGLAG_EQ)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(G_MLSL)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(G_MLSL_LDS)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_SLSQP)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(LD_CCSAQ)
    NLOPTSOLVER_ALGORITHM_NLOPT_TO_DART(GN_ESCH)
    default:
      dtwarn << "[NloptSolver] Attempt to convert unsupported algorithm '"
             << algorithm
             << "'. Use NloptSolver::Algorithm::LN_COBYLA instead. \n";
      return LN_COBYLA;
  }
}

//==============================================================================
double NloptSolver::_nlopt_func(
    unsigned n, const double* x, double* gradient, void* func_data)
{
  Function* fn = static_cast<Function*>(func_data);

  Eigen::Map<const Eigen::VectorXd> mapX(x, n);

  if (gradient)
  {
    Eigen::Map<Eigen::VectorXd> grad(gradient, n);
    fn->evalGradient(mapX, grad);
  }

  return fn->eval(mapX);
}

//==============================================================================
void NloptSolver::_nlopt_mfunc(
    unsigned m,
    double* result,
    unsigned n,
    const double* x,
    double* gradient,
    void* func_data)
{
  Eigen::Map<const Eigen::VectorXd> mapX(x, n);
  Eigen::Map<Eigen::VectorXd> f(result, m);
  Eigen::Map<Eigen::MatrixXd> grad(gradient, 0, 0);
  if (gradient)
    new (&grad) Eigen::Map<Eigen::MatrixXd, Eigen::RowMajor>(gradient, m, n);

  return (*static_cast<MultiFunction*>(func_data))(mapX, f, grad);
}

} // namespace optimizer
} // namespace dart

max_stars_repo_name: dartsim/dart

max_issues_repo_name: dartsim/dart

max_forks_repo_name: dartsim/dart


text:
/*
 * Copyright 2020 OmniSci, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// project headers
#include "DBEngine.h"
#include "Shared/ArrowUtil.h"
#include "Shared/InlineNullValues.h"
#include "Shared/scope.h"

// boost headers
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>

// std headers
#include <array>
#include <cstdlib>
#include <exception>
#include <filesystem>
#include <iostream>
#include <limits>

// arrow headers
#include <arrow/api.h>
#include <arrow/csv/reader.h>
#include <arrow/io/file.h>

// Google Test
#include <gtest/gtest.h>

// Global variables controlling execution
extern bool g_enable_columnar_output;
extern bool g_enable_lazy_fetch;

// DBEngine Instance
static std::shared_ptr<EmbeddedDatabase::DBEngine> g_dbe;

// Input files' names
static const char* TABLE6x4_CSV_FILE =
    "../../Tests/EmbeddedDataFiles/embedded_db_test_6x4table.csv";
static const char* JOIN_TABLE_CSV_FILE =
    "../../Tests/EmbeddedDataFiles/embedded_db_test_join_table.csv";
static const char* NULLSTABLE6x4_CSV_FILE =
    "../../Tests/EmbeddedDataFiles/embedded_db_test_nulls_table.csv";

//  Content of the table stored in $TABLE6x4_CSV_FILE file
static std::array<int32_t, 6> table6x4_col_i32 = {0, 0, 0, 1, 1, 1};
static std::array<int64_t, 6> table6x4_col_i64 = {0, 0, 0, 1, 1, 1};
static std::array<int64_t, 6> table6x4_col_bi = {1, 2, 3, 4, 5, 6};
static std::array<double, 6> table6x4_col_d = {10.1, 20.2, 30.3, 40.4, 50.5, 60.6};

//  HELPERS
namespace {

// Processes command line args, return options string
std::string parse_cli_args(int argc, char* argv[]) {
  namespace po = boost::program_options;

  int calcite_port = 5555;
  bool columnar_output = true;
  std::string catalog_path = "tmp";
  std::string opt_str;

  po::options_description desc("Options");

  desc.add_options()("help,h", "Print help messages ");

  desc.add_options()("data,d",
                     po::value<std::string>(&catalog_path)->default_value(catalog_path),
                     "Directory path to OmniSci catalogs");

  desc.add_options()("calcite-port,p",
                     po::value<int>(&calcite_port)->default_value(calcite_port),
                     "Calcite port");

  desc.add_options()("enable-columnar-output,o",
                     po::value<bool>(&columnar_output)->default_value(columnar_output),
                     "Enable columnar_output");

  po::variables_map vm;

  try {
    po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);

    if (vm.count("help")) {
      std::cout << desc;
      std::exit(EXIT_SUCCESS);
    }
    po::notify(vm);

    opt_str = catalog_path + " --calcite-port " + std::to_string(calcite_port);
    if (columnar_output) {
      opt_str += " --enable-columnar-output";
    }
  } catch (boost::program_options::error& e) {
    std::cerr << "Usage Error: " << e.what() << std::endl;
    std::cout << desc;
    std::exit(EXIT_FAILURE);
  }

  return opt_str;
}

// Creates a table from TABLE6x4_CSV_FILE file with provided
// parameters:
//  + table_name -- name of the table
//  + fragment_size -- fragment size for the table; 0 to use the default value
void load_table6x4_csv(std::string csv_file,
                       std::string table_name,
                       size_t fragment_size) {
  if (!g_dbe) {
    throw std::runtime_error("DBEngine is not initialized.  Aborting.\n");
  }
  std::string frag_size_param =
      fragment_size > 0 ? ", fragment_size=" + std::to_string(fragment_size) : "";

  std::string create_query = "CREATE TEMPORARY TABLE " + table_name +
                             " (t TEXT, i INT, bi BIGINT, d DOUBLE) "
                             "WITH (storage_type='CSV:" +
                             csv_file + "'" + frag_size_param + ");";

  g_dbe->executeDDL(create_query);
}

// Load CSV file into a temporary table using Arrow import
void import_file(const std::string& file_name,
                 const std::string& table_name,
                 size_t fragment_size = 0) {
  arrow::io::IOContext io_context = arrow::io::default_io_context();
  auto arrow_parse_options = arrow::csv::ParseOptions::Defaults();
  auto arrow_read_options = arrow::csv::ReadOptions::Defaults();
  auto arrow_convert_options = arrow::csv::ConvertOptions::Defaults();
  std::shared_ptr<arrow::io::ReadableFile> inp;
  auto file_result = arrow::io::ReadableFile::Open(file_name);
  ARROW_THROW_NOT_OK(file_result.status());
  inp = file_result.ValueOrDie();
  auto table_reader_result = arrow::csv::TableReader::Make(
      io_context, inp, arrow_read_options, arrow_parse_options, arrow_convert_options);

  ARROW_THROW_NOT_OK(table_reader_result.status());
  auto table_reader = table_reader_result.ValueOrDie();
  std::shared_ptr<arrow::Table> arrowTable;
  auto arrow_table_result = table_reader->Read();
  ARROW_THROW_NOT_OK(arrow_table_result.status());
  arrowTable = arrow_table_result.ValueOrDie();

  g_dbe->importArrowTable(table_name, arrowTable, fragment_size);
}

template <typename TYPE>
constexpr TYPE null_builder() {
  static_assert(std::is_floating_point_v<TYPE> || std::is_integral_v<TYPE>,
                "Unsupported type");

  if constexpr (std::is_floating_point_v<TYPE>) {
    return inline_fp_null_value<TYPE>();
  } else if constexpr (std::is_integral_v<TYPE>) {
    return inline_int_null_value<TYPE>();
  }
}

template <typename TYPE, size_t len>
void compare_columns(const std::array<TYPE, len>& expected,
                     const std::shared_ptr<arrow::ChunkedArray>& actual) {
  using ArrowColType = arrow::NumericArray<typename arrow::CTypeTraits<TYPE>::ArrowType>;
  const arrow::ArrayVector& chunks = actual->chunks();

  TYPE null_val = null_builder<TYPE>();

  for (int i = 0, k = 0; i < actual->num_chunks(); i++) {
    auto chunk = chunks[i];
    auto arrow_row_array = std::static_pointer_cast<ArrowColType>(chunk);

    const TYPE* chunk_data = arrow_row_array->raw_values();
    for (int64_t j = 0; j < arrow_row_array->length(); j++, k++) {
      if (expected[k] == null_val) {
        CHECK(chunk->IsNull(j));
      } else {
        CHECK(chunk->IsValid(j));
        ASSERT_EQ(expected[k], chunk_data[j]);
      }
    }
  }
};

//  Testing helper functions for 6x4 table contained in $TABLE6x4_CSV_FILE file.
//  NOTE: load_table6x4_csv() creates table with Arrow types <dictionary, int32, int64,
//  float64> so we check that.
void test_arrow_table_conversion_table6x4(size_t fragment_size) {
  auto cursor = g_dbe->executeDML("select * from test_chunked");

  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();

  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 4);
  ASSERT_EQ(table->num_rows(), (int64_t)6);

  auto column = table->column(1);
  size_t expected_chunk_count =
      (g_enable_lazy_fetch || !g_enable_columnar_output)
          ? 1
          : (table->num_rows() + fragment_size - 1) / fragment_size;

  size_t actual_chunks_count = column->num_chunks();
  ASSERT_EQ(actual_chunks_count, expected_chunk_count);

  ASSERT_NE(table->column(0), nullptr);
  ASSERT_NE(table->column(1), nullptr);
  ASSERT_NE(table->column(2), nullptr);
  ASSERT_NE(table->column(3), nullptr);

  auto schema = table->schema();
  ASSERT_EQ(schema->num_fields(), 4);
  ASSERT_EQ(schema->GetFieldByName("i")->type()->Equals(arrow::int32()), true);
  ASSERT_EQ(schema->GetFieldByName("bi")->type()->Equals(arrow::int64()), true);
  ASSERT_EQ(schema->GetFieldByName("d")->type()->Equals(arrow::float64()), true);

  compare_columns(table6x4_col_i32, table->column(1));
  compare_columns(table6x4_col_bi, table->column(2));
  compare_columns(table6x4_col_d, table->column(3));
}

//  Performs tests for 6x4 table contained in $TABLE6x4_CSV_FILE for various
//  values of enable_columnar_output, enable_lazy_fetch, fragment_size
void test_chunked_conversion(bool enable_columnar_output,
                             bool enable_lazy_fetch,
                             size_t fragment_size) {
  bool prev_enable_columnar_output = g_enable_columnar_output;
  bool prev_enable_lazy_fetch = g_enable_lazy_fetch;

  ScopeGuard reset = [prev_enable_columnar_output, prev_enable_lazy_fetch] {
    g_enable_columnar_output = prev_enable_columnar_output;
    g_enable_lazy_fetch = prev_enable_lazy_fetch;
  };

  g_enable_columnar_output = enable_columnar_output;
  g_enable_lazy_fetch = enable_lazy_fetch;
  test_arrow_table_conversion_table6x4(/*fragment_size=*/fragment_size);
}

}  // anonymous namespace

//  Tests `DBEngine::importArrowTable()'  functionality
//  NOTE: For $TABLE6x4_CSV_FILE file, import_file() creates a table with Arrow
//  types <dictionary, int64, int64, float64> rather than <dictionary, int32, int64,
//  float64> (the latter is obtained via SQL query loading), so we perform checks
//  accordingly
TEST(DBEngine, ImportArrowTable) {
  import_file(TABLE6x4_CSV_FILE, "loading_test", /*fragment_size=*/4);
  auto cursor = g_dbe->executeDML("select * from loading_test");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 4);
  ASSERT_EQ(table->num_rows(), (int64_t)6);

  std::shared_ptr<arrow::Schema> schema = table->schema();

  ASSERT_EQ(schema->num_fields(), 4);
  ASSERT_EQ(schema->GetFieldByName("i")->type()->Equals(arrow::int64()), true);
  ASSERT_EQ(schema->GetFieldByName("bi")->type()->Equals(arrow::int64()), true);
  ASSERT_EQ(schema->GetFieldByName("d")->type()->Equals(arrow::float64()), true);

  compare_columns(table6x4_col_i64, table->column(1));
  compare_columns(table6x4_col_bi, table->column(2));
  compare_columns(table6x4_col_d, table->column(3));
}

TEST(DBEngine, SelectBool) {
  GTEST_SKIP();  //  `DBEngine::importArrowTable()' incorrectly imports bools so until
                 //  that is fixed, this test will fail. Remove GTEST_SKIP(); after that
                 //  bug is fixed.
                 //  Cf: https://github.com/intel-ai/omniscidb/issues/218#issue-1070805451
  auto cursor = g_dbe->executeDML("SELECT * FROM join_table;");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 3);
  ASSERT_EQ(table->num_rows(), 2);
  compare_columns(std::array<int64_t, 2>{0, 1}, table->column(0));
  compare_columns(std::array<bool, 2>{false, true}, table->column(1));
  compare_columns(std::array<int64_t, 2>{100, 200}, table->column(2));
}

//  Tests getArrowRecordBatch() for all columns selection
TEST(DBEngine, ArrowRecordBatch_SimpleSelectAll) {
  auto cursor = g_dbe->executeDML("select * from test");
  ASSERT_NE(cursor, nullptr);
  auto rbatch = cursor->getArrowRecordBatch();
  ASSERT_NE(rbatch, nullptr);
  ASSERT_EQ(rbatch->num_columns(), 4);
  ASSERT_EQ(rbatch->num_rows(), (int64_t)6);
}

//  Tests getArrowRecordBatch() for two columns (TEXT "t", INT "i") selection
TEST(DBEngine, ArrowRecordBatch_TextIntSelect) {
  auto cursor = g_dbe->executeDML("select t, i from test");
  ASSERT_NE(cursor, nullptr);
  auto rbatch = cursor->getArrowRecordBatch();
  ASSERT_NE(rbatch, nullptr);
  ASSERT_EQ(rbatch->num_columns(), 2);
  ASSERT_EQ(rbatch->num_rows(), (int64_t)6);
}

//  Tests getArrowTable() for three columns (TEXT "t", INT "i", BIGINT "bi") selection
TEST(DBEngine, ArrowTable_TextIntBigintSelect) {
  auto cursor = g_dbe->executeDML("select t, i, bi from test");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 3);
  ASSERT_EQ(table->num_rows(), (int64_t)6);
}

//  Two column selection with filtering. The result is empty table.
TEST(DBEngine, ArrowTable_EmptySelection) {
  auto cursor = g_dbe->executeDML("select i, d from test where d > 1000");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 2);
  ASSERT_EQ(table->num_rows(), (int64_t)0);
}

//  Chunked Arrow Table Conversion Tests
TEST(DBEngine, ArrowTableChunked_Conversion1) {
  test_chunked_conversion(/*columnar_output=*/false, /*lazy_fetch=*/false, 4);
}

TEST(DBEngine, ArrowTableChunked_Conversion2) {
  test_chunked_conversion(/*columnar_output=*/true, /*lazy_fetch=*/false, 4);
}

TEST(DBEngine, ArrowTableChunked_Conversion3) {
  test_chunked_conversion(/*columnar_output=*/false, /*lazy_fetch=*/true, 4);
}

TEST(DBEngine, ArrowTableChunked_Conversion4) {
  test_chunked_conversion(/*columnar_output=*/true, /*lazy_fetch=*/true, 4);
}

//  Projection operation test (column "d")
TEST(DBEngine, ArrowTableChunked_SingleColumnConversion) {
  auto cursor = g_dbe->executeDML("select 2*d from test_chunked");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 1);
  ASSERT_EQ(table->num_rows(), (int64_t)6);
}

//  Tests for GROUP BY query
TEST(DBEngine, ArrowTableChunked_GROUPBY1) {
  auto cursor = g_dbe->executeDML(
      "SELECT COUNT(d),COUNT(bi),COUNT(t),i FROM test_chunked GROUP BY i;");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 4);
  ASSERT_EQ(table->num_rows(), (int64_t)2);

  compare_columns(std::array<int32_t, 2>{3, 3}, table->column(0));
  compare_columns(std::array<int32_t, 2>{3, 3}, table->column(1));
  compare_columns(std::array<int32_t, 2>{3, 3}, table->column(2));
  compare_columns(std::array<int32_t, 2>{0, 1}, table->column(3));
}

//  Tests for JOIN query

//  This test performs a simple JOIN test for supplied boolean
//  values of enable_columnar_output, enable_lazy_fetch
void JoinTest(bool enable_columnar_output, bool enable_lazy_fetch) {
  bool prev_enable_columnar_output = g_enable_columnar_output;
  bool prev_enable_lazy_fetch = g_enable_lazy_fetch;

  ScopeGuard reset = [prev_enable_columnar_output, prev_enable_lazy_fetch] {
    g_enable_columnar_output = prev_enable_columnar_output;
    g_enable_lazy_fetch = prev_enable_lazy_fetch;
  };

  g_enable_columnar_output = enable_columnar_output;
  g_enable_lazy_fetch = enable_lazy_fetch;

  auto cursor = g_dbe->executeDML(
      "SELECT * FROM test_chunked "
      "INNER JOIN join_table "
      "ON test_chunked.i=join_table.i;");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);
  ASSERT_EQ(table->num_columns(), 7);
  ASSERT_EQ(table->num_rows(), (int64_t)6);

  compare_columns(table6x4_col_i32, table->column(1));
  compare_columns(table6x4_col_bi, table->column(2));
  compare_columns(table6x4_col_d, table->column(3));
  compare_columns(table6x4_col_i64, table->column(4));
  // TODO: Enable the line below after SelectBool test works OK
  // compare_columns(std::array<bool,6>{false, false, false, true, true, true},
  // table->column(5));
  compare_columns(std::array<int64_t, 6>{100, 100, 100, 200, 200, 200}, table->column(6));
}

TEST(DBEngine, ArrowTableChunked_JOIN1) {
  JoinTest(false, false);
}
TEST(DBEngine, ArrowTableChunked_JOIN2) {
  JoinTest(false, true);
}
TEST(DBEngine, ArrowTableChunked_JOIN3) {
  JoinTest(true, false);
}
TEST(DBEngine, ArrowTableChunked_JOIN4) {
  JoinTest(true, true);
}

//  Tests with NULLs
TEST(DBEngine, ArrowTableChunked_NULLS1) {
  auto cursor = g_dbe->executeDML("select * from chunked_nulls;");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);

  ASSERT_EQ(table->num_columns(), 4);
  ASSERT_EQ(table->num_rows(), (int64_t)6);

  int32_t i32_null = null_builder<int32_t>();
  int64_t i64_null = null_builder<int64_t>();
  double f64_null = null_builder<double>();
  compare_columns(std::array<int32_t, 6>{i32_null, 0, i32_null, i32_null, 1, 1},
                  table->column(1));
  compare_columns(std::array<int64_t, 6>{i64_null, 2, 3, 4, i64_null, 6},
                  table->column(2));
  compare_columns(std::array<double, 6>{10.1, f64_null, f64_null, 40.4, 50.5, f64_null},
                  table->column(3));
}

TEST(DBEngine, ArrowTableChunked_NULLS2) {
  auto cursor = g_dbe->executeDML("select 2*i,3*bi,4*d from chunked_nulls;");
  ASSERT_NE(cursor, nullptr);
  auto table = cursor->getArrowTable();
  ASSERT_NE(table, nullptr);

  ASSERT_EQ(table->num_columns(), 3);
  ASSERT_EQ(table->num_rows(), (int64_t)6);

  int32_t i32_null = null_builder<int32_t>();
  int64_t i64_null = null_builder<int64_t>();
  double f64_null = null_builder<double>();
  compare_columns(std::array<int32_t, 6>{i32_null, 0, i32_null, i32_null, 2 * 1, 2 * 1},
                  table->column(0));
  compare_columns(std::array<int64_t, 6>{i64_null, 3 * 2, 3 * 3, 3 * 4, i64_null, 3 * 6},
                  table->column(1));
  compare_columns(
      std::array<double, 6>{4 * 10.1, f64_null, f64_null, 4 * 40.4, 4 * 50.5, f64_null},
      table->column(2));
}

int main(int argc, char* argv[]) try {
  namespace fs = std::filesystem;

  testing::InitGoogleTest(&argc, argv);

  auto options_str = parse_cli_args(argc, argv);

  auto check_file_existence = [](std::string fname) {
    if (!fs::exists(fs::path{fname})) {
      throw std::runtime_error("File not found: " + fname + ". Aborting\n");
    }
  };

  check_file_existence(TABLE6x4_CSV_FILE);
  check_file_existence(NULLSTABLE6x4_CSV_FILE);
  check_file_existence(JOIN_TABLE_CSV_FILE);

  g_dbe = EmbeddedDatabase::DBEngine::create(options_str);

  load_table6x4_csv(TABLE6x4_CSV_FILE, "test", /*fragment_size=*/0);
  load_table6x4_csv(TABLE6x4_CSV_FILE, "test_chunked", /*fragment_size=*/4);
  load_table6x4_csv(NULLSTABLE6x4_CSV_FILE, "chunked_nulls", /*fragment_size=*/3);
  import_file(JOIN_TABLE_CSV_FILE, "join_table", /*fragment_size=*/2);

  int err = RUN_ALL_TESTS();

  g_dbe.reset();
  return err;
} catch (const boost::system::system_error& e) {
  std::cout << e.what();
  LOG(ERROR) << e.what();
  return EXIT_FAILURE;
} catch (const std::exception& e) {
  std::cout << e.what();
  LOG(ERROR) << e.what();
  return EXIT_FAILURE;
}

max_stars_repo_name: intel-ai/omniscidb

max_issues_repo_name: intel-ai/omniscidb

max_forks_repo_name: intel-ai/omniscidb


text:
/*
 * #%L
 * %%
 * Copyright (C) 2011 - 2017 BMW Car IT GmbH
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

#include <cassert>
#include <chrono>
#include <memory>
#include <string>
#include <stdexcept>

#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/program_options.hpp>

#include "joynr/JoynrRuntime.h"
#include "joynr/Logger.h"
#include "joynr/Semaphore.h"
#include "joynr/types/ProviderQos.h"
#include "joynr/types/ProviderScope.h"

#include "SystemIntegrationTestProvider.h"
#include "SitUtil.h"

#ifdef JOYNR_ENABLE_DLT_LOGGING
#include <dlt/dlt.h>
#endif // JOYNR_ENABLE_DLT_LOGGING

using namespace joynr;

int main(int argc, char* argv[])
{
#ifdef JOYNR_ENABLE_DLT_LOGGING
    // Register app at the dlt-daemon for logging
    DLT_REGISTER_APP("JYSP", argv[0]);
#endif // JOYNR_ENABLE_DLT_LOGGING

    Logger logger("ProviderApplication");

    namespace po = boost::program_options;

    po::positional_options_description positionalCmdLineOptions;
    positionalCmdLineOptions.add("domain", 1);
    positionalCmdLineOptions.add("runForever", 1);

    std::string providerDomain;
    bool runForever = false;
    std::string pathToSettings;
    std::string sslCertFilename;
    std::string sslPrivateKeyFilename;
    std::string sslCaCertFilename;

    po::options_description cmdLineOptions("Available options");
    cmdLineOptions.add_options()(
            "domain,d", po::value(&providerDomain)->required(), "joynr domain to be used")(
            "runForever,r",
            po::value(&runForever)->default_value(false),
            "If not set the provider will terminate"
            "after answering the first request.")("pathtosettings,p",
                                                  po::value(&pathToSettings),
                                                  "Absolute path to a non-default setting file.")(
            "ssl-cert-pem",
            po::value(&sslCertFilename),
            "Absolute path to public certificate for this application.")(
            "ssl-privatekey-pem",
            po::value(&sslPrivateKeyFilename),
            "Absolute path to private key for this application.")(
            "ssl-ca-cert-pem",
            po::value(&sslCaCertFilename),
            "Absolute path to certificate of CA.")("help,h", "Print help message");

    try {
        po::variables_map variablesMap;
        po::store(po::command_line_parser(argc, argv)
                          .options(cmdLineOptions)
                          .positional(positionalCmdLineOptions)
                          .run(),
                  variablesMap);

        if (variablesMap.count("help")) {
            std::cout << cmdLineOptions << std::endl;
            return EXIT_SUCCESS;
        }

        po::notify(variablesMap);
    } catch (const std::exception& e) {
        std::cerr << e.what() << std::endl;
        std::cerr << cmdLineOptions << std::endl;
        return EXIT_FAILURE;
    }

    if (pathToSettings.empty()) {
        boost::filesystem::path appFilename = boost::filesystem::path(argv[0]);
        std::string appDirectory =
                boost::filesystem::system_complete(appFilename).parent_path().string();
        pathToSettings = appDirectory + "/resources/systemintegrationtest-provider.settings";
    }

    std::shared_ptr<JoynrRuntime> runtime;
    try {
        runtime = joynr::sitUtil::createRuntime(
                pathToSettings, sslCertFilename, sslPrivateKeyFilename, sslCaCertFilename);
    } catch (const std::invalid_argument& e) {
        JOYNR_LOG_FATAL(logger, e.what());
        runtime.reset();
    }

    if (!runtime) {
        return EXIT_FAILURE;
    }

    try {
        joynr::Semaphore semaphore;

        JOYNR_LOG_INFO(logger, "Registering provider on domain {}", providerDomain);
        auto provider =
                std::make_shared<SystemIntegrationTestProvider>([&]() { semaphore.notify(); });

        joynr::types::ProviderQos providerQos;
        std::chrono::milliseconds millisSinceEpoch =
                std::chrono::duration_cast<std::chrono::milliseconds>(
                        std::chrono::system_clock::now().time_since_epoch());
        providerQos.setPriority(millisSinceEpoch.count());
        providerQos.setScope(joynr::types::ProviderScope::GLOBAL);

        runtime->registerProvider<test::SystemIntegrationTestProvider>(
                providerDomain, provider, providerQos);

        if (runForever) {
            while (true) {
                semaphore.wait();
            }
        } else {
            bool successful = semaphore.waitFor(std::chrono::milliseconds(30000));
            runtime->unregisterProvider<test::SystemIntegrationTestProvider>(
                    providerDomain, provider);

            return successful ? EXIT_SUCCESS : EXIT_FAILURE;
        }
    } catch (exceptions::JoynrRuntimeException& e) {
        JOYNR_LOG_FATAL(logger, e.what());
        return EXIT_FAILURE;
    }
}


text:
// author: jinliang

#pragma once

#include <stdint.h>
#include <map>
#include <vector>
#include <condition_variable>
#include <boost/unordered_map.hpp>

#include <petuum_ps_common/util/thread.hpp>
#include <petuum_ps/thread/row_request_oplog_mgr.hpp>
#include <petuum_ps/thread/bg_oplog.hpp>
#include <petuum_ps/thread/ps_msgs.hpp>
#include <petuum_ps_common/comm_bus/comm_bus.hpp>
#include <petuum_ps_common/include/configs.hpp>
#include <petuum_ps/client/client_table.hpp>
#include <petuum_ps/thread/append_only_row_oplog_buffer.hpp>
#include <petuum_ps/thread/row_oplog_serializer.hpp>

namespace petuum {
class AbstractBgWorker : public Thread {
public:
  AbstractBgWorker(int32_t id, int32_t comm_channel_idx,
           std::map<int32_t, ClientTable* > *tables,
           pthread_barrier_t *init_barrier,
           pthread_barrier_t *create_table_barrier);
  virtual ~AbstractBgWorker();

  void ShutDown();

  void AppThreadRegister();
  void AppThreadDeregister();

  bool CreateTable(int32_t table_id,
                   const ClientTableConfig& table_config);

  bool RequestRow(int32_t table_id, int32_t row_id, int32_t clock);
  void RequestRowAsync(int32_t table_id, int32_t row_id, int32_t clock,
                       bool forced);
  void GetAsyncRowRequestReply();
  void SignalHandleAppendOnlyBuffer(int32_t table_id);

  void ClockAllTables();
  void SendOpLogsAllTables();

  virtual void *operator() ();

protected:
  virtual void InitWhenStart();

  virtual void SetWaitMsg();
  virtual void CreateRowRequestOpLogMgr() = 0;

  static bool WaitMsgBusy(int32_t *sender_id, zmq::message_t *zmq_msg,
                          long timeout_milli = -1);
  static bool WaitMsgSleep(int32_t *sender_id, zmq::message_t *zmq_msg,
                           long timeout_milli  = -1);
  static bool WaitMsgTimeOut(int32_t *sender_id,
                             zmq::message_t *zmq_msg,
                             long timeout_milli);

  CommBus::WaitMsgTimeOutFunc WaitMsg_;

  typedef size_t (*GetSerializedRowOpLogSizeFunc)(AbstractRowOpLog *row_oplog);
  static size_t GetDenseSerializedRowOpLogSize(AbstractRowOpLog *row_oplog);
  static size_t GetSparseSerializedRowOpLogSize(AbstractRowOpLog *row_oplog);

  /* Functions Called From Main Loop -- BEGIN */
  void InitCommBus();
  void BgServerHandshake();
  void HandleCreateTables();

  // get connection from init thread
  void RecvAppInitThreadConnection(int32_t *num_connected_app_threads);

  virtual void PrepareBeforeInfiniteLoop();
  // invoked after all tables have been created
  virtual void FinalizeTableStats();
  virtual long ResetBgIdleMilli();
  virtual long BgIdleWork();

  /* Functions Called From Main Loop -- END */

  virtual void HandleAppendOpLogMsg(int32_t table_id);

  void HandleAppendOpLogAndApply(
      int32_t table_id, ClientTable *table, AbstractAppendOnlyBuffer *buff);
  void HandleAppendOpLogAndNotApply(
      int32_t table_id, ClientTable *table, AbstractAppendOnlyBuffer *buff);

  AppendOnlyRowOpLogBuffer *CreateAppendOnlyRowOpLogBufferIfNotExist(
      int32_t table_id, ClientTable *table);

  /* Handles Sending OpLogs -- BEGIN */
  virtual long HandleClockMsg(bool clock_advanced);

  virtual BgOpLog *PrepareOpLogsToSend() = 0;
  void CreateOpLogMsgs(const BgOpLog *bg_oplog);
  size_t SendOpLogMsgs(bool clock_advanced) ;

  size_t CountRowOpLogToSend(
      int32_t row_id, AbstractRowOpLog *row_oplog,
      std::map<int32_t, size_t> *table_num_bytes_by_server,
      BgOpLogPartition *bg_table_oplog,
      GetSerializedRowOpLogSizeFunc GetSerializedRowOpLogSize);

  virtual void TrackBgOpLog(BgOpLog *bg_oplog) = 0;

  void FinalizeOpLogMsgStats(
      int32_t table_id,
      std::map<int32_t, size_t> *table_num_bytes_by_server,
      std::map<int32_t, std::map<int32_t, size_t> >
      *server_table_oplog_size_map);
  /* Handles Sending OpLogs -- END */

  /* Handles Row Requests -- BEGIN */
  void CheckForwardRowRequestToServer(int32_t app_thread_id,
                                      RowRequestMsg &row_request_msg);
  void HandleServerRowRequestReply(
      int32_t server_id,
      ServerRowRequestReplyMsg &server_row_request_reply_msg);

  virtual void CheckAndApplyOldOpLogsToRowData(int32_t table_id,
                                               int32_t row_id, uint32_t row_version,
                                               AbstractRow *row_data) = 0;
  /* Handles Row Requests -- END */

  // Handles server pushed rows
  virtual void HandleServerPushRow(int32_t sender_id, void *msg_mem);

  /* Helper Functions */
  size_t SendMsg(MsgBase *msg);
  void RecvMsg(zmq::message_t &zmq_msg);
  void ConnectToNameNodeOrServer(int32_t server_id);

  virtual ClientRow *CreateClientRow(int32_t clock, AbstractRow *row_data) = 0;

  virtual void UpdateExistingRow(int32_t table_id, int32_t row_id,
                                 ClientRow *clien_row, ClientTable *client_table,
                                 const void *data, size_t row_size, uint32_t version);

  virtual void InsertNonexistentRow(int32_t table_id,
                                    int32_t row_id, ClientTable *client_table, const void *data,
                                    size_t row_size, uint32_t version, int32_t clock);

  int32_t my_id_;
  int32_t my_comm_channel_idx_;
  std::map<int32_t, ClientTable* > *tables_;
  std::vector<int32_t> server_ids_;

  uint32_t version_;
  int32_t client_clock_;
  int32_t clock_has_pushed_;
  RowRequestOpLogMgr *row_request_oplog_mgr_;
  CommBus* const comm_bus_;

  pthread_barrier_t *init_barrier_;
  pthread_barrier_t *create_table_barrier_;

  // initialized at Creation time, used in CreateSendOpLogs()
  // For server x, table y, the size of serialized OpLog is ...
  std::map<int32_t, std::map<int32_t, size_t> > server_table_oplog_size_map_;
  // The OpLog msg to each server
  std::map<int32_t, ClientSendOpLogMsg* > server_oplog_msg_map_;
  // size of oplog per table, reused across multiple tables
  std::map<int32_t, size_t> table_num_bytes_by_server_;

  std::unordered_map<int32_t, AppendOnlyRowOpLogBuffer*> append_only_row_oplog_buffer_map_;
  std::unordered_map<int32_t, int32_t> append_only_buff_proc_count_;

  std::unordered_map<int32_t, RowOpLogSerializer*> row_oplog_serializer_map_;
};

}

ext: hpp