text:
#ifdef _OPENMP
#include <omp.h>
#endif
#include <Python.h>
#include <numpy/ndarrayobject.h>
#include <iostream>
#include <boost/assert.hpp>
#include <boost/python.hpp>
#include <kgraph.h>
#include <kgraph-data.h>

#ifdef USE_BLAS
extern "C" {
    enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102};
    enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113};
    void cblas_sgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA,
                 const enum CBLAS_TRANSPOSE TransB, const int M, const int N,
                 const int K, const float alpha, const float *A,
                 const int lda, const float *B, const int ldb,
                 const float beta, float *C, const int ldc);
    void cblas_dgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA,
                 const enum CBLAS_TRANSPOSE TransB, const int M, const int N,
                 const int K, const double alpha, const double *A,
                 const int lda, const double *B, const int ldb,
                 const double beta, double *C, const int ldc);
}
#endif

using namespace std;
namespace python = boost::python;

/// Oracle for matrix data.
/** DATA_TYPE can be Matrix or MatrixProxy,
* DIST_TYPE should be one class within the namespace kgraph.metric.
*/
namespace kgraph {

#ifdef USE_BLAS
    void blas_prod (MatrixProxy<float> const &p1, float const *p2, int n2, int l2, Matrix<float> *r) {
        r->zero();
        cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, p1.size(), n2, p1.dim(),
                    -2.0, p1[0], p1[1]-p1[0], p2, l2, 0, (*r)[0], (*r)[1]-(*r)[0]);
    }

    void blas_prod (MatrixProxy<double> const &p1, double const *p2, int n2, int l2, Matrix<double> *r) {
        r->zero();
        cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, p1.size(), n2, p1.dim(),
                    -2.0, p1[0], p1[1]-p1[0], p2, l2, 0, (*r)[0], (*r)[1]-(*r)[0]);
    }
#endif

template <typename DATA_TYPE>
class MatrixOracleL2SQR: public kgraph::IndexOracle {
    MatrixProxy<DATA_TYPE> proxy;
    vector<DATA_TYPE> n2;
public:
    class SearchOracle: public kgraph::SearchOracle {
        MatrixProxy<DATA_TYPE> proxy;
        DATA_TYPE const *proxy_n2;
        DATA_TYPE const *query;
        DATA_TYPE n2;
    public:
        SearchOracle (MatrixProxy<DATA_TYPE> const &p, DATA_TYPE const *p_n2, DATA_TYPE const *q): proxy(p), proxy_n2(p_n2), query(q) {
            n2 = kgraph::metric::l2sqr::norm2(q, proxy.dim());
        }
        virtual unsigned size () const {
            return proxy.size();
        }
        virtual float operator () (unsigned i) const {
            return proxy_n2[i] + n2 - 2 * kgraph::metric::l2sqr::dot(proxy[i], query, proxy.dim());
        }
    };
    template <typename MATRIX_TYPE>
    MatrixOracleL2SQR (MATRIX_TYPE const &m): proxy(m), n2(proxy.size()) {
        for (unsigned i = 0; i < proxy.size(); ++i) {
            n2[i] = kgraph::metric::l2sqr::norm2(proxy[i], proxy.dim());
        }
    }
    virtual unsigned size () const {
        return proxy.size();
    }
    virtual float operator () (unsigned i, unsigned j) const {
        return n2[i] + n2[j] - 2 * kgraph::metric::l2sqr::dot(proxy[i], proxy[j], proxy.dim());
    }
    SearchOracle query (DATA_TYPE const *query) const {
        return SearchOracle(proxy, &n2[0], query);
    }

#ifdef USE_BLAS
    template <typename MATRIX_TYPE>
    void blasQuery (MATRIX_TYPE const &q, unsigned K, float epsilon,
            kgraph::MatrixProxy<unsigned, 1> *ids, kgraph::MatrixProxy<float, 1> *dists) {
        BOOST_VERIFY(dists);

        static unsigned constexpr BLOCK_SIZE = 1024;
        MatrixProxy<DATA_TYPE> q_proxy(q);
        unsigned block = BLOCK_SIZE;
        if (block < K) {
            block = K;
        }
        if (block >= proxy.size()) {
            block = proxy.size();
        }
        BOOST_VERIFY(block >= K);
        Matrix<DATA_TYPE> dot(q_proxy.size(), block);

        vector<float> qn2s(q_proxy.size());
        for (unsigned i = 0; i < q_proxy.size(); ++i) {
            qn2s[i] = kgraph::metric::l2sqr::norm2(q_proxy[i], q_proxy.dim());
        }

        unsigned begin = 0; // divide all data into blocks
        while (begin < proxy.size()) {
            unsigned end = begin + block;
            if (end > proxy.size()) {
                end = proxy.size();
            }
            blas_prod(q_proxy, proxy[begin], end-begin, proxy[1]-proxy[0], &dot);
            // do one block
            if (begin == 0) {
                // first block
#pragma omp parallel for
                for (unsigned i = 0; i < q_proxy.size(); ++i) {
                    DATA_TYPE *row = dot[i];
                    DATA_TYPE qn2 = qn2s[i];
                    vector<pair<DATA_TYPE, unsigned>> rank(end-begin);
                    for (unsigned j = 0; j < rank.size(); ++j) {
                        rank[j] = std::make_pair(qn2 + n2[j] + row[j], j);
                    }
                    std::sort(rank.begin(), rank.end());
                    unsigned *pid = (*ids)[i];
                    float *pdist = (*dists)[i];
                    for (unsigned j = 0; j < K; ++j) {
                        pid[j] = rank[j].second;
                        pdist[j] = rank[j].first;
                    }
                }
            }
            else { // subsequent blocks, using inserting instead of sort
#pragma omp parallel for
                for (unsigned i = 0; i < q_proxy.size(); ++i) {
                    DATA_TYPE *row = dot[i];
                    DATA_TYPE qn2 = qn2s[i];
                    unsigned *pid = (*ids)[i];
                    float *pdist = (*dists)[i];
                    for (unsigned j = 0; j < end-begin; ++j) {
                        // insert
                        unsigned id = begin + j;
                        float d = qn2 + n2[id] + row[j];
                        unsigned c = K-1;
                        if (d >= pdist[c]) continue;
                        while ((c > 0) && (d < pdist[c-1])) {
                            pid[c] = pid[c-1];
                            pdist[c] = pdist[c-1];
                            --c;
                        }
                        pid[c] = id;
                        pdist[c] = d;
                    }
                }
            }
            begin = end;
        }
    }
#endif
};

}

class KGraph {
    kgraph::KGraph *index;
    bool hasIndex;

    template <typename TYPE>
    void checkArray (python::object const &data) {
        PyArrayObject *array = reinterpret_cast<PyArrayObject *>(data.ptr());
        BOOST_VERIFY(array->nd == 2);
        //cerr << "dims: " << array->dimensions[0] << ' ' << array->dimensions[1] << endl;
        //cerr << "stride: " << array->strides[0] << ' ' << array->strides[1] << endl;
        PyArray_Descr *descr = array->descr;
        /*
        BOOST_VERIFY(descr->type_num == NPY_FLOAT);
        */
        BOOST_VERIFY(descr->elsize == sizeof(TYPE));
        //cerr << "type: " << descr->type_num << endl;
        //cerr << "size: " << descr->elsize << endl;
        //cerr << "alignment: " << descr->alignment << endl;
    }

    template <typename TYPE>
    void buildImpl (python::object const &data,
                               kgraph::KGraph::IndexParams params) {
        checkArray<TYPE>(data);
        kgraph::MatrixProxy<TYPE> dmatrix(reinterpret_cast<PyArrayObject *>(data.ptr()));
        kgraph::MatrixOracleL2SQR<TYPE> oracle(dmatrix);
        if (dmatrix.dim() % 4) {
            cerr << "Dimension has to be multiples of 4." << endl;
            BOOST_VERIFY(0);
        }
        index->build(oracle, params, NULL);
        hasIndex = true;
    }

    template <typename TYPE>
    python::object searchImpl (python::object const &data,
                               python::object const &query,
                               kgraph::KGraph::SearchParams params,
                               unsigned threads,
                               bool withDistance,
                               bool blas) {
        checkArray<TYPE>(data);
        checkArray<TYPE>(query);
        kgraph::MatrixProxy<TYPE> dmatrix(reinterpret_cast<PyArrayObject *>(data.ptr()));
        if (dmatrix.dim() % 4) {
            cerr << "Dimension has to be multiples of 4." << endl;
            BOOST_VERIFY(0);
        }
        kgraph::MatrixProxy<TYPE> qmatrix(reinterpret_cast<PyArrayObject *>(query.ptr()));
        kgraph::MatrixOracleL2SQR<TYPE> oracle(dmatrix);
        npy_intp dims[] = {qmatrix.size(), params.K};
        PyObject *result =  PyArray_SimpleNew(2, dims, NPY_UINT32);
        PyObject *distance =  PyArray_SimpleNew(2, dims, NPY_FLOAT);
        kgraph::MatrixProxy<unsigned, 1> rmatrix(reinterpret_cast<PyArrayObject *>(result));
        kgraph::MatrixProxy<float, 1> distmatrix(reinterpret_cast<PyArrayObject *>(distance));
#ifdef _OPENMP
        if (threads) ::omp_set_num_threads(threads);
#endif
#ifdef USE_BLAS
        if (blas) {
            oracle.blasQuery(qmatrix, params.K, params.epsilon, &rmatrix, &distmatrix);
        }
        else
#endif
            if (hasIndex) {
#pragma omp parallel for
            for (unsigned i = 0; i < qmatrix.size(); ++i) {
                if (withDistance) {
                    index->search(oracle.query(qmatrix[i]), params, const_cast<unsigned *>(rmatrix[i]),
                                  const_cast<float *>(distmatrix[i]),NULL);
                }
                else {
                    index->search(oracle.query(qmatrix[i]), params, const_cast<unsigned *>(rmatrix[i]), NULL);
                }
            }
        }
        else {
#pragma omp parallel for
            for (unsigned i = 0; i < qmatrix.size(); ++i) {
                if (withDistance) {
                    oracle.query(qmatrix[i]).search(params.K, params.epsilon, const_cast<unsigned *>(rmatrix[i]),
                                 const_cast<float *>(distmatrix[i]));
                }
                else {
                    oracle.query(qmatrix[i]).search(params.K, params.epsilon, const_cast<unsigned *>(rmatrix[i]), NULL);
                }
            }
        }

        if (withDistance) {
            PyObject* tup = PyTuple_New(2);
            PyTuple_SetItem(tup,0,result);
            PyTuple_SetItem(tup,1,distance);
            return python::object(python::handle<>(tup));
        }
        else {
            Py_DECREF(distance);
            return python::object(python::handle<>(result));
        }
    }

public:
    KGraph (): index(kgraph::KGraph::create()), hasIndex(false) {
        if (!index) throw runtime_error("error creating kgraph instance");
    }
    ~KGraph () {
        if (index) delete index;
    }
    void load (char const *path) {
        index->load(path);
        hasIndex = true;
    }
    void save (char const *path) const {
        index->save(path);
    }
    void build (python::object const &data,
               unsigned iterations,
               unsigned L,
               unsigned K,
               unsigned S,
               unsigned controls,
               float delta,
               float recall,
               unsigned prune) {
        kgraph::KGraph::IndexParams params;
        params.iterations = iterations;
        params.L = L;
        params.K = K;
        params.S = S;
        params.controls = controls;
        params.delta = delta;
        params.recall = recall;
        params.prune = prune;
        PyArrayObject *pd = reinterpret_cast<PyArrayObject *>(data.ptr());
        switch (pd->descr->type_num) {
            case NPY_FLOAT: buildImpl<float>(data, params); return;
            case NPY_DOUBLE: buildImpl<double>(data, params); return;
        }
        throw runtime_error("data type not supported.");
    }
    python::object search (python::object const &data,
                           python::object const &query,
                unsigned K,
                unsigned P,
                unsigned M,
                unsigned T,
                unsigned threads,
                bool withDistance,
                bool blas) {
        kgraph::KGraph::SearchParams params;
        params.K = K;
        params.P = P;
        params.M = M;
        params.T = T;
        PyArrayObject *pd = reinterpret_cast<PyArrayObject *>(data.ptr());
        PyArrayObject *pq = reinterpret_cast<PyArrayObject *>(query.ptr());
        if (pd->descr->type_num != pq->descr->type_num) throw runtime_error("data and query have different types");
        switch (pd->descr->type_num) {
            case NPY_FLOAT: return searchImpl<float>(data, query, params, threads, withDistance, blas);
            case NPY_DOUBLE: return searchImpl<double>(data, query, params, threads, withDistance, blas);
        }
        throw runtime_error("data type not supported.");
        return python::object();
    }
};

BOOST_PYTHON_MODULE(pykgraph)
{
    import_array();
    python::numeric::array::set_module_and_type("numpy", "ndarray");
    python::class_<KGraph>("KGraph")
        .def("load", &KGraph::load)
        .def("save", &KGraph::save)
        .def("build", &KGraph::build,
             (python::arg("data"),
              python::arg("iterations") = kgraph::default_iterations,
              python::arg("L") = kgraph::default_L,
              python::arg("K") = kgraph::default_K,
              python::arg("S") = kgraph::default_S,
              python::arg("controls") = kgraph::default_controls,
              python::arg("delta") = kgraph::default_delta,
              python::arg("recall") = kgraph::default_recall,
              python::arg("prune") = kgraph::default_prune))
        .def("search", &KGraph::search,
            (python::arg("data"),
             python::arg("query"),
             python::arg("K") = kgraph::default_K,
             python::arg("P") = kgraph::default_P,
             python::arg("M") = kgraph::default_M,
             python::arg("T") = kgraph::default_T,
             python::arg("threads") = 0,
             python::arg("withDistance") = false,
             python::arg("blas") = false))
        ;
}

max_stars_repo_name: huahaiy/docker-kgraph

max_issues_repo_name: huahaiy/docker-kgraph

max_forks_repo_name: huahaiy/docker-kgraph


text:
//////////////////////////////////////////////////////////////////////////////
//
// (C) Copyright Ion Gaztanaga 2008-2012. Distributed under the Boost
// Software License, Version 1.0. (See accompanying file
// LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// See http://www.boost.org/libs/interprocess for documentation.
//
//////////////////////////////////////////////////////////////////////////////

#ifndef BOOST_INTERPROCESS_DETAIL_PREPROCESSOR_HPP
#define BOOST_INTERPROCESS_DETAIL_PREPROCESSOR_HPP

#if (defined _MSC_VER) && (_MSC_VER >= 1200)
#  pragma once
#endif

#include <boost/interprocess/detail/config_begin.hpp>

#ifdef BOOST_INTERPROCESS_PERFECT_FORWARDING
#error "This file is not needed when perfect forwarding is available"
#endif

#include <boost/preprocessor/iteration/local.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>
#include <boost/preprocessor/cat.hpp>
#include <boost/preprocessor/repetition/enum.hpp>
#include <boost/preprocessor/repetition/repeat.hpp>

#define BOOST_INTERPROCESS_MAX_CONSTRUCTOR_PARAMETERS 10

//Note:
//We define template parameters as const references to
//be able to bind temporaries. After that we will un-const them.
//This cast is ugly but it is necessary until "perfect forwarding"
//is achieved in C++0x. Meanwhile, if we want to be able to
//bind rvalues with non-const references, we have to be ugly
#ifndef BOOST_NO_RVALUE_REFERENCES
   #define BOOST_INTERPROCESS_PP_PARAM_LIST(z, n, data) \
   BOOST_PP_CAT(P, n) && BOOST_PP_CAT(p, n) \
   //!
#else
   #define BOOST_INTERPROCESS_PP_PARAM_LIST(z, n, data) \
   const BOOST_PP_CAT(P, n) & BOOST_PP_CAT(p, n) \
   //!
#endif

#ifndef BOOST_NO_RVALUE_REFERENCES
   #define BOOST_INTERPROCESS_PP_PARAM(U, u) \
   U && u \
   //!
#else
   #define BOOST_INTERPROCESS_PP_PARAM(U, u) \
   const U & u \
   //!
#endif

#ifndef BOOST_NO_RVALUE_REFERENCES

   #define BOOST_INTERPROCESS_PP_PARAM_INIT(z, n, data) \
     BOOST_PP_CAT(m_p, n) (::boost::forward< BOOST_PP_CAT(P, n) >( BOOST_PP_CAT(p, n) ))  \
   //!

#else //#ifndef BOOST_NO_RVALUE_REFERENCES

   #define BOOST_INTERPROCESS_PP_PARAM_INIT(z, n, data) \
     BOOST_PP_CAT(m_p, n) (const_cast<BOOST_PP_CAT(P, n) &>(BOOST_PP_CAT(p, n))) \
   //!
#endif

#ifndef BOOST_NO_RVALUE_REFERENCES

   #if defined(BOOST_MOVE_MSVC_10_MEMBER_RVALUE_REF_BUG)

      namespace boost {
      namespace interprocess {
      namespace ipcdetail {
         template<class T>
         struct ref_holder;

         template<class T>
         struct ref_holder<T &>
         {
            ref_holder(T &t)
               : t_(t)
            {}
            T &t_;
            T & get() {  return t_;   }
            T & get_lvalue() {  return t_;   }
         };

         template<class T>
         struct ref_holder<const T>
         {
            ref_holder(const T &t)
               : t_(t)
            {}
            const T &t_;
            const T & get() {  return t_;   }
            const T & get_lvalue() {  return t_;   }
         };

         template<class T>
         struct ref_holder<const T &&>
         {
            ref_holder(const T &t)
               : t_(t)
            {}
            const T &t_;
            const T & get() {  return t_;   }
            const T & get_lvalue() {  return t_;   }
         };

         template<class T>
         struct ref_holder
         {
            ref_holder(T &&t)
               : t_(t)
            {}
            T &t_;
            T && get() {  return ::boost::move(t_);   }
            T & get_lvalue() {  return t_;   }
         };

         template<class T>
         struct ref_holder<T &&>
         {
            ref_holder(T &&t)
               : t(t)
            {}
            T &t;
            T && get()  { return ::boost::move(t_); }
            T & get_lvalue() {  return t_;   }
         };

      }  //namespace ipcdetail {
      }  //namespace interprocess {
      }  //namespace boost {

      #define BOOST_INTERPROCESS_PP_PARAM_DEFINE(z, n, data)  \
         ::boost::interprocess::ipcdetail::ref_holder<BOOST_PP_CAT(P, n)> BOOST_PP_CAT(m_p, n);  \
      //!

      #define BOOST_INTERPROCESS_PP_PARAM_INC(z, n, data)   \
         BOOST_PP_CAT(++m_p, n).get_lvalue()                \
      //!

   #else //BOOST_MOVE_MSVC_10_MEMBER_RVALUE_REF_BUG

      #define BOOST_INTERPROCESS_PP_PARAM_DEFINE(z, n, data)\
      BOOST_PP_CAT(P, n) && BOOST_PP_CAT(m_p, n);           \
      //!

      #define BOOST_INTERPROCESS_PP_PARAM_INC(z, n, data)   \
         BOOST_PP_CAT(++m_p, n)                             \
      //!

   #endif //defined(BOOST_MOVE_MSVC_10_MEMBER_RVALUE_REF_BUG)

#else
   #define BOOST_INTERPROCESS_PP_PARAM_DEFINE(z, n, data)   \
   BOOST_PP_CAT(P, n) & BOOST_PP_CAT(m_p, n);               \
   //!

   #define BOOST_INTERPROCESS_PP_PARAM_INC(z, n, data)      \
      BOOST_PP_CAT(++m_p, n)                                \
   //!

#endif

#define BOOST_INTERPROCESS_PP_PARAM_FORWARD(z, n, data) \
::boost::forward< BOOST_PP_CAT(P, n) >( BOOST_PP_CAT(p, n) ) \
//!

#if !defined(BOOST_NO_RVALUE_REFERENCES) && defined(BOOST_MOVE_MSVC_10_MEMBER_RVALUE_REF_BUG)

   #define BOOST_INTERPROCESS_PP_MEMBER_FORWARD(z, n, data) BOOST_PP_CAT(this->m_p, n).get() \
   //!

   #define BOOST_INTERPROCESS_PP_MEMBER_IT_FORWARD(z, n, data) \
   BOOST_PP_CAT(*m_p, n).get_lvalue()                          \
   //!

#else

   #define BOOST_INTERPROCESS_PP_MEMBER_FORWARD(z, n, data)       \
   ::boost::forward< BOOST_PP_CAT(P, n) >( BOOST_PP_CAT(m_p, n) ) \
   //!

   #define BOOST_INTERPROCESS_PP_MEMBER_IT_FORWARD(z, n, data)    \
   BOOST_PP_CAT(*m_p, n)                                          \
   //!

#endif   //!defined(BOOST_NO_RVALUE_REFERENCES) && defined(BOOST_MOVE_MSVC_10_MEMBER_RVALUE_REF_BUG)

#include <boost/interprocess/detail/config_end.hpp>

#else
#ifdef BOOST_INTERPROCESS_PERFECT_FORWARDING
#error "This file is not needed when perfect forwarding is available"
#endif
#endif //#ifndef BOOST_INTERPROCESS_DETAIL_PREPROCESSOR_HPP

ext: hpp

max_stars_repo_name: randolphwong/mcsema

max_issues_repo_name: randolphwong/mcsema

max_forks_repo_name: randolphwong/mcsema


text:
#include <iostream>
#include <Eigen/Dense>

using namespace Eigen;

int main(){
    using std::cout;
    using std::endl;

    MatrixXd A = MatrixXd::Zero(2,2);
    MatrixXd B = MatrixXd::Zero(2,2);

    A(0, 0) = 1;
    A(0, 1) = 2;
    A(1, 0) = 3;
    A(1, 1) = 4;

    B(0, 0) = 5;
    B(0, 1) = 6;
    B(1, 0) = 7;
    B(1, 1) = 8;

    cout << "Same Dimension Matrix Multiply" << endl;
    cout << A.rows() << " " << A.cols() << endl;
    cout << A*B << endl; // 行列の積
    cout << A.array() * B.array() << endl; // 要素積

    MatrixXd X = MatrixXd::Zero(2,3);
    MatrixXd Y = MatrixXd::Zero(3,2);

    X << 1,2,3, 4,5,6;
    Y << 1,2, 3,4, 5,6;

    cout << "Different Dimension Matrix Multiply" << endl;
    cout << "X shape: " << X.rows() << " " << X.cols() << endl;
    cout << "Y shape: " << Y.rows() << " " << Y.cols() << endl;
    cout << X * Y << endl;
    cout << X.array() * Y.transpose().array() << endl;

    return 0;
}

ext: cpp


text:
#include <steem/chain/steem_fwd.hpp>

#include <steem/plugins/account_history_rocksdb/account_history_rocksdb_plugin.hpp>

#include <steem/chain/database.hpp>
#include <steem/chain/history_object.hpp>
#include <steem/chain/index.hpp>
#include <steem/chain/util/impacted.hpp>

#include <steem/plugins/chain/chain_plugin.hpp>

#include <steem/utilities/benchmark_dumper.hpp>
#include <steem/utilities/plugin_utilities.hpp>

#include <appbase/application.hpp>

#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/write_batch_with_index.h>

#include <boost/type.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/container/flat_set.hpp>

#include <limits>
#include <string>
#include <typeindex>
#include <typeinfo>

namespace bpo = boost::program_options;

#define STEEM_NAMESPACE_PREFIX "steem::protocol::"
#define OPEN_FILE_LIMIT 750

#define DIAGNOSTIC(s)
//#define DIAGNOSTIC(s) s

#define CURRENT_LIB 1
#define OPERATION_BY_ID 2
#define OPERATION_BY_BLOCK 3
#define AH_INFO_BY_NAME 4
#define AH_OPERATION_BY_ID 5

#define WRITE_BUFFER_FLUSH_LIMIT     10
#define ACCOUNT_HISTORY_LENGTH_LIMIT 30
#define ACCOUNT_HISTORY_TIME_LIMIT   30
#define VIRTUAL_OP_FLAG              0x8000000000000000

/** Because localtion_id_pair stores block_number paired with (VIRTUAL_OP_FLAG|operation_id),
 *  max allowed operation-id is max_int (instead of max_uint).
 */
#define MAX_OPERATION_ID             std::numeric_limits<int64_t>::max()

#define STORE_MAJOR_VERSION          1
#define STORE_MINOR_VERSION          0

namespace steem { namespace plugins { namespace account_history_rocksdb {

using steem::protocol::account_name_type;
using steem::protocol::block_id_type;
using steem::protocol::operation;
using steem::protocol::signed_block;
using steem::protocol::signed_block_header;
using steem::protocol::signed_transaction;

using steem::chain::operation_notification;
using steem::chain::transaction_id_type;

using steem::utilities::benchmark_dumper;

using ::rocksdb::DB;
using ::rocksdb::DBOptions;
using ::rocksdb::Options;
using ::rocksdb::PinnableSlice;
using ::rocksdb::ReadOptions;
using ::rocksdb::Slice;
using ::rocksdb::Comparator;
using ::rocksdb::ColumnFamilyDescriptor;
using ::rocksdb::ColumnFamilyOptions;
using ::rocksdb::ColumnFamilyHandle;
using ::rocksdb::WriteBatch;

/** Represents an AH entry in mapped to account name.
 *  Holds additional informations, which are needed to simplify pruning process.
 *  All operations specific to given account, are next mapped to ID of given object.
 */
class account_history_info
{
public:
   int64_t        id = 0;
   uint32_t       oldestEntryId = 0;
   uint32_t       newestEntryId = 0;
   /// Timestamp of oldest operation, just to quickly decide if start detail prune checking at all.
   time_point_sec oldestEntryTimestamp;

   uint32_t getAssociatedOpCount() const
   {
      return newestEntryId - oldestEntryId + 1;
   }
};

namespace
{
   template <class T>
   serialize_buffer_t dump(const T& obj)
   {
      serialize_buffer_t serializedObj;
      auto size = fc::raw::pack_size(obj);
      serializedObj.resize(size);
      fc::datastream<char*> ds(serializedObj.data(), size);
      fc::raw::pack(ds, obj);
      return serializedObj;
   }

   template <class T>
   void load(T& obj, const char* data, size_t size)
   {
      fc::datastream<const char*> ds(data, size);
      fc::raw::unpack(ds, obj);
   }

   template <class T>
   void load(T& obj, const serialize_buffer_t& source)
   {
      load(obj, source.data(), source.size());
   }

/** Helper class to simplify construction of Slice objects holding primitive type values.
 *
 */
template <typename T>
class PrimitiveTypeSlice final : public Slice
{
public:
   explicit PrimitiveTypeSlice(T value) : _value(value)
   {
      data_ = reinterpret_cast<const char*>(&_value);
      size_ = sizeof(T);
   }

   static const T& unpackSlice(const Slice& s)
   {
      assert(sizeof(T) == s.size());
      return *reinterpret_cast<const T*>(s.data());
   }

   static const T& unpackSlice(const std::string& s)
   {
      assert(sizeof(T) == s.size());
      return *reinterpret_cast<const T*>(s.data());
   }

private:
   T _value;
};

/** Helper base class to cover all common functionality across defined comparators.
 *
 */
class AComparator : public Comparator
{
public:
   virtual const char* Name() const override final
   {
      static const std::string name = boost::core::demangle(typeid(this).name());
      return name.c_str();
   }

  virtual void FindShortestSeparator(std::string* start, const Slice& limit) const override final
  {
     /// Nothing to do.
  }

  virtual void FindShortSuccessor(std::string* key) const override final
  {
     /// Nothing to do.
  }

protected:
   AComparator() = default;
};

/// Pairs account_name storage type and the ID to make possible nonunique index definition over names.
typedef std::pair<account_name_type::Storage, size_t> account_name_storage_id_pair;

template <typename T>
class PrimitiveTypeComparatorImpl final : public AComparator
{
public:
  virtual int Compare(const Slice& a, const Slice& b) const override
  {
      if(a.size() != sizeof(T) || b.size() != sizeof(T))
         return a.compare(b);

      const T& id1 = retrieveKey(a);
      const T& id2 = retrieveKey(b);

      if(id1 < id2)
         return -1;

      if(id1 > id2)
         return 1;

      return 0;
  }

  virtual bool Equal(const Slice& a, const Slice& b) const override
  {
      if(a.size() != sizeof(T) || b.size() != sizeof(T))
         return a == b;

      const auto& id1 = retrieveKey(a);
      const auto& id2 = retrieveKey(b);

      return id1 == id2;
  }

private:
   const T& retrieveKey(const Slice& slice) const
   {
      assert(sizeof(T) == slice.size());
      const char* rawData = slice.data();
      return *reinterpret_cast<const T*>(rawData);
   }
};

typedef PrimitiveTypeComparatorImpl<uint32_t> by_id_ComparatorImpl;

typedef PrimitiveTypeComparatorImpl<account_name_type::Storage> by_account_name_ComparatorImpl;

typedef PrimitiveTypeSlice< uint32_t > lib_id_slice_t;
typedef PrimitiveTypeSlice< uint32_t > lib_slice_t;

#define LIB_ID lib_id_slice_t( 0 )

/** Location index is nonunique. Since RocksDB supports only unique indexes, it must be extended
 *  by some unique part (ie ID).
 *
 */
typedef std::pair< uint32_t, uint64_t > block_op_id_pair;
typedef PrimitiveTypeComparatorImpl< block_op_id_pair > op_by_block_num_ComparatorImpl;

/// Compares account_history_info::id and rocksdb_operation_object::id pair
typedef std::pair< int64_t, uint32_t > ah_op_id_pair;
typedef PrimitiveTypeComparatorImpl< ah_op_id_pair > ah_op_by_id_ComparatorImpl;

typedef PrimitiveTypeSlice< int64_t > id_slice_t;
typedef PrimitiveTypeSlice< block_op_id_pair > op_by_block_num_slice_t;
typedef PrimitiveTypeSlice< uint32_t > by_block_slice_t;
typedef PrimitiveTypeSlice< account_name_type::Storage > ah_info_by_name_slice_t;
typedef PrimitiveTypeSlice< ah_op_id_pair > ah_op_by_id_slice_t;

const Comparator* by_id_Comparator()
{
   static by_id_ComparatorImpl c;
   return &c;
}

const Comparator* op_by_block_num_Comparator()
{
   static op_by_block_num_ComparatorImpl c;
   return &c;
}

const Comparator* by_account_name_Comparator()
{
   static by_account_name_ComparatorImpl c;
   return &c;
}

const Comparator* ah_op_by_id_Comparator()
{
   static ah_op_by_id_ComparatorImpl c;
   return &c;
}

#define checkStatus(s) FC_ASSERT((s).ok(), "Data access failed: ${m}", ("m", (s).ToString()))

class operation_name_provider
{
public:
   typedef void result_type;

   static std::string getName(const operation& op)
   {
      operation_name_provider provider;
      op.visit(provider);
      return provider._name;
   }

   template<typename Op>
   void operator()( const Op& ) const
   {
      _name = fc::get_typename<Op>::name();
   }

private:
   operation_name_provider() = default;

/// Class attributes:
private:
   mutable std::string _name;
};

class CachableWriteBatch : public WriteBatch
{
public:
   CachableWriteBatch(const std::unique_ptr<DB>& storage, const std::vector<ColumnFamilyHandle*>& columnHandles) :
      _storage(storage), _columnHandles(columnHandles) {}

   bool getAHInfo(const account_name_type& name, account_history_info* ahInfo) const
   {
      auto fi = _ahInfoCache.find(name);
      if(fi != _ahInfoCache.end())
      {
         *ahInfo = fi->second;
         return true;
      }

      ah_info_by_name_slice_t key(name.data);
      PinnableSlice buffer;
      auto s = _storage->Get(ReadOptions(), _columnHandles[AH_INFO_BY_NAME], key, &buffer);
      if(s.ok())
      {
         load(*ahInfo, buffer.data(), buffer.size());
         return true;
      }

      FC_ASSERT(s.IsNotFound());
      return false;
   }

   void putAHInfo(const account_name_type& name, const account_history_info& ahInfo)
   {
      _ahInfoCache[name] = ahInfo;
      auto serializeBuf = dump(ahInfo);
      ah_info_by_name_slice_t nameSlice(name.data);
      auto s = Put(_columnHandles[AH_INFO_BY_NAME], nameSlice, Slice(serializeBuf.data(), serializeBuf.size()));
      checkStatus(s);
   }

   void Clear()
   {
      _ahInfoCache.clear();
      WriteBatch::Clear();
   }

private:
   const std::unique_ptr<DB>&                        _storage;
   const std::vector<ColumnFamilyHandle*>&           _columnHandles;
   std::map<account_name_type, account_history_info> _ahInfoCache;
};

} /// anonymous

class account_history_rocksdb_plugin::impl final
{
public:
   impl( account_history_rocksdb_plugin& self, const bpo::variables_map& options, const bfs::path& storagePath) :
      _self(self),
      _mainDb(appbase::app().get_plugin<steem::plugins::chain::chain_plugin>().db()),
      _storagePath(storagePath),
      _writeBuffer(_storage, _columnHandles)
      {
      collectOptions(options);

      _mainDb.add_pre_reindex_handler([&]( const steem::chain::reindex_notification& note ) -> void
         {
            on_pre_reindex( note );
         }, _self, 0);

      _mainDb.add_post_reindex_handler([&]( const steem::chain::reindex_notification& note ) -> void
         {
            on_post_reindex( note );
         }, _self, 0);

      add_plugin_index< volatile_operation_index >( _mainDb );
      }

   ~impl()
   {
      shutdownDb();
   }

   void openDb()
   {
      createDbSchema(_storagePath);

      auto columnDefs = prepareColumnDefinitions(true);

      DB* storageDb = nullptr;
      auto strPath = _storagePath.string();
      Options options;
      /// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
      options.IncreaseParallelism();
      options.OptimizeLevelStyleCompaction();

      DBOptions dbOptions(options);
      options.max_open_files = OPEN_FILE_LIMIT;

      auto status = DB::Open(dbOptions, strPath, columnDefs, &_columnHandles, &storageDb);

      if(status.ok())
      {
         ilog("RocksDB opened successfully storage at location: `${p}'.", ("p", strPath));
         verifyStoreVersion(storageDb);
         loadSeqIdentifiers(storageDb);
         _storage.reset(storageDb);

         const auto& rocksdb_plugin = appbase::app().get_plugin< account_history_rocksdb_plugin >();

         // I do not like using exceptions for control paths, but column definitions are set multiple times
         // opening the db, so that is not a good place to write the initial lib.
         try
         {
            get_lib();
         }
         catch( fc::assert_exception& )
         {
            update_lib( 0 );
         }

         _on_post_apply_operation_con = _mainDb.add_post_apply_operation_handler(
            [&]( const operation_notification& note )
            {
               on_post_apply_operation(note);
            },
            rocksdb_plugin
         );

         _on_irreversible_block_conn = _mainDb.add_irreversible_block_handler(
            [&]( uint32_t block_num )
            {
               on_irreversible_block( block_num );
            },
            rocksdb_plugin
         );
      }
      else
      {
         elog("RocksDB cannot open database at location: `${p}'.\nReturned error: ${e}",
            ("p", strPath)("e", status.ToString()));
      }
   }

   void printReport(uint32_t blockNo, const char* detailText) const;
   void on_pre_reindex( const steem::chain::reindex_notification& note );
   void on_post_reindex( const steem::chain::reindex_notification& note );

   /// Allows to start immediate data import (outside replay process).
   void importData(unsigned int blockLimit);

   void find_account_history_data(const account_name_type& name, uint64_t start, uint32_t limit,
      std::function<void(unsigned int, const rocksdb_operation_object&)> processor) const;
   bool find_operation_object(size_t opId, rocksdb_operation_object* op) const;
   /// Allows to look for all operations present in given block and call `processor` for them.
   void find_operations_by_block(size_t blockNum,
      std::function<void(const rocksdb_operation_object&)> processor) const;
   /// Allows to enumerate all operations registered in given block range.
   uint32_t enumVirtualOperationsFromBlockRange(uint32_t blockRangeBegin,
      uint32_t blockRangeEnd, std::function<void(const rocksdb_operation_object&)> processor) const;

   void shutdownDb()
   {
      chain::util::disconnect_signal(_on_post_apply_operation_con);
      chain::util::disconnect_signal(_on_irreversible_block_conn);
      flushStorage();
      cleanupColumnHandles();
      _storage.reset();
   }

private:
   uint32_t get_lib();
   void update_lib( uint32_t );

   typedef std::vector<ColumnFamilyDescriptor> ColumnDefinitions;
   ColumnDefinitions prepareColumnDefinitions(bool addDefaultColumn);

   /// Returns true if database will need data import.
   bool createDbSchema(const bfs::path& path);

   void cleanupColumnHandles()
   {
      for(auto* h : _columnHandles)
         delete h;
      _columnHandles.clear();
   }

   template< typename T >
   void importOperation( rocksdb_operation_object& obj, const T& impacted )
   {
      if(_lastTx != obj.trx_id)
      {
         ++_txNo;
         _lastTx = obj.trx_id;
      }

      obj.id = _operationSeqId++;

      serialize_buffer_t serializedObj;
      auto size = fc::raw::pack_size(obj);
      serializedObj.resize(size);
      {
         fc::datastream<char*> ds(serializedObj.data(), size);
         fc::raw::pack(ds, obj);
      }

      id_slice_t idSlice(obj.id);
      auto s = _writeBuffer.Put(_columnHandles[OPERATION_BY_ID], idSlice, Slice(serializedObj.data(), serializedObj.size()));
      checkStatus(s);

      // uint64_t location = ( (uint64_t) obj.trx_in_block << 32 ) | ( (uint64_t) obj.op_in_trx << 16 ) | ( obj.virtual_op );

      uint64_t encoded_id = (uint64_t) obj.id;
      if( obj.virtual_op > 0 )
      {
         encoded_id |= VIRTUAL_OP_FLAG;
      }

      op_by_block_num_slice_t blockLocSlice( block_op_id_pair( obj.block, encoded_id ) );
      s = _writeBuffer.Put(_columnHandles[OPERATION_BY_BLOCK], blockLocSlice, idSlice);
      checkStatus(s);

      for(const auto& name : impacted)
         buildAccountHistoryRecord( name, obj );

      if(++_collectedOps >= _collectedOpsWriteLimit)
         flushWriteBuffer();

      ++_totalOps;
}

   void buildAccountHistoryRecord( const account_name_type& name, const rocksdb_operation_object& obj );
   void prunePotentiallyTooOldItems(account_history_info* ahInfo, const account_name_type& name,
      const fc::time_point_sec& now);

   void saveStoreVersion()
   {
      PrimitiveTypeSlice<uint32_t> majorVSlice(STORE_MAJOR_VERSION);
      PrimitiveTypeSlice<uint32_t> minorVSlice(STORE_MINOR_VERSION);

      auto s = _writeBuffer.Put(Slice("STORE_MAJOR_VERSION"), majorVSlice);
      checkStatus(s);
      s = _writeBuffer.Put(Slice("STORE_MINOR_VERSION"), minorVSlice);
      checkStatus(s);
   }

   void verifyStoreVersion(DB* storageDb)
   {
      ReadOptions rOptions;

      std::string buffer;
      auto s = storageDb->Get(rOptions, "STORE_MAJOR_VERSION", &buffer);
      checkStatus(s);
      const auto major = PrimitiveTypeSlice<uint32_t>::unpackSlice(buffer);

      FC_ASSERT(major == STORE_MAJOR_VERSION, "Store major version mismatch");

      s = storageDb->Get(rOptions, "STORE_MINOR_VERSION", &buffer);
      checkStatus(s);
      const auto minor = PrimitiveTypeSlice<uint32_t>::unpackSlice(buffer);

      FC_ASSERT(minor == STORE_MINOR_VERSION, "Store minor version mismatch");
   }

   void storeSequenceIds()
   {
      Slice opSeqIdName("OPERATION_SEQ_ID");
      Slice ahSeqIdName("AH_SEQ_ID");

      id_slice_t opId(_operationSeqId);
      id_slice_t ahId(_accountHistorySeqId);

      auto s = _writeBuffer.Put(opSeqIdName, opId);
      checkStatus(s);
      s = _writeBuffer.Put(ahSeqIdName, ahId);
      checkStatus(s);
   }

   void loadSeqIdentifiers(DB* storageDb)
   {
      Slice opSeqIdName("OPERATION_SEQ_ID");
      Slice ahSeqIdName("AH_SEQ_ID");

      ReadOptions rOptions;

      std::string buffer;
      auto s = storageDb->Get(rOptions, opSeqIdName, &buffer);
      checkStatus(s);
      _operationSeqId = id_slice_t::unpackSlice(buffer);

      s = storageDb->Get(rOptions, ahSeqIdName, &buffer);
      checkStatus(s);
      _accountHistorySeqId = id_slice_t::unpackSlice(buffer);

      ilog("Loaded OperationObject seqId: ${o}, AccountHistoryObject seqId: ${ah}.",
         ("o", _operationSeqId)("ah", _accountHistorySeqId));
   }

   void flushWriteBuffer(DB* storage = nullptr)
   {
      storeSequenceIds();

      if(storage == nullptr)
         storage = _storage.get();

      ::rocksdb::WriteOptions wOptions;
      auto s = storage->Write(wOptions, _writeBuffer.GetWriteBatch());
      checkStatus(s);
      _writeBuffer.Clear();
      _collectedOps = 0;
   }

   void flushStorage()
   {
      if(_storage == nullptr)
         return;

      /// If there are still not yet saved changes let's do it now.
      if(_collectedOps != 0)
         flushWriteBuffer();

      ::rocksdb::FlushOptions fOptions;
      for(const auto& cf : _columnHandles)
      {
         auto s = _storage->Flush(fOptions, cf);
         checkStatus(s);
      }
   }

   void on_post_apply_operation(const operation_notification& opNote);

   void on_irreversible_block( uint32_t block_num );

   void collectOptions(const bpo::variables_map& options);

   /** Returns true if given account is tracked.
    *  Depends on `"account-history-whitelist-ops"`, `account-history-blacklist-ops` option usage.
    *  Only some accounts can be chosen for tracking operation history.
    */
   bool isTrackedAccount(const account_name_type& name) const;

   /** Returns a collection of the accounts being impacted by given `op` and
    *  also tracked, because of passed options.
    *
    *  \see isTrackedAccount.
    */
   std::vector<account_name_type> getImpactedAccounts(const operation& op) const;

   /** Returns true if given operation should be collected.
    *  Depends on `account-history-blacklist-ops`, `account-history-whitelist-ops`.
    */
   bool isTrackedOperation(const operation& op) const;

   void storeOpFilteringParameters(const std::vector<std::string>& opList,
      flat_set<std::string>* storage) const;

/// Class attributes:
private:
   typedef flat_map< account_name_type, account_name_type > account_name_range_index;

   account_history_rocksdb_plugin&  _self;
   chain::database&                 _mainDb;
   bfs::path                        _storagePath;
   std::unique_ptr<DB>              _storage;
   std::vector<ColumnFamilyHandle*> _columnHandles;
   CachableWriteBatch               _writeBuffer;

   boost::signals2::connection      _on_post_apply_operation_con;
   boost::signals2::connection      _on_irreversible_block_conn;

   /// Helper member to be able to detect another incomming tx and increment tx-counter.
   transaction_id_type              _lastTx;
   size_t                           _txNo = 0;
   /// Total processed ops in this session (counts every operation, even excluded by filtering).
   size_t                           _totalOps = 0;
   /// Total number of ops being skipped by filtering options.
   size_t                           _excludedOps = 0;
   /// Total number of accounts (impacted by ops) excluded from processing because of filtering.
   mutable size_t                   _excludedAccountCount = 0;
   /// IDs to be assigned to object.id field.
   uint64_t                         _operationSeqId = 0;
   uint64_t                         _accountHistorySeqId = 0;

   /// Number of data-chunks for ops being stored inside _writeBuffer. To decide when to flush.
   unsigned int                     _collectedOps = 0;
   /** Limit which value depends on block data source:
    *    - if blocks come from network, there is no need for delaying write, becasue they appear quite rare (limit == 1)
    *    - if reindex process or direct import has been spawned, this massive operation can need reduction of direct
           writes (limit == WRITE_BUFFER_FLUSH_LIMIT).
    */
   unsigned int                     _collectedOpsWriteLimit = 1;

   account_name_range_index         _tracked_accounts;
   flat_set<std::string>            _op_list;
   flat_set<std::string>            _blacklisted_op_list;

   bool                             _reindexing = false;

   bool                             _prune = false;
};

void account_history_rocksdb_plugin::impl::collectOptions(const boost::program_options::variables_map& options)
{
    fc::mutable_variant_object state_opts;

   typedef std::pair< account_name_type, account_name_type > pairstring;
   STEEM_LOAD_VALUE_SET(options, "account-history-rocksdb-track-account-range", _tracked_accounts, pairstring);

   state_opts[ "account-history-rocksdb-track-account-range" ] = _tracked_accounts;

   if(options.count("account-history-rocksdb-whitelist-ops"))
   {
      const auto& args = options.at("account-history-rocksdb-whitelist-ops").as<std::vector<std::string>>();
      storeOpFilteringParameters(args, &_op_list);

      if( _op_list.size() )
      {
         state_opts["account-history-rocksdb-whitelist-ops"] = _op_list;
      }
   }

   if(_op_list.empty() == false)
      ilog( "Account History: whitelisting ops ${o}", ("o", _op_list) );

   if(options.count("account-history-rocksdb-blacklist-ops"))
   {
      const auto& args = options.at("account-history-rocksdb-blacklist-ops").as<std::vector<std::string>>();
      storeOpFilteringParameters(args, &_blacklisted_op_list);

      if( _blacklisted_op_list.size() )
      {
         state_opts["account-history-rocksdb-blacklist-ops"] = _blacklisted_op_list;
      }
   }

   if(_blacklisted_op_list.empty() == false)
      ilog( "Account History: blacklisting ops ${o}", ("o", _blacklisted_op_list) );

   appbase::app().get_plugin< chain::chain_plugin >().report_state_options( _self.name(), state_opts );
}

inline bool account_history_rocksdb_plugin::impl::isTrackedAccount(const account_name_type& name) const
{
   if(_tracked_accounts.empty())
      return true;

   /// Code below is based on original contents of account_history_plugin_impl::on_post_apply_operation
   auto itr = _tracked_accounts.lower_bound(name);

   /*
      * The map containing the ranges uses the key as the lower bound and the value as the upper bound.
      * Because of this, if a value exists with the range (key, value], then calling lower_bound on
      * the map will return the key of the next pair. Under normal circumstances of those ranges not
      * intersecting, the value we are looking for will not be present in range that is returned via
      * lower_bound.
      *
      * Consider the following example using ranges ["a","c"], ["g","i"]
      * If we are looking for "bob", it should be tracked because it is in the lower bound.
      * However, lower_bound( "bob" ) returns an iterator to ["g","i"]. So we need to decrement the iterator
      * to get the correct range.
      *
      * If we are looking for "g", lower_bound( "g" ) will return ["g","i"], so we need to make sure we don't
      * decrement.
      *
      * If the iterator points to the end, we should check the previous (equivalent to rbegin)
      *
      * And finally if the iterator is at the beginning, we should not decrement it for obvious reasons
      */
   if(itr != _tracked_accounts.begin() &&
      ((itr != _tracked_accounts.end() && itr->first != name ) || itr == _tracked_accounts.end()))
   {
      --itr;
   }

   bool inRange = itr != _tracked_accounts.end() && itr->first <= name && name <= itr->second;

   _excludedAccountCount += inRange;

   DIAGNOSTIC(
      if(inRange)
         ilog("Account: ${a} matched to defined account range: [${b},${e}]", ("a", name)("b", itr->first)("e", itr->second) );
      else
         ilog("Account: ${a} ignored due to defined tracking filters.", ("a", name));
   );

   return inRange;
}

std::vector<account_name_type> account_history_rocksdb_plugin::impl::getImpactedAccounts(const operation& op) const
{
   flat_set<account_name_type> impacted;
   steem::app::operation_get_impacted_accounts(op, impacted);
   std::vector<account_name_type> retVal;

   if(impacted.empty())
      return retVal;

   if(_tracked_accounts.empty())
   {
      retVal.insert(retVal.end(), impacted.begin(), impacted.end());
      return retVal;
   }

   retVal.reserve(impacted.size());

   for(const auto& name : impacted)
   {
      if(isTrackedAccount(name))
         retVal.push_back(name);
   }

   return retVal;
}

inline bool account_history_rocksdb_plugin::impl::isTrackedOperation(const operation& op) const
{
   if(_op_list.empty() && _blacklisted_op_list.empty())
      return true;

   auto opName = operation_name_provider::getName(op);

   if(_blacklisted_op_list.find(opName) != _blacklisted_op_list.end())
      return false;

   bool isTracked = (_op_list.empty() || (_op_list.find(opName) != _op_list.end()));
   return isTracked;
}

void account_history_rocksdb_plugin::impl::storeOpFilteringParameters(const std::vector<std::string>& opList,
   flat_set<std::string>* storage) const
   {
      for(const auto& arg : opList)
      {
         std::vector<std::string> ops;
         boost::split(ops, arg, boost::is_any_of(" \t,"));

         for(const string& op : ops)
         {
            if( op.empty() == false )
               storage->insert( STEEM_NAMESPACE_PREFIX + op );
         }
      }
   }

void account_history_rocksdb_plugin::impl::find_account_history_data(const account_name_type& name, uint64_t start,
   uint32_t limit, std::function<void(unsigned int, const rocksdb_operation_object&)> processor) const
{
   ReadOptions rOptions;

   ah_info_by_name_slice_t nameSlice(name.data);
   PinnableSlice buffer;
   auto s = _storage->Get(rOptions, _columnHandles[AH_INFO_BY_NAME], nameSlice, &buffer);

   if(s.IsNotFound())
      return;

   checkStatus(s);

   account_history_info ahInfo;
   load(ahInfo, buffer.data(), buffer.size());

   ah_op_by_id_slice_t lowerBoundSlice(std::make_pair(ahInfo.id, ahInfo.oldestEntryId));
   ah_op_by_id_slice_t upperBoundSlice(std::make_pair(ahInfo.id, ahInfo.newestEntryId+1));

   rOptions.iterate_lower_bound = &lowerBoundSlice;
   rOptions.iterate_upper_bound = &upperBoundSlice;

   ah_op_by_id_slice_t key(std::make_pair(ahInfo.id, start));
   id_slice_t ahIdSlice(ahInfo.id);

   std::unique_ptr<::rocksdb::Iterator> it(_storage->NewIterator(rOptions, _columnHandles[AH_OPERATION_BY_ID]));

   it->SeekForPrev(key);

   if(it->Valid() == false)
      return;

   auto keySlice = it->key();
   auto keyValue = ah_op_by_id_slice_t::unpackSlice(keySlice);

   auto lowerBound = keyValue.second > limit ? keyValue.second - limit : 0;

   for(; it->Valid(); it->Prev())
   {
      auto keySlice = it->key();
      if(keySlice.starts_with(ahIdSlice) == false)
         break;

      keyValue = ah_op_by_id_slice_t::unpackSlice(keySlice);

      auto valueSlice = it->value();
      const auto& opId = id_slice_t::unpackSlice(valueSlice);
      rocksdb_operation_object oObj;
      bool found = find_operation_object(opId, &oObj);
      FC_ASSERT(found, "Missing operation?");

      processor(keyValue.second, oObj);

      if(keyValue.second <= lowerBound)
        break;
   }
}

bool account_history_rocksdb_plugin::impl::find_operation_object(size_t opId, rocksdb_operation_object* op) const
{
   std::string data;
   id_slice_t idSlice(opId);
   ::rocksdb::Status s = _storage->Get(ReadOptions(), _columnHandles[OPERATION_BY_ID], idSlice, &data);

   if(s.ok())
   {
      load(*op, data.data(), data.size());
      return true;
   }

   FC_ASSERT(s.IsNotFound());

   return false;
}

void account_history_rocksdb_plugin::impl::find_operations_by_block(size_t blockNum,
   std::function<void(const rocksdb_operation_object&)> processor) const
{
   std::unique_ptr<::rocksdb::Iterator> it(_storage->NewIterator(ReadOptions(), _columnHandles[OPERATION_BY_BLOCK]));
   by_block_slice_t blockNumSlice(blockNum);
   op_by_block_num_slice_t key(block_op_id_pair(blockNum, 0));

   for(it->Seek(key); it->Valid() && it->key().starts_with(blockNumSlice); it->Next())
   {
      auto valueSlice = it->value();
      const auto& opId = id_slice_t::unpackSlice(valueSlice);

      rocksdb_operation_object op;
      bool found = find_operation_object(opId, &op);
      FC_ASSERT(found);

      processor(op);
   }
}

uint32_t account_history_rocksdb_plugin::impl::enumVirtualOperationsFromBlockRange(uint32_t blockRangeBegin,
   uint32_t blockRangeEnd, std::function<void(const rocksdb_operation_object&)> processor) const
{
   FC_ASSERT(blockRangeEnd > blockRangeBegin, "Block range must be upward");

   op_by_block_num_slice_t upperBoundSlice(block_op_id_pair(blockRangeEnd, 0));

   op_by_block_num_slice_t rangeBeginSlice(block_op_id_pair(blockRangeBegin, 0));

   ReadOptions rOptions;
   rOptions.iterate_upper_bound = &upperBoundSlice;

   std::unique_ptr<::rocksdb::Iterator> it(_storage->NewIterator(rOptions, _columnHandles[OPERATION_BY_BLOCK]));

   uint32_t lastFoundBlock = 0;

   for(it->Seek(rangeBeginSlice); it->Valid(); it->Next())
   {
      auto keySlice = it->key();
      const auto& key = op_by_block_num_slice_t::unpackSlice(keySlice);

      /// Accept only virtual operations
      if(key.second & VIRTUAL_OP_FLAG)
      {
         auto valueSlice = it->value();
         const auto& opId = id_slice_t::unpackSlice(valueSlice);

         rocksdb_operation_object op;
         bool found = find_operation_object(opId, &op);
         FC_ASSERT(found);

         processor(op);
         lastFoundBlock = op.block;
      }
   }

   op_by_block_num_slice_t lowerBoundSlice(block_op_id_pair(lastFoundBlock, 0));
   rOptions = ReadOptions();
   rOptions.iterate_lower_bound = &lowerBoundSlice;
   it.reset(_storage->NewIterator(rOptions, _columnHandles[OPERATION_BY_BLOCK]));

   op_by_block_num_slice_t nextRangeBeginSlice(block_op_id_pair(lastFoundBlock + 1, 0));
   for(it->Seek(nextRangeBeginSlice); it->Valid(); it->Next())
   {
      auto keySlice = it->key();
      const auto& key = op_by_block_num_slice_t::unpackSlice(keySlice);

      if(key.second & VIRTUAL_OP_FLAG)
         return key.first;
   }

   return 0;
}

uint32_t account_history_rocksdb_plugin::impl::get_lib()
{
   std::string data;
   auto s = _storage->Get(ReadOptions(), _columnHandles[CURRENT_LIB], LIB_ID, &data );

   FC_ASSERT( s.ok(), "Could not find last irreversible block." );

   uint32_t lib = 0;
   load( lib, data.data(), data.size() );
   return lib;
}

void account_history_rocksdb_plugin::impl::update_lib( uint32_t lib )
{
   auto s = _writeBuffer.Put( _columnHandles[ CURRENT_LIB ], LIB_ID, lib_slice_t( lib ) );
   checkStatus( s );
}

account_history_rocksdb_plugin::impl::ColumnDefinitions account_history_rocksdb_plugin::impl::prepareColumnDefinitions(bool addDefaultColumn)
{
   ColumnDefinitions columnDefs;
   if(addDefaultColumn)
      columnDefs.emplace_back(::rocksdb::kDefaultColumnFamilyName, ColumnFamilyOptions());

   columnDefs.emplace_back("current_lib", ColumnFamilyOptions());

   columnDefs.emplace_back("operation_by_id", ColumnFamilyOptions());
   auto& byIdColumn = columnDefs.back();
   byIdColumn.options.comparator = by_id_Comparator();

   columnDefs.emplace_back("operation_by_block", ColumnFamilyOptions());
   auto& byLocationColumn = columnDefs.back();
   byLocationColumn.options.comparator = op_by_block_num_Comparator();

   columnDefs.emplace_back("account_history_info_by_name", ColumnFamilyOptions());
   auto& byAccountNameColumn = columnDefs.back();
   byAccountNameColumn.options.comparator = by_account_name_Comparator();

   columnDefs.emplace_back("ah_operation_by_id", ColumnFamilyOptions());
   auto& byAHInfoColumn = columnDefs.back();
   byAHInfoColumn.options.comparator = ah_op_by_id_Comparator();

   return columnDefs;
}

bool account_history_rocksdb_plugin::impl::createDbSchema(const bfs::path& path)
{
   DB* db = nullptr;

   auto columnDefs = prepareColumnDefinitions(true);
   auto strPath = path.string();
   Options options;
   /// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
   options.IncreaseParallelism();
   options.OptimizeLevelStyleCompaction();

   auto s = DB::OpenForReadOnly(options, strPath, columnDefs, &_columnHandles, &db);

   if(s.ok())
   {
      cleanupColumnHandles();
      delete db;
      return false; /// DB does not need data import.
   }

   options.create_if_missing = true;

   s = DB::Open(options, strPath, &db);
   if(s.ok())
   {
      columnDefs = prepareColumnDefinitions(false);
      s = db->CreateColumnFamilies(columnDefs, &_columnHandles);
      if(s.ok())
      {
         ilog("RockDB column definitions created successfully.");
         saveStoreVersion();
         /// Store initial values of Seq-IDs for held objects.
         flushWriteBuffer(db);
         cleanupColumnHandles();
      }
      else
      {
         elog("RocksDB can not create column definitions at location: `${p}'.\nReturned error: ${e}",
            ("p", strPath)("e", s.ToString()));
      }

      delete db;

      return true; /// DB needs data import
   }
   else
   {
      elog("RocksDB can not create storage at location: `${p}'.\nReturned error: ${e}",
         ("p", strPath)("e", s.ToString()));

      return false;
   }
}

void account_history_rocksdb_plugin::impl::buildAccountHistoryRecord( const account_name_type& name, const rocksdb_operation_object& obj )
{
   std::string strName = name;

   ReadOptions rOptions;
   //rOptions.tailing = true;

   ah_info_by_name_slice_t nameSlice(name.data);

   account_history_info ahInfo;
   bool found = _writeBuffer.getAHInfo(name, &ahInfo);

   if(found)
   {
      auto count = ahInfo.getAssociatedOpCount();

      if(_prune && count > ACCOUNT_HISTORY_LENGTH_LIMIT &&
         ((obj.timestamp - ahInfo.oldestEntryTimestamp) > fc::days(ACCOUNT_HISTORY_TIME_LIMIT)))
         {
            prunePotentiallyTooOldItems(&ahInfo, name, obj.timestamp);
         }

      auto nextEntryId = ++ahInfo.newestEntryId;
       _writeBuffer.putAHInfo(name, ahInfo);

      ah_op_by_id_slice_t ahInfoOpSlice(std::make_pair(ahInfo.id, nextEntryId));
      id_slice_t valueSlice(obj.id);
      auto s = _writeBuffer.Put(_columnHandles[AH_OPERATION_BY_ID], ahInfoOpSlice, valueSlice);
      checkStatus(s);
   }
   else
   {
      /// New entry must be created - there is first operation recorded.
      ahInfo.id = _accountHistorySeqId++;
      ahInfo.newestEntryId = ahInfo.oldestEntryId = 0;
      ahInfo.oldestEntryTimestamp = obj.timestamp;

      _writeBuffer.putAHInfo(name, ahInfo);

      ah_op_by_id_slice_t ahInfoOpSlice(std::make_pair(ahInfo.id, 0));
      id_slice_t valueSlice(obj.id);
      auto s = _writeBuffer.Put(_columnHandles[AH_OPERATION_BY_ID], ahInfoOpSlice, valueSlice);
      checkStatus(s);
   }
}

void account_history_rocksdb_plugin::impl::prunePotentiallyTooOldItems(account_history_info* ahInfo, const account_name_type& name,
   const fc::time_point_sec& now)
{
   std::string strName = name;

   auto ageLimit =  fc::days(ACCOUNT_HISTORY_TIME_LIMIT);

   ah_op_by_id_slice_t oldestEntrySlice(
      std::make_pair(ahInfo->id, ahInfo->oldestEntryId));
   auto lookupUpperBound = std::make_pair(ahInfo->id + 1,
      ahInfo->newestEntryId - ACCOUNT_HISTORY_LENGTH_LIMIT + 1);

   ah_op_by_id_slice_t newestEntrySlice(lookupUpperBound);

   ReadOptions rOptions;
   //rOptions.tailing = true;
   rOptions.iterate_lower_bound = &oldestEntrySlice;
   rOptions.iterate_upper_bound = &newestEntrySlice;

   auto s = _writeBuffer.SingleDelete(_columnHandles[AH_OPERATION_BY_ID], oldestEntrySlice);
   checkStatus(s);

   std::unique_ptr<::rocksdb::Iterator> dataItr(_storage->NewIterator(rOptions, _columnHandles[AH_OPERATION_BY_ID]));

   /** To clean outdated records we have to iterate over all AH records having subsequent number greater than limit
    *  and additionally verify date of operation, to clean up only these exceeding a date limit.
    *  So just operations having a list position > ACCOUNT_HISTORY_LENGTH_LIMIT and older that ACCOUNT_HISTORY_TIME_LIMIT
    *  shall be removed.
    */
   dataItr->Seek(oldestEntrySlice);

   /// Boundaries of keys to be removed
   //uint32_t leftBoundary = ahInfo->oldestEntryId;
   uint32_t rightBoundary = ahInfo->oldestEntryId;

   for(; dataItr->Valid(); dataItr->Next())
   {
      auto key = dataItr->key();
      auto foundEntry = ah_op_by_id_slice_t::unpackSlice(key);

      if(foundEntry.first != ahInfo->id || foundEntry.second >= lookupUpperBound.second)
         break;

      auto value = dataItr->value();

      auto pointedOpId = id_slice_t::unpackSlice(value);
      rocksdb_operation_object op;
      find_operation_object(pointedOpId, &op);

      auto age = now - op.timestamp;

      if(age > ageLimit)
      {
         rightBoundary = foundEntry.second;
         ah_op_by_id_slice_t rightBoundarySlice(
            std::make_pair(ahInfo->id, rightBoundary));
         s = _writeBuffer.SingleDelete(_columnHandles[4], rightBoundarySlice);
         checkStatus(s);
      }
      else
      {
         ahInfo->oldestEntryId = foundEntry.second;
         ahInfo->oldestEntryTimestamp = op.timestamp;
         FC_ASSERT(ahInfo->oldestEntryId <= ahInfo->newestEntryId);

         break;
      }
   }
}

void account_history_rocksdb_plugin::impl::on_pre_reindex(const steem::chain::reindex_notification& note)
{
   ilog("Received onReindexStart request, attempting to clean database storage.");

   shutdownDb();
   std::string strPath = _storagePath.string();

   auto s = ::rocksdb::DestroyDB(strPath, ::rocksdb::Options());
   checkStatus(s);

   openDb();

   ilog("Setting write limit to massive level");

   _collectedOpsWriteLimit = WRITE_BUFFER_FLUSH_LIMIT;

   _lastTx = transaction_id_type();
   _txNo = 0;
   _totalOps = 0;
   _excludedOps = 0;
   _reindexing = true;

   ilog("onReindexStart request completed successfully.");
}

void account_history_rocksdb_plugin::impl::on_post_reindex(const steem::chain::reindex_notification& note)
{
   ilog("Reindex completed up to block: ${b}. Setting back write limit to non-massive level.",
      ("b", note.last_block_number));

   flushStorage();
   _collectedOpsWriteLimit = 1;
   _reindexing = false;
   update_lib( note.last_block_number ); // We always reindex irreversible blocks.

   printReport( note.last_block_number, "RocksDB data reindex finished." );
}

void account_history_rocksdb_plugin::impl::printReport(uint32_t blockNo, const char* detailText) const
{
   ilog("${t}Processed blocks: ${n}, containing: ${tx} transactions and ${op} operations.\n"
        "${ep} operations have been filtered out due to configured options.\n"
        "${ea} accounts have been filtered out due to configured options.",
      ("t", detailText)
      ("n", blockNo)
      ("tx", _txNo)
      ("op", _totalOps)
      ("ep", _excludedOps)
      ("ea", _excludedAccountCount)
      );
}

void account_history_rocksdb_plugin::impl::importData(unsigned int blockLimit)
{
   if(_storage == nullptr)
   {
      ilog("RocksDB has no opened storage. Skipping data import...");
      return;
   }

   ilog("Starting data import...");

   block_id_type lastBlock;
   size_t blockNo = 0;

   _lastTx = transaction_id_type();
   _txNo = 0;
   _totalOps = 0;
   _excludedOps = 0;

   benchmark_dumper dumper;
   dumper.initialize([](benchmark_dumper::database_object_sizeof_cntr_t&){}, "rocksdb_data_import.json");

   _mainDb.foreach_operation([blockLimit, &blockNo, &lastBlock, this](
      const signed_block_header& prevBlockHeader, const signed_block& block, const signed_transaction& tx,
      uint32_t txInBlock, const operation& op, uint16_t opInTx) -> bool
   {
      if(lastBlock != block.previous)
      {
         blockNo = block.block_num();
         lastBlock = block.previous;

         if(blockLimit != 0 && blockNo > blockLimit)
         {
            ilog( "RocksDb data import stopped because of block limit reached.");
            return false;
         }
      }

      auto impacted = getImpactedAccounts( op );

      if( impacted.empty() )
         return true;

      rocksdb_operation_object obj;
      obj.trx_id = tx.id();
      obj.block = blockNo;
      obj.trx_in_block = txInBlock;
      obj.op_in_trx = opInTx;
      obj.timestamp = _mainDb.head_block_time();
      auto size = fc::raw::pack_size( op );
      obj.serialized_op.resize( size );
      fc::datastream< char* > ds( obj.serialized_op.data(), size );
      fc::raw::pack( ds, op );

      importOperation( obj, impacted );

      return true;
   }
   );

   if(_collectedOps != 0)
      flushWriteBuffer();

   const auto& measure = dumper.measure(blockNo, [](benchmark_dumper::index_memory_details_cntr_t&, bool){});
   ilog( "RocksDb data import - Performance report at block ${n}. Elapsed time: ${rt} ms (real), ${ct} ms (cpu). Memory usage: ${cm} (current), ${pm} (peak) kilobytes.",
      ("n", blockNo)
      ("rt", measure.real_ms)
      ("ct", measure.cpu_ms)
      ("cm", measure.current_mem)
      ("pm", measure.peak_mem) );

   printReport(blockNo, "RocksDB data import finished. ");
}

void account_history_rocksdb_plugin::impl::on_post_apply_operation(const operation_notification& n)
{
   if( n.block % 10000 == 0 && n.trx_in_block == 0 && n.op_in_trx == 0 && n.virtual_op == 0 )
   {
      ilog("RocksDb data import processed blocks: ${n}, containing: ${tx} transactions and ${op} operations.\n"
           " ${ep} operations have been filtered out due to configured options.\n"
           " ${ea} accounts have been filtered out due to configured options.",
         ("n", n.block)
         ("tx", _txNo)
         ("op", _totalOps)
         ("ep", _excludedOps)
         ("ea", _excludedAccountCount)
         );
   }

   if( !isTrackedOperation(n.op) )
   {
      ++_excludedOps;
      return;
   }

   auto impacted = getImpactedAccounts(n.op);

   if( impacted.empty() )
      return; // Ignore operations not impacting any account (according to original implementation)

   if( _reindexing )
   {
      rocksdb_operation_object obj;
      obj.trx_id = n.trx_id;
      obj.block = n.block;
      obj.trx_in_block = n.trx_in_block;
      obj.op_in_trx = n.op_in_trx;
      obj.virtual_op = n.virtual_op;
      obj.timestamp = _mainDb.head_block_time();
      auto size = fc::raw::pack_size( n.op );
      obj.serialized_op.resize( size );
      fc::datastream< char* > ds( obj.serialized_op.data(), size );
      fc::raw::pack( ds, n.op );

      importOperation( obj, impacted );
   }
   else
   {
      _mainDb.create< volatile_operation_object >( [&]( volatile_operation_object& o )
      {
         o.trx_id = n.trx_id;
         o.block = n.block;
         o.trx_in_block = n.trx_in_block;
         o.op_in_trx = n.op_in_trx;
         o.virtual_op = n.virtual_op;
         o.timestamp = _mainDb.head_block_time();
         auto size = fc::raw::pack_size( n.op );
         o.serialized_op.resize( size );
         fc::datastream< char* > ds( o.serialized_op.data(), size );
         fc::raw::pack( ds, n.op );
         o.impacted.insert( o.impacted.end(), impacted.begin(), impacted.end() );
      });
   }
}

void account_history_rocksdb_plugin::impl::on_irreversible_block( uint32_t block_num )
{
   if( _reindexing ) return;

   if( block_num <= get_lib() ) return;

   const auto& volatile_idx = _mainDb.get_index< volatile_operation_index, by_block >();
   auto itr = volatile_idx.begin();

   vector< const volatile_operation_object* > to_delete;

   while( itr != volatile_idx.end() && itr->block <= block_num )
   {
      rocksdb_operation_object obj( *itr );
      importOperation( obj , itr->impacted );
      to_delete.push_back( &(*itr) );
      ++itr;
   }

   for( const volatile_operation_object* o : to_delete )
   {
      _mainDb.remove( *o );
   }

   update_lib( block_num );
}

account_history_rocksdb_plugin::account_history_rocksdb_plugin()
{
}

account_history_rocksdb_plugin::~account_history_rocksdb_plugin()
{

}

void account_history_rocksdb_plugin::set_program_options(
   boost::program_options::options_description &command_line_options,
   boost::program_options::options_description &cfg)
{
   cfg.add_options()
      ("account-history-rocksdb-path", bpo::value<bfs::path>()->default_value("blockchain/account-history-rocksdb-storage"),
         "The location of the rocksdb database for account history. By default it is $DATA_DIR/blockchain/account-history-rocksdb-storage")
      ("account-history-rocksdb-track-account-range", boost::program_options::value< std::vector<std::string> >()->composing()->multitoken(), "Defines a range of accounts to track as a json pair [\"from\",\"to\"] [from,to] Can be specified multiple times.")
      ("account-history-rocksdb-whitelist-ops", boost::program_options::value< std::vector<std::string> >()->composing(), "Defines a list of operations which will be explicitly logged.")
      ("account-history-rocksdb-blacklist-ops", boost::program_options::value< std::vector<std::string> >()->composing(), "Defines a list of operations which will be explicitly ignored.")

   ;
   command_line_options.add_options()
      ("account-history-rocksdb-immediate-import", bpo::bool_switch()->default_value(false),
         "Allows to force immediate data import at plugin startup. By default storage is supplied during reindex process.")
      ("account-history-rocksdb-stop-import-at-block", bpo::value<uint32_t>()->default_value(0),
         "Allows to specify block number, the data import process should stop at.")
   ;
}

void account_history_rocksdb_plugin::plugin_initialize(const boost::program_options::variables_map& options)
{
   if(options.count("account-history-rocksdb-stop-import-at-block"))
      _blockLimit = options.at("account-history-rocksdb-stop-import-at-block").as<uint32_t>();

   _doImmediateImport = options.at("account-history-rocksdb-immediate-import").as<bool>();

   bfs::path dbPath;

   if(options.count("account-history-rocksdb-path"))
      dbPath = options.at("account-history-rocksdb-path").as<bfs::path>();

   if(dbPath.is_absolute() == false)
   {
      auto basePath = appbase::app().data_dir();
      auto actualPath = basePath / dbPath;
      dbPath = actualPath;
   }

   _my = std::make_unique<impl>( *this, options, dbPath );

   _my->openDb();
}

void account_history_rocksdb_plugin::plugin_startup()
{
   ilog("Starting up account_history_rocksdb_plugin...");

   if(_doImmediateImport)
      _my->importData(_blockLimit);
}

void account_history_rocksdb_plugin::plugin_shutdown()
{
   ilog("Shutting down account_history_rocksdb_plugin...");
   _my->shutdownDb();
}

void account_history_rocksdb_plugin::find_account_history_data(const account_name_type& name, uint64_t start, uint32_t limit,
   std::function<void(unsigned int, const rocksdb_operation_object&)> processor) const
{
   _my->find_account_history_data(name, start, limit, processor);
}

bool account_history_rocksdb_plugin::find_operation_object(size_t opId, rocksdb_operation_object* op) const
{
   return _my->find_operation_object(opId, op);
}

void account_history_rocksdb_plugin::find_operations_by_block(size_t blockNum,
   std::function<void(const rocksdb_operation_object&)> processor) const
{
   _my->find_operations_by_block(blockNum, processor);
}

uint32_t account_history_rocksdb_plugin::enum_operations_from_block_range(uint32_t blockRangeBegin, uint32_t blockRangeEnd,
   std::function<void(const rocksdb_operation_object&)> processor) const
{
   return _my->enumVirtualOperationsFromBlockRange(blockRangeBegin, blockRangeEnd, processor);
}

} } }

FC_REFLECT( steem::plugins::account_history_rocksdb::account_history_info,
   (id)(oldestEntryId)(newestEntryId)(oldestEntryTimestamp) )

max_stars_repo_name: ausbitbank/steem

max_issues_repo_name: ausbitbank/steem

max_forks_repo_name: blocktradesdevs/steem


text:
// Copyright (C) 2011-2012 by the Bem++ Authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#ifndef fiber_default_local_assembler_for_potential_operators_on_surfaces_hpp
#define fiber_default_local_assembler_for_potential_operators_on_surfaces_hpp

#include "../common/common.hpp"

#include "local_assembler_for_potential_operators.hpp"

#include "_2d_array.hpp"
#include "accuracy_options.hpp"
#include "default_local_assembler_for_operators_on_surfaces_utilities.hpp"
#include "element_pair_topology.hpp"
#include "numerical_quadrature.hpp"
#include "parallelization_options.hpp"
#include "shared_ptr.hpp"
#include "kernel_trial_integrator.hpp"
#include "verbosity_level.hpp"

#include <boost/static_assert.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include <tbb/concurrent_unordered_map.h>
#include <cstring>
#include <climits>
#include <set>
#include <utility>
#include <vector>

namespace Fiber {

/** \cond FORWARD_DECL */
class OpenClHandler;
template <typename CoordinateType> class CollectionOfShapesetTransformations;
template <typename ValueType> class CollectionOfKernels;
template <typename BasisFunctionType, typename KernelType, typename ResultType>
class KernelTrialIntegral;
template <typename CoordinateType> class RawGridGeometry;
template <typename BasisFunctionType>
class QuadratureDescriptorSelectorForPotentialOperators;
template <typename CoordinateType> class SingleQuadratureRuleFamily;
/** \endcond */

template <typename BasisFunctionType, typename KernelType, typename ResultType,
          typename GeometryFactory>
class DefaultLocalAssemblerForPotentialOperatorsOnSurfaces
    : public LocalAssemblerForPotentialOperators<ResultType> {
public:
  typedef typename ScalarTraits<ResultType>::RealType CoordinateType;

  DefaultLocalAssemblerForPotentialOperatorsOnSurfaces(
      const arma::Mat<CoordinateType> &points,
      const shared_ptr<const GeometryFactory> &geometryFactory,
      const shared_ptr<const RawGridGeometry<CoordinateType>> &rawGeometry,
      const shared_ptr<const std::vector<const Shapeset<BasisFunctionType> *>> &
          trialShapesets,
      const shared_ptr<const CollectionOfKernels<KernelType>> &kernels,
      const shared_ptr<const CollectionOfShapesetTransformations<
          CoordinateType>> &trialTransformations,
      const shared_ptr<const KernelTrialIntegral<BasisFunctionType, KernelType,
                                                 ResultType>> &integral,
      const ParallelizationOptions &parallelizationOptions,
      VerbosityLevel::Level verbosityLevel,
      const shared_ptr<const QuadratureDescriptorSelectorForPotentialOperators<
          BasisFunctionType>> &quadDescSelector,
      const shared_ptr<const SingleQuadratureRuleFamily<CoordinateType>> &
          quadRuleFamily);
  virtual ~DefaultLocalAssemblerForPotentialOperatorsOnSurfaces();

  virtual void
  evaluateLocalContributions(const std::vector<int> &pointIndices,
                             int trialElementIndex,
                             LocalDofIndex localTrialDofIndex,
                             std::vector<arma::Mat<ResultType>> &result,
                             CoordinateType nominalDistance = -1.);

  virtual void
  evaluateLocalContributions(int pointIndex, int componentIndex,
                             const std::vector<int> &trialElementIndices,
                             std::vector<arma::Mat<ResultType>> &result,
                             CoordinateType nominalDistance = -1.);

  virtual void
  evaluateLocalContributions(const std::vector<int> &pointIndices,
                             const std::vector<int> &trialElementIndices,
                             Fiber::_2dArray<arma::Mat<ResultType>> &result,
                             CoordinateType nominalDistance = -1.);

  virtual int resultDimension() const;

  virtual CoordinateType estimateRelativeScale(CoordinateType minDist) const;

private:
  /** \cond PRIVATE */
  typedef KernelTrialIntegrator<BasisFunctionType, KernelType, ResultType>
  Integrator;
  typedef DefaultLocalAssemblerForOperatorsOnSurfacesUtilities<
      BasisFunctionType> Utilities;

  const Integrator &selectIntegrator(int testElementIndex,
                                     int trialElementIndex,
                                     CoordinateType nominalDistance = -1.);

  int order(int pointIndex, int trialElementIndex,
            CoordinateType nominalDistance) const;

  const Integrator &getIntegrator(const SingleQuadratureDescriptor &index);

  CoordinateType pointElementDistanceSquared(int pointIndex,
                                             int trialElementIndex) const;

  void precalculateElementSizesAndCenters();

private:
  arma::Mat<CoordinateType> m_points;
  shared_ptr<const GeometryFactory> m_geometryFactory;
  shared_ptr<const RawGridGeometry<CoordinateType>> m_rawGeometry;
  shared_ptr<const std::vector<const Shapeset<BasisFunctionType> *>>
  m_trialShapesets;
  shared_ptr<const CollectionOfKernels<KernelType>> m_kernels;
  shared_ptr<const CollectionOfShapesetTransformations<CoordinateType>>
  m_trialTransformations;
  shared_ptr<const KernelTrialIntegral<BasisFunctionType, KernelType,
                                       ResultType>> m_integral;
  ParallelizationOptions m_parallelizationOptions;
  VerbosityLevel::Level m_verbosityLevel;
  shared_ptr<const QuadratureDescriptorSelectorForPotentialOperators<
      BasisFunctionType>> m_quadDescSelector;
  shared_ptr<const SingleQuadratureRuleFamily<CoordinateType>> m_quadRuleFamily;

  typedef tbb::concurrent_unordered_map<SingleQuadratureDescriptor,
                                        Integrator *> IntegratorMap;
  IntegratorMap m_kernelTrialIntegrators;

  enum {
    INVALID_INDEX = INT_MAX
  };
  /** \endcond */
};

} // namespace Fiber

#include "default_local_assembler_for_potential_operators_on_surfaces_imp.hpp"

#endif

max_stars_repo_name: mdavezac/bempp

max_issues_repo_name: mdavezac/bempp

max_forks_repo_name: mdavezac/bempp


text:
#ifndef Integration_ModifiedArrheniusIntegralBase_hpp
#define Integration_ModifiedArrheniusIntegralBase_hpp

/** @file ModifiedArrheniusIntegralBase.hpp
  * @brief
  * @author C.D. Clark III
  * @date 06/27/17
  */

#include <boost/math/tools/roots.hpp>
using boost::math::tools::bracket_and_solve_root;
using boost::math::tools::eps_tolerance;
#include "ArrheniusIntegralBase.hpp"

namespace libArrhenius {

/** @class ModifiedArrheniusIntegralBase
  * @brief Base class for common modified Arrhenius integral data.
  * @author C.D. Clark III
  */
template<typename Real>
class ModifiedArrheniusIntegralBase : public ArrheniusIntegralBase<Real>
{
  protected:
    Real n;

  public:

    template<typename T>
    void setExponent( T n_ ) { n = n_; }
    Real getExponent( ) { return n; }

    Real getCriticalTemperature() const {
      Real Tcrit = ArrheniusIntegralBase<Real>::getCriticalTemperature();
      if( n == 0 )
        return Tcrit;

      // if n != 0, then we have a transcendental equation we need to solve.
      //
      // ln T^n - Ea/R/T + ln A = 0  (solve for T)
      //
      // use the n = 0 case as a first gues. this will be too high
      Real factor = 1.1;
      eps_tolerance<Real> tol( std::numeric_limits<Real>::digits - 3 );
      boost::uintmax_t maxit = 100;
      auto Tcrit_range =  bracket_and_solve_root(
               [&](Real T){
               return Constants::MKS::R*T*log(this->A*pow(T,this->n)) - this->Ea; },
               Tcrit, factor, true, tol, maxit );
      Tcrit = Tcrit_range.first;
      return Tcrit;
    }

    Real rate( Real T ) const { return pow(T,n)*this->A*exp( -this->Ea / Constants::MKS::R /T ); }

  protected:
};

}

#endif // include protector


text:
#if !BOOST_PP_IS_ITERATING
//==============================================================================
//         Copyright 2003 - 2012   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2012   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef NT2_CORE_FUNCTIONS_GENERIC_SUB2IND_HPP_INCLUDED
#define NT2_CORE_FUNCTIONS_GENERIC_SUB2IND_HPP_INCLUDED

#include <nt2/core/functions/sub2ind.hpp>
#include <nt2/sdk/parameters.hpp>
#include <nt2/core/container/dsl.hpp>
#include <nt2/core/utility/as_index.hpp>
#include <nt2/include/functions/plus.hpp>
#include <nt2/include/functions/minus.hpp>
#include <nt2/include/functions/splat.hpp>
#include <nt2/include/functions/touint.hpp>
#include <nt2/include/functions/as_size.hpp>
#include <boost/fusion/include/make_vector.hpp>
#include <boost/preprocessor/arithmetic/inc.hpp>
#include <boost/preprocessor/repetition/repeat.hpp>
#include <boost/preprocessor/iteration/iterate.hpp>
#include <boost/preprocessor/repetition/enum.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>

namespace nt2 { namespace ext
{
   #define BOOST_PP_FILENAME_1 "nt2/core/functions/generic/sub2ind.hpp"
   #define BOOST_PP_ITERATION_LIMITS (2, BOOST_PP_INC(NT2_MAX_DIMENSIONS))
   #include BOOST_PP_ITERATE()
} }

#endif

#else

  #define N BOOST_PP_ITERATION()

  #define M0(z,n,t) (t<arithmetic_<A1> >)
  #define M1(z,n,t) nt2::touint(a##n - one)

  /// INTERNAL ONLY - sub2ind(ettent,D1,D2)
  BOOST_DISPATCH_IMPLEMENT  ( sub2ind_, tag::cpu_
                            , (SZ)(A1)
                            , (fusion_sequence_<SZ>)
                              BOOST_PP_REPEAT(N,M0,generic_)
                            )
  {
    typedef A1 result_type;

    BOOST_FORCEINLINE
    result_type operator()(SZ const& sz, BOOST_PP_ENUM_PARAMS(N,A1 const& a) ) const
    {
      A1 one  = One<A1>();
      A1 that = one
              + splat<A1> ( as_index ( sz, boost::fusion::make_vector
                                        ( BOOST_PP_ENUM(N, M1, ~) )
                                    )
                          );
      return that;
    }
  };

  /// INTERNAL ONLY - ind2sub(size,IND)
  BOOST_DISPATCH_IMPLEMENT  ( sub2ind_, tag::cpu_
                            , (SZ)(A1)
                            , ((ast_<SZ,nt2::container::domain>))
                              BOOST_PP_REPEAT(N,M0,scalar_)
                            )
  {
    BOOST_DISPATCH_RETURNS( BOOST_PP_INC(N)
                          , (SZ const& sz, BOOST_PP_ENUM_PARAMS(N,A1 a) )
                          , ( nt2::sub2ind( nt2::as_size(sz)
                                          , BOOST_PP_ENUM_PARAMS(N,a)
                                          )
                            )
                          )
  };

  #undef M1
  #undef M0
  #undef N

#endif


text:
/*
 *  heuristics.hpp
 *
 *
 *  Created by Andrea Bedini on 24/Nov/2011.
 *  Copyright (c) 2011-2014, Andrea Bedini <andrea.bedini@gmail.com>.
 *
 *  Distributed under the terms of the Modified BSD License.
 *  The full license is in the file COPYING, distributed as part of
 *  this software.
 *
 */

#ifndef HEURISTICS_HPP
#define HEURISTICS_HPP

#include <boost/range/algorithm.hpp>

namespace heuristics {
  using namespace boost;

  template<class Vertex, class Graph>
  void eliminate_vertex(Vertex v, Graph& g) {
    for (auto a : as_range(adjacent_vertices(v, g))) {
      for (auto b : as_range(adjacent_vertices(v, g))) {
        if (a != b and not edge(a, b, g).second) {
          add_edge(a, b, g);
        }
      }
    }
    clear_vertex(v, g);
    remove_vertex(v, g);
  }

  template<class Graph, class Vertex>
  unsigned int num_non_adjacent_neighbors(Vertex v, Graph const& g)
  {
    unsigned int n = 0;
    for (auto u : as_range(adjacent_vertices(v, g))) {
      auto adj = adjacent_vertices(u, g);
      for (auto z : as_range(adjacent_vertices(v, g))) {
        if (boost::find(adj, z) != end(adj))
          n ++;
      }
    }
    return n;
  }

  template<class Graph, class OutputIterator>
  void greedy_degree_order(Graph g, OutputIterator out) {
    using vertex = typename Graph::vertex_descriptor;
    while (num_vertices(g) > 0) {
      auto v = *min_element(vertices(g), [&](vertex v, vertex u) {
        return degree(v, g) < degree(u, g);
      });
      *out++ = get(vertex_index, g, v);
      eliminate_vertex(v, g);
    }
  }

  //////////////////////////////////////////////////////////////////////

  template<class Graph, class OutputIterator>
  void greedy_fillin_order(Graph g, OutputIterator out) {
    using vertex = typename Graph::vertex_descriptor;
    while (num_vertices(g) > 0) {
      auto v = *min_element(vertices(g), [&](vertex v, vertex u) {
        return num_non_adjacent_neighbors(v, g) < num_non_adjacent_neighbors(u, g);
      });
      *out++ = get(vertex_index, g, v);
      eliminate_vertex(v, g);
    }
  }

  //////////////////////////////////////////////////////////////////////

  template<class Graph, class OutputIterator>
  void greedy_local_degree_order(Graph g, OutputIterator out) {
    using vertex = typename Graph::vertex_descriptor;
    auto compare = [&](vertex v, vertex u) {
      return degree(v, g) < degree(u, g);
    };
    vertex current = *min_element(vertices(g), compare);
    while (num_vertices(g) > 0) {
      *out++ = get(vertex_index, g, current);
      vertex next = *min_element(vertices(g), compare);
      eliminate_vertex(current, g);
      current = next;
    }
  }

  //////////////////////////////////////////////////////////////////////

  template<class Graph, class OutputIterator>
  void greedy_local_fillin_order(Graph g, OutputIterator out) {
    using vertex = typename Graph::vertex_descriptor;
    auto compare =  [&](vertex v, vertex u) {
      return num_non_adjacent_neighbors(v, g) < num_non_adjacent_neighbors(u, g);
    };
    vertex current = *min_element(vertices(g), compare);
    while (num_vertices(g) > 0) {
      *out++ = get(vertex_index, g, current);
      vertex next = *min_element(adjacent_vertices(current, g), compare);
      eliminate_vertex(current, g);
      current = next;
    }
  }
}

#endif

max_stars_repo_name: andreabedini/tutte

max_issues_repo_name: andreabedini/tutte

max_forks_repo_name: andreabedini/tutte


text:
#include <tacopie/tacopie>

#include <iostream>
#include <stdio.h>
#include <fstream>
#include "ros/ros.h"
#include <tf2_msgs/TFMessage.h>
#include <math.h>
#include <ros/console.h>
#include <tf/transform_listener.h>
#include <boost/thread.hpp>
#include <tf/tf.h>
#include "TCPPackageConstants.h"
#include <image_transport/image_transport.h>

#include "baxter_core_msgs/EndpointState.h"
#include "action_manager/execAction.h"

using namespace std;

// flags
int PROCESS_TF_DATA = 1;
int PROCESS_GRIPPER_DATA = 1;
int PROCESS_IMAGE_DATA = 0;

tacopie::tcp_client client;

// to count the seq of data points
// use the first 100 to find offset
const int num_to_find_offset = 40;
int left_data_offset_counter = 0;

//store the last 4 data points, running average
double last_n_left_data_average[6] = {0,0,0,0,0,0};

// use the average of data from the 1st second to calculate offset (100 data points)
// store the offset in a array
// left: force.x, force.y, force.z, torque.x, torque.y, torque.z
double force_torque_offset_left[6] = {0,0,0,0,0,0};

int imageCounter = 0;

ros::ServiceClient action_client;

// ros::WallTime begin;
// ros::WallTime now;

void write_cb(const tacopie::tcp_client::write_result& res) {
    if(res.success) {
        // cout << "sent package " << res.size << " bytes" << endl;
    } else {
        cout << "sending failed!" << endl;
    }
}

void on_new_message(const tacopie::tcp_client::read_result& res) {
    if (res.success){
    	// here print the action name
        for (auto i: res.buffer){
            cout << i ;
        }
        cout << endl;
        cout << res.buffer.size() << endl;

        const char* p = res.buffer.data(); // p
        if (res.buffer.size() <= 3){
            //const char* p = res.buffer.data(); // p
            switch(p[0]){
                case 'i':
                case 'I':
                    if (PROCESS_IMAGE_DATA){
                        PROCESS_IMAGE_DATA = 0;
                    } else {
                        PROCESS_IMAGE_DATA = 1;
                    }
                    break;

                case 'f':
                case 'F':
                    if (PROCESS_GRIPPER_DATA){
                        PROCESS_GRIPPER_DATA = 0;
                        left_data_offset_counter = 0;
                        for (int i = 0; i < 6; i ++){
                            last_n_left_data_average[i] = 0;
                            force_torque_offset_left[i] = 0;
                        }
                    } else {
                        PROCESS_GRIPPER_DATA = 1;
                    }
                    break;

                case 't':
                case 'T':
                    if (PROCESS_TF_DATA){
                        PROCESS_TF_DATA = 0;
                    } else {
                        PROCESS_TF_DATA = 1;
                    }
                    break;
            }
        }
        else {
        	if (p[0] == 'a') {
        		std::string action_string(p+1);
                //cout << str(p[1]) << endl; // get action name
        		// to add service call function
        		//action_srv.request.action = str(p[1]);
        		//action_client.call(action_srv);
        		//sleep(1.5);

                action_manager::execAction action_srv;

                if(action_string == "grasp") {
                    action_string = "grasp_right";
                    action_srv.request.action = action_string;
                }
                else if(action_string == "ungrasp") {
                    action_string = "ungrasp_right";
                    action_srv.request.action = action_string;
                }
                else if(action_string == "push") {
                    action_string = "push";
                    action_client.call(action_srv);
                    action_string = "twist";
                }
                else {
                    action_srv.request.action = action_string;
                }
                action_client.call(action_srv);
        	}
        }

    }
    else {
        cout << "read failed, client disconnected" << endl;
    }
    client.async_read({30, bind(&on_new_message, placeholders::_1)});
}

void startConnection() {
	client.connect("192.168.1.176", 12345);
    cout << "connection established" << endl;

}

void leftEndPointCallBack(const baxter_core_msgs::EndpointState end_point_state) {
    if (!PROCESS_GRIPPER_DATA){
        return;
    }
    double fx = end_point_state.wrench.force.x;
    double fy = end_point_state.wrench.force.y;
    double fz = end_point_state.wrench.force.z;
    double tx = end_point_state.wrench.torque.x;
    double ty = end_point_state.wrench.torque.y;
    double tz = end_point_state.wrench.torque.z;
    if (left_data_offset_counter < num_to_find_offset)
    {
        force_torque_offset_left[0] += fx;
        force_torque_offset_left[1] += fy;
        force_torque_offset_left[2] += fz;
        force_torque_offset_left[3] += tx;
        force_torque_offset_left[4] += ty;
        force_torque_offset_left[5] += tz;
        left_data_offset_counter ++;
    }
    else if (left_data_offset_counter == num_to_find_offset)
    {
        for (int i = 0; i < 6; i++){
            force_torque_offset_left[i] /= num_to_find_offset;
        }
        left_data_offset_counter ++;
        // now=ros::WallTime::now();
        // ros::WallDuration diff=now-begin;

        // cout<<"diff: "<<diff.toSec() <<endl;
    }
    else
    {
        // because sensor feels the force/torque on the robot
        // the force/torque robot exerts is in the other direction
        fx = force_torque_offset_left[0] - fx;
        fy = force_torque_offset_left[1] - fy;
        fz = force_torque_offset_left[2] - fz;
        tx = force_torque_offset_left[3] - tx;
        ty = force_torque_offset_left[4] - ty;
        tz = force_torque_offset_left[5] - tz;

        // calculate new running average
        last_n_left_data_average[0] = ((last_n_left_data_average[0] * 3) + fx)/4;
        last_n_left_data_average[1] = ((last_n_left_data_average[1] * 3) + fy)/4;
        last_n_left_data_average[2] = ((last_n_left_data_average[2] * 3) + fz)/4;
        last_n_left_data_average[3] = ((last_n_left_data_average[3] * 3) + tx)/4;
        last_n_left_data_average[4] = ((last_n_left_data_average[4] * 3) + ty)/4;
        last_n_left_data_average[5] = ((last_n_left_data_average[5] * 3) + tz)/4;

        //calculate the magnitude for force and torque
        double force_mag = sqrt(pow(last_n_left_data_average[0],2.0) +
                                pow(last_n_left_data_average[1],2.0) +
                                pow(last_n_left_data_average[2],2.0));
        double torque_mag = sqrt(pow(last_n_left_data_average[3],2.0) +
                                pow(last_n_left_data_average[4],2.0) +
                                pow(last_n_left_data_average[5],2.0));

        int num_of_elements = 10;
        float* msg = new float[num_of_elements];
        size_t msg_length = sizeof(float) * num_of_elements;

        // first 4 byte encodes the size of the following package
        ((int*) msg)[0] = msg_length - 4;
        ((int*) msg)[1] = LEFT_FORCE_TORQUE;
        // to change
        for (int i = 0; i < 6; i++){
            msg[i+2] = (float) last_n_left_data_average[i];
        }
        msg[8] = (float) force_mag;
        msg[9] = (float) torque_mag;

        char* c_ptr = reinterpret_cast<char*>(msg);
        vector<char> vec(c_ptr, c_ptr + msg_length);
        client.async_write({vec, bind(&write_cb, placeholders::_1)});

        delete[] msg;
    }
}

void getTransform(tf::TransformListener& listener, tf::StampedTransform& transform,
                const int target_frame, float* msg, int index) {
    try {
        listener.waitForTransform("screen", frame_name_arr[target_frame], ros::Time(0), ros::Duration(5.0));
        listener.lookupTransform("screen", frame_name_arr[target_frame], ros::Time(0), transform);
    }
    catch (tf::TransformException ex){
        ROS_ERROR("%s",ex.what());
        // ros::Duration(1.0).sleep();
    }

    tf::Vector3 translation = transform.getOrigin();
    tf::Quaternion rotation = transform.getRotation();

    ((int*) msg)[index] = target_frame;
    msg[index + 1] = translation.x();
    msg[index + 2] = translation.y();
    msg[index + 3] = translation.z();
    msg[index + 4] = rotation.getX();
    msg[index + 5] = rotation.getY();
    msg[index + 6] = rotation.getZ();
    msg[index + 7] = rotation.getW();
}

void tf_thread()
{
    tf::TransformListener listener;

    ros::Rate rate(15.0);

    while (ros::ok()){
        rate.sleep();

        if (!PROCESS_TF_DATA){
            continue;
        }
        tf::StampedTransform transform;
        int num_of_elements = TOTAL_NUM_TF * 8 + 1;
        size_t msg_length = sizeof(float) * num_of_elements;

        float* msg = new float[num_of_elements];
        ((int*) msg)[0] = msg_length - 4;

        int index = 1;

        for (int i = 0; i < TOTAL_NUM_TF; i++) {
            getTransform(listener, transform, i, msg, index);
            index += 8;
        }

        char* c_ptr = reinterpret_cast<char*>(msg);
        vector<char> vec(c_ptr, c_ptr + msg_length);
        client.async_write({vec, bind(&write_cb, placeholders::_1)});

        delete[] msg;
    }
}

// void tcpListener() {
//     size_t read_size = 20;
//     client.async_read({read_size, bind(&on_new_message, placeholders::_1)});
//     boost::this_thread::interruption_point();
//     // while (ros::ok()){
//     //     boost::this_thread::interruption_point();
//     //     client.async_read({read_size, bind(&on_new_message, placeholders::_1)});
//     // }
// }

void imageCallback(const sensor_msgs::ImageConstPtr& msg) {
    if (!PROCESS_IMAGE_DATA){
        return;
    }
    if (imageCounter < 3){
        imageCounter ++;
        return;
    }
    imageCounter = 0;

    // cout << "height: " << msg->height << endl;
    // cout << "width: " << msg->width << endl;
    // cout << typeid(msg->data).name() << endl;   // it is vector<unsigned char>
    int msg_length = (msg->data).size();
    // cout << msg_length << endl;

    vector<char> vec((char*)&msg_length, (char*)&msg_length + 4);
    vec.insert(vec.end(), reinterpret_cast<const char*>((msg->data).data()), reinterpret_cast<const char*>((msg->data).data()) + msg_length);

    client.async_write({vec, bind(&write_cb, placeholders::_1)});

}

int main(int argc, char **argv) {
    ros::init(argc,argv,"send_to_hololens");
    ros::NodeHandle n;

    startConnection();

    if (ros::console::set_logger_level(ROSCONSOLE_DEFAULT_NAME, ros::console::levels::Debug))
    {
        // Change the level to fit your needs
        ros::console::notifyLoggerLevelsChanged();
    }

    image_transport::ImageTransport it(n);

    ros::Subscriber subLeft = n.subscribe("/robot/limb/left/endpoint_state", 1, leftEndPointCallBack);
    // image_transport::Subscriber subImage = it.subscribe("/cameras/right_hand_camera/image", 1, imageCallback);
    image_transport::Subscriber subImage = it.subscribe("/simtrack/image", 1, imageCallback);
    action_client = n.serviceClient<action_manager::execAction>("/action_manager/exec");

    client.async_read({30, bind(&on_new_message, placeholders::_1)});

    boost::thread t1(tf_thread);
    // boost::thread t2(tcpListener);

    ros::Rate r1(15);

    // begin = ros::WallTime::now();

    while (ros::ok()){
        ros::spinOnce();
        r1.sleep();
    }

    t1.join();
    // t2.interrupt();

 	// close(sock);

	return 0;
}