text:
//
// connection.cpp
// ~~~~~~~~~~~~~~
//
// Copyright (c) 2003-2008 Christopher M. Kohlhoff (chris at kohlhoff dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//

#include "stdafx.h"

#include "connection.hpp"

#include <vector>
#include <boost/bind.hpp>

#include "connection_manager.hpp"
#include "request_handler.hpp"
#include "AppLog.h"
#include "Settings.h"
#include "TokenBucket.h"

using namespace boost::posix_time;

namespace http {
	namespace server {

		connection::connection(boost::asio::io_service& io_service,
			connection_manager& manager, request_handler& handler)
			: socket_(io_service),
			connection_manager_(manager),
			request_handler_(handler),
			_closeAfterResponseWritten(false),
			_connKillTimer(io_service),
			_requestInProgress(false),
			_tokenBucketTimer(io_service)
		{
			_debug = (aperture::Settings::instance().config())["debug"].as<bool>();

			if (_debug)
			{
				aperture::AppLog::instance().out()
					<< "[HTTP] Starting new connection "
					<< std::endl;
			}
		}

		boost::asio::ip::tcp::socket& connection::socket()
		{
			return socket_;
		}

		void connection::init_new_request()
		{
			request::ptr newRequest(new request);
			newRequest->is_bad = false;
			newRequest->is_ready = false;

			requests_.push(newRequest);
		}

		void connection::start()
		{
			_lastActiveTime = second_clock::local_time();
			this->reset_timeout_timer();

			this->init_new_request();
			socket_.async_read_some(boost::asio::buffer(buffer_),
				boost::bind(&connection::handle_read, shared_from_this(),
				boost::asio::placeholders::error,
				boost::asio::placeholders::bytes_transferred));
		}

		void connection::stop()
		{
			if (_debug)
			{
				aperture::AppLog::instance().out() << "[HTTP] Closing socket" << std::endl;
			}

			socket_.close();
			_connKillTimer.cancel();
			_tokenBucketTimer.cancel();
		}

		void connection::reset_timeout_timer()
		{
			_connKillTimer.expires_from_now(seconds(MAXIMUM_LIVE_TIME));

			_connKillTimer.async_wait(
				boost::bind(&connection::on_timeout, shared_from_this(),
				boost::asio::placeholders::error));
		}

		void connection::process_next_request()
		{
			_lastActiveTime = second_clock::local_time();

			if (requests_.front()->is_bad)
			{
				_requestInProgress = true;

				reply_ = reply::stock_reply(reply::bad_request);
				header connClose;
				connClose.name = "Connection";
				connClose.value = "close";

				reply_.headers.push_back(connClose);

				_closeAfterResponseWritten = true;

				boost::asio::async_write(socket_, reply_.to_buffers(),
					boost::bind(&connection::handle_write, shared_from_this(),
					boost::asio::placeholders::error));
			}
			else if (requests_.front()->is_ready)
			{
				_requestInProgress = true;

				reply_.reset();
				request_handler_.handle_request(*requests_.front(), reply_,
					boost::bind(&connection::reply_ready, this));
			}
			else
			{
				//nothing is ready, begin a recv
				_requestInProgress = false;

				socket_.async_read_some(boost::asio::buffer(buffer_),
				boost::bind(&connection::handle_read, shared_from_this(),
					boost::asio::placeholders::error,
					boost::asio::placeholders::bytes_transferred));
			}
		}

		void connection::handle_read(const boost::system::error_code& e,
			std::size_t bytes_transferred)
		{
			if (!e)
			{
				_lastActiveTime = second_clock::local_time();

				boost::array<char, 8192>::iterator bufferPos = buffer_.data();

				while (bufferPos != buffer_.data() + bytes_transferred)
				{
					boost::tribool result;
					boost::tie(result, bufferPos) = request_parser_.parse(
						*requests_.back(), bufferPos, buffer_.data() + bytes_transferred);

					if (result)
					{
						//reset the parser
						request_parser_.reset();
						//push a new starting request to begin building for the next iteration
						this->init_new_request();
					}
					else if (!result)
					{
						//reset the parser
						request_parser_.reset();
						//the connection will be terminated once this request is processed
						break;
					}
					else
					{
						if (! requests_.front()->is_ready)
						{
							//continue receiving until we have at least one completed request
							socket_.async_read_some(boost::asio::buffer(buffer_),
							boost::bind(&connection::handle_read, shared_from_this(),
								boost::asio::placeholders::error,
								boost::asio::placeholders::bytes_transferred));

							break;
						}
					}
				}

				if (requests_.front()->is_ready || requests_.front()->is_bad)
				{
					this->process_next_request();
				}
			}
			else
			{
				if (_debug)
				{
					aperture::AppLog::instance().out()
						<< "[HTTP] Read error, Stopping connection "
						<< std::endl;
				}

				connection_manager_.stop(shared_from_this());
			}
		}

		void connection::handle_write(const boost::system::error_code& e)
		{
			if (!e)
			{
				if (_closeAfterResponseWritten)
				{
					requests_.pop();

					if (_debug)
					{
						aperture::AppLog::instance().out()
							<< "[HTTP] Closing connection: closeAfterResponseWritten:  "
							<< _closeAfterResponseWritten << std::endl;
					}

					// Initiate graceful connection closure.
					boost::system::error_code ignored_ec;
					socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec);
					connection_manager_.stop(shared_from_this());
				}
				else
				{
					requests_.pop();

					//handle next request
					this->process_next_request();
				}
			}
			else
			{
				connection_manager_.stop(shared_from_this());
			}
		}

		void connection::reply_ready()
		{
			if (requests_.front()->header_value("Connection") == "close"
				|| _closeAfterResponseWritten)
			{
				header connClose;
				connClose.name = "Connection";
				connClose.value = "close";

				reply_.headers.push_back(connClose);

				_closeAfterResponseWritten = true;
			}

			boost::shared_ptr<TokenBucket> bucket = request_handler_.getBucket(reply_.token);

			if (bucket)
			{
				reply_.token_bucket = bucket;
				boost::asio::async_write(socket_, reply_.header_buffers(),
					boost::bind(&connection::handle_tb_header_write, shared_from_this(),
					boost::asio::placeholders::error));
			}
			else
			{
				boost::asio::async_write(socket_, reply_.to_buffers(),
					boost::bind(&connection::handle_write, shared_from_this(),
					boost::asio::placeholders::error));
			}
		}

		void connection::handle_tb_header_write(const boost::system::error_code& e)
		{
			if (!e)
			{
				this->try_send_next_chunk();
			}
			else
			{
				connection_manager_.stop(shared_from_this());
			}
		}

		void connection::try_send_next_chunk()
		{
			_lastActiveTime = second_clock::local_time();

			if (reply_.bytes_sent == reply_.content.size())
			{
				//done
				this->handle_write(boost::system::error_code());
				return;
			}

			//otherwise, check to see if we have enough tokens to send now
			int transmissionSize = std::min(reply_.remaining_bytes(), (size_t)reply_.token_bucket->getMaxBurst());
			if (reply_.token_bucket->removeTokens(transmissionSize))
			{
				//we have enough tokens, send out a chunk
				boost::asio::async_write(socket_, reply_.get_next_chunk(transmissionSize),
						boost::bind(&connection::handle_tb_write, shared_from_this(),
						boost::asio::placeholders::error,
						boost::asio::placeholders::bytes_transferred));
			}
			else
			{
				//we do not have enough tokens, schedule a retry
				_tokenBucketTimer.expires_from_now(std::chrono::milliseconds((int)(TokenBucket::DRIP_PERIOD_PERIOD_MS)));
				_tokenBucketTimer.async_wait(boost::bind(&connection::on_tb_wait_timeout, this, boost::asio::placeholders::error));
			}
		}

		void connection::on_tb_wait_timeout(const boost::system::error_code& e)
		{
			if (e != boost::asio::error::operation_aborted)
			{
				this->try_send_next_chunk();
			}
		}

		void connection::handle_tb_write(const boost::system::error_code& e,
			size_t bytes_transferred)
		{
			if (!e)
			{
				reply_.bytes_sent += bytes_transferred;
				this->try_send_next_chunk();
			}
			else
			{
				this->handle_write(e);
			}
		}

		void connection::on_timeout(const boost::system::error_code& e)
		{
			if (e != boost::asio::error::operation_aborted)
			{
				if (second_clock::local_time() - _lastActiveTime > seconds(MAXIMUM_LIVE_TIME) &&
					! _requestInProgress)
				{
					if (_debug)
					{
						aperture::AppLog::instance().out()
							<< "[HTTP] Timeout waiting for command, disconnecting"
							<< std::endl;
					}

					// Initiate graceful connection closure.
					// this will cause a read error and close out the socket
					boost::system::error_code ignored_ec;
					socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec);
					socket_.close();
					//connection_manager_.stop(shared_from_this());
				}
				else
				{
					this->reset_timeout_timer();
				}
			}
		}

	} // namespace server
} // namespace http

max_stars_repo_name: llaxton/M-O-S-E-S-aperture

max_issues_repo_name: llaxton/M-O-S-E-S-aperture

max_forks_repo_name: llaxton/M-O-S-E-S-aperture


text:
//==============================================================================
//         Copyright 2003 - 2012   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2012   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#define NT2_UNIT_MODULE "nt2::meta::as_real"

#include <nt2/sdk/config/types.hpp>
#include <nt2/sdk/meta/as_floating.hpp>
#include <boost/type_traits/is_same.hpp>

#include <nt2/sdk/unit/module.hpp>
#include <nt2/sdk/unit/tests/basic.hpp>

////////////////////////////////////////////////////////////////////////////////
// Test that as_floating is correct w/r to original sign
////////////////////////////////////////////////////////////////////////////////
NT2_TEST_CASE(as_floating)
{
  using nt2::meta::as_floating;
  using boost::is_same;

  NT2_TEST( (is_same<as_floating<double       >::type,double  >::value ));
  NT2_TEST( (is_same<as_floating<float        >::type,float   >::value ));
  NT2_TEST( (is_same<as_floating<nt2::int64_t >::type,double  >::value ));
  NT2_TEST( (is_same<as_floating<nt2::int32_t >::type,float   >::value ));
  NT2_TEST( (is_same<as_floating<nt2::uint64_t>::type,double  >::value ));
  NT2_TEST( (is_same<as_floating<nt2::uint32_t>::type,float   >::value ));
}

ext: cpp


text:
#pragma once

#define FMT_HEADER_ONLY

#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/websocket.hpp>
#include <fmt/color.h>
#include <fmt/format.h>
#include <fmt/ostream.h>
#include <nlohmann/json.hpp>

namespace project
{
    namespace beast = boost::beast;

    namespace http = beast::http;

    namespace websocket = beast::websocket;

    namespace net = boost::asio;

    namespace ssl = boost::asio::ssl;

    using tcp = boost::asio::ip::tcp;

    using error_code = beast::error_code;

    using system_error = beast::system_error;

    using json = nlohmann::json;

}   // namespace project

ext: hpp


text:
//------------------------------------------------------------------------------
/// \file Listen_tests.cpp
/// \author Ernest Yeung
/// \email  ernestyalumni@gmail.com
//------------------------------------------------------------------------------
#include "IPC/Sockets/Listen.h"

#include "IPC/Sockets/Bind.h"
#include "IPC/Sockets/InternetAddress.h"
#include "IPC/Sockets/ParameterFamilies.h"
#include "IPC/Sockets/Socket.h"

#include <boost/test/unit_test.hpp>

using IPC::Sockets::Bind;
using IPC::Sockets::Domain;
using IPC::Sockets::InternetSocketAddress;
using IPC::Sockets::MakeListen;
using IPC::Sockets::Socket;
using IPC::Sockets::Type;

BOOST_AUTO_TEST_SUITE(IPC)
BOOST_AUTO_TEST_SUITE(Sockets)
BOOST_AUTO_TEST_SUITE(Listen_tests)

//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
BOOST_AUTO_TEST_CASE(BindBindsSocketToAnyAvailablePort)
{
  {
    Socket socket {Domain::ipv4, Type::stream};
    BOOST_TEST_REQUIRE(socket.domain() == AF_INET);
    BOOST_TEST_REQUIRE(socket.type() == SOCK_STREAM);

    const InternetSocketAddress internet_socket_address {21234};

    Bind bind_f {internet_socket_address};
    const auto bind_result = bind_f(socket);

    BOOST_TEST_REQUIRE(!static_cast<bool>(bind_result));

    // Setup the socket for listening with a queue length of 5.
    MakeListen listen_f {5};
    const auto listen_result = listen_f(socket);

    BOOST_TEST(!static_cast<bool>(listen_result));
  }
}

BOOST_AUTO_TEST_SUITE_END() // Listen_tests
BOOST_AUTO_TEST_SUITE_END() // Sockets
BOOST_AUTO_TEST_SUITE_END() // IPC

max_stars_repo_name: ernestyalumni/HrdwCCppCUDA

max_issues_repo_name: ernestyalumni/HrdwCCppCUDA

max_forks_repo_name: ernestyalumni/HrdwCCppCUDA


text:
///////////////////////////////////////////////////////////////////////////////
// BSD 3-Clause License
//
// Copyright (C) 2019-2020, LAAS-CNRS, University of Edinburgh
// Copyright note valid unless otherwise stated in individual files.
// All rights reserved.
///////////////////////////////////////////////////////////////////////////////

#ifndef CROCODDYL_CORE_COST_BASE_HPP_
#define CROCODDYL_CORE_COST_BASE_HPP_

#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>

#include "crocoddyl/core/fwd.hpp"
#include "crocoddyl/core/state-base.hpp"
#include "crocoddyl/core/data-collector-base.hpp"
#include "crocoddyl/core/activation-base.hpp"
#include "crocoddyl/core/activations/quadratic.hpp"

namespace crocoddyl {

/**
 * @brief Abstract class for cost models
 *
 * In Crocoddyl, a cost model is defined by the scalar activation function \f$a(\cdot)\f$ and by the residual function
 * \f$\mathbf{r}(\cdot)\f$ as follows: \f[ cost = a(\mathbf{r}(\mathbf{x}, \mathbf{u})), \f] where
 * the residual function depends on the state point \f$\mathbf{x}\in\mathcal{X}\f$, which lies in the state manifold
 * described with a `nq`-tuple, its velocity \f$\dot{\mathbf{x}}\in T_{\mathbf{x}}\mathcal{X}\f$ that belongs to
 * the tangent space with `nv` dimension, and the control input \f$\mathbf{u}\in\mathbb{R}^{nu}\f$. The residual vector
 * is defined by \f$\mathbf{r}\in\mathbb{R}^{nr}\f$ where `nr` describes its dimension in the Euclidean space. On the
 * other hand, the activation function builds a cost value based on the definition of the residual vector. The residual
 * vector has to be specialized in a derived classes.
 *
 * The main computations are carring out in `calc` and `calcDiff` routines. `calc` computes the cost (and its residual)
 * and `calcDiff` computes the derivatives of the cost function (and its residual). Concretely speaking, `calcDiff`
 * builds a linear-quadratic approximation of the cost function with the form: \f$\mathbf{l_x}\in\mathbb{R}^{ndx}\f$,
 * \f$\mathbf{l_u}\in\mathbb{R}^{nu}\f$, \f$\mathbf{l_{xx}}\in\mathbb{R}^{ndx\times ndx}\f$,
 * \f$\mathbf{l_{xu}}\in\mathbb{R}^{ndx\times nu}\f$, \f$\mathbf{l_{uu}}\in\mathbb{R}^{nu\times nu}\f$ are the
 * Jacobians and Hessians, respectively.
 * Additionally, it is important remark that `calcDiff()` computes the derivates using the latest stored values by
 * `calc()`. Thus, we need to run first `calc()`.
 *
 * \sa `StateAbstractTpl`, `ActivationModelAbstractTpl`, `calc()`, `calcDiff()`, `createData()`
 */
template <typename _Scalar>
class CostModelAbstractTpl {
 public:
  EIGEN_MAKE_ALIGNED_OPERATOR_NEW

  typedef _Scalar Scalar;
  typedef MathBaseTpl<Scalar> MathBase;
  typedef CostDataAbstractTpl<Scalar> CostDataAbstract;
  typedef StateAbstractTpl<Scalar> StateAbstract;
  typedef ActivationModelAbstractTpl<Scalar> ActivationModelAbstract;
  typedef ActivationModelQuadTpl<Scalar> ActivationModelQuad;
  typedef DataCollectorAbstractTpl<Scalar> DataCollectorAbstract;
  typedef typename MathBase::VectorXs VectorXs;
  typedef typename MathBase::MatrixXs MatrixXs;

  /**
   * @brief Initialize the cost model
   *
   * @param[in] state       State of the multibody system
   * @param[in] activation  Activation model
   * @param[in] nu          Dimension of control vector
   */
  CostModelAbstractTpl(boost::shared_ptr<StateAbstract> state, boost::shared_ptr<ActivationModelAbstract> activation,
                       const std::size_t nu);

  /**
   * @copybrief CostModelAbstractTpl()
   *
   * The default `nu` value is obtained from `StateAbstractTpl::get_nv()`.
   *
   * @param[in] state       State of the multibody system
   * @param[in] activation  Activation model
   */
  CostModelAbstractTpl(boost::shared_ptr<StateAbstract> state, boost::shared_ptr<ActivationModelAbstract> activation);

  /**
   * @copybrief CostModelAbstractTpl()
   *
   * We use `ActivationModelQuadTpl` as a default activation model (i.e. \f$a=\frac{1}{2}\|\mathbf{r}\|^2\f$)
   *
   * @param[in] state  State of the multibody system
   * @param[in] nr     Dimension of residual vector
   * @param[in] nu     Dimension of control vector
   */
  CostModelAbstractTpl(boost::shared_ptr<StateAbstract> state, const std::size_t nr, const std::size_t nu);

  /**
   * @copybrief CostModelAbstractTpl()
   *
   * We use `ActivationModelQuadTpl` as a default activation model (i.e. \f$a=\frac{1}{2}\|\mathbf{r}\|^2\f$).
   * Furthermore, the default `nu` value is obtained from `StateAbstractTpl::get_nv()`.
   *
   * @param[in] state  State of the multibody system
   * @param[in] nr     Dimension of residual vector
   * @param[in] nu     Dimension of control vector
   */
  CostModelAbstractTpl(boost::shared_ptr<StateAbstract> state, const std::size_t nr);
  virtual ~CostModelAbstractTpl();

  /**
   * @brief Compute the cost value and its residual vector
   *
   * @param[in] data  Cost data
   * @param[in] x     State point \f$\mathbf{x}\in\mathbb{R}^{ndx}\f$
   * @param[in] u     Control input \f$\mathbf{u}\in\mathbb{R}^{nu}\f$
   */
  virtual void calc(const boost::shared_ptr<CostDataAbstract>& data, const Eigen::Ref<const VectorXs>& x,
                    const Eigen::Ref<const VectorXs>& u) = 0;

  /**
   * @brief Compute the Jacobian and Hessian of cost and its residual vector
   *
   * It computes the Jacobian and Hessian of the cost function. It assumes that `calc()` has been run first.
   *
   * @param[in] data  Cost data
   * @param[in] x     State point \f$\mathbf{x}\in\mathbb{R}^{ndx}\f$
   * @param[in] u     Control input \f$\mathbf{u}\in\mathbb{R}^{nu}\f$
   */
  virtual void calcDiff(const boost::shared_ptr<CostDataAbstract>& data, const Eigen::Ref<const VectorXs>& x,
                        const Eigen::Ref<const VectorXs>& u) = 0;

  /**
   * @brief Create the cost data
   *
   * The default data contains objects to store the values of the cost, residual vector and their derivatives (first
   * and second order derivatives). However, it is possible to specialized this function is we need to create
   * additional data, for instance, to avoid dynamic memory allocation.
   *
   * @param data  Data collector
   * @return the cost data
   */
  virtual boost::shared_ptr<CostDataAbstract> createData(DataCollectorAbstract* const data);

  /**
   * @copybrief calc()
   *
   * @param[in] data  Cost data
   * @param[in] x     State point
   */
  void calc(const boost::shared_ptr<CostDataAbstract>& data, const Eigen::Ref<const VectorXs>& x);

  /**
   * @copybrief calcDiff()
   *
   * @param[in] data  Cost data
   * @param[in] x     State point
   */
  void calcDiff(const boost::shared_ptr<CostDataAbstract>& data, const Eigen::Ref<const VectorXs>& x);

  /**
   * @brief Return the state
   */
  const boost::shared_ptr<StateAbstract>& get_state() const;

  /**
   * @brief Return the activation model
   */
  const boost::shared_ptr<ActivationModelAbstract>& get_activation() const;

  /**
   * @brief Return the dimension of the control input
   */
  std::size_t get_nu() const;

  /**
   * @brief Modify the cost reference
   */
  template <class ReferenceType>
  void set_reference(ReferenceType ref);

  /**
   * @brief Return the cost reference
   */
  template <class ReferenceType>
  ReferenceType get_reference() const;

 protected:
  /**
   * @copybrief set_reference()
   */
  virtual void set_referenceImpl(const std::type_info&, const void*);

  /**
   * @copybrief get_reference()
   */
  virtual void get_referenceImpl(const std::type_info&, void*) const;

  boost::shared_ptr<StateAbstract> state_;                 //!< State description
  boost::shared_ptr<ActivationModelAbstract> activation_;  //!< Activation model
  std::size_t nu_;                                         //!< Control dimension
  VectorXs unone_;                                         //!< No control vector
};

template <typename _Scalar>
struct CostDataAbstractTpl {
  EIGEN_MAKE_ALIGNED_OPERATOR_NEW

  typedef _Scalar Scalar;
  typedef MathBaseTpl<Scalar> MathBase;
  typedef ActivationDataAbstractTpl<Scalar> ActivationDataAbstract;
  typedef DataCollectorAbstractTpl<Scalar> DataCollectorAbstract;
  typedef typename MathBase::VectorXs VectorXs;
  typedef typename MathBase::MatrixXs MatrixXs;

  template <template <typename Scalar> class Model>
  CostDataAbstractTpl(Model<Scalar>* const model, DataCollectorAbstract* const data)
      : shared(data),
        activation(model->get_activation()->createData()),
        cost(Scalar(0.)),
        Lx(model->get_state()->get_ndx()),
        Lu(model->get_nu()),
        Lxx(model->get_state()->get_ndx(), model->get_state()->get_ndx()),
        Lxu(model->get_state()->get_ndx(), model->get_nu()),
        Luu(model->get_nu(), model->get_nu()),
        r(model->get_activation()->get_nr()),
        Rx(model->get_activation()->get_nr(), model->get_state()->get_ndx()),
        Ru(model->get_activation()->get_nr(), model->get_nu()) {
    Lx.setZero();
    Lu.setZero();
    Lxx.setZero();
    Lxu.setZero();
    Luu.setZero();
    r.setZero();
    Rx.setZero();
    Ru.setZero();
  }
  virtual ~CostDataAbstractTpl() {}

  DataCollectorAbstract* shared;
  boost::shared_ptr<ActivationDataAbstract> activation;
  Scalar cost;
  VectorXs Lx;
  VectorXs Lu;
  MatrixXs Lxx;
  MatrixXs Lxu;
  MatrixXs Luu;
  VectorXs r;
  MatrixXs Rx;
  MatrixXs Ru;
};

}  // namespace crocoddyl

/* --- Details -------------------------------------------------------------- */
/* --- Details -------------------------------------------------------------- */
/* --- Details -------------------------------------------------------------- */
#include "crocoddyl/core/cost-base.hxx"

#endif  // CROCODDYL_CORE_COST_BASE_HPP_

max_stars_repo_name: wxmerkt/crocoddyl

max_issues_repo_name: wxmerkt/crocoddyl

max_forks_repo_name: wxmerkt/crocoddyl


text:
#include <koinos/mempool/mempool.hpp>

#include <functional>
#include <tuple>

#include <boost/multiprecision/cpp_int.hpp>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/composite_key.hpp>
#include <boost/multi_index/mem_fun.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/sequenced_index.hpp>

#include <koinos/util/base58.hpp>
#include <koinos/util/conversion.hpp>
#include <koinos/util/hex.hpp>

namespace koinos::mempool {

namespace detail {

using namespace boost;
using int128_t = boost::multiprecision::int128_t;

struct pending_transaction_object
{
   crypto::multihash     id;
   protocol::transaction transaction;
   block_height_type     last_update;            //< Chain height at the time of submission
   account_type          payer;                  //< Payer at the time of submission
   uint64_t              rc_limit;               //< Max resources at the time of submission
   uint64_t              disk_storage_used;
   uint64_t              network_bandwidth_used;
   uint64_t              compute_bandwidth_used;
};

struct by_id;
struct by_height;

using pending_transaction_index = multi_index_container<
   pending_transaction_object,
   multi_index::indexed_by<
      multi_index::sequenced<>,
      multi_index::ordered_unique< multi_index::tag< by_id >,
         multi_index::member< pending_transaction_object, crypto::multihash, &pending_transaction_object::id >
      >,
      multi_index::ordered_non_unique< multi_index::tag< by_height >,
         multi_index::member< pending_transaction_object, block_height_type, &pending_transaction_object::last_update >
      >
   >
>;

struct account_resources_object
{
   account_type      account;
   uint64_t          resources;
   uint64_t          max_resources;
   block_height_type last_update;
};

struct by_account;

using account_resources_index = multi_index_container<
   account_resources_object,
   multi_index::indexed_by<
      multi_index::ordered_unique< multi_index::tag< by_account >,
         multi_index::member< account_resources_object, account_type, &account_resources_object::account >
      >
   >
>;

class mempool_impl final
{
private:
   account_resources_index          _account_resources_idx;
   mutable std::mutex               _account_resources_mutex;

   pending_transaction_index        _pending_transaction_idx;
   mutable std::mutex               _pending_transaction_mutex;

public:
   mempool_impl();
   virtual ~mempool_impl();

   bool has_pending_transaction( const crypto::multihash& id ) const;
   std::vector< rpc::mempool::pending_transaction > get_pending_transactions( std::size_t limit );
   bool check_pending_account_resources(
      const account_type& payer,
      uint64_t max_payer_resources,
      uint64_t trx_resource_limit ) const;
   void add_pending_transaction(
      const protocol::transaction& transaction,
      block_height_type height,
      const account_type& payer,
      uint64_t max_payer_rc,
      uint64_t rc_limit,
      uint64_t disk_storaged_used,
      uint64_t network_bandwidth_used,
      uint64_t compute_bandwidth_used );
   void remove_pending_transaction( const crypto::multihash& id );
   void prune( block_height_type h );
   std::size_t payer_entries_size() const;
   void cleanup_account_resources( const pending_transaction_object& pending_trx );

private:
   bool check_pending_account_resources_lockfree(
         const account_type& payer,
         uint64_t max_payer_rc,
         uint64_t rc_limit
      )const;
};

mempool_impl::mempool_impl() {}
mempool_impl::~mempool_impl() = default;

bool mempool_impl::has_pending_transaction( const crypto::multihash& id ) const
{
   std::lock_guard< std::mutex > guard( _pending_transaction_mutex );
   auto& id_idx = _pending_transaction_idx.get< by_id >();

   auto it = id_idx.find( id );

   return it != id_idx.end();
}

std::vector< rpc::mempool::pending_transaction > mempool_impl::get_pending_transactions( std::size_t limit )
{
   KOINOS_ASSERT( limit <= MAX_PENDING_TRANSACTION_REQUEST, pending_transaction_request_overflow, "Requested too many pending transactions. Max: ${max}", ("max", MAX_PENDING_TRANSACTION_REQUEST) );

   std::lock_guard< std::mutex > guard( _pending_transaction_mutex );

   std::vector< rpc::mempool::pending_transaction > pending_transactions;
   pending_transactions.reserve(limit);

   auto itr = _pending_transaction_idx.begin();

   while ( itr != _pending_transaction_idx.end() && pending_transactions.size() < limit )
   {
      rpc::mempool::pending_transaction ptx;
      *ptx.mutable_transaction() = itr->transaction;
      ptx.set_disk_storage_used( itr->disk_storage_used );
      ptx.set_network_bandwidth_used( itr->network_bandwidth_used );
      ptx.set_compute_bandwidth_used( itr->compute_bandwidth_used );

      pending_transactions.push_back( ptx );
      ++itr;
   }

   return pending_transactions;
}

bool mempool_impl::check_pending_account_resources_lockfree(
   const account_type& payer,
   uint64_t max_payer_resources,
   uint64_t trx_resource_limit ) const
{
   auto& account_idx = _account_resources_idx.get< by_account >();
   auto it = account_idx.find( payer );

   if ( it == account_idx.end() )
   {
      return trx_resource_limit <= max_payer_resources;
   }

   int128_t max_resource_delta = int128_t( max_payer_resources ) - int128_t( it->max_resources );
   int128_t new_resources = int128_t( it->resources ) + max_resource_delta - int128_t( trx_resource_limit );
   return new_resources >= 0;
}

bool mempool_impl::check_pending_account_resources(
   const account_type& payer,
   uint64_t max_payer_rc,
   uint64_t rc_limit ) const
{
   std::lock_guard< std::mutex > guard( _account_resources_mutex );
   return check_pending_account_resources_lockfree( payer, max_payer_rc, rc_limit );
}

void mempool_impl::add_pending_transaction(
   const protocol::transaction& transaction,
   block_height_type height,
   const account_type& payer,
   uint64_t max_payer_rc,
   uint64_t rc_limit,
   uint64_t disk_storaged_used,
   uint64_t network_bandwidth_used,
   uint64_t compute_bandwidth_used )
{
   {
      std::lock_guard< std::mutex > guard( _account_resources_mutex );

      KOINOS_ASSERT(
         check_pending_account_resources_lockfree( payer, max_payer_rc, rc_limit ),
         pending_transaction_exceeds_resources,
         "transaction would exceed maximum resources for account: ${a}", ("a", util::encode_base58( util::converter::as< std::vector< std::byte > >( payer )))
      );

      {
         auto id = util::converter::to< crypto::multihash >( transaction.id() );

         std::lock_guard< std::mutex > guard( _pending_transaction_mutex );

         auto rval = _pending_transaction_idx.emplace_back( pending_transaction_object {
            .id                     = id,
            .transaction            = transaction,
            .last_update            = height,
            .payer                  = payer,
            .rc_limit               = rc_limit,
            .disk_storage_used      = disk_storaged_used,
            .network_bandwidth_used = network_bandwidth_used,
            .compute_bandwidth_used = compute_bandwidth_used
         } );

         KOINOS_ASSERT( rval.second, pending_transaction_insertion_failure, "failed to insert transaction with id: ${id}", ("id", id) );
      }

      auto& account_idx = _account_resources_idx.get< by_account >();
      auto it = account_idx.find( payer );

      if ( it == account_idx.end() )
      {
         _account_resources_idx.insert( account_resources_object {
            .account       = payer,
            .resources     = max_payer_rc - rc_limit,
            .max_resources = max_payer_rc,
            .last_update   = height
         } );
      }
      else
      {
         int128_t max_resource_delta = int128_t( max_payer_rc ) - int128_t( it->max_resources );
         int128_t new_resources = int128_t( it->resources ) + max_resource_delta - int128_t( rc_limit );

         account_idx.modify( it, [&]( account_resources_object& aro )
         {
            aro.max_resources = max_payer_rc;
            aro.resources = new_resources.convert_to< uint64_t >();
            aro.last_update = height;
         } );
      }
   }

   LOG(info) << "Transaction added to mempool: " << util::to_hex( transaction.id() );
}

void mempool_impl::remove_pending_transaction( const crypto::multihash& id )
{
   std::lock_guard< std::mutex > account_guard( _account_resources_mutex );
   std::lock_guard< std::mutex > trx_guard( _pending_transaction_mutex );

   auto& id_idx = _pending_transaction_idx.get< by_id >();

   auto it = id_idx.find( id );
   if ( it != id_idx.end() )
   {
      cleanup_account_resources( *it );
      LOG(info) << "Removing included transaction from mempool: " << util::to_hex( it->transaction.id() );
      id_idx.erase( it );
   }
}

void mempool_impl::prune( block_height_type h )
{
   std::lock_guard< std::mutex > account_guard( _account_resources_mutex );
   std::lock_guard< std::mutex > trx_guard( _pending_transaction_mutex );

   auto& by_block_idx = _pending_transaction_idx.get< by_height >();
   auto itr = by_block_idx.begin();

   while( itr != by_block_idx.end() && itr->last_update <= h )
   {
      cleanup_account_resources( *itr );
      LOG(info) << "Pruning transaction from mempool: " << util::to_hex( itr->transaction.id() );
      itr = by_block_idx.erase( itr );
   }
}

std::size_t mempool_impl::payer_entries_size() const
{
   std::lock_guard< std::mutex > guard( _account_resources_mutex );
   return _account_resources_idx.size();
}

void mempool_impl::cleanup_account_resources( const pending_transaction_object& pending_trx )
{
   auto itr = _account_resources_idx.find( pending_trx.payer );
   if ( itr != _account_resources_idx.end() )
   {
      if ( itr->resources + pending_trx.rc_limit >= itr->max_resources )
      {
         _account_resources_idx.erase( itr );
      }
      else
      {
         _account_resources_idx.modify( itr, [&]( account_resources_object& aro )
         {
            aro.resources += pending_trx.rc_limit;
         } );
      }
   }
}

} // detail

mempool::mempool() : _my( std::make_unique< detail::mempool_impl >() ) {}
mempool::~mempool() = default;

bool mempool::has_pending_transaction( const crypto::multihash& id ) const
{
   return _my->has_pending_transaction( id );
}

std::vector< rpc::mempool::pending_transaction > mempool::get_pending_transactions( std::size_t limit )
{
   return _my->get_pending_transactions( limit );
}

bool mempool::check_pending_account_resources(
   const account_type& payer,
   uint64_t max_payer_resources,
   uint64_t trx_resource_limit ) const
{
   return _my->check_pending_account_resources( payer, max_payer_resources, trx_resource_limit );
}

void mempool::add_pending_transaction(
   const protocol::transaction& transaction,
   block_height_type height,
   const account_type& payer,
   uint64_t max_payer_rc,
   uint64_t rc_limit,
   uint64_t disk_storaged_used,
   uint64_t network_bandwidth_used,
   uint64_t compute_bandwidth_used )
{
   _my->add_pending_transaction( transaction, height, payer, max_payer_rc, rc_limit, disk_storaged_used, network_bandwidth_used, compute_bandwidth_used );
}

void mempool::remove_pending_transaction( const crypto::multihash& id )
{
   _my->remove_pending_transaction( id );
}

void mempool::prune( block_height_type h )
{
   _my->prune( h );
}

std::size_t mempool::payer_entries_size() const
{
   return _my->payer_entries_size();
}

} // koinos::mempool

max_stars_repo_name: koinos/koinos-mempool

max_issues_repo_name: koinos/koinos-mempool

max_forks_repo_name: koinos/koinos-mempool


text:
#include "base/SRC_FIRST.hpp"
#include "testing/testing.hpp"
#include "3party/lodepng/lodepng_io.hpp"
#include "platform/platform.hpp"

#include <boost/gil/gil_all.hpp>
#include <boost/mpl/vector_c.hpp>
#include "std/iostream.hpp"
#include "std/iomanip.hpp"

#include "graphics/texture.hpp"

namespace gil = boost::gil;
namespace mpl = boost::mpl;

typedef yg::gl::RGBA4Texture::pixel_t pixel_t;
typedef yg::gl::RGBA4Texture::view_t view_t;
typedef yg::gl::RGBA4Texture::image_t image_t;
//typedef yg::gl::Texture::iterator_t iterator_t;
//typedef yg::gl::Texture::locator_t locator_t;

UNIT_TEST(ABGR4_GIL_COPY)
{
  image_t image(4, 4);
  view_t view = gil::view(image);

  gil::fill_pixels(view, pixel_t(0x0, 0x0, 0x0, 0x0));

  for (size_t y = 0; y < 4; ++y)
  {
    for (size_t x = 0; x < 4; ++x)
    {
      unsigned short val = *((unsigned short *)&view(x, y));
      cout << hex << val << " ";
    }
    cout << endl;
  }

  pixel_t cl(0xF, 0, 0xF, 0);

  view(1, 1) = cl;
  view(1, 2) = cl;
  view(2, 1) = cl;
  view(2, 2) = cl;

  for (size_t y = 0; y < 4; ++y)
  {
    for (size_t x = 0; x < 4; ++x)
    {
      unsigned short val = *((unsigned short *)&view(x, y));

      cout << hex << val << " ";
    }
    cout << endl;
  }
}

UNIT_TEST(FormatABGR4_Layout)
{
  pixel_t px[2] = {pixel_t(0xF, 0xE, 0xD, 0xC), pixel_t(0xF, 0xE, 0xD, 0xC)};

  uint32_t raw = *(uint32_t *)(px);

  cout << "packed 2 pixels " << hex << raw << std::endl;

  TEST_EQUAL(raw, 0xCDEFCDEF, ());
}

UNIT_TEST(FormatRGBA4444_Layout)
{
  pixel_t px(15, 14, 13, 12);
  uint16_t * pxp = (uint16_t*)&px;
  uint16_t val = *pxp;

  cout << "shifting 1 << 4 = " << (1 << 4) << endl;
  cout << "shifting 1 >> 4 = " << (1 >> 4) << endl;

  cout << "value : " << hex << val
       << ", sh1 : " << hex << ((val) & 0xF)
       << ", sh2 : " << hex << ((val >> 4) & 0xF)
       << ", sh3 : " << hex << ((val >> 8) & 0xF)
       << ", sh4 : " << hex << ((val >> 12) & 0xF) << endl;

  TEST_EQUAL(gil::get_color(px, gil::alpha_t()), 15, ());
  TEST_EQUAL(gil::get_color(px, gil::blue_t()), 14, ());
  TEST_EQUAL(gil::get_color(px, gil::green_t()), 13, ());
  TEST_EQUAL(gil::get_color(px, gil::red_t()), 12, ());
}

UNIT_TEST(FormatRGBA4444_SaveLoad)
{
  image_t image;

//  gil::lodepng_read_image((GetPlatform().ResourcesDir() + "basic.png").c_str(), image);
}

max_stars_repo_name: bowlofstew/omim

max_issues_repo_name: bowlofstew/omim

max_forks_repo_name: bowlofstew/omim


text:
#include "core/core_module.h"
#include "core/file/file_subsystem.h"
#include "core/asset/asset_subsystem.h"
#include "lua_bind/lua_subsystem.h"

#include <boost/function.hpp>
#include <boost/bind.hpp>
#include "core/config/config.h"

using namespace luna;

#include <gtest/gtest.h>
#include "lua/lua.h"
#include <lua/lauxlib.h>
#include "lua/lualib.h"

int printMsg(lua_State* L)
{
	return 0;
}
class NativeClass
{
public:
	int Print(int i)
	{
		std::cout << i;
		return 0;
	}
};

template<typename TCls, typename TRet, typename ...ARGS>
lua_CFunction MemberFunctionCaller(TRet(TCls::* func)(ARGS...))
{
	typedef decltype(func) FTYPE; (void)(func);
	struct HelperClass
	{
		static int Invoke(lua_State* state)
		{
			void* calleePtr = lua_touserdata(state, lua_upvalueindex(1));
			luaL_argcheck(state, calleePtr, 1, "cpp closure function not found.");
			if (calleePtr)
			{
				volatile int idx = sizeof...(ARGS) + 1; (void)(idx);
				LuaStackReturn<TRet>(state, (LuaStack<TCls>::get(state, 1).* * (FTYPE*)(calleePtr))(LuaStack<ARGS>::get(state, idx--)...));
				return 1;
			}
			return 0;
		}
	};
	return HelperClass::Invoke;
}

lua_State* L;
int main(int argc, const char *argv[])
{
	L = luaL_newstate();
	luaL_openlibs(L); /* opens the basic library */
	// reg func
	lua_pushcfunction(L, printMsg);
	lua_setglobal(L, "my_printMsg");
	return 1;
}

max_stars_repo_name: LunarGameTeam/Lunar-GameEngine

max_issues_repo_name: LunarGameTeam/Lunar-GameEngine

max_forks_repo_name: LunarGameTeam/Lunar-GameEngine


text:
///
/// @file   DocumentManager.cpp
/// @brief  Manages properties and rawtexts
/// @date   2009-10-21 12:16:36
/// @author Deepesh Shrestha, Peiseng Wang,
///

#include "DocumentManager.h"
#include "DocContainer.h"
#include "highlighter/Highlighter.h"
#include "snippet-generation-submanager/SnippetGeneratorSubManager.h"
#include "text-summarization-submanager/TextSummarizationSubManager.h"

#include <configuration-manager/ConfigurationTool.h>
#include <util/profiler/ProfilerGroup.h>
#include <common/SFLogger.h>
#include <common/NumericPropertyTable.h>
#include <common/NumericRangePropertyTable.h>
#include <common/RTypeStringPropTable.h>
#include <la/analyzer/MultiLanguageAnalyzer.h>
#include <am/sequence_file/ssfr.h>

#include <langid/langid.h>

#include <fstream>
#include <boost/archive/xml_oarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/archive/archive_exception.hpp>
#include <boost/algorithm/string.hpp>

#include <protect/RestrictMacro.h>

namespace sf1r
{

const std::string DocumentManager::ACL_FILE = "ACLTable";
const std::string DocumentManager::PROPERTY_LENGTH_FILE = "PropertyLengthDb.xml";
const std::string DocumentManager::PROPERTY_BLOCK_SUFFIX = ".blocks";

namespace
{
const std::string DOCID("DOCID");
const std::string DATE("DATE");
}

DocumentManager::DocumentManager(
        const std::string& path,
        const IndexBundleSchema& indexSchema,
        const izenelib::util::UString::EncodingType encodingType,
        size_t documentCacheNum)
    : path_(path)
    , delfilter_count_(0)
    , documentCache_(100)
    , indexSchema_(indexSchema)
    , encodingType_(encodingType)
    , maxSnippetLength_(200)
{
    propertyValueTable_ = new DocContainer(path);
    propertyValueTable_->open();
    //buildPropertyIdMapper_();
    restorePropertyLengthDb_();
    loadDelFilter_();
    //aclTable_.open();
    snippetGenerator_ = new SnippetGeneratorSubManager;
    highlighter_ = new Highlighter;

    // Normal index
    for (IndexBundleSchema::const_iterator it = indexSchema_.begin();
            it != indexSchema_.end(); ++it)
    {
        if(it->isRTypeString())
        {
            initRTypeStringPropTable(it->getName());
        }
        else if (it->isRTypeNumeric())
        {
            initNumericPropertyTable_(it->getName(), it->getType(), it->getIsRange());
        }
    }
}

void DocumentManager::setZambeziConfig(const ZambeziConfig& zambeziConfig)
{
    zambeziConfig_ = zambeziConfig;

    for (IndexBundleSchema::const_iterator it = zambeziConfig_.zambeziIndexSchema.begin();
         it != zambeziConfig_.zambeziIndexSchema.end(); ++it)
    {
        if(it->isRTypeString())
        {
            initRTypeStringPropTable(it->getName());
        }
        else if (it->isRTypeNumeric())
        {
            initNumericPropertyTable_(it->getName(), it->getType(), it->getIsRange());
        }
    }

    buildPropertyIdMapper_();
}

DocumentManager::~DocumentManager()
{
    flush();

    if (propertyValueTable_) delete propertyValueTable_;
    if (snippetGenerator_) delete snippetGenerator_;
    if (highlighter_) delete highlighter_;
}

bool DocumentManager::flush()
{
    propertyValueTable_->flush();
    for (RTypeStringPropTableMap::iterator it = rtype_string_proptable_.begin();
            it != rtype_string_proptable_.end(); ++it)
    {
        it->second->flush();
    }

    for (NumericPropertyTableMap::iterator it = numericPropertyTables_.begin();
            it != numericPropertyTables_.end(); ++it)
    {
        it->second->flush();
    }
    saveDelFilter_();
    return savePropertyLengthDb_();
}

bool DocumentManager::insertDocument(const Document& document)
{

    typedef Document::property_const_iterator iterator;
    for (iterator it = document.propertyBegin(), itEnd = document.propertyEnd(); it
            != itEnd; ++it)
    {
        const propertyid_t* pid = propertyIdMapper_.findIdByValue(it->first);
        if (!pid)
        {
            continue;
        }

        const Document::doc_prop_value_strtype* stringValue =
            izenelib::get<Document::doc_prop_value_strtype>(&it->second);
        if (stringValue)
        {
            if (propertyLengthDb_.size() <= *pid)
            {
                boost::unique_lock<boost::shared_mutex> lock(shared_mutex_);
                if (propertyLengthDb_.size() <= *pid)
                    propertyLengthDb_.resize(*pid + 1);
            }
            boost::shared_lock<boost::shared_mutex> lock(shared_mutex_);
            propertyLengthDb_[*pid] += stringValue->length();
        }
    }

    if (!propertyValueTable_->insert(document.getId(), document))
    {
        return false;
    }

    return true;
}

bool DocumentManager::updateDocument(const Document& document)
{
    if (propertyValueTable_->update(document.getId(), document))
    {
        documentCache_.del(document.getId());
        return true;
    }

    return false;
}

bool DocumentManager::updatePartialDocument(const Document& document) // ok, right ...
{
    docid_t docId = document.getId();
    Document oldDoc;

    if (!getDocument(docId, oldDoc))
    {
        return false;
    }

    typedef Document::property_const_iterator iterator;

    for (iterator it = document.propertyBegin(), itEnd = document.propertyEnd(); it
            != itEnd; ++it)
    {
        const propertyid_t* pid = propertyIdMapper_.findIdByValue(it->first);
        if (!pid)
        {
            // not in config, skip
            continue;
        }

        if (! boost::iequals(it->first,DOCID) && ! boost::iequals(it->first,DATE))
        {
            oldDoc.updateProperty(it->first, it->second);
        }
    }

    return updateDocument(oldDoc);
}

bool DocumentManager::isDeleted(docid_t docId) const
{
    if (docId-- == 0) return false;

    size_t segment = docId / DELFILTER_SEGMENT_SIZE;
    if (delfilter_[segment].empty()) return false;
    size_t offset = docId % DELFILTER_SEGMENT_SIZE;

    return delfilter_[segment].test(offset);
}

bool DocumentManager::removeDocument(docid_t docId)
{
    if (docId-- == 0) return false;

    size_t segment = docId / DELFILTER_SEGMENT_SIZE;
    size_t offset = docId % DELFILTER_SEGMENT_SIZE;
    for (int i = segment; i >= 0 && delfilter_[i].empty(); --i)
    {
        delfilter_[i].resize(DELFILTER_SEGMENT_SIZE);
    }

    if (delfilter_[segment].test(offset))
        return false;

    delfilter_[segment].set(offset);
    ++delfilter_count_;
    documentCache_.del(docId + 1);

    return true;
}

std::size_t DocumentManager::getTotalPropertyLength(const std::string& property)
{
    boost::unordered_map<std::string, unsigned int>::const_iterator iter =
        propertyAliasMap_.find(property);

    boost::shared_lock<boost::shared_mutex> lock(shared_mutex_);
    if (iter != propertyAliasMap_.end() && iter->second < propertyLengthDb_.size())
        return propertyLengthDb_[ iter->second ];
    else
    {
        const unsigned int* id = propertyIdMapper_.findIdByValue(property);
        if (id != 0 && *id < propertyLengthDb_.size())
            return propertyLengthDb_[*id];
    }

    return 0;
}

bool DocumentManager::getPropertyValue(
        docid_t docId,
        const std::string& propertyName,
        PropertyValue& result)
{
    Document doc;
    const std::string* realPropertyName = &propertyName;

    //restore alias property name to original
    boost::unordered_map<std::string, unsigned int>::iterator aIter =
        propertyAliasMap_.find(propertyName);
    if (aIter != propertyAliasMap_.end())
    {
        realPropertyName = propertyIdMapper_.findValueById(aIter->second);
    }

    NumericPropertyTableMap::const_iterator it = numericPropertyTables_.find(*realPropertyName);
    if (it != numericPropertyTables_.end())
    {
        std::string tempStr;
        if (!it->second->getStringValue(docId, tempStr))
        {
//          return false;
        }
        result = str_to_propstr(tempStr, encodingType_);
        return true;
    }

    RTypeStringPropTableMap::const_iterator rtype_table_cit = rtype_string_proptable_.find(*realPropertyName);
    if( rtype_table_cit != rtype_string_proptable_.end() )
    {
        std::string tempStr;
        rtype_table_cit->second->getRTypeString(docId, tempStr);
        result = str_to_propstr(tempStr, encodingType_);
        return true;
    }

    if (documentCache_.getValue(docId, doc))
    {
        result = doc.property(*realPropertyName);
    }
    else
    {
        if (getDocument(docId, doc) == false)
            return false;
        documentCache_.insertValue(docId, doc);
        result = doc.property(*realPropertyName);
    }
//  if (getDocument(docId, doc))
//      result = doc.property(*realPropertyName);
    return true;
}

bool DocumentManager::getDocument(docid_t docId, Document& document, bool forceget)
{
    CREATE_SCOPED_PROFILER ( getDocument, "DocumentManager", "DocumentManager::getDocument");
    return (forceget || !isDeleted(docId)) && propertyValueTable_->get(docId, document);
}

void DocumentManager::getRTypePropertiesForDocument(docid_t docId, Document& document)
{
    for (NumericPropertyTableMap::const_iterator it = numericPropertyTables_.begin();
            it != numericPropertyTables_.end(); ++it)
    {
        std::string tempStr;
        if (it->second->getStringValue(docId, tempStr))
            document.property(it->first) = str_to_propstr(tempStr, encodingType_);
    }
    for(RTypeStringPropTableMap::const_iterator cit = rtype_string_proptable_.begin();
        cit != rtype_string_proptable_.end(); ++cit)
    {
        std::string tempStr;
        if(cit->second->getRTypeString(docId, tempStr))
            document.property(cit->first) = str_to_propstr(tempStr, encodingType_);
    }
}

bool DocumentManager::existDocument(docid_t docId)
{
    return propertyValueTable_->exist(docId);
}

bool DocumentManager::getDocumentByCache(
        docid_t docId,
        Document& document,
        bool forceget)
{
    if (documentCache_.getValue(docId, document))
    {
        return true;
    }
    if ((forceget || !isDeleted(docId)) && propertyValueTable_->get(docId, document))
    {
        documentCache_.insertValue(docId, document);
        return true;
    }
    return false;
}

bool DocumentManager::getDocuments(
        const std::vector<unsigned int>& ids,
        std::vector<Document>& docs,
        bool forceget)
{
    docs.resize(ids.size());
    bool ret = true;
    for (size_t i=0; i<ids.size(); i++)
    {
        ret &= getDocumentByCache(ids[i], docs[i], forceget);
    }
    return ret;
}

docid_t DocumentManager::getMaxDocId() const
{
    return propertyValueTable_->getMaxDocId();
}

uint32_t DocumentManager::getNumDocs()
{
    return getMaxDocId() - delfilter_count_;
}

bool DocumentManager::getDeletedDocIdList(std::vector<docid_t>& docid_list) const
{
    docid_list.clear();
    docid_list.reserve(delfilter_count_);

    for (size_t segment = 0; segment < 32 && !delfilter_[segment].empty(); ++segment)
    {
        docid_t base = segment * DELFILTER_SEGMENT_SIZE + 1;
        DelFilterType::size_type find = delfilter_[segment].find_first();
        while (find != DelFilterType::npos)
        {
            docid_list.push_back(base + (docid_t)find);
            find = delfilter_[segment].find_next(find);
        }
    }

    return true;
}

bool DocumentManager::loadDelFilter_()
{
    const std::string filter_file = (boost::filesystem::path(path_) / "del_filter").string();
    std::vector<DelFilterBlockType> filter_data;

    if (!izenelib::am::ssf::Util<>::Load(filter_file, filter_data))
        return false;

    size_t segsize = DELFILTER_SEGMENT_SIZE / sizeof(DelFilterBlockType) / 8;
    size_t segment = filter_data.size() / segsize;
    std::vector<DelFilterBlockType>::const_iterator it = filter_data.begin();
    for (size_t i = 0; i < segment; ++i, it += segsize)
    {
        delfilter_[i].clear();
        delfilter_[i].append(it, it + segsize);
    }

    return true;
}

bool DocumentManager::saveDelFilter_() const
{
    size_t segment = 0;
    while (segment < 32 && !delfilter_[segment].empty()) ++segment;

    size_t segsize = DELFILTER_SEGMENT_SIZE / sizeof(DelFilterBlockType) / 8;
    std::vector<DelFilterBlockType> filter_data(segment * segsize);

    std::vector<DelFilterBlockType>::iterator it = filter_data.begin();
    for (size_t i = 0; i < segment; ++i, it += segsize)
    {
        boost::to_block_range(delfilter_[i], it);
    }

    const std::string filter_file = (boost::filesystem::path(path_) / "del_filter").string();
    if (!izenelib::am::ssf::Util<>::Save(filter_file, filter_data))
    {
        std::cout << "DocumentManager::saveDelFilter_() failed" << std::endl;
        return false;
    }

    return true;
}

void DocumentManager::buildPropertyIdMapper_()
{
    config_tool::PROPERTY_ALIAS_MAP_T propertyAliasMap;
    config_tool::buildPropertyAliasMap(indexSchema_, propertyAliasMap); // for the second map ...

    for (IndexBundleSchema::const_iterator it = indexSchema_.begin(), itEnd = indexSchema_.end(); //
            it != itEnd; ++it)
    {
        propertyid_t originalPropertyId(0), originalBlockId(0);

        bool hasSummary(it->getIsSummary());
        bool hasSnippet(it->getIsSnippet());

        if (hasSnippet || hasSummary)
        {
            originalBlockId = propertyIdMapper_.insert(it->getName()
                              + PROPERTY_BLOCK_SUFFIX);

            boost::unordered_map<std::string, unsigned int>::iterator dispIter;
            dispIter = displayLengthMap_.find(it->getName());
            if (dispIter == displayLengthMap_.end())
            {
                unsigned int dispLength = it->getDisplayLength();
                if ((dispLength < 50) || (dispLength > 250)) //50, 250 as min and max DisplayLength
                    dispLength = maxSnippetLength_;
                displayLengthMap_.insert(std::make_pair(it->getName(),
                                                        dispLength));
            }
        }
        originalPropertyId = propertyIdMapper_.insert(it->getName()); // only this is ok;..

        // For alias property
        config_tool::PROPERTY_ALIAS_MAP_T::iterator aliasIter =
            propertyAliasMap.find(it->getName());
        if (aliasIter != propertyAliasMap.end())
        {

            for (std::vector<PropertyConfig>::iterator vecIter =
                    aliasIter->second.begin(); vecIter
                    != aliasIter->second.end(); vecIter++)
            {
                if (hasSnippet)
                {

                    propertyAliasMap_.insert(make_pair(vecIter->getName()
                                                       + PROPERTY_BLOCK_SUFFIX, originalBlockId));

                }
                propertyAliasMap_.insert(make_pair(vecIter->getName(),
                                                   originalPropertyId)) ;
            }
        }
    }

    // for zambezi index;
    for (IndexBundleSchema::const_iterator it = zambeziConfig_.zambeziIndexSchema.begin(),
        itEnd = zambeziConfig_.zambeziIndexSchema.end(); it != itEnd; ++it)
    {
        propertyIdMapper_.insert(it->getName());
    }

} // end - buildPropertyIdMapper_()

bool DocumentManager::savePropertyLengthDb_() const
{
    try
    {
        const std::string kDbPath = path_ + PROPERTY_LENGTH_FILE;
        std::ofstream ofs(kDbPath.c_str());
        if (ofs)
        {
            boost::archive::xml_oarchive oa(ofs);
            oa << boost::serialization::make_nvp("PropertyLength", propertyLengthDb_);
        }

        return ofs;
    }
    catch (boost::archive::archive_exception& e)
    {
        DLOG(ERROR)<<"Serialization Error while saving property length."<<e.what()<<endl;
        return false;
    }
}

bool DocumentManager::restorePropertyLengthDb_()
{
    try
    {
        const std::string kDbPath = path_ + PROPERTY_LENGTH_FILE;
        std::ifstream ifs(kDbPath.c_str());
        if (ifs)
        {
            boost::archive::xml_iarchive ia(ifs);
            ia >> boost::serialization::make_nvp("PropertyLength", propertyLengthDb_);
        }
        return ifs;
    }
    catch (boost::archive::archive_exception& e)
    {
        DLOG(ERROR)<<"Serialization Error while restoring property length."<<e.what()<<endl;
        propertyLengthDb_.clear();
        return false;
    }
}

bool DocumentManager::getRawTextOfDocuments(
        const std::vector<docid_t>& docIdList, const string& propertyName,
        const bool summaryOn, const unsigned int summaryNum,
        const unsigned int option,
        const std::vector<izenelib::util::UString>& queryTerms,
        std::vector<Document::doc_prop_value_strtype>& outSnippetList,
        std::vector<Document::doc_prop_value_strtype>& outRawSummaryList,
        std::vector<Document::doc_prop_value_strtype>& outFullTextList)
{
    try
    {
        unsigned int docListSize = docIdList.size();
        std::vector<Document::doc_prop_value_strtype> snippetList(docListSize);
        std::vector<Document::doc_prop_value_strtype> rawSummaryList(docListSize);
        std::vector<Document::doc_prop_value_strtype> fullTextList(docListSize);

        Document::doc_prop_value_strtype rawText; // raw text
        izenelib::util::UString rawUText; // raw text
        Document::doc_prop_value_strtype result; // output variable to store return value
        izenelib::util::UString resultU;

        bool ret = false;
        for (unsigned int listId = 0; listId != docListSize; ++listId)
        {
            docid_t docId = docIdList[listId];
            result.clear();
            resultU.clear();

            if (!getPropertyValue(docId, propertyName, rawText))
                continue;

            rawUText = propstr_to_ustr(rawText, encodingType_);
            fullTextList[listId] = rawText;

            std::string sentenceProperty = propertyName + PROPERTY_BLOCK_SUFFIX;
            std::vector<CharacterOffset> sentenceOffsets;
            if (!getPropertyValue(docId, sentenceProperty, sentenceOffsets))
                continue;

            ret = true;

            maxSnippetLength_ = getDisplayLength_(propertyName);
            processOptionForRawText(option, queryTerms, rawUText,
                                    sentenceOffsets, resultU);

            result = ustr_to_propstr(resultU);
            if (result.size() > 0)
                snippetList[listId] = result;
            else
                snippetList[listId] = rawText;

            //process only if summary is ON
            unsigned int numSentences = summaryNum <= 0 ? 1 : summaryNum;
            if (summaryOn)
            {
                izenelib::util::UString summary;
                getSummary(rawUText, sentenceOffsets, numSentences,
                           option, queryTerms, summary);

                rawSummaryList[listId] = ustr_to_propstr(summary);
            }
        }
        outSnippetList.swap(snippetList);
        outFullTextList.swap(fullTextList);
        if (summaryOn)
            outRawSummaryList.swap(rawSummaryList);

        return ret;
    }
    catch (std::exception& e)
    {
        return false;
    }
}

bool DocumentManager::getRawTextOfOneDocument(
        const docid_t docId,
        Document& document,
        const string& propertyName,
        const unsigned int option,
        const std::vector<izenelib::util::UString>& queryTerms,
        Document::doc_prop_value_strtype& outSnippet,
        Document::doc_prop_value_strtype& rawText)
{
    rawText.clear();
    NumericPropertyTableMap::const_iterator it = numericPropertyTables_.find(propertyName);
    RTypeStringPropTableMap::const_iterator rtype_table_cit = rtype_string_proptable_.find(propertyName);
    if (it != numericPropertyTables_.end())
    {
        std::string tempStr;
        if (!it->second->getStringValue(docId, tempStr))
        {
//          return false;
        }
        rawText = str_to_propstr(tempStr, encodingType_);
    }
    else if( rtype_table_cit != rtype_string_proptable_.end() )
    {
        std::string tempStr;
        rtype_table_cit->second->getRTypeString(docId, tempStr);
        rawText = str_to_propstr(tempStr, encodingType_);
    }
    else
    {
        Document::doc_prop_value_strtype propValue;
        if(document.getProperty(propertyName,propValue))
            rawText = propValue;
    }

    if (rawText.empty())
    {
        //DLOG(ERROR)<<"No RawText For This Property. Property Name "<<propertyName<<endl;
        return true;
    }

    std::vector<CharacterOffset> sentenceOffsets;
    if (option > 1)
    {
        if (!getPropertyValue(docId, propertyName + PROPERTY_BLOCK_SUFFIX,
                              sentenceOffsets))
        {
            outSnippet = rawText;
            return false;
        }
    }

    maxSnippetLength_ = getDisplayLength_(propertyName);
    izenelib::util::UString tempustr;
    processOptionForRawText(option, queryTerms, propstr_to_ustr(rawText), sentenceOffsets,
                            tempustr);

    outSnippet = ustr_to_propstr(tempustr);
    //put raw text to outSnippet if it is empty
    if (outSnippet.size() <= 0)
        outSnippet = rawText;

    return true;

}

bool DocumentManager::processOptionForRawText(
        const unsigned int option,
        const std::vector<izenelib::util::UString>& queryTerms,
        const izenelib::util::UString& rawText,
        const std::vector<CharacterOffset>& sentenceOffsets,
        izenelib::util::UString& result)
{
    switch (option)
    {
    case X_RAWTEXT:
        result = rawText;
        break;

    case O_RAWTEXT:
        highlighter_->getHighlightedText(rawText, queryTerms, encodingType_, result);
        break;

    case X_SNIPPET:
        snippetGenerator_->getSnippet(rawText, sentenceOffsets, queryTerms,
                                      maxSnippetLength_, false, encodingType_, result);
        break;

    case O_SNIPPET:
        snippetGenerator_->getSnippet(rawText, sentenceOffsets, queryTerms,
                                      maxSnippetLength_, true, encodingType_, result);
        break;
    }
    return true;
}

bool DocumentManager::getSummary(
        const izenelib::util::UString& rawText,
        const std::vector<CharacterOffset>& sentenceOffsets,
        unsigned int numSentences,
        const unsigned int option,
        const std::vector<izenelib::util::UString>& queryTerms,
        izenelib::util::UString& summary)
{
    uint32_t offsetIndex = 1;
    uint32_t summaryCount = 0;
    uint32_t indexedSummaryCount = 0;

    //if stored summaries is less than the requested summary number
    if (numSentences > sentenceOffsets[indexedSummaryCount])
        numSentences = sentenceOffsets[indexedSummaryCount];

    while (offsetIndex < sentenceOffsets[indexedSummaryCount] * 2 && summaryCount < numSentences)
    {
        uint32_t length = sentenceOffsets[offsetIndex + 1]
                          - sentenceOffsets[offsetIndex];
        izenelib::util::UString result;
        izenelib::util::UString sentence;
        rawText.substr(sentence, sentenceOffsets[offsetIndex], length);
        offsetIndex += 2;
        switch (option)
        {
        case X_RAWTEXT: //don't highlight
        case X_SNIPPET:
            result = sentence;
            break;
        case O_RAWTEXT: //Do highlight
        case O_SNIPPET:
            if (highlighter_->getHighlightedText(sentence, queryTerms,
                                                encodingType_, result) == false)
            {
                result = sentence;
            }
            break;
        }
        if (result.length()> 0)
            summary += result;

        ++summaryCount;
    }
    return true;
}

unsigned int DocumentManager::getDisplayLength_(const string& propertyName)
{
    boost::unordered_map< std::string, unsigned int>::iterator dispIter =
        displayLengthMap_.find(propertyName);
    if (dispIter != displayLengthMap_.end())
        return dispIter->second;

    return 0;
}

bool DocumentManager::highlightText(
        const izenelib::util::UString& text,
        const std::vector<izenelib::util::UString> queryTerms,
        izenelib::util::UString& outText)
{
    izenelib::util::UString highlightedText;
    highlighter_->getHighlightedText(text, queryTerms, encodingType_,
                                    highlightedText);
    outText.swap(highlightedText);
    return true;
}

boost::shared_ptr<NumericPropertyTableBase>& DocumentManager::getNumericPropertyTable(const std::string& propertyName)
{
    static boost::shared_ptr<NumericPropertyTableBase> emptyNumericPropertyTable;
    NumericPropertyTableMap::iterator it = numericPropertyTables_.find(propertyName);
    if (it != numericPropertyTables_.end())
        return it->second;
    else
        return emptyNumericPropertyTable;
}

void DocumentManager::initNumericPropertyTable_(
        const std::string& propertyName,
        PropertyDataType propertyType,
        bool isRange)
{
    if (numericPropertyTables_.find(propertyName) != numericPropertyTables_.end())
        return;

    boost::shared_ptr<NumericPropertyTableBase>& numericPropertyTable = numericPropertyTables_[propertyName];
    switch (propertyType)
    {
    case INT8_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<int8_t>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<int8_t>(propertyType));
        break;

    case INT16_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<int16_t>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<int16_t>(propertyType));
        break;

    case INT32_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<int32_t>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<int32_t>(propertyType));
        break;

    case INT64_PROPERTY_TYPE:
    case DATETIME_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<int64_t>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<int64_t>(propertyType));
        break;

    case FLOAT_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<float>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<float>(propertyType));
        break;

    case DOUBLE_PROPERTY_TYPE:
        if (isRange)
            numericPropertyTable.reset(new NumericRangePropertyTable<double>(propertyType));
        else
            numericPropertyTable.reset(new NumericPropertyTable<double>(propertyType));
        break;

    default:
        numericPropertyTables_.erase(propertyName);
        return;
    }
    numericPropertyTable->init(path_ + propertyName + ".rtype_data");
}

boost::shared_ptr<RTypeStringPropTable>& DocumentManager::getRTypeStringPropTable(const std::string& propertyName)
{
    static boost::shared_ptr<RTypeStringPropTable> emptyPropertyTable;
    RTypeStringPropTableMap::iterator it = rtype_string_proptable_.find(propertyName);
    if (it != rtype_string_proptable_.end())
        return it->second;
    else
        return emptyPropertyTable;
}

void DocumentManager::initRTypeStringPropTable(
        const std::string& propertyName)
{
    if (rtype_string_proptable_.find(propertyName) != rtype_string_proptable_.end())
        return;

    boost::shared_ptr<RTypeStringPropTable>& rtype_string_prop = rtype_string_proptable_[propertyName];
    rtype_string_prop.reset(new RTypeStringPropTable(STRING_PROPERTY_TYPE));
    rtype_string_prop->init(path_ + propertyName + ".rtypestring_data");
}

void DocumentManager::moveRTypeValues(docid_t oldId, docid_t newId)
{
    for (NumericPropertyTableMap::iterator it = numericPropertyTables_.begin();
            it != numericPropertyTables_.end(); ++it)
    {
        it->second->copyValue(oldId, newId);
    }
    for(RTypeStringPropTableMap::iterator rtype_it = rtype_string_proptable_.begin();
        rtype_it != rtype_string_proptable_.end(); ++rtype_it)
    {
        rtype_it->second->copyValue(oldId, newId);
    }
}

// this is used for Rebuild collection ...

void DocumentManager::copyRTypeValues(
    boost::shared_ptr<DocumentManager>& source,
    docid_t from, docid_t to)
{
    std::set<std::string> doneProperty;
    for (IndexBundleSchema::const_iterator it = indexSchema_.begin();
            it != indexSchema_.end(); ++it)
    {
        if(it->isRTypeString())
        {
            doneProperty.insert(it->getName());
            std::string fieldValue;
            boost::shared_ptr<RTypeStringPropTable> sourceTable = source->getRTypeStringPropTable(it->getName());
            if(!sourceTable) continue;
            boost::shared_ptr<RTypeStringPropTable> rtypeStringPropertyTable = rtype_string_proptable_[it->getName()];
            bool ret = sourceTable->getRTypeString(from, fieldValue);
            if(ret) rtypeStringPropertyTable->updateRTypeString(to, fieldValue);
        }
        else if (it->isRTypeNumeric())
        {
            doneProperty.insert(it->getName());
            boost::shared_ptr<NumericPropertyTableBase> sourceTable = source->getNumericPropertyTable(it->getName());
            if(!sourceTable) continue;
            boost::shared_ptr<NumericPropertyTableBase> numericPropertyTable = numericPropertyTables_[it->getName()];
            if( (it->getType() == DATETIME_PROPERTY_TYPE) &&
                (it->getIsFilter() && !it->getIsMultiValue()) )
            {
                time_t fieldValue;
                bool ret = sourceTable->getInt64Value(from, fieldValue);
                if(ret) numericPropertyTable->setInt64Value(to, fieldValue);
            }
            else
            {
                std::string fieldValue;
                bool ret = sourceTable->getStringValue(from, fieldValue);
                if(ret) numericPropertyTable->setStringValue(to, fieldValue);
            }
        }
    }

    // for zambezi index
    for (IndexBundleSchema::const_iterator it = zambeziConfig_.zambeziIndexSchema.begin();
            it != zambeziConfig_.zambeziIndexSchema.end(); ++it)
    {
        if (it->isRTypeNumeric())
        {
            if (!doneProperty.insert(it->getName()).second)
                continue;

            boost::shared_ptr<NumericPropertyTableBase> sourceTable = source->getNumericPropertyTable(it->getName());
            if(!sourceTable) continue;
            boost::shared_ptr<NumericPropertyTableBase> numericPropertyTable = numericPropertyTables_[it->getName()];
            if( (it->getType() == DATETIME_PROPERTY_TYPE) &&
                (it->getIsFilter() && !it->getIsMultiValue()) )
            {
                time_t fieldValue;
                bool ret = sourceTable->getInt64Value(from, fieldValue);
                if(ret) numericPropertyTable->setInt64Value(to, fieldValue);
            }
            else
            {
                std::string fieldValue;
                bool ret = sourceTable->getStringValue(from, fieldValue);
                if(ret) numericPropertyTable->setStringValue(to, fieldValue);
            }
        }
    }
}

DocumentManager::NumericPropertyTableMap& DocumentManager::getNumericPropertyTableMap()
{
    return numericPropertyTables_;
}

DocumentManager::RTypeStringPropTableMap& DocumentManager::getRTypeStringPropTableMap()
{
    return rtype_string_proptable_;
}

}


text:
#include <boost/bind/bind.hpp>

int
main ()
{
  return 0;
}


text:
/*
	This file is part of solidity.

	solidity is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	solidity is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <test/yulPhaser/TestHelpers.h>

#include <tools/yulPhaser/FitnessMetrics.h>
#include <tools/yulPhaser/GeneticAlgorithms.h>
#include <tools/yulPhaser/Population.h>

#include <libsolutil/CommonIO.h>

#include <boost/test/unit_test.hpp>

#include <algorithm>
#include <vector>

using namespace std;
using namespace boost::unit_test::framework;
using namespace boost::test_tools;

namespace solidity::phaser::test
{

class GeneticAlgorithmFixture
{
protected:
	shared_ptr<FitnessMetric> m_fitnessMetric = make_shared<ChromosomeLengthMetric>();
};

BOOST_AUTO_TEST_SUITE(Phaser)
BOOST_AUTO_TEST_SUITE(GeneticAlgorithmsTest)
BOOST_AUTO_TEST_SUITE(RandomAlgorithmTest)

BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite_and_randomise_rest_of_population, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert((chromosomeLengths(population) == vector<size_t>{3, 3, 3, 3, 5, 5, 5, 5}));
	RandomAlgorithm algorithm({0.5, 1, 1});

	Population newPopulation = algorithm.runNextRound(population);
	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{1, 1, 1, 1, 3, 3, 3, 3}));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_elite_with_worse_individuals, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert((chromosomeLengths(population) == vector<size_t>{3, 3, 3, 3, 5, 5, 5, 5}));
	RandomAlgorithm algorithm({0.5, 7, 7});

	Population newPopulation = algorithm.runNextRound(population);
	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{3, 3, 3, 3, 7, 7, 7, 7}));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_replace_all_chromosomes_if_zero_size_elite, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert((chromosomeLengths(population) == vector<size_t>{3, 3, 3, 3, 5, 5, 5, 5}));
	RandomAlgorithm algorithm({0.0, 1, 1});

	Population newPopulation = algorithm.runNextRound(population);
	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{1, 1, 1, 1, 1, 1, 1, 1}));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_any_chromosomes_if_whole_population_is_the_elite, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 4, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert((chromosomeLengths(population) == vector<size_t>{3, 3, 3, 3, 5, 5, 5, 5}));
	RandomAlgorithm algorithm({1.0, 1, 1});

	Population newPopulation = algorithm.runNextRound(population);
	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{3, 3, 3, 3, 5, 5, 5, 5}));
}

BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE(GenerationalElitistWithExclusivePoolsTest)

BOOST_FIXTURE_TEST_CASE(runNextRound_should_preserve_elite_and_regenerate_rest_of_population, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 6, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert((chromosomeLengths(population) == vector<size_t>{3, 3, 3, 3, 3, 3, 5, 5, 5, 5}));

	GenerationalElitistWithExclusivePools::Options options = {
		/* mutationPoolSize = */ 0.2,
		/* crossoverPoolSize = */ 0.2,
		/* randomisationChance = */ 0.0,
		/* deletionVsAdditionChance = */ 1.0,
		/* percentGenesToRandomise = */ 0.0,
		/* percentGenesToAddOrDelete = */ 1.0,
	};
	GenerationalElitistWithExclusivePools algorithm(options);

	Population newPopulation = algorithm.runNextRound(population);

	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{0, 0, 3, 3, 3, 3, 3, 3, 3, 3}));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_not_replace_elite_with_worse_individuals, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 6, 3, 3) + Population::makeRandom(m_fitnessMetric, 4, 5, 5);
	assert(chromosomeLengths(population) == (vector<size_t>{3, 3, 3, 3, 3, 3, 5, 5, 5, 5}));

	GenerationalElitistWithExclusivePools::Options options = {
		/* mutationPoolSize = */ 0.2,
		/* crossoverPoolSize = */ 0.2,
		/* randomisationChance = */ 0.0,
		/* deletionVsAdditionChance = */ 0.0,
		/* percentGenesToRandomise = */ 0.0,
		/* percentGenesToAddOrDelete = */ 1.0,
	};
	GenerationalElitistWithExclusivePools algorithm(options);

	Population newPopulation = algorithm.runNextRound(population);

	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{3, 3, 3, 3, 3, 3, 3, 3, 7, 7}));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_generate_individuals_in_the_crossover_pool_by_mutating_the_elite, GeneticAlgorithmFixture)
{
	auto population = Population::makeRandom(m_fitnessMetric, 20, 5, 5);

	GenerationalElitistWithExclusivePools::Options options = {
		/* mutationPoolSize = */ 0.8,
		/* crossoverPoolSize = */ 0.0,
		/* randomisationChance = */ 0.5,
		/* deletionVsAdditionChance = */ 0.5,
		/* percentGenesToRandomise = */ 1.0,
		/* percentGenesToAddOrDelete = */ 1.0,
	};
	GenerationalElitistWithExclusivePools algorithm(options);

	SimulationRNG::reset(1);
	Population newPopulation = algorithm.runNextRound(population);

	BOOST_TEST((
		chromosomeLengths(newPopulation) ==
		vector<size_t>{0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 11, 11, 11}
	));
}

BOOST_FIXTURE_TEST_CASE(runNextRound_should_generate_individuals_in_the_crossover_pool_by_crossing_over_the_elite, GeneticAlgorithmFixture)
{
	auto population = (
		Population(m_fitnessMetric, {Chromosome("aa"), Chromosome("ff")}) +
		Population::makeRandom(m_fitnessMetric, 8, 6, 6)
	);
	assert((chromosomeLengths(population) == vector<size_t>{2, 2, 6, 6, 6, 6, 6, 6, 6, 6}));

	GenerationalElitistWithExclusivePools::Options options = {
		/* mutationPoolSize = */ 0.0,
		/* crossoverPoolSize = */ 0.8,
		/* randomisationChance = */ 0.0,
		/* deletionVsAdditionChance = */ 0.0,
		/* percentGenesToRandomise = */ 0.0,
		/* percentGenesToAddOrDelete = */ 0.0,
	};
	GenerationalElitistWithExclusivePools algorithm(options);

	SimulationRNG::reset(1);
	Population newPopulation = algorithm.runNextRound(population);

	vector<Individual> const& newIndividuals = newPopulation.individuals();
	BOOST_TEST((chromosomeLengths(newPopulation) == vector<size_t>{2, 2, 2, 2, 2, 2, 2, 2, 2, 2}));
	for (auto& individual: newIndividuals)
		BOOST_TEST((
			individual.chromosome == Chromosome("aa") ||
			individual.chromosome == Chromosome("af") ||
			individual.chromosome == Chromosome("fa") ||
			individual.chromosome == Chromosome("ff")
		));
	BOOST_TEST(any_of(newIndividuals.begin() + 2, newIndividuals.end(), [](auto& individual){
		return individual.chromosome != Chromosome("aa") && individual.chromosome != Chromosome("ff");
	}));
}

BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()

}

max_stars_repo_name: MrChico/solidity

max_issues_repo_name: MrChico/solidity

max_forks_repo_name: MrChico/solidity


text:
/*
 * @CopyRight:
 * FISCO-BCOS is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * FISCO-BCOS is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with FISCO-BCOS.  If not, see <http://www.gnu.org/licenses/>
 * (c) 2016-2019 fisco-dev contributors.
 */
/** @file SQLBasicAccess.cpp
 *  @author darrenyin
 *  @date 2019-04-24
 */

#include "SQLBasicAccess.h"
#include "SQLConnectionPool.h"
#include "StorageException.h"
#include "libconfig/GlobalConfigure.h"
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/dynamic_bitset.hpp>

using namespace dev::storage;
using namespace std;

SQLBasicAccess::SQLBasicAccess()
{
    if (g_BCOSConfig.version() >= V2_6_0)
    {
        // supported_version >= v2.6.0, enable compress
        m_rowFormat = " ROW_FORMAT=COMPRESSED ";
    }
}

int SQLBasicAccess::Select(int64_t, const string& _table, const string&, Condition::Ptr _condition,
    vector<map<string, string>>& _values)
{
    string sql = this->BuildQuerySql(_table, _condition);
    Connection_T conn = m_connPool->GetConnection();
    uint32_t retryCnt = 0;
    uint32_t retryMax = 10;
    while (conn == NULL && retryCnt++ < retryMax)
    {
        SQLBasicAccess_LOG(WARNING)
            << "table:" << _table << "sql:" << sql << " get connection failed";
        sleep(1);
        conn = m_connPool->GetConnection();
    }

    if (conn == NULL)
    {
        SQLBasicAccess_LOG(ERROR) << "table:" << _table << "sql:" << sql
                                  << " get connection failed";
        return -1;
    }
    TRY
    {
        PreparedStatement_T _prepareStatement =
            Connection_prepareStatement(conn, "%s", sql.c_str());
        if (_condition)
        {
            uint32_t index = 0;
            for (auto& it : *(_condition))
            {
                PreparedStatement_setString(
                    _prepareStatement, ++index, it.second.right.second.c_str());
            }
        }
        ResultSet_T result = PreparedStatement_executeQuery(_prepareStatement);
        int32_t columnCnt = ResultSet_getColumnCount(result);

        bool tableWithBlobField = isBlobType(_table);
        while (ResultSet_next(result))
        {
            map<string, string> value;
            for (int32_t index = 1; index <= columnCnt; ++index)
            {
                auto fieldName = ResultSet_getColumnName(result, index);
                if (tableWithBlobField)
                {
                    int size;
                    auto bytes = ResultSet_getBlob(result, index, &size);
                    // Note: Since the field may not be set, it must be added here to determine
                    //       whether the value of the obtained field is nullptr
                    if (bytes)
                    {
                        value[fieldName] = string((char*)bytes, size);
                    }
                }
                else
                {
                    auto selectResult = ResultSet_getString(result, index);
                    if (selectResult)
                    {
                        value[fieldName] = selectResult;
                    }
                }
            }
            _values.push_back(move(value));
        }
    }
    CATCH(SQLException)
    {
        m_connPool->ReturnConnection(conn);
        // Note: when select exception caused by table doesn't exist and sql error,
        //       here must return 0, in case of the DB is normal but the sql syntax error or the
        //       table will be created later
        SQLBasicAccess_LOG(ERROR) << "select exception for sql error:" << Exception_frame.message;
        return 0;
    }
    END_TRY;
    m_connPool->ReturnConnection(conn);
    return 0;
}

string SQLBasicAccess::BuildQuerySql(string _table, Condition::Ptr _condition)
{
    _table = boost::algorithm::replace_all_copy(_table, "\\", "\\\\");
    _table = boost::algorithm::replace_all_copy(_table, "`", "\\`");
    string sql = "select * from `";
    sql.append(_table).append("`");
    if (_condition)
    {
        bool hasWhereClause = false;
        for (auto it = _condition->begin(); it != _condition->end(); ++it)
        {
            if (!hasWhereClause)
            {
                hasWhereClause = true;
                sql.append(BuildConditionSql(" where", it, _condition));
            }
            else
            {
                sql.append(BuildConditionSql(" and", it, _condition));
            }
        }
    }
    return sql;
}
string SQLBasicAccess::BuildConditionSql(const string& _strPrefix,
    map<string, Condition::Range>::const_iterator& _it, Condition::Ptr _condition)
{
    string strConditionSql = _strPrefix;
    if (_it->second.left.second == _it->second.right.second && _it->second.left.first &&
        _it->second.right.first)
    {
        strConditionSql.append(" `").append(_it->first).append("`=").append("?");
    }
    else
    {
        if (_it->second.left.second != _condition->unlimitedField())
        {
            if (_it->second.left.first)
            {
                strConditionSql.append(" `").append(_it->first).append("`>=").append("?");
            }
            else
            {
                strConditionSql.append(" `").append(_it->first).append("`>").append("?");
            }
        }
        if (_it->second.right.second != _condition->unlimitedField())
        {
            if (_it->second.right.first)
            {
                strConditionSql.append(" `").append(_it->first).append("`<=").append("?");
            }
            else
            {
                strConditionSql.append(" `").append(_it->first).append("`<").append("?");
            }
        }
    }
    return strConditionSql;
}

SQLFieldType SQLBasicAccess::getFieldType(std::string const& _tableName)
{
    // _sys_hash_2_block_ or _sys_block_2_nonce, the SYS_VALUE field is blob
    if (_tableName == SYS_HASH_2_BLOCK || _tableName == SYS_BLOCK_2_NONCES || _tableName == SYS_CNS)
    {
        if (g_BCOSConfig.version() >= V2_6_0)
        {
            return SQLFieldType::LongBlobType;
        }
        return SQLFieldType::LongStringType;
    }
    // _sys_hash_2_blockheader_, the SYS_VALUE and SYS_SIG_LIST are blob
    if (_tableName == SYS_HASH_2_BLOCKHEADER)
    {
        return SQLFieldType::LongBlobType;
    }
    // contract table, all the fields are blob type
    if (boost::starts_with(_tableName, string("c_")))
    {
        if (g_BCOSConfig.version() >= V2_5_0)
        {
            return SQLFieldType::MediumBlobType;
        }
    }
    // supported_version >= v2.6.0, all the field type of user created table is mediumblob
    if (g_BCOSConfig.version() >= V2_6_0)
    {
        return SQLFieldType::MediumBlobType;
    }
    // supported_version < v2.6.0, the user created table is mediumtext
    return SQLFieldType::MediumStringType;
}

string SQLBasicAccess::BuildCreateTableSql(
    const string& _tableName, const string& _keyField, const string& _valueField)
{
    stringstream ss;
    ss << "CREATE TABLE IF NOT EXISTS `" << _tableName << "`(\n";
    ss << " `_id_` BIGINT unsigned auto_increment,\n";
    if (g_BCOSConfig.version() <= V2_1_0)
    {
        ss << " `_hash_` varchar(128) DEFAULT NULL,\n";
    }
    ss << " `_num_` BIGINT not null,\n";
    ss << " `_status_` int not null,\n";
    ss << "`" << _keyField << "` varchar(255) default '',\n";

    SQLBasicAccess_LOG(DEBUG) << "valuefield:" << _valueField;
    vector<string> vecSplit;
    boost::split(vecSplit, _valueField, boost::is_any_of(","));
    auto it = vecSplit.begin();

    auto fieldType = getFieldType(_tableName);
    std::string fieldTypeName = SQLFieldTypeName[fieldType];
    for (; it != vecSplit.end(); ++it)
    {
        *it = boost::algorithm::replace_all_copy(*it, "\\", "\\\\");
        *it = boost::algorithm::replace_all_copy(*it, "`", "\\`");
        ss << "`" << *it << "` " << fieldTypeName << ",\n";
    }
    ss << " PRIMARY KEY( `_id_` ),\n";
    ss << " KEY(`" << _keyField << "`(191)),\n";
    ss << " KEY(`_num_`)\n";
    // supported_version >= v2.6.0, enable compress
    ss << ") " << m_rowFormat << " ENGINE=InnoDB default charset=utf8mb4;";
    return ss.str();
}

string SQLBasicAccess::BuildCreateTableSql(const Entry::Ptr& _entry)
{
    string tableName(_entry->getField("table_name"));
    string keyField(_entry->getField("key_field"));
    string valueField(_entry->getField("value_field"));

    tableName = boost::algorithm::replace_all_copy(tableName, "\\", "\\\\");
    tableName = boost::algorithm::replace_all_copy(tableName, "`", "\\`");

    keyField = boost::algorithm::replace_all_copy(keyField, "\\", "\\\\");
    keyField = boost::algorithm::replace_all_copy(keyField, "`", "\\`");

    string sql = BuildCreateTableSql(tableName, keyField, valueField);
    SQLBasicAccess_LOG(DEBUG) << "create table:" << tableName << " keyfield:" << keyField
                              << " value field:" << valueField << " sql:" << sql;
    return sql;
}

void SQLBasicAccess::GetCommitFieldNameAndValueEachTable(const string& _num,
    const Entries::Ptr& _data, const vector<size_t>& _indexList, string& _fieldStr,
    vector<string>& _valueList)
{
    bool isFirst = true;
    for (auto index : _indexList)
    {
        const auto& entry = _data->get(index);
        for (auto fieldIt : *entry)
        {
            if (fieldIt.first == NUM_FIELD || fieldIt.first == "_hash_" ||
                fieldIt.first == ID_FIELD || fieldIt.first == STATUS)
            {
                continue;
            }
            if (isFirst)
            {
                _fieldStr.append("`").append(fieldIt.first).append("`,");
            }
            _valueList.push_back(fieldIt.second);
        }

        if (g_BCOSConfig.version() <= V2_1_0)
        {
            _valueList.push_back("0");
        }

        _valueList.push_back(_num);
        _valueList.push_back(boost::lexical_cast<string>(entry->getID()));
        _valueList.push_back(boost::lexical_cast<string>(entry->getStatus()));
        isFirst = false;
    }

    if (!_fieldStr.empty())
    {
        if (g_BCOSConfig.version() <= V2_1_0)
        {
            _fieldStr.append("`_hash_`,");
        }
        _fieldStr.append("`").append(NUM_FIELD).append("`,");
        _fieldStr.append("`").append(ID_FIELD).append("`,");
        _fieldStr.append("`").append(STATUS).append("`");
    }
}

void SQLBasicAccess::GetCommitFieldNameAndValue(
    const Entries::Ptr& _data, const string& _num, map<string, vector<string>>& _field2Values)
{
    set<string> keySet;
    for (size_t i = 0; i < _data->size(); ++i)
    {
        const auto& entry = _data->get(i);
        for (auto fieldIt : *entry)
        {
            if (fieldIt.first == NUM_FIELD || fieldIt.first == "_hash_" ||
                fieldIt.first == ID_FIELD || fieldIt.first == STATUS)
            {
                continue;
            }
            keySet.insert(fieldIt.first);
        }
    }

    map<string, vector<size_t>> groupedEntries;
    map<string, uint32_t> field2Position;
    uint32_t currentFieldPostion = 0;
    for (size_t i = 0; i < _data->size(); ++i)
    {
        boost::dynamic_bitset<> keyVec(keySet.size(), 0);
        const auto& entry = _data->get(i);
        for (const auto& fieldIt : *entry)
        {
            if (fieldIt.first == NUM_FIELD || fieldIt.first == "_hash_" ||
                fieldIt.first == ID_FIELD || fieldIt.first == STATUS)
            {
                continue;
            }

            if (field2Position.find(fieldIt.first) == field2Position.end())
            {
                field2Position[fieldIt.first] = currentFieldPostion;
                keyVec[currentFieldPostion] = 1;
                ++currentFieldPostion;
            }
            else
            {
                keyVec[field2Position[fieldIt.first]] = 1;
            }
        }
        string key;
        boost::to_string(keyVec, key);
        groupedEntries[key].push_back(i);
    }

    for (const auto& it : groupedEntries)
    {
        string fieldStr;
        vector<string> valueList;
        GetCommitFieldNameAndValueEachTable(_num, _data, it.second, fieldStr, valueList);
        _field2Values[fieldStr].insert(_field2Values[fieldStr].end(),
            make_move_iterator(valueList.begin()), make_move_iterator(valueList.end()));
    }
}

int SQLBasicAccess::Commit(int64_t _num, const vector<TableData::Ptr>& _datas)
{
    string errorMsg;
    volatile uint32_t retryCnt = 0;
    uint32_t retryMax = 10;
    volatile int ret = 0;
    TRY
    {
        ret = CommitDo(_num, _datas, errorMsg);
        while (ret < 0 && ++retryCnt < retryMax)
        {
            sleep(1);
            ret = CommitDo(_num, _datas, errorMsg);
        }
        if (ret < 0)
        {
            SQLBasicAccess_LOG(ERROR) << "commit failed error message:" << errorMsg;
            return -1;
        }
    }
    ELSE
    {
        SQLBasicAccess_LOG(ERROR) << "commit failed just return";
        return -1;
    }
    END_TRY;
    return ret;
}

int SQLBasicAccess::CommitDo(int64_t _num, const vector<TableData::Ptr>& _datas, string& _errorMsg)
{
    SQLBasicAccess_LOG(INFO) << " commit num:" << _num;
    string strNum = to_string(_num);
    if (_datas.size() == 0)
    {
        SQLBasicAccess_LOG(DEBUG) << "empty data just return";
        return 0;
    }

    /*execute commit operation*/

    Connection_T conn = m_connPool->GetConnection();
    TRY
    {
        for (auto tableDataPtr : _datas)
        {
            if (tableDataPtr->info->name == SYS_TABLES)
            {
                for (size_t i = 0; i < tableDataPtr->newEntries->size(); ++i)
                {
                    Entry::Ptr entry = tableDataPtr->newEntries->get(i);
                    string sql = BuildCreateTableSql(entry);
                    Connection_execute(conn, "%s", sql.c_str());
                }
            }
        }
    }
    CATCH(SQLException)
    {
        _errorMsg = Exception_frame.message;
        SQLBasicAccess_LOG(ERROR) << "create table exception:" << _errorMsg;
        m_connPool->ReturnConnection(conn);
        return -1;
    }
    END_TRY;

    volatile int32_t rowCount = 0;
    m_connPool->BeginTransaction(conn);
    TRY
    {
        for (auto data : _datas)
        {
            auto tableInfo = data->info;
            string tableName = tableInfo->name;
            map<string, vector<string>> field2Values;
            this->GetCommitFieldNameAndValue(data->dirtyEntries, strNum, field2Values);
            this->GetCommitFieldNameAndValue(data->newEntries, strNum, field2Values);

            SQLBasicAccess_LOG(DEBUG) << "table:" << tableName << " split to "
                                      << field2Values.size() << " parts to commit";

            tableName = boost::algorithm::replace_all_copy(tableName, "\\", "\\\\");
            tableName = boost::algorithm::replace_all_copy(tableName, "`", "\\`");

            auto tableWithBlobField = isBlobType(tableName);
            for (auto item : field2Values)
            {
                const auto& name = item.first;
                const auto& values = item.second;
                vector<SQLPlaceholderItem> sqlPlaceholders =
                    this->BuildCommitSql(tableName, name, values);
                auto itValue = values.begin();
                for (auto& placeholder : sqlPlaceholders)
                {
                    PreparedStatement_T preStatement =
                        Connection_prepareStatement(conn, "%s", placeholder.sql.c_str());
                    SQLBasicAccess_LOG(TRACE)
                        << "table:" << tableName << " sql:" << placeholder.sql;

                    uint32_t index = 0;
                    for (; itValue != values.end(); ++itValue)
                    {
                        if (tableWithBlobField)
                        {
                            PreparedStatement_setBlob(
                                preStatement, ++index, itValue->c_str(), itValue->size());
                        }
                        else
                        {
                            PreparedStatement_setString(preStatement, ++index, itValue->c_str());
                            SQLBasicAccess_LOG(TRACE) << " index:" << index << " num:" << _num
                                                      << " setString:" << itValue->c_str();
                        }

                        if (index == placeholder.placeholderCnt)
                        {
                            PreparedStatement_execute(preStatement);
                            rowCount += (int32_t)PreparedStatement_rowsChanged(preStatement);
                            ++itValue;
                            break;
                        }
                    }
                }
            }
        }
    }
    CATCH(SQLException)
    {
        _errorMsg = Exception_frame.message;
        SQLBasicAccess_LOG(ERROR) << "insert data exception:" << _errorMsg;
        SQLBasicAccess_LOG(DEBUG) << "active connections:" << m_connPool->GetActiveConnections()
                                  << " max connetions:" << m_connPool->GetMaxConnections()
                                  << " now connections:" << m_connPool->GetTotalConnections();
        m_connPool->RollBack(conn);
        m_connPool->ReturnConnection(conn);
        return -1;
    }
    END_TRY;

    SQLBasicAccess_LOG(INFO) << "commit now active connections:"
                             << m_connPool->GetActiveConnections()
                             << " max connections:" << m_connPool->GetMaxConnections();
    m_connPool->Commit(conn);
    m_connPool->ReturnConnection(conn);
    return rowCount;
}

vector<SQLPlaceholderItem> SQLBasicAccess::BuildCommitSql(
    const string& _table, const string& _fieldStr, const vector<string>& _fieldValues)
{
    vector<string> fieldNames;
    boost::split(fieldNames, _fieldStr, boost::is_any_of(","));
    vector<SQLPlaceholderItem> sqlPlaceholders;
    if (fieldNames.size() == 0 || _fieldValues.size() == 0 ||
        (_fieldValues.size() % fieldNames.size()))
    {
        /*throw exception*/
        SQLBasicAccess_LOG(ERROR) << "table name:" << _table << "field size:" << fieldNames.size()
                                  << " value size:" << _fieldValues.size()
                                  << " field size and value should be greate than 0";
        THROW(SQLException, "PreparedStatement_executeQuery");
    }

    string sqlHeader = "replace into `";
    sqlHeader.append(_table).append("`(");
    sqlHeader.append(_fieldStr);
    sqlHeader.append(") values");

    SQLBasicAccess_LOG(INFO) << "table name:" << _table << "field size:" << fieldNames.size()
                             << " value size:" << _fieldValues.size();

    string sql = sqlHeader;
    uint32_t placeholderCnt = 0;
    uint32_t valueSize = _fieldValues.size();
    uint32_t columnSize = fieldNames.size();
    for (uint32_t index = 0; index < valueSize; ++index)
    {
        ++placeholderCnt;
        if (index % columnSize == 0)
        {
            sql.append("(?,");
        }
        else
        {
            sql.append("?,");
        }
        if (index % columnSize == (columnSize - 1))
        {
            sql = sql.substr(0, sql.size() - 1);
            sql.append("),");
            /*
                if placeholders count is great than 65535 sql will execute failed
                so we need to execute in multiple sqls
            */
            if (placeholderCnt >= maxPlaceHolderCnt)
            {
                sql = sql.substr(0, sql.size() - 1);
                SQLPlaceholderItem item;
                item.sql = sql;
                item.placeholderCnt = placeholderCnt;
                sqlPlaceholders.emplace_back(item);
                sql = sqlHeader;
                placeholderCnt = 0;
            }
        }
    }
    if (placeholderCnt > 0)
    {
        sql = sql.substr(0, sql.size() - 1);
        SQLPlaceholderItem item;
        item.sql = sql;
        item.placeholderCnt = placeholderCnt;
        sqlPlaceholders.emplace_back(item);
    }
    return sqlPlaceholders;
}

void SQLBasicAccess::setConnPool(SQLConnectionPool::Ptr& _connPool)
{
    this->m_connPool = _connPool;
}

void SQLBasicAccess::ExecuteSql(const string& _sql)
{
    Connection_T conn = m_connPool->GetConnection();
    if (conn == NULL)
    {
        SQLBasicAccess_LOG(ERROR) << "get connection failed sql:" << _sql;
        THROW(SQLException, "PreparedStatement_executeQuery");
    }

    TRY { Connection_execute(conn, "%s", _sql.c_str()); }
    CATCH(SQLException)
    {
        SQLBasicAccess_LOG(ERROR) << "execute sql failed sql:" << _sql;
        m_connPool->ReturnConnection(conn);
        throw StorageException(-1, "execute sql failed sql:" + _sql);
    }
    END_TRY;
    SQLBasicAccess_LOG(INFO) << "execute sql success sql:" << _sql
                             << " now active connection:" << m_connPool->GetActiveConnections()
                             << " max connection :" << m_connPool->GetMaxConnections();

    m_connPool->ReturnConnection(conn);
}

max_stars_repo_name: imtypist/FISCO-BCOS

max_issues_repo_name: imtypist/FISCO-BCOS

max_forks_repo_name: imtypist/FISCO-BCOS