text:
/*
 * Copyright (c) 2018 Peerplays Blockchain Standards Association, and contributors.
 *
 * The MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#define DEFAULT_LOGGER "betting"
#include <graphene/chain/betting_market_object.hpp>
#include <graphene/chain/database.hpp>
#include <boost/integer/common_factor_rt.hpp>

#include <boost/msm/back/state_machine.hpp>
#include <boost/msm/front/state_machine_def.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/msm/back/tools.hpp>

namespace graphene { namespace chain {
   enum class betting_market_state {
      unresolved,
      frozen,
      closed,
      graded,
      canceled,
      settled
   };
} }
FC_REFLECT_ENUM(graphene::chain::betting_market_state,
                (unresolved)
                (frozen)
                (closed)
                (graded)
                (canceled)
                (settled))

namespace graphene { namespace chain {

namespace msm = boost::msm;
namespace mpl = boost::mpl;

/* static */ share_type bet_object::get_approximate_matching_amount(share_type bet_amount, bet_multiplier_type backer_multiplier, bet_type back_or_lay, bool round_up /* = false */)
{
   fc::uint128_t amount_to_match_128 = bet_amount.value;

   if (back_or_lay == bet_type::back)
   {
       amount_to_match_128 *= backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION;
       if (round_up)
         amount_to_match_128 += GRAPHENE_BETTING_ODDS_PRECISION - 1;
       amount_to_match_128 /= GRAPHENE_BETTING_ODDS_PRECISION;
   }
   else
   {
       amount_to_match_128 *= GRAPHENE_BETTING_ODDS_PRECISION;
       if (round_up)
         amount_to_match_128 += backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION - 1;
       amount_to_match_128 /= backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION;
   }
   return amount_to_match_128.to_uint64();
}

share_type bet_object::get_approximate_matching_amount(bool round_up /* = false */) const
{
   return get_approximate_matching_amount(amount_to_bet.amount, backer_multiplier, back_or_lay, round_up);
}

/* static */ share_type bet_object::get_exact_matching_amount(share_type bet_amount, bet_multiplier_type backer_multiplier, bet_type back_or_lay)
{
   share_type back_ratio;
   share_type lay_ratio;
   std::tie(back_ratio, lay_ratio) = get_ratio(backer_multiplier);
   if (back_or_lay == bet_type::back)
      return bet_amount / back_ratio * lay_ratio;
   else
      return bet_amount / lay_ratio * back_ratio;
}

share_type bet_object::get_exact_matching_amount() const
{
   return get_exact_matching_amount(amount_to_bet.amount, backer_multiplier, back_or_lay);
}

/* static */ std::pair<share_type, share_type> bet_object::get_ratio(bet_multiplier_type backer_multiplier)
{
   share_type gcd = boost::integer::gcd(GRAPHENE_BETTING_ODDS_PRECISION, static_cast<int32_t>(backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION));
   return std::make_pair(GRAPHENE_BETTING_ODDS_PRECISION / gcd, (backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION) / gcd);
}

std::pair<share_type, share_type> bet_object::get_ratio() const
{
   return get_ratio(backer_multiplier);
}

share_type bet_object::get_minimum_matchable_amount() const
{
   share_type gcd = boost::integer::gcd(GRAPHENE_BETTING_ODDS_PRECISION, static_cast<int32_t>(backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION));
   return (back_or_lay == bet_type::back ? GRAPHENE_BETTING_ODDS_PRECISION : backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION) / gcd;
}

share_type bet_object::get_minimum_matching_amount() const
{
   share_type gcd = boost::integer::gcd(GRAPHENE_BETTING_ODDS_PRECISION, static_cast<int32_t>(backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION));
   return (back_or_lay == bet_type::lay ? GRAPHENE_BETTING_ODDS_PRECISION : backer_multiplier - GRAPHENE_BETTING_ODDS_PRECISION) / gcd;
}

share_type betting_market_position_object::reduce()
{
   share_type additional_not_cancel_balance = std::min(pay_if_payout_condition, pay_if_not_payout_condition);
   if (additional_not_cancel_balance == 0)
      return 0;
   pay_if_payout_condition -= additional_not_cancel_balance;
   pay_if_not_payout_condition -= additional_not_cancel_balance;
   pay_if_not_canceled += additional_not_cancel_balance;

   share_type immediate_winnings = std::min(pay_if_canceled, pay_if_not_canceled);
   if (immediate_winnings == 0)
      return 0;
   pay_if_canceled -= immediate_winnings;
   pay_if_not_canceled -= immediate_winnings;
   return immediate_winnings;
}

// betting market object implementation
namespace
{
   // Events -- most events happen when the witnesses publish an update operation with a new
   // status, so if they publish an event with the status set to `frozen`, we'll generate a `frozen_event`
   struct unresolved_event
   {
      database& db;
      unresolved_event(database& db) : db(db) {}
   };
   struct frozen_event
   {
      database& db;
      frozen_event(database& db) : db(db) {}
   };
   struct closed_event
   {
      database& db;
      closed_event(database& db) : db(db) {}
   };
   struct graded_event
   {
      database& db;
      betting_market_resolution_type new_grading;
      graded_event(database& db, betting_market_resolution_type new_grading) : db(db), new_grading(new_grading) {}
   };
   struct settled_event
   {
      database& db;
      settled_event(database& db) : db(db) {}
   };
   struct canceled_event
   {
      database& db;
      canceled_event(database& db) : db(db) {}
   };

   // Events
   struct betting_market_state_machine_ : public msm::front::state_machine_def<betting_market_state_machine_>
   {
      // disable a few state machine features we don't use for performance
      typedef int no_exception_thrown;
      typedef int no_message_queue;

      // States
      struct unresolved : public msm::front::state<>{
         template <class Event>
         void on_entry(const Event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> unresolved", ("id", fsm.betting_market_obj->id));
         }
      };
      struct frozen : public msm::front::state<>{
         template <class Event>
         void on_entry(const Event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> frozen", ("id", fsm.betting_market_obj->id));
         }
      };
      struct closed : public msm::front::state<>{
         template <class Event>
         void on_entry(const Event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> closed", ("id", fsm.betting_market_obj->id));
         }
      };
      struct graded : public msm::front::state<>{
         void on_entry(const graded_event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> graded", ("id", fsm.betting_market_obj->id));
            fsm.betting_market_obj->resolution = event.new_grading;
         }
      };
      struct settled : public msm::front::state<>{
         template <class Event>
         void on_entry(const Event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> settled", ("id", fsm.betting_market_obj->id));
         }
      };
      struct canceled : public msm::front::state<>{
         void on_entry(const canceled_event& event, betting_market_state_machine_& fsm) {
            dlog("betting market ${id} -> canceled", ("id", fsm.betting_market_obj->id));
            fsm.betting_market_obj->resolution = betting_market_resolution_type::cancel;
            fsm.betting_market_obj->cancel_all_bets(event.db);
         }
      };

      typedef unresolved initial_state;
      typedef betting_market_state_machine_ x; // makes transition table cleaner

      // Transition table for betting market
      struct transition_table : mpl::vector<
      //    Start                       Event                         Next                       Action               Guard
      //  +---------------------------+-----------------------------+----------------------------+---------------------+----------------------+
      _row < unresolved,               frozen_event,                 frozen >,
      _row < unresolved,               closed_event,                 closed >,
      _row < unresolved,               canceled_event,               canceled >,
      //  +---------------------------+-----------------------------+----------------------------+---------------------+----------------------+
      _row < frozen,                   unresolved_event,             unresolved >,
      _row < frozen,                   closed_event,                 closed >,
      _row < frozen,                   canceled_event,               canceled >,
      //  +---------------------------+-----------------------------+----------------------------+---------------------+----------------------+
      _row < closed,                   graded_event,                 graded >,
      _row < closed,                   canceled_event,               canceled >,
      //  +---------------------------+-----------------------------+----------------------------+---------------------+----------------------+
      _row < graded,                   settled_event,                settled >,
      _row < graded,                   canceled_event,               canceled >
      //  +---------------------------+-----------------------------+----------------------------+---------------------+----------------------+
      > {};

      template <class Fsm, class Event>
      void no_transition(Event const& e, Fsm& ,int state)
      {
         FC_THROW_EXCEPTION(graphene::chain::no_transition, "No transition");
      }

      template <class Fsm>
      void no_transition(canceled_event const& e, Fsm&, int state)
      {
         //ignore transitions from settled to canceled state
         //and from canceled to canceled state
      }

      betting_market_object* betting_market_obj;
      betting_market_state_machine_(betting_market_object* betting_market_obj) : betting_market_obj(betting_market_obj) {}
   };
   typedef msm::back::state_machine<betting_market_state_machine_> betting_market_state_machine;

} // end anonymous namespace

class betting_market_object::impl {
public:
   betting_market_state_machine state_machine;

   impl(betting_market_object* self) : state_machine(self) {}
};

betting_market_object::betting_market_object() :
   my(new impl(this))
{
}

betting_market_object::betting_market_object(const betting_market_object& rhs) :
   graphene::db::abstract_object<betting_market_object>(rhs),
   group_id(rhs.group_id),
   description(rhs.description),
   payout_condition(rhs.payout_condition),
   resolution(rhs.resolution),
   my(new impl(this))
{
   my->state_machine = rhs.my->state_machine;
   my->state_machine.betting_market_obj = this;
}

betting_market_object& betting_market_object::operator=(const betting_market_object& rhs)
{
   //graphene::db::abstract_object<betting_market_object>::operator=(rhs);
   id = rhs.id;
   group_id = rhs.group_id;
   description = rhs.description;
   payout_condition = rhs.payout_condition;
   resolution = rhs.resolution;

   my->state_machine = rhs.my->state_machine;
   my->state_machine.betting_market_obj = this;

   return *this;
}

betting_market_object::~betting_market_object()
{
}

namespace {

   bool verify_betting_market_status_constants()
   {
      unsigned error_count = 0;
      typedef msm::back::generate_state_set<betting_market_state_machine::stt>::type all_states;
      static char const* filled_state_names[mpl::size<all_states>::value];
      mpl::for_each<all_states,boost::msm::wrap<mpl::placeholders::_1> >
         (msm::back::fill_state_names<betting_market_state_machine::stt>(filled_state_names));
      for (unsigned i = 0; i < mpl::size<all_states>::value; ++i)
      {
         try
         {
            // this is an approximate test, the state name provided by typeinfo will be mangled, but should
            // at least contain the string we're looking for
            const char* fc_reflected_value_name = fc::reflector<betting_market_state>::to_string((betting_market_state)i);
            if (!strstr(filled_state_names[i], fc_reflected_value_name))
            {
               fc_elog(fc::logger::get("default"),
                       "Error, state string mismatch between fc and boost::msm for int value ${int_value}: "
                       "boost::msm -> ${boost_string}, fc::reflect -> ${fc_string}",
                       ("int_value", i)("boost_string", filled_state_names[i])("fc_string", fc_reflected_value_name));
               ++error_count;
            }
         }
         catch (const fc::bad_cast_exception&)
         {
            fc_elog(fc::logger::get("default"),
                    "Error, no reflection for value ${int_value} in enum betting_market_status",
                    ("int_value", i));
            ++error_count;
         }
      }
      if (error_count == 0)
         dlog("Betting market status constants are correct");
      else
         wlog("There were ${count} errors in the betting market status constants", ("count", error_count));

      return error_count == 0;
   }
} // end anonymous namespace

betting_market_status betting_market_object::get_status() const
{
   static bool state_constants_are_correct = verify_betting_market_status_constants();
   (void)&state_constants_are_correct;
   betting_market_state state = (betting_market_state)my->state_machine.current_state()[0];

   edump((state));

   switch (state)
   {
      case betting_market_state::unresolved:
         return betting_market_status::unresolved;
      case betting_market_state::frozen:
         return betting_market_status::frozen;
      case betting_market_state::closed:
         return betting_market_status::unresolved;
      case betting_market_state::canceled:
         return betting_market_status::canceled;
      case betting_market_state::graded:
         return betting_market_status::graded;
      case betting_market_state::settled:
         return betting_market_status::settled;
      default:
         FC_THROW("Unexpected betting market state");
   };
}

void betting_market_object::cancel_all_unmatched_bets(database& db) const
{
   const auto& bet_odds_idx = db.get_index_type<bet_object_index>().indices().get<by_odds>();

   // first, cancel all bets on the active books
   auto book_itr = bet_odds_idx.lower_bound(std::make_tuple(id));
   auto book_end = bet_odds_idx.upper_bound(std::make_tuple(id));
   while (book_itr != book_end)
   {
      auto old_book_itr = book_itr;
      ++book_itr;
      db.cancel_bet(*old_book_itr, true);
   }

   // then, cancel any delayed bets on that market.  We don't have an index for
   // that, so walk through all delayed bets
   book_itr = bet_odds_idx.begin();
   while (book_itr != bet_odds_idx.end() &&
          book_itr->end_of_delay)
   {
      auto old_book_itr = book_itr;
      ++book_itr;
      if (old_book_itr->betting_market_id == id)
         db.cancel_bet(*old_book_itr, true);
   }
}

void betting_market_object::cancel_all_bets(database& db) const
{
    const auto& bets_by_market_id = db.get_index_type<bet_object_index>().indices().get<by_betting_market_id>();

    auto bet_it = bets_by_market_id.lower_bound(id);
    auto bet_it_end = bets_by_market_id.upper_bound(id);
    while (bet_it != bet_it_end)
    {
        auto old_bet_it = bet_it;
        ++bet_it;
        db.cancel_bet(*old_bet_it, true);
    }
}

void betting_market_object::pack_impl(std::ostream& stream) const
{
   boost::archive::binary_oarchive oa(stream, boost::archive::no_header|boost::archive::no_codecvt|boost::archive::no_xml_tag_checking);
   oa << my->state_machine;
}

void betting_market_object::unpack_impl(std::istream& stream)
{
   boost::archive::binary_iarchive ia(stream, boost::archive::no_header|boost::archive::no_codecvt|boost::archive::no_xml_tag_checking);
   ia >> my->state_machine;
}

void betting_market_object::on_unresolved_event(database& db)
{
   my->state_machine.process_event(unresolved_event(db));
}

void betting_market_object::on_frozen_event(database& db)
{
   my->state_machine.process_event(frozen_event(db));
}

void betting_market_object::on_closed_event(database& db)
{
   my->state_machine.process_event(closed_event(db));
}

void betting_market_object::on_graded_event(database& db, betting_market_resolution_type new_grading)
{
   my->state_machine.process_event(graded_event(db, new_grading));
}

void betting_market_object::on_settled_event(database& db)
{
   my->state_machine.process_event(settled_event(db));
}

void betting_market_object::on_canceled_event(database& db)
{
   my->state_machine.process_event(canceled_event(db));
}

} } // graphene::chain

namespace fc {
   // Manually reflect betting_market_object to variant to properly reflect "state"
   void to_variant(const graphene::chain::betting_market_object& event_obj, fc::variant& v)
   {
      fc::mutable_variant_object o;
      o("id", event_obj.id)
       ("group_id", event_obj.group_id)
       ("description", event_obj.description)
       ("payout_condition", event_obj.payout_condition)
       ("resolution", event_obj.resolution)
       ("status", event_obj.get_status());

      v = o;
   }

   // Manually reflect betting_market_object to variant to properly reflect "state"
   void from_variant(const fc::variant& v, graphene::chain::betting_market_object& event_obj)
   {
      event_obj.id = v["id"].as<graphene::chain::betting_market_id_type>();
      event_obj.group_id = v["name"].as<graphene::chain::betting_market_group_id_type>();
      event_obj.description = v["description"].as<graphene::chain::internationalized_string_type>();
      event_obj.payout_condition = v["payout_condition"].as<graphene::chain::internationalized_string_type>();
      event_obj.resolution = v["resolution"].as<fc::optional<graphene::chain::betting_market_resolution_type>>();
      graphene::chain::betting_market_status status = v["status"].as<graphene::chain::betting_market_status>();
      const_cast<int*>(event_obj.my->state_machine.current_state())[0] = (int)status;
   }
} //end namespace fc


text:
#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE LibmeshEntitySetTester
#include <boost/test/included/unit_test.hpp>

#include <algorithm>
#include <cassert>
#include <cmath>
#include <cstdlib>
#include <iostream>
#include <sstream>
#include <vector>

#include <LibmeshEntityExtraData.hpp>
#include <LibmeshEntitySet.hpp>

#include <Teuchos_Array.hpp>
#include <Teuchos_ArrayRCP.hpp>
#include <Teuchos_CommHelpers.hpp>
#include <Teuchos_DefaultComm.hpp>
#include <Teuchos_DefaultMpiComm.hpp>
#include <Teuchos_OpaqueWrapper.hpp>
#include <Teuchos_RCP.hpp>
#include <Teuchos_Tuple.hpp>
#include <Teuchos_TypeTraits.hpp>

#include <libmesh/cell_hex8.h>
#include <libmesh/libmesh.h>
#include <libmesh/mesh.h>
#include <libmesh/node.h>
#include <libmesh/parallel.h>
#include <libmesh/point.h>

//---------------------------------------------------------------------------//
// MPI Setup
//---------------------------------------------------------------------------//

template <class Ordinal>
Teuchos::RCP<const Teuchos::Comm<Ordinal>> getDefaultComm()
{
#ifdef HAVE_MPI
    return Teuchos::DefaultComm<Ordinal>::getComm();
#else
    return Teuchos::rcp( new Teuchos::SerialComm<Ordinal>() );
#endif
}

//---------------------------------------------------------------------------//
// Hex-8 test.
BOOST_AUTO_TEST_CASE( checkLibmeshEntitySet )
{
	const std::string argv_string = "unit_test --keep-cout";
	const char *argv_char = argv_string.c_str();
	Teuchos::GlobalMPISession mpiSession(
			&boost::unit_test::framework::master_test_suite().argc,
			&boost::unit_test::framework::master_test_suite().argv);
	auto comm = Teuchos::DefaultComm<int>::getComm();
	auto mpi_comm = Teuchos::rcp_dynamic_cast<const Teuchos::MpiComm<int>>(
			comm);
	auto opaque_comm = mpi_comm->getRawMpiComm();
	auto raw_comm = (*opaque_comm)();

    // Create the mesh.
    int space_dim = 3;
    libMesh::LibMeshInit libmesh_init( 1, &argv_char, raw_comm );
    BOOST_VERIFY( libMesh::initialized() );
    BOOST_VERIFY( (int)libmesh_init.comm().rank() == comm->getRank() );
    Teuchos::RCP<libMesh::Mesh> mesh =
        Teuchos::rcp( new libMesh::Mesh( libmesh_init.comm(), space_dim ) );

    // Create the nodes.
    int rank = comm->getRank();
    Teuchos::Array<libMesh::Node *> nodes( 8 );
    double node_coords[3];
    node_coords[0] = 0.0;
    node_coords[1] = 0.0;
    node_coords[2] = 0.0;
    nodes[0] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 0,
        rank );

    node_coords[0] = 1.0;
    node_coords[1] = 0.0;
    node_coords[2] = 0.0;
    nodes[1] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 1,
        rank );

    node_coords[0] = 1.0;
    node_coords[1] = 1.0;
    node_coords[2] = 0.0;
    nodes[2] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 2,
        rank );

    node_coords[0] = 0.0;
    node_coords[1] = 1.0;
    node_coords[2] = 0.0;
    nodes[3] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 3,
        rank );

    node_coords[0] = 0.0;
    node_coords[1] = 0.0;
    node_coords[2] = 1.0;
    nodes[4] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 4,
        rank );

    node_coords[0] = 1.0;
    node_coords[1] = 0.0;
    node_coords[2] = 1.0;
    nodes[5] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 5,
        rank );

    node_coords[0] = 1.0;
    node_coords[1] = 1.0;
    node_coords[2] = 1.0;
    nodes[6] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 6,
        rank );

    node_coords[0] = 0.0;
    node_coords[1] = 1.0;
    node_coords[2] = 1.0;
    nodes[7] = mesh->add_point(
        libMesh::Point( node_coords[0], node_coords[1], node_coords[2] ), 7,
        rank );

    // Make a hex-8.
    libMesh::Elem *hex_elem = mesh->add_elem( new libMesh::Hex8 );
    hex_elem->processor_id() = rank;
    hex_elem->set_id() = 2 * rank;
    for ( int i = 0; i < 8; ++i )
        hex_elem->set_node( i ) = nodes[i];

    // Make 2 subdomains and put the hex-8 in the first subdomain.
    int subdomain_1_id = 1;
    int subdomain_2_id = 2;
    std::set<libMesh::subdomain_id_type> subdomain_ids;
    subdomain_ids.insert( subdomain_1_id );
    subdomain_ids.insert( subdomain_2_id );
    hex_elem->subdomain_id() = subdomain_1_id;

    // Make 2 boundaries and add the first elem side to one and first node to
    // the second.
    int boundary_1_id = 1;
    int boundary_2_id = 2;
    mesh->get_boundary_info().add_side( hex_elem, 0, boundary_1_id );
    mesh->get_boundary_info().add_node( nodes[0], boundary_2_id );

    // Check libmesh validity.
    mesh->libmesh_assert_valid_parallel_ids();

    // Create an entity set.
    auto entity_set =
        Teuchos::rcp( new LibmeshAdapter::LibmeshEntitySet( mesh ) );

    // Test the set.
    Teuchos::RCP<const Teuchos::Comm<int>> set_comm =
        entity_set->communicator();
    BOOST_VERIFY( set_comm->getRank() == comm->getRank() );
    BOOST_VERIFY( set_comm->getSize() == comm->getSize() );
    BOOST_VERIFY( space_dim == entity_set->physicalDimension() );

    // Make an iterator for the hex.
    LibmeshAdapter::ElemPredicateFunction all_pred =
        [=]( LibmeshAdapter::LibmeshEntity<libMesh::Elem> ) { return true; };
    auto volume_iterator =
        entity_set->entityIterator( all_pred );

    // Test the volume iterator.
    BOOST_VERIFY( volume_iterator.size() == 1 );
    BOOST_VERIFY( volume_iterator == volume_iterator.begin() );
    BOOST_VERIFY( volume_iterator != volume_iterator.end() );

    // Test the volume under the iterator.
    BOOST_VERIFY( hex_elem->id() == volume_iterator->id() );
    BOOST_VERIFY( comm->getRank() == volume_iterator->ownerRank() );
    BOOST_VERIFY( space_dim == volume_iterator->topologicalDimension() );
    BOOST_VERIFY( space_dim == volume_iterator->physicalDimension() );

   auto extra_data_1 =
        volume_iterator->extraData();
   BOOST_VERIFY( hex_elem ==
                   Teuchos::rcp_dynamic_cast<
                       LibmeshAdapter::LibmeshEntityExtraData<libMesh::Elem>>(
                       extra_data_1 )
                       ->d_libmesh_geom.getRawPtr() );

    Teuchos::Tuple<double, 6> hex_bounds_1;
    volume_iterator->boundingBox( hex_bounds_1 );
    BOOST_VERIFY( 0.0 == hex_bounds_1[0] );
    BOOST_VERIFY( 0.0 == hex_bounds_1[1] );
    BOOST_VERIFY( 0.0 == hex_bounds_1[2] );
    BOOST_VERIFY( 1.0 == hex_bounds_1[3] );
    BOOST_VERIFY( 1.0 == hex_bounds_1[4] );
    BOOST_VERIFY( 1.0 == hex_bounds_1[5] );

    // Test the end of the iterator.
    volume_iterator++;
    BOOST_VERIFY( volume_iterator != volume_iterator.begin() );
    BOOST_VERIFY( volume_iterator == volume_iterator.end() );

    // Make an iterator for the nodes.
    LibmeshAdapter::NodePredicateFunction all_node_pred =
            [=]( LibmeshAdapter::LibmeshEntity<libMesh::Node> ) { return true; };
   auto node_iterator =
        entity_set->entityIterator( all_node_pred );

    // Test the node iterator.
    unsigned num_nodes = 8;
    BOOST_VERIFY( node_iterator.size() == num_nodes );
    BOOST_VERIFY( node_iterator == node_iterator.begin() );
    BOOST_VERIFY( node_iterator != node_iterator.end() );
    auto node_begin = node_iterator.begin();
    auto node_end = node_iterator.end();
    auto node_id_it = nodes.begin();
    for ( node_iterator = node_begin; node_iterator != node_end;
          ++node_iterator, ++node_id_it )
    {
    	BOOST_VERIFY( node_iterator->id() == ( *node_id_it )->id() );
    }

    // Get each entity and check.
    LibmeshAdapter::LibmeshEntity<libMesh::Elem> set_hex;
    entity_set->getEntity( hex_elem->id(), set_hex );
    BOOST_VERIFY( set_hex.id() == hex_elem->id() );

    for ( unsigned i = 0; i < num_nodes; ++i )
    {
    	LibmeshAdapter::LibmeshEntity<libMesh::Node> set_node;
        entity_set->getEntity( nodes[i]->id(), set_node );
        BOOST_VERIFY( set_node.id() == nodes[i]->id() );
    }

    // Check the adjacency function.
    Teuchos::Array<LibmeshAdapter::LibmeshEntity<libMesh::Elem>> hex_adjacent_volumes;
    entity_set->getAdjacentEntities( set_hex, hex_adjacent_volumes );
    BOOST_VERIFY( 0 == hex_adjacent_volumes.size() );

    Teuchos::Array<LibmeshAdapter::LibmeshEntity<libMesh::Node>> hex_adjacent_nodes;
    entity_set->getAdjacentEntities( set_hex, hex_adjacent_nodes );
    BOOST_VERIFY( num_nodes == hex_adjacent_nodes.size() );
    for ( unsigned i = 0; i < num_nodes; ++i )
    {
    	BOOST_VERIFY( hex_adjacent_nodes[i].id() == nodes[i]->id() );
    }

    for ( unsigned i = 0; i < num_nodes; ++i )
    {
        Teuchos::Array<LibmeshAdapter::LibmeshEntity<libMesh::Elem>> node_adjacent_volumes;
        entity_set->getAdjacentEntities( hex_adjacent_nodes[i],
                                         node_adjacent_volumes );
        BOOST_VERIFY( 1 == node_adjacent_volumes.size() );
        BOOST_VERIFY( node_adjacent_volumes[0].id() == hex_elem->id() );
    }

}

//---------------------------------------------------------------------------//
// end tstLibmeshEntitySet.cpp
//---------------------------------------------------------------------------//

ext: cpp


text:
//==================================================================================================
/*!
  @file

  @copyright 2012-2015 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#ifndef BOOST_SIMD_CONSTANT_DEFINITION_POWLARGELIM_HPP_INCLUDED
#define BOOST_SIMD_CONSTANT_DEFINITION_POWLARGELIM_HPP_INCLUDED

#include <boost/simd/config.hpp>
#include <boost/simd/detail/brigand.hpp>
#include <boost/simd/detail/dispatch.hpp>
#include <boost/simd/detail/constant_traits.hpp>
#include <boost/dispatch/function/make_callable.hpp>
#include <boost/dispatch/hierarchy/functions.hpp>
#include <boost/dispatch/as.hpp>

namespace boost { namespace simd
{
  namespace tag
  {
    struct powlargelim_ : boost::dispatch::constant_value_<powlargelim_>
    {
      BOOST_DISPATCH_MAKE_CALLABLE(ext,powlargelim_,boost::dispatch::constant_value_<powlargelim_>);
      BOOST_SIMD_REGISTER_CONSTANT(0, 0x44ffe000UL, 0x40cfff8000000000ULL);
    };
  }

  namespace ext
  {
    BOOST_DISPATCH_FUNCTION_DECLARATION(tag,powlargelim_);
  }

  namespace detail
  {
    BOOST_DISPATCH_CALLABLE_DEFINITION(tag::powlargelim_,powlargelim);
  }

  template<typename T> BOOST_FORCEINLINE auto Powlargelim()
  BOOST_NOEXCEPT_DECLTYPE(detail::powlargelim( boost::dispatch::as_<T>{}))
  {
    return detail::powlargelim( boost::dispatch::as_<T>{} );
  }
} }

#endif

ext: hpp


text:
//    Copyright 2017 Rainer Gemulla
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
// TODO: add all kinds of arguments

#include <iostream>

#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <boost/random/uniform_real.hpp>

#include <util/evaluation.h>
#include <mf/mf.h>

using namespace std;
using namespace mf;
using namespace rg;
using namespace boost::program_options;

log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("main"));

int main(int argc, char *argv[]) {
	string traceFile;
	string traceVar;
	string inputMatrixFile;
	mf_size_type epochs;

	// read command line
	options_description desc("Options");
	desc.add_options()
		("help", "produce help message")
		("epochs", value<mf_size_type>(&epochs)->default_value(10), "number of epochs to run [10]")
		("trace", value<string>(&traceFile)->default_value("trace.R"), "filename of trace [trace.R]")
		("traceVar", value<string>(&traceVar)->default_value("trace"), "variable name for trace [traceVar]")
	    ("input-file", value<string>(&inputMatrixFile), "input matrix")
	    ;

	positional_options_description pdesc;
	pdesc.add("input-file", 1);

	variables_map vm;
	store(command_line_parser(argc, argv).options(desc).positional(pdesc).run(), vm);
	notify(vm);

	if (vm.count("help") || vm.count("input-file")==0) {
		cout << "mfsgd [options] <input-file> <sample-file>" << endl << endl;
	    cout << desc << endl;
	    return 1;
	}

	// read input matrix
	SparseMatrix v;
	readMatrix(inputMatrixFile, v);
	LOG4CXX_INFO(logger, "Data matrix: "
		<< v.size1() << " x " << v.size2() << ", " << v.nnz() << " nonzeros");
	mf_size_type size1 = v.size1();
	mf_size_type size2 = v.size2();
	mf_size_type r = 50;

	// compute sample matrix
	Random32 random; // note: this takes a default seed (not randomized!)
	ProjectedSparseMatrix Vsample;
	projectRandomSubmatrix(random, v, Vsample, v.size1()/10, v.size2()/10);
	projectFrequent(Vsample, 0);
	LOG4CXX_INFO(logger, "Sample matrix: "
		<< Vsample.data.size1() << " x " << Vsample.data.size2()
		<< ", " << Vsample.data.nnz() << " nonzeros");

	// generate initial factors by sampling from a uniform[-0.5,0.5] distribution
	DenseMatrix w(size1, r);
	DenseMatrixCM h(r, size2);
	generateRandom(w, random, boost::uniform_real<>(-0.5, 0.5));
	generateRandom(h, random, boost::uniform_real<>(-0.5, 0.5));

	// parameters for SGD
	double epsMax = 0.1;
	double lambda = 50;
	typedef UpdateTruncate<UpdateNzsl> Update;
	typedef RegularizeL2 Regularize;
	typedef SumLoss<NzslLoss, L2Loss> Loss;
	Update update = Update(UpdateNzsl(), -1000, 10100); // truncate for numerical stability
	Regularize regularize = Regularize(lambda);
	Loss loss((NzslLoss()), L2Loss(lambda));

	// initialize the SGD
	Timer t;
	SgdRunner sgdRunner(random);
	SgdJob<Update,Regularize> job(v, w, h, update, regularize, SGD_ORDER_WOR);
	DecayAuto<Update,Regularize,Loss> decay(job, loss, Vsample, epsMax, 7);
	Trace trace;

	// run SGD
	t.start();
	sgdRunner.run(job, loss, epochs, decay, trace, BALANCE_NONE, BALANCE_SIMPLE);
	t.stop();
	LOG4CXX_INFO(logger, "Total time: " << t);

	// write trace to an R file
	LOG4CXX_INFO(logger, "Writing trace to " << traceFile);
	trace.toRfile(traceFile, traceVar);

	return 0;
}

ext: cc

max_stars_repo_name: Hui-Li/DSGDPP

max_issues_repo_name: Hui-Li/DSGDPP

max_forks_repo_name: Hui-Li/DSGDPP


text:
/**********************************************************************
 * LeechCraft - modular cross-platform feature rich internet client.
 * Copyright (C) 2010-2011  Oleg Linkin
 *
 * Boost Software License - Version 1.0 - August 17th, 2003
 *
 * Permission is hereby granted, free of charge, to any person or organization
 * obtaining a copy of the software and accompanying documentation covered by
 * this license (the "Software") to use, reproduce, display, distribute,
 * execute, and transmit the Software, and to prepare derivative works of the
 * Software, and to permit third-parties to whom the Software is furnished to
 * do so, all subject to the following:
 *
 * The copyright notices in the Software and this entire statement, including
 * the above license grant, this restriction and the following disclaimer,
 * must be included in all copies of the Software, in whole or in part, and
 * all derivative works of the Software, unless such copies or derivative
 * works are solely in the form of machine-executable object code generated by
 * a source language processor.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 **********************************************************************/

#include "serverinfowidget.h"
#include <boost/bind.hpp>
#include "ircserverclentry.h"
#include <QtDebug>

namespace LeechCraft
{
namespace Azoth
{
namespace Acetamide
{
	ServerInfoWidget::ServerInfoWidget (IrcServerCLEntry *isEntry , QWidget *parent)
	: QWidget (parent)
	, ISCLEntry_ (isEntry)
	{
		Ui_.setupUi (this);
		Init ();
		SetISupport ();
	}

	void ServerInfoWidget::Init ()
	{
		Parameter2Command_ ["casemapping"] = boost::bind (&QLineEdit::setText,
				Ui_.CaseMapping_, _1);
		Parameter2Command_ ["chanlimit"] = boost::bind (&QLineEdit::setText,
				Ui_.ChanLimit_, _1);
		Parameter2Command_ ["chanmodes"] = boost::bind (&ServerInfoWidget::SetChanModes,
				this, _1);
		Parameter2Command_ ["channellen"] = boost::bind (&QLineEdit::setText,
				Ui_.ChannelLen_, _1);
		Parameter2Command_ ["chantypes"] = boost::bind (&QLineEdit::setText,
				Ui_.ChanTypes_, _1);
		Parameter2Command_ ["excepts"] = boost::bind (&ServerInfoWidget::SetExcepts,
				this, _1);
		Parameter2Command_ ["idchan"] = boost::bind (&QLineEdit::setText,
				Ui_.IdChan_, _1);
		Parameter2Command_ ["kicklen"] = boost::bind (&QLineEdit::setText,
				Ui_.KickLen_, _1);
		Parameter2Command_ ["maxlist"] = boost::bind (&QLineEdit::setText,
				Ui_.MaxList_, _1);
		Parameter2Command_ ["modes"] = boost::bind (&QLineEdit::setText,
				Ui_.Modes_, _1);
		Parameter2Command_ ["network"] = boost::bind (&QLineEdit::setText,
				Ui_.NetworkName_, _1);
		Parameter2Command_ ["nicklen"] = boost::bind (&QLineEdit::setText,
				Ui_.NickLength_, _1);
		Parameter2Command_ ["prefix"] = boost::bind (&ServerInfoWidget::SetPrefix,
				this, _1);
		Parameter2Command_ ["safelist"] = boost::bind (&ServerInfoWidget::SetSafeList,
				this, _1);
		Parameter2Command_ ["statusmsg"] = boost::bind (&QLineEdit::setText,
				Ui_.StatusMsg_, _1);
		Parameter2Command_ ["std"] = boost::bind (&QLineEdit::setText,
				Ui_.Std_, _1);
		Parameter2Command_ ["targmax"] = boost::bind (&ServerInfoWidget::SetTargMax,
				this, _1);
		Parameter2Command_ ["topiclen"] = boost::bind (&QLineEdit::setText,
				Ui_.TopicLen_, _1);
		Parameter2Command_ ["invex"] = boost::bind (&ServerInfoWidget::SetInvEx,
				this, _1);
	}

	void ServerInfoWidget::SetISupport ()
	{
		const QMap<QString, QString>& info = ISCLEntry_->GetISupport ();

		for (QMap<QString, QString>::const_iterator it_begin = info.begin (),
				it_end = info.end (); it_begin != it_end; ++it_begin)
			if (Parameter2Command_.contains (it_begin.key ().toLower ()))
				Parameter2Command_ [it_begin.key ().toLower ()] (it_begin.value ());
	}

	void ServerInfoWidget::SetChanModes (const QString& modes)
	{
		const QStringList& list = modes.split (',');

		Ui_.ChanModesA_->setText (list.at (0));
		Ui_.ChanModesB_->setText (list.at (1));
		Ui_.ChanModesC_->setText (list.at (2));
		Ui_.ChanModesD_->setText (list.at (3));
	}

	void ServerInfoWidget::SetExcepts (const QString& str)
	{
		Ui_.Excepts_->setChecked (GetBoolFromString (str));
	}

	void ServerInfoWidget::SetPrefix (const QString& str)
	{
		const int mode_end = str.indexOf (')');
		const QString& modeStr = str.mid (1, mode_end - 1);
		const QString& prefixStr = str.mid (mode_end + 1);
		const int rowCount = qMin (modeStr.length (), prefixStr.length ());

		if (modeStr.length () != prefixStr.length ())
			qWarning () << "number of modes is not equal to number of prefixes";

		Ui_.tableWidget->clear ();
		Ui_.tableWidget->setRowCount (rowCount);

		for (int i = 0; i < rowCount; ++i)
		{
			QTableWidgetItem *newMode = new QTableWidgetItem (modeStr [i]);
			QTableWidgetItem *newPrefix = new QTableWidgetItem (prefixStr [i]);
			Ui_.tableWidget->setItem (i, 0, newMode);
			Ui_.tableWidget->setItem (i, 1, newPrefix);
		}
	}

	void ServerInfoWidget::SetSafeList (const QString& str)
	{
		Ui_.SafeList_->setChecked (GetBoolFromString (str));
	}

	void ServerInfoWidget::SetTargMax (const QString& str)
	{
		const QStringList& list = str.split (',');

		Ui_.TargetMax_->clear ();
		Ui_.TargetMax_->setRowCount (list.count ());

		int row = 0;
		Q_FOREACH (const QString& param, list)
		{
			const int index = param.indexOf (':');
			QTableWidgetItem *target = new QTableWidgetItem (param.mid (0, index));
			QTableWidgetItem *count = new QTableWidgetItem (param.mid (index + 1));
			Ui_.TargetMax_->setItem (row, 0, target);
			Ui_.TargetMax_->setItem (row, 1, count);
			++row;
		}
	}

	void ServerInfoWidget::SetInvEx (const QString& str)
	{
		Ui_.InvEx_->setChecked (GetBoolFromString (str));
	}

	bool ServerInfoWidget::GetBoolFromString (const QString& str)
	{
		return str == "true";
	}

	void ServerInfoWidget::accept ()
	{
	}
}
}
}

max_stars_repo_name: MellonQ/leechcraft

max_issues_repo_name: MellonQ/leechcraft

max_forks_repo_name: MellonQ/leechcraft


text:
/* vim: set sw=4 sts=4 et foldmethod=syntax : */

#include "dimacs.hh"
#include "graph.hh"
#include "graph_file_error.hh"

#include <boost/regex.hpp>
#include <fstream>

using std::getline;
using std::ifstream;
using std::stoi;
using std::string;

using boost::regex;
using boost::smatch;

auto read_dimacs(const string & filename) -> Graph
{
    Graph result{ 0 };

    ifstream infile{ filename };
    if (! infile)
        throw GraphFileError{ filename, "unable to open file" };

    string line;
    while (getline(infile, line)) {
        if (line.empty())
            continue;

        /* Lines are comments, a problem description (contains the number of
         * vertices), or an edge. */
        static const regex
            comment{ R"(c(\s.*)?)" },
            problem{ R"(p\s+(edge|col)\s+(\d+)\s+(\d+)?\s*)" },
            edge{ R"(e\s+(\d+)\s+(\d+)\s*)" };

        smatch match;
        if (regex_match(line, match, comment)) {
            /* Comment, ignore */
        }
        else if (regex_match(line, match, problem)) {
            /* Problem. Specifies the size of the graph. Must happen exactly
             * once. */
            if (0 != result.size())
                throw GraphFileError{ filename, "multiple 'p' lines encountered" };
            result.resize(stoi(match.str(2)));
        }
        else if (regex_match(line, match, edge)) {
            /* An edge. DIMACS files are 1-indexed. We assume we've already had
             * a problem line (if not our size will be 0, so we'll throw). */
            int a{ stoi(match.str(1)) }, b{ stoi(match.str(2)) };
            if (0 == a || 0 == b || a > result.size() || b > result.size())
                throw GraphFileError{ filename, "line '" + line + "' edge index out of bounds" };
            result.add_edge(a - 1, b - 1);
        }
        else
            throw GraphFileError{ filename, "cannot parse line '" + line + "'" };
    }

    if (! infile.eof())
        throw GraphFileError{ filename, "error reading file" };

    return result;
}

max_stars_repo_name: zxcvsop/YewPar

max_issues_repo_name: zxcvsop/YewPar

max_forks_repo_name: zxcvsop/YewPar


text:
//[ LazyVector
///////////////////////////////////////////////////////////////////////////////
//  Copyright 2008 Eric Niebler. Distributed under the Boost
//  Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// This example constructs a mini-library for linear algebra, using
// expression templates to eliminate the need for temporaries when
// adding vectors of numbers.
//
// This example uses a domain with a grammar to prune the set
// of overloaded operators. Only those operators that produce
// valid lazy vector expressions are allowed.

#include <vector>
#include <iostream>
#include <boost/mpl/int.hpp>
#include <boost/proto/core.hpp>
#include <boost/proto/context.hpp>
namespace mpl = boost::mpl;
namespace proto = boost::proto;
using proto::_;

// This grammar describes which lazy vector expressions
// are allowed; namely, vector terminals and addition
// and subtraction of lazy vector expressions.
struct LazyVectorGrammar
  : proto::or_<
        proto::terminal< std::vector<_> >
      , proto::plus< LazyVectorGrammar, LazyVectorGrammar >
      , proto::minus< LazyVectorGrammar, LazyVectorGrammar >
    >
{};

// Expressions in the lazy vector domain must conform
// to the lazy vector grammar
struct lazy_vector_domain;

// Here is an evaluation context that indexes into a lazy vector
// expression, and combines the result.
template<typename Size = std::size_t>
struct lazy_subscript_context
{
    lazy_subscript_context(Size subscript)
      : subscript_(subscript)
    {}

    // Use default_eval for all the operations ...
    template<typename Expr, typename Tag = typename Expr::proto_tag>
    struct eval
      : proto::default_eval<Expr, lazy_subscript_context>
    {};

    // ... except for terminals, which we index with our subscript
    template<typename Expr>
    struct eval<Expr, proto::tag::terminal>
    {
        typedef typename proto::result_of::value<Expr>::type::value_type result_type;

        result_type operator ()( Expr const & expr, lazy_subscript_context & ctx ) const
        {
            return proto::value( expr )[ ctx.subscript_ ];
        }
    };

    Size subscript_;
};

// Here is the domain-specific expression wrapper, which overrides
// operator [] to evaluate the expression using the lazy_subscript_context.
template<typename Expr>
struct lazy_vector_expr
  : proto::extends<Expr, lazy_vector_expr<Expr>, lazy_vector_domain>
{
    typedef proto::extends<Expr, lazy_vector_expr<Expr>, lazy_vector_domain> base_type;

    lazy_vector_expr( Expr const & expr = Expr() )
      : base_type( expr )
    {}

    // Use the lazy_subscript_context<> to implement subscripting
    // of a lazy vector expression tree.
    template< typename Size >
    typename proto::result_of::eval< Expr, lazy_subscript_context<Size> >::type
    operator []( Size subscript ) const
    {
        lazy_subscript_context<Size> ctx(subscript);
        return proto::eval(*this, ctx);
    }
};

// Here is our lazy_vector terminal, implemented in terms of lazy_vector_expr
template< typename T >
struct lazy_vector
  : lazy_vector_expr< typename proto::terminal< std::vector<T> >::type >
{
    typedef typename proto::terminal< std::vector<T> >::type expr_type;

    lazy_vector( std::size_t size = 0, T const & value = T() )
      : lazy_vector_expr<expr_type>( expr_type::make( std::vector<T>( size, value ) ) )
    {}

    // Here we define a += operator for lazy vector terminals that
    // takes a lazy vector expression and indexes it. expr[i] here
    // uses lazy_subscript_context<> under the covers.
    template< typename Expr >
    lazy_vector &operator += (Expr const & expr)
    {
        std::size_t size = proto::value(*this).size();
        for(std::size_t i = 0; i < size; ++i)
        {
            proto::value(*this)[i] += expr[i];
        }
        return *this;
    }
};

// Tell proto that in the lazy_vector_domain, all
// expressions should be wrapped in laxy_vector_expr<>
struct lazy_vector_domain
  : proto::domain<proto::generator<lazy_vector_expr>, LazyVectorGrammar>
{};

int main()
{
    // lazy_vectors with 4 elements each.
    lazy_vector< double > v1( 4, 1.0 ), v2( 4, 2.0 ), v3( 4, 3.0 );

    // Add two vectors lazily and get the 2nd element.
    double d1 = ( v2 + v3 )[ 2 ];   // Look ma, no temporaries!
    std::cout << d1 << std::endl;

    // Subtract two vectors and add the result to a third vector.
    v1 += v2 - v3;                  // Still no temporaries!
    std::cout << '{' << v1[0] << ',' << v1[1]
              << ',' << v1[2] << ',' << v1[3] << '}' << std::endl;

    // This expression is disallowed because it does not conform
    // to the LazyVectorGrammar
    //(v2 + v3) += v1;

    return 0;
}
//]

max_issues_repo_name: boost-cmake/vintage

max_forks_repo_name: boost-cmake/vintage


text: #include <boost/geometry/geometries/concepts/linestring_concept.hpp>

max_stars_repo_name: miathedev/BoostForArduino

max_issues_repo_name: miathedev/BoostForArduino

max_forks_repo_name: miathedev/BoostForArduino


text:
//------------------------------------------------------------------------------
/*

    Copyright (c) 2012, 2017 Ripple Labs Inc.

    Permission to use, copy, modify, and/or distribute this software for any
    purpose  with  or without fee is hereby granted, provided that the above
    copyright notice and this permission notice appear in all copies.

    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================

#include <ripple/nodestore/impl/DatabaseShardImp.h>
#include <ripple/app/ledger/InboundLedgers.h>
#include <ripple/app/ledger/LedgerMaster.h>
#include <ripple/basics/ByteUtilities.h>
#include <ripple/basics/chrono.h>
#include <ripple/basics/random.h>
#include <ripple/core/ConfigSections.h>
#include <ripple/nodestore/DummyScheduler.h>
#include <ripple/nodestore/Manager.h>
#include <ripple/overlay/Overlay.h>
#include <ripple/overlay/predicates.h>
#include <ripple/protocol/HashPrefix.h>

#include <boost/algorithm/string/predicate.hpp>

namespace ripple {
namespace NodeStore {

DatabaseShardImp::DatabaseShardImp(
    Application& app,
    Stoppable& parent,
    std::string const& name,
    Scheduler& scheduler,
    int readThreads,
    beast::Journal j)
    : DatabaseShard(
        name,
        parent,
        scheduler,
        readThreads,
        app.config().section(ConfigSection::shardDatabase()),
        j)
    , app_(app)
    , earliestShardIndex_(seqToShardIndex(earliestSeq()))
    , avgShardFileSz_(ledgersPerShard_ * kilobytes(192))
{
}

DatabaseShardImp::~DatabaseShardImp()
{
    // Stop threads before data members are destroyed
    stopThreads();

    // Close backend databases before destroying the context
    std::lock_guard lock(m_);
    complete_.clear();
    if (incomplete_)
        incomplete_.reset();
    preShards_.clear();
    ctx_.reset();
}

bool
DatabaseShardImp::init()
{
    using namespace boost::filesystem;

    std::lock_guard lock(m_);
    auto fail = [j = j_](std::string const& msg)
    {
        JLOG(j.error()) <<
            "[" << ConfigSection::shardDatabase() << "] " << msg;
        return false;
    };

    if (init_)
        return fail("already initialized");

    Config const& config {app_.config()};
    Section const& section {config.section(ConfigSection::shardDatabase())};
    if (section.empty())
        return fail("missing configuration");

    {
        // Node and shard stores must use same earliest ledger sequence
        std::uint32_t seq;
        if (get_if_exists<std::uint32_t>(
                config.section(ConfigSection::nodeDatabase()),
                "earliest_seq",
                seq))
        {
            std::uint32_t seq2;
            if (get_if_exists<std::uint32_t>(section, "earliest_seq", seq2) &&
                seq != seq2)
            {
                return fail("and [" + ConfigSection::shardDatabase() +
                    "] both define 'earliest_seq'");
            }
        }
    }

    if (!get_if_exists<boost::filesystem::path>(section, "path", dir_))
        return fail("'path' missing");

    if (boost::filesystem::exists(dir_))
    {
        if (!boost::filesystem::is_directory(dir_))
            return fail("'path' must be a directory");
    }
    else
        boost::filesystem::create_directories(dir_);

    {
        std::uint64_t sz;
        if (!get_if_exists<std::uint64_t>(section, "max_size_gb", sz))
            return fail("'max_size_gb' missing");

        if ((sz << 30) < sz)
            return fail("'max_size_gb' overflow");

        // Minimum storage space required (in gigabytes)
        if (sz < 10)
            return fail("'max_size_gb' must be at least 10");

        // Convert to bytes
        maxFileSz_ = sz << 30;
    }

    if (section.exists("ledgers_per_shard"))
    {
        // To be set only in standalone for testing
        if (!config.standalone())
            return fail("'ledgers_per_shard' only honored in stand alone");

        ledgersPerShard_ = get<std::uint32_t>(section, "ledgers_per_shard");
        if (ledgersPerShard_ == 0 || ledgersPerShard_ % 256 != 0)
            return fail("'ledgers_per_shard' must be a multiple of 256");
    }

    // NuDB is the default and only supported permanent storage backend
    // "Memory" and "none" types are supported for tests
    backendName_ = get<std::string>(section, "type", "nudb");
    if (!boost::iequals(backendName_, "NuDB") &&
        !boost::iequals(backendName_, "Memory") &&
        !boost::iequals(backendName_, "none"))
    {
        return fail("'type' value unsupported");
    }

    // Check if backend uses permanent storage
    if (auto factory = Manager::instance().find(backendName_))
    {
        auto backend {factory->createInstance(
            NodeObject::keyBytes, section, scheduler_, j_)};
        backed_ = backend->backed();
        if (!backed_)
        {
            setFileStats(lock);
            init_ = true;
            return true;
        }
    }
    else
        return fail(backendName_ + " backend unsupported");

    try
    {
        ctx_ = std::make_unique<nudb::context>();
        ctx_->start();

        // Find shards
        for (auto const& d : directory_iterator(dir_))
        {
            if (!is_directory(d))
                continue;

            // Validate shard directory name is numeric
            auto dirName = d.path().stem().string();
            if (!std::all_of(
                dirName.begin(),
                dirName.end(),
                [](auto c) {
                return ::isdigit(static_cast<unsigned char>(c));
            }))
            {
                continue;
            }

            auto const shardIndex {std::stoul(dirName)};
            if (shardIndex < earliestShardIndex())
            {
                return fail("shard " + std::to_string(shardIndex) +
                    " comes before earliest shard index " +
                    std::to_string(earliestShardIndex()));
            }

            // Check if a previous import failed
            if (is_regular_file(
                dir_ / std::to_string(shardIndex) / importMarker_))
            {
                JLOG(j_.warn()) <<
                    "shard " << shardIndex <<
                    " previously failed import, removing";
                remove_all(dir_ / std::to_string(shardIndex));
                continue;
            }

            auto shard {std::make_unique<Shard>(app_, *this, shardIndex, j_)};
            if (!shard->open(scheduler_, *ctx_))
                return false;

            if (shard->complete())
                complete_.emplace(shard->index(), std::move(shard));
            else
            {
                if (incomplete_)
                    return fail("more than one control file found");
                incomplete_ = std::move(shard);
            }
        }
    }
    catch (std::exception const& e)
    {
        return fail(std::string("exception ") +
            e.what() + " in function " + __func__);
    }

    setFileStats(lock);
    updateStatus(lock);
    init_ = true;
    return true;
}

boost::optional<std::uint32_t>
DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq)
{
    std::lock_guard lock(m_);
    assert(init_);

    if (incomplete_)
        return incomplete_->prepare();
    if (!canAdd_)
        return boost::none;
    if (backed_)
    {
        // Check available storage space
        if (fileSz_ + avgShardFileSz_ > maxFileSz_)
        {
            JLOG(j_.debug()) << "maximum storage size reached";
            canAdd_ = false;
            return boost::none;
        }
        if (avgShardFileSz_ > available())
        {
            JLOG(j_.error()) << "insufficient storage space available";
            canAdd_ = false;
            return boost::none;
        }
    }

    auto const shardIndex {findShardIndexToAdd(validLedgerSeq, lock)};
    if (!shardIndex)
    {
        JLOG(j_.debug()) << "no new shards to add";
        canAdd_ = false;
        return boost::none;
    }
    // With every new shard, clear family caches
    app_.shardFamily()->reset();

    incomplete_ = std::make_unique<Shard>(app_, *this, *shardIndex, j_);
    if (!incomplete_->open(scheduler_, *ctx_))
    {
        incomplete_.reset();
        return boost::none;
    }

    return incomplete_->prepare();
}

bool
DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
{
    std::lock_guard lock(m_);
    assert(init_);

    auto fail = [j = j_, shardIndex](std::string const& msg)
    {
        JLOG(j.error()) << "shard " << shardIndex << " " << msg;
        return false;
    };

    if (!canAdd_)
        return fail("cannot be stored at this time");

    if (shardIndex < earliestShardIndex())
    {
        return fail("comes before earliest shard index " +
            std::to_string(earliestShardIndex()));
    }

    // If we are synced to the network, check if the shard index
    // is greater or equal to the current shard.
    auto seqCheck = [&](std::uint32_t seq)
    {
        // seq will be greater than zero if valid
        if (seq > earliestSeq() && shardIndex >= seqToShardIndex(seq))
            return fail("has an invalid index");
        return true;
    };
    if (!seqCheck(app_.getLedgerMaster().getValidLedgerIndex()) ||
        !seqCheck(app_.getLedgerMaster().getCurrentLedgerIndex()))
    {
        return false;
    }

    if (complete_.find(shardIndex) != complete_.end())
    {
        JLOG(j_.debug()) << "shard " << shardIndex << " is already stored";
        return false;
    }
    if (incomplete_ && incomplete_->index() == shardIndex)
    {
        JLOG(j_.debug()) << "shard " << shardIndex << " is being acquired";
        return false;
    }
    if (preShards_.find(shardIndex) != preShards_.end())
    {
        JLOG(j_.debug()) <<
            "shard " << shardIndex << " is already prepared for import";
        return false;
    }

    // Check limit and space requirements
    if (backed_)
    {
        std::uint64_t const sz {
            (preShards_.size() + 1 + (incomplete_ ? 1 : 0)) * avgShardFileSz_};
        if (fileSz_ + sz > maxFileSz_)
        {
            JLOG(j_.debug()) <<
                "shard " << shardIndex << " exceeds the maximum storage size";
            return false;
        }
        if (sz > available())
            return fail("insufficient storage space available");
    }

    // Add to shards prepared
    preShards_.emplace(shardIndex, nullptr);
    return true;
}

void
DatabaseShardImp::removePreShard(std::uint32_t shardIndex)
{
    std::lock_guard lock(m_);
    assert(init_);

    preShards_.erase(shardIndex);
}

std::string
DatabaseShardImp::getPreShards()
{
    RangeSet<std::uint32_t> rs;
    {
        std::lock_guard lock(m_);
        assert(init_);

        if (preShards_.empty())
            return {};
        for (auto const& ps : preShards_)
            rs.insert(ps.first);
    }
    return to_string(rs);
};

bool
DatabaseShardImp::importShard(std::uint32_t shardIndex,
    boost::filesystem::path const& srcDir, bool validate)
{
    using namespace boost::filesystem;
    try
    {
        if (!is_directory(srcDir) || is_empty(srcDir))
        {
            JLOG(j_.error()) << "invalid source directory " << srcDir.string();
            return false;
        }
    }
    catch (std::exception const& e)
    {
        JLOG(j_.error()) <<
            "exception " << e.what() << " in function " << __func__;
        return false;
    }

    auto move = [&](path const& src, path const& dst)
    {
        try
        {
            rename(src, dst);
        }
        catch (std::exception const& e)
        {
            JLOG(j_.error()) <<
                "exception " << e.what() << " in function " << __func__;
            return false;
        }
        return true;
    };

    std::unique_lock<std::mutex> lock(m_);
    assert(init_);

    // Check shard is prepared
    auto it {preShards_.find(shardIndex)};
    if(it == preShards_.end())
    {
        JLOG(j_.error()) << "shard " << shardIndex << " is an invalid index";
        return false;
    }

    // Move source directory to the shard database directory
    auto const dstDir {dir_ / std::to_string(shardIndex)};
    if (!move(srcDir, dstDir))
        return false;

    // Create the new shard
    auto shard {std::make_unique<Shard>(app_, *this, shardIndex, j_)};
    auto fail = [&](std::string const& msg)
    {
        if (!msg.empty())
        {
            JLOG(j_.error()) << "shard " << shardIndex << " " << msg;
        }
        shard.reset();
        move(dstDir, srcDir);
        return false;
    };

    if (!shard->open(scheduler_, *ctx_))
        return fail({});
    if (!shard->complete())
        return fail("is incomplete");

    try
    {
        // Verify database integrity
        shard->getBackend()->verify();
    }
    catch (std::exception const& e)
    {
        return fail(std::string("exception ") +
            e.what() + " in function " + __func__);
    }

    // Validate shard ledgers
    if (validate)
    {
        // Shard validation requires releasing the lock
        // so the database can fetch data from it
        it->second = shard.get();
        lock.unlock();
        auto const valid {shard->validate()};
        lock.lock();
        if (!valid)
        {
            it = preShards_.find(shardIndex);
            if(it != preShards_.end())
                it->second = nullptr;
            return fail("failed validation");
        }
    }

    // Add the shard
    complete_.emplace(shardIndex, std::move(shard));
    preShards_.erase(shardIndex);

    std::lock_guard lockg(*lock.release(), std::adopt_lock);
    setFileStats(lockg);
    updateStatus(lockg);
    return true;
}

std::shared_ptr<Ledger>
DatabaseShardImp::fetchLedger(uint256 const& hash, std::uint32_t seq)
{
    if (!contains(seq))
        return {};

    auto nObj = fetch(hash, seq);
    if (!nObj)
        return {};

    auto fail = [this, seq](std::string const& msg)-> std::shared_ptr<Ledger>
    {
        JLOG(j_.error()) << "shard " << seqToShardIndex(seq) << " " << msg;
        return {};
    };

    auto ledger {std::make_shared<Ledger>(
        InboundLedger::deserializeHeader(makeSlice(nObj->getData()), true),
        app_.config(),
        *app_.shardFamily())};

    if (ledger->info().seq != seq)
    {
        return fail("encountered invalid ledger sequence " +
            std::to_string(seq));
    }
    if (ledger->info().hash != hash)
    {
        return fail("encountered invalid ledger hash " +
            to_string(hash) + " on sequence " + std::to_string(seq));
    }

    ledger->setFull();
    if (!ledger->stateMap().fetchRoot(
        SHAMapHash {ledger->info().accountHash}, nullptr))
    {
        return fail("is missing root STATE node on hash " +
            to_string(hash) + " on sequence " + std::to_string(seq));
    }

    if (ledger->info().txHash.isNonZero())
    {
        if (!ledger->txMap().fetchRoot(
            SHAMapHash {ledger->info().txHash}, nullptr))
        {
            return fail("is missing root TXN node on hash " +
                to_string(hash) + " on sequence " + std::to_string(seq));
        }
    }
    return ledger;
}

void
DatabaseShardImp::setStored(std::shared_ptr<Ledger const> const& ledger)
{
    auto const shardIndex {seqToShardIndex(ledger->info().seq)};
    auto fail = [j = j_, shardIndex](std::string const& msg)
    {
        JLOG(j.error()) << "shard " << shardIndex << " " << msg;
    };

    if (ledger->info().hash.isZero())
    {
        return fail("encountered a zero ledger hash on sequence " +
            std::to_string(ledger->info().seq));
    }
    if (ledger->info().accountHash.isZero())
    {
        return fail("encountered a zero account hash on sequence " +
            std::to_string(ledger->info().seq));
    }

    std::lock_guard lock(m_);
    assert(init_);

    if (!incomplete_ || shardIndex != incomplete_->index())
    {
        return fail("ledger sequence " + std::to_string(ledger->info().seq) +
            " is not being acquired");
    }
    if (!incomplete_->setStored(ledger))
        return;
    if (incomplete_->complete())
    {
        complete_.emplace(incomplete_->index(), std::move(incomplete_));
        incomplete_.reset();
        updateStatus(lock);

        // Update peers with new shard index
        protocol::TMPeerShardInfo message;
        PublicKey const& publicKey {app_.nodeIdentity().first};
        message.set_nodepubkey(publicKey.data(), publicKey.size());
        message.set_shardindexes(std::to_string(shardIndex));
        app_.overlay().foreach(send_always(
            std::make_shared<Message>(message, protocol::mtPEER_SHARD_INFO)));
    }

    setFileStats(lock);
}

bool
DatabaseShardImp::contains(std::uint32_t seq)
{
    auto const shardIndex {seqToShardIndex(seq)};
    std::lock_guard lock(m_);
    assert(init_);

    if (complete_.find(shardIndex) != complete_.end())
        return true;
    if (incomplete_ && incomplete_->index() == shardIndex)
        return incomplete_->contains(seq);
    return false;
}

std::string
DatabaseShardImp::getCompleteShards()
{
    std::lock_guard lock(m_);
    assert(init_);

    return status_;
}

void
DatabaseShardImp::validate()
{
    std::vector<std::shared_ptr<Shard>> completeShards;
    {
        std::lock_guard lock(m_);
        assert(init_);

        if (complete_.empty())
        {
            JLOG(j_.error()) << "no shards found to validate";
            return;
        }

        JLOG(j_.debug()) << "Validating shards " << status_;

        completeShards.reserve(complete_.size());
        for (auto const& shard : complete_)
            completeShards.push_back(shard.second);
    }

    // Verify each complete stored shard
    for (auto const& shard : completeShards)
        shard->validate();

    app_.shardFamily()->reset();
}

void
DatabaseShardImp::import(Database& source)
{
    {
        std::lock_guard lock(m_);
        assert(init_);

        // Only the application local node store can be imported
        if (&source != &app_.getNodeStore())
        {
            assert(false);
            JLOG(j_.error()) << "invalid source database";
            return;
        }

        std::uint32_t earliestIndex;
        std::uint32_t latestIndex;
        {
            auto loadLedger = [&](bool ascendSort = true) ->
                boost::optional<std::uint32_t>
            {
                std::shared_ptr<Ledger> ledger;
                std::uint32_t seq;
                std::tie(ledger, seq, std::ignore) = loadLedgerHelper(
                    "WHERE LedgerSeq >= " + std::to_string(earliestSeq()) +
                    " order by LedgerSeq " + (ascendSort ? "asc" : "desc") +
                    " limit 1", app_, false);
                if (!ledger || seq == 0)
                {
                    JLOG(j_.error()) <<
                        "no suitable ledgers were found in"
                        " the SQLite database to import";
                    return boost::none;
                }
                return seq;
            };

            // Find earliest ledger sequence stored
            auto seq {loadLedger()};
            if (!seq)
                return;
            earliestIndex = seqToShardIndex(*seq);

            // Consider only complete shards
            if (seq != firstLedgerSeq(earliestIndex))
                ++earliestIndex;

            // Find last ledger sequence stored
            seq = loadLedger(false);
            if (!seq)
                return;
            latestIndex = seqToShardIndex(*seq);

            // Consider only complete shards
            if (seq != lastLedgerSeq(latestIndex))
                --latestIndex;

            if (latestIndex < earliestIndex)
            {
                JLOG(j_.error()) <<
                    "no suitable ledgers were found in"
                    " the SQLite database to import";
                return;
            }
        }

        // Import the shards
        for (std::uint32_t shardIndex = earliestIndex;
            shardIndex <= latestIndex; ++shardIndex)
        {
            if (fileSz_ + avgShardFileSz_ > maxFileSz_)
            {
                JLOG(j_.error()) << "maximum storage size reached";
                canAdd_ = false;
                break;
            }
            if (avgShardFileSz_ > available())
            {
                JLOG(j_.error()) << "insufficient storage space available";
                canAdd_ = false;
                break;
            }

            // Skip if already stored
            if (complete_.find(shardIndex) != complete_.end() ||
                (incomplete_ && incomplete_->index() == shardIndex))
            {
                JLOG(j_.debug()) << "shard " << shardIndex << " already exists";
                continue;
            }

            // Verify SQLite ledgers are in the node store
            {
                auto const firstSeq {firstLedgerSeq(shardIndex)};
                auto const lastSeq {
                    std::max(firstSeq, lastLedgerSeq(shardIndex))};
                auto const numLedgers {shardIndex == earliestShardIndex()
                    ? lastSeq - firstSeq + 1 : ledgersPerShard_};
                auto ledgerHashes{getHashesByIndex(firstSeq, lastSeq, app_)};
                if (ledgerHashes.size() != numLedgers)
                    continue;

                bool valid {true};
                for (std::uint32_t n = firstSeq; n <= lastSeq; n += 256)
                {
                    if (!source.fetch(ledgerHashes[n].first, n))
                    {
                        JLOG(j_.warn()) <<
                            "SQLite ledger sequence " << n <<
                            " mismatches node store";
                        valid = false;
                        break;
                    }
                }
                if (!valid)
                    continue;
            }

            // Create the new shard
            app_.shardFamily()->reset();
            auto const shardDir {dir_ / std::to_string(shardIndex)};
            auto shard {std::make_unique<Shard>(app_, *this, shardIndex, j_)};
            if (!shard->open(scheduler_, *ctx_))
            {
                shard.reset();
                continue;
            }

            // Create a marker file to signify an import in progress
            auto const markerFile {shardDir / importMarker_};
            std::ofstream ofs {markerFile.string()};
            if (!ofs.is_open())
            {
                JLOG(j_.error()) <<
                    "shard " << shardIndex <<
                    " is unable to create temp marker file";
                shard.reset();
                removeAll(shardDir, j_);
                continue;
            }
            ofs.close();

            // Copy the ledgers from node store
            while (auto seq = shard->prepare())
            {
                auto ledger = loadByIndex(*seq, app_, false);
                if (!ledger || ledger->info().seq != seq ||
                    !Database::copyLedger(*shard->getBackend(), *ledger,
                        nullptr, nullptr, shard->lastStored()))
                    break;

                if (!shard->setStored(ledger))
                    break;
                if (shard->complete())
                {
                    JLOG(j_.debug()) <<
                        "shard " << shardIndex << " was successfully imported";
                    removeAll(markerFile, j_);
                    break;
                }
            }

            if (!shard->complete())
            {
                JLOG(j_.error()) <<
                    "shard " << shardIndex << " failed to import";
                shard.reset();
                removeAll(shardDir, j_);
            }
            else
                setFileStats(lock);
        }

        // Re initialize the shard store
        init_ = false;
        complete_.clear();
        incomplete_.reset();
    }

    if (!init())
        Throw<std::runtime_error>("import: failed to initialize");
}

std::int32_t
DatabaseShardImp::getWriteLoad() const
{
    std::int32_t wl {0};
    {
        std::lock_guard lock(m_);
        assert(init_);

        for (auto const& e : complete_)
            wl += e.second->getBackend()->getWriteLoad();
        if (incomplete_)
            wl += incomplete_->getBackend()->getWriteLoad();
    }
    return wl;
}

void
DatabaseShardImp::store(NodeObjectType type,
    Blob&& data, uint256 const& hash, std::uint32_t seq)
{
#if RIPPLE_VERIFY_NODEOBJECT_KEYS
    assert(hash == sha512Hash(makeSlice(data)));
#endif
    std::shared_ptr<NodeObject> nObj;
    auto const shardIndex {seqToShardIndex(seq)};
    {
        std::lock_guard lock(m_);
        assert(init_);

        if (!incomplete_ || shardIndex != incomplete_->index())
        {
            JLOG(j_.warn()) <<
                "shard " << shardIndex <<
                " ledger sequence " << seq <<
                " is not being acquired";
            return;
        }
        nObj = NodeObject::createObject(
            type, std::move(data), hash);
        incomplete_->pCache()->canonicalize(hash, nObj, true);
        incomplete_->getBackend()->store(nObj);
        incomplete_->nCache()->erase(hash);
    }
    storeStats(nObj->getData().size());
}

std::shared_ptr<NodeObject>
DatabaseShardImp::fetch(uint256 const& hash, std::uint32_t seq)
{
    auto cache {selectCache(seq)};
    if (cache.first)
        return doFetch(hash, seq, *cache.first, *cache.second, false);
    return {};
}

bool
DatabaseShardImp::asyncFetch(uint256 const& hash,
    std::uint32_t seq, std::shared_ptr<NodeObject>& object)
{
    auto cache {selectCache(seq)};
    if (cache.first)
    {
        // See if the object is in cache
        object = cache.first->fetch(hash);
        if (object || cache.second->touch_if_exists(hash))
            return true;
        // Otherwise post a read
        Database::asyncFetch(hash, seq, cache.first, cache.second);
    }
    return false;
}

bool
DatabaseShardImp::copyLedger(std::shared_ptr<Ledger const> const& ledger)
{
    auto const shardIndex {seqToShardIndex(ledger->info().seq)};
    std::lock_guard lock(m_);
    assert(init_);

    if (!incomplete_ || shardIndex != incomplete_->index())
    {
        JLOG(j_.warn()) <<
            "shard " << shardIndex <<
            " source ledger sequence " << ledger->info().seq <<
            " is not being acquired";
        return false;
    }

    if (!Database::copyLedger(*incomplete_->getBackend(), *ledger,
        incomplete_->pCache(), incomplete_->nCache(),
        incomplete_->lastStored()))
    {
        return false;
    }

    if (!incomplete_->setStored(ledger))
        return false;
    if (incomplete_->complete())
    {
        complete_.emplace(incomplete_->index(), std::move(incomplete_));
        incomplete_.reset();
        updateStatus(lock);
    }

    setFileStats(lock);
    return true;
}

int
DatabaseShardImp::getDesiredAsyncReadCount(std::uint32_t seq)
{
    auto const shardIndex {seqToShardIndex(seq)};
    {
        std::lock_guard lock(m_);
        assert(init_);

        auto it = complete_.find(shardIndex);
        if (it != complete_.end())
            return it->second->pCache()->getTargetSize() / asyncDivider;
        if (incomplete_ && incomplete_->index() == shardIndex)
            return incomplete_->pCache()->getTargetSize() / asyncDivider;
    }
    return cacheTargetSize / asyncDivider;
}

float
DatabaseShardImp::getCacheHitRate()
{
    float sz, f {0};
    {
        std::lock_guard lock(m_);
        assert(init_);

        sz = complete_.size();
        for (auto const& e : complete_)
            f += e.second->pCache()->getHitRate();
        if (incomplete_)
        {
            f += incomplete_->pCache()->getHitRate();
            ++sz;
        }
    }
    return f / std::max(1.0f, sz);
}

void
DatabaseShardImp::sweep()
{
    std::lock_guard lock(m_);
    assert(init_);

    for (auto const& e : complete_)
        e.second->sweep();

    if (incomplete_)
        incomplete_->sweep();
}

std::shared_ptr<NodeObject>
DatabaseShardImp::fetchFrom(uint256 const& hash, std::uint32_t seq)
{
    auto const shardIndex {seqToShardIndex(seq)};
    std::unique_lock<std::mutex> lock(m_);
    assert(init_);
    {
        auto it = complete_.find(shardIndex);
        if (it != complete_.end())
        {
            lock.unlock();
            return fetchInternal(hash, *it->second->getBackend());
        }
    }
    if (incomplete_ && incomplete_->index() == shardIndex)
    {
        lock.unlock();
        return fetchInternal(hash, *incomplete_->getBackend());
    }

    // Used to validate import shards
    auto it = preShards_.find(shardIndex);
    if (it != preShards_.end() && it->second)
    {
        lock.unlock();
        return fetchInternal(hash, *it->second->getBackend());
    }
    return {};
}

boost::optional<std::uint32_t>
DatabaseShardImp::findShardIndexToAdd(
    std::uint32_t validLedgerSeq, std::lock_guard<std::mutex>&)
{
    auto const maxShardIndex {[this, validLedgerSeq]()
    {
        auto shardIndex {seqToShardIndex(validLedgerSeq)};
        if (validLedgerSeq != lastLedgerSeq(shardIndex))
            --shardIndex;
        return shardIndex;
    }()};
    auto const numShards {complete_.size() +
        (incomplete_ ? 1 : 0) + preShards_.size()};

    // Check if the shard store has all shards
    if (numShards >= maxShardIndex)
        return boost::none;

    if (maxShardIndex < 1024 ||
        static_cast<float>(numShards) / maxShardIndex > 0.5f)
    {
        // Small or mostly full index space to sample
        // Find the available indexes and select one at random
        std::vector<std::uint32_t> available;
        available.reserve(maxShardIndex - numShards + 1);

        for (auto shardIndex = earliestShardIndex();
            shardIndex <= maxShardIndex;
            ++shardIndex)
        {
            if (complete_.find(shardIndex) == complete_.end() &&
                (!incomplete_ || incomplete_->index() != shardIndex) &&
                preShards_.find(shardIndex) == preShards_.end())
            {
                available.push_back(shardIndex);
            }
        }

        if (available.empty())
            return boost::none;

        if (available.size() == 1)
            return available.front();

        return available[rand_int(0u,
            static_cast<std::uint32_t>(available.size() - 1))];
    }

    // Large, sparse index space to sample
    // Keep choosing indexes at random until an available one is found
    // chances of running more than 30 times is less than 1 in a billion
    for (int i = 0; i < 40; ++i)
    {
        auto const shardIndex {rand_int(earliestShardIndex(), maxShardIndex)};
        if (complete_.find(shardIndex) == complete_.end() &&
            (!incomplete_ || incomplete_->index() != shardIndex) &&
            preShards_.find(shardIndex) == preShards_.end())
        {
            return shardIndex;
        }
    }

    assert(false);
    return boost::none;
}

void
DatabaseShardImp::setFileStats(std::lock_guard<std::mutex>&)
{
    fileSz_ = 0;
    fdRequired_ = 0;
    if (!complete_.empty())
    {
        for (auto const& e : complete_)
        {
            fileSz_ += e.second->fileSize();
            fdRequired_ += e.second->fdRequired();
        }
        avgShardFileSz_ = fileSz_ / complete_.size();
    }
    else
        avgShardFileSz_ = 0;

    if (incomplete_)
    {
        fileSz_ += incomplete_->fileSize();
        fdRequired_ += incomplete_->fdRequired();
    }

    if (!backed_)
        return;

    // Require at least 15 file descriptors
    fdRequired_ = std::max(fdRequired_, 15);

    if (fileSz_ >= maxFileSz_)
    {
        JLOG(j_.warn()) << "maximum storage size reached";
        canAdd_ = false;
    }
    else if (maxFileSz_ - fileSz_ > available())
    {
        JLOG(j_.warn()) <<
            "maximum shard store size exceeds available storage space";
    }
}

void
DatabaseShardImp::updateStatus(std::lock_guard<std::mutex>&)
{
    if (!complete_.empty())
    {
        RangeSet<std::uint32_t> rs;
        for (auto const& e : complete_)
            rs.insert(e.second->index());
        status_ = to_string(rs);
    }
    else
        status_.clear();
}

std::pair<std::shared_ptr<PCache>, std::shared_ptr<NCache>>
DatabaseShardImp::selectCache(std::uint32_t seq)
{
    auto const shardIndex {seqToShardIndex(seq)};
    std::lock_guard lock(m_);
    assert(init_);

    {
        auto it = complete_.find(shardIndex);
        if (it != complete_.end())
        {
            return std::make_pair(it->second->pCache(),
                it->second->nCache());
        }
    }

    if (incomplete_ && incomplete_->index() == shardIndex)
    {
        return std::make_pair(incomplete_->pCache(),
            incomplete_->nCache());
    }

    // Used to validate import shards
    auto it = preShards_.find(shardIndex);
    if (it != preShards_.end() && it->second)
        return std::make_pair(it->second->pCache(), it->second->nCache());
    return {};
}

std::uint64_t
DatabaseShardImp::available() const
{
    try
    {
        return boost::filesystem::space(dir_).available;
    }
    catch (std::exception const& e)
    {
        JLOG(j_.error()) << "exception " << e.what() <<
            " in function " << __func__;
        return 0;
    }
}

//------------------------------------------------------------------------------

std::unique_ptr<DatabaseShard>
make_ShardStore(
    Application& app,
    Stoppable& parent,
    Scheduler& scheduler,
    int readThreads,
    beast::Journal j)
{
    // The shard store is optional. Future changes will require it.
    Section const& section {
        app.config().section(ConfigSection::shardDatabase())};
    if (section.empty())
        return nullptr;

    auto shardStore = std::make_unique<DatabaseShardImp>(
        app,
        parent,
        "ShardStore",
        scheduler,
        readThreads,
        j);
    if (shardStore->init())
        shardStore->setParent(parent);
    else
        shardStore.reset();

    return shardStore;
}

} // NodeStore
} // ripple

max_stars_repo_name: ripplealpha/ripple-alpha-core

max_issues_repo_name: ripplealpha/ripple-alpha-core

max_forks_repo_name: ripplealpha/ripple-alpha-core