text:
//			Copyright Gavin Band 2008 - 2012.
// Distributed under the Boost Software License, Version 1.0.
//	  (See accompanying file LICENSE_1_0.txt or copy at
//			http://www.boost.org/LICENSE_1_0.txt)

#include <string>
#include <memory>
#include <boost/shared_ptr.hpp>
#include <boost/tuple/tuple.hpp>
#include <boost/thread/thread.hpp>
#include <boost/format.hpp>
#include "genfile/VariantIdentifyingData.hpp"
#include "genfile/VariantEntry.hpp"
#include "genfile/Error.hpp"
#include "genfile/db/Connection.hpp"
#include "genfile/db/SQLStatement.hpp"
#include "appcontext/get_current_time_as_string.hpp"
#include "qcdb/FlatTableMultiVariantDBOutputter.hpp"

// #define DEBUG_FLATTABLEDBOUTPUTTER 1

namespace qcdb {
	namespace {
		std::vector< std::string > generate_key_entry_names( std::size_t n ) {
			std::vector< std::string > result(n) ;
			for( std::size_t i = 0; i < n; ++i ) {
				result[i] = (boost::format( "variant%d" ) % (i+1)).str() ;
			}
			return result ;
		}
	}

	FlatTableMultiVariantDBOutputter::UniquePtr FlatTableMultiVariantDBOutputter::create(
		std::string const& filename,
		std::size_t number_of_key_fields,
		std::string const& analysis_name,
		std::string const& analysis_description,
		Metadata const& metadata,
		std::string const& snp_match_fields,
		boost::optional< genfile::db::Connection::RowId > analysis_id
	) {
		return UniquePtr(
			new FlatTableMultiVariantDBOutputter( filename, number_of_key_fields, analysis_name, analysis_description, metadata, snp_match_fields, analysis_id )
		) ;
	}

	FlatTableMultiVariantDBOutputter::FlatTableMultiVariantDBOutputter(
		std::string const& filename,
		std::size_t number_of_key_fields,
		std::string const& analysis_name,
		std::string const& analysis_description,
		Metadata const& metadata,
		std::string const& snp_match_fields,
		boost::optional< genfile::db::Connection::RowId > analysis_id
	):
		m_outputter( filename, analysis_name, analysis_description, metadata, analysis_id, snp_match_fields ),
		m_key_entry_names( generate_key_entry_names( number_of_key_fields )),
		m_table_name( "analysis" + genfile::string_utils::to_string( m_outputter.analysis_id() ) ),
		m_without_rowid( false ),
		m_max_variants_per_block( 1000 )
	{}

	FlatTableMultiVariantDBOutputter::~FlatTableMultiVariantDBOutputter() {
	}

	void FlatTableMultiVariantDBOutputter::set_table_name( std::string const& table_name ) {
		if( table_name.find_first_not_of( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_" ) != std::string::npos ) {
			throw genfile::BadArgumentError( "qcdb::FlatTableMultiVariantDBOutputter::set_table_name()", "table_name=\"" + table_name + "\"" ) ;
		}
		m_table_name = table_name ;
	}

	void FlatTableMultiVariantDBOutputter::set_without_rowid() {
		m_without_rowid = true ;
	}

	FlatTableMultiVariantDBOutputter::AnalysisId FlatTableMultiVariantDBOutputter::analysis_id() const {
		return m_outputter.analysis_id() ;
	}

	void FlatTableMultiVariantDBOutputter::finalise( long options ) {
		store_block() ;
		m_keys.clear() ;
		m_values.clear() ;

		if( (options & qcdb::eCreateIndices) && (!m_without_rowid) && (m_key_entry_names.size() > 0) ) {
			genfile::db::Connection::ScopedTransactionPtr transaction = m_outputter.connection().open_transaction( 7200 ) ;
			std::string sql = "CREATE INDEX IF NOT EXISTS " + m_table_name + "_index ON `" + m_table_name + "` (";
			for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
				sql += (i>0 ? ", `" : " `" ) + (m_key_entry_names[i] + "_id`" ) ;
			}
			sql += ")" ;
			m_outputter.connection().run_statement( sql ) ;
		}

		m_outputter.finalise( options ) ;
	}

	void FlatTableMultiVariantDBOutputter::set_variant_names( std::vector< std::string > const& names ) {
		// Ensure we don't change names once we have begun writing data
		if( names != m_key_entry_names && m_insert_data_sql.get() ) {
			throw genfile::BadArgumentError(
				"qcdb::FlatTableMultiVariantDBOutputter::set_variant_names()",
				"names",
				"Can't change table column names because data has already been written."
			) ;
		}
		m_key_entry_names = names ;
	}

	void FlatTableMultiVariantDBOutputter::add_variable( std::string const& variable ) {
		VariableMap::left_const_iterator where = m_variables.left.find( variable ) ;
		if( where == m_variables.left.end() ) {
			if( m_insert_data_sql.get() ) {
				// Uh-oh, table columns are already fixed.
				// TODO: alter to put this data in SummaryData table?
				throw genfile::BadArgumentError( "qcdb::FlatTableMultiVariantDBOutputter::add_variable()", "variable=\"" + variable + "\"" ) ;
			}
			else {
				// Still have time to add the variable to our list of variables, retaining the order of addition.
				m_variables.left.insert( VariableMap::left_value_type( variable, m_variables.size() ) ) ;
			}
		}
	}

	void FlatTableMultiVariantDBOutputter::create_new_key(
		Key const& key
	 ) {
		if( key.size() > m_key_entry_names.size() ) {
			throw genfile::BadArgumentError(
				"qcdb::FlatTableMultiVariantDBOutputter::create_new_key()",
				"key",
				(
					boost::format(
						"number of fields is larger than expected (%d, expected at most %d)"
					) % key.size() % m_key_entry_names.size()
				).str()
			) ;
		}
 		if( m_keys.size() == m_max_variants_per_block ) {
 			store_block() ;
 			m_keys.clear() ;
 			m_values.clear() ;
 		}
		m_keys.push_back( key ) ;
	}

	void FlatTableMultiVariantDBOutputter::store_data_for_key(
		Key const& key,
		std::string const& variable,
		genfile::VariantEntry const& value
	) {
		bool const new_key = m_keys.empty() || key != m_keys.back() ;
		if( new_key ) {
			create_new_key( key ) ;
		}

		VariableMap::left_const_iterator where = m_variables.left.find( variable ) ;
		if( where == m_variables.left.end() ) {
			if( m_insert_data_sql.get() ) {
				// Uh-oh, table columns are already fixed.
				// TODO: alter to put this data in SummaryData table?
				throw genfile::BadArgumentError( "qcdb::FlatTableMultiVariantDBOutputter::store_per_variant_pair_data()", "variable=\"" + variable + "\"" ) ;
			}
			else {
				// Still have time to add the variable to our list of variables, retaining the order of addition.
				where = m_variables.left.insert( VariableMap::left_value_type( variable, m_variables.size() ) ).first ;
			}
		}

		// Store the value of this variable
		m_values[ std::make_pair( m_keys.size() - 1, where->second ) ] = value ;
	}

	void FlatTableMultiVariantDBOutputter::store_block() {
		genfile::db::Connection::ScopedTransactionPtr transaction = m_outputter.connection().open_transaction( 7200 ) ;

		if( !m_insert_data_sql.get() ) {
#if DEBUG_FLATTABLEDBOUTPUTTER
			std::cerr << "FlatTableMultiVariantDBOutputter::store_block(): creating schema...\n" ;
#endif
			create_schema() ;
#if DEBUG_FLATTABLEDBOUTPUTTER
			std::cerr << "FlatTableMultiVariantDBOutputter::store_block(): ok.\n" ;
#endif
			// create_variables() ;
		}
		std::vector< genfile::db::Connection::RowId > variant_ids ;
		for( std::size_t key_i = 0; key_i < m_keys.size(); ++key_i ) {
			variant_ids.resize( m_keys[key_i].size() ) ;
			for( std::size_t i = 0; i < m_keys[key_i].size(); ++i ) {
				variant_ids[i] = m_outputter.get_or_create_variant( m_keys[key_i][i] ) ;
			}
			store_data_for_variants( key_i, m_outputter.analysis_id(), variant_ids ) ;
		}
	}

	std::string FlatTableMultiVariantDBOutputter::get_table_name() const {
		return m_table_name ;
	}

	void FlatTableMultiVariantDBOutputter::create_schema() {
		using genfile::string_utils::to_string ;
		std::ostringstream schema_sql ;
		std::ostringstream index_sql ;
		std::ostringstream insert_data_sql ;
		std::ostringstream insert_data_sql_columns ;
		std::ostringstream insert_data_sql_values ;
		std::string const& table_name = m_table_name ;

		schema_sql << "CREATE TABLE IF NOT EXISTS "
			<< table_name
			<< " ( "
			"analysis_id INT NOT NULL REFERENCES Entity( id )"
				;
		for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
			schema_sql
				<< ", `" << m_key_entry_names[i] << "_id` INT NOT NULL REFERENCES Variant( id )" ;
		}

		insert_data_sql << "INSERT INTO "
			<< table_name ;
		insert_data_sql_columns << "( analysis_id" ;
		insert_data_sql_values << "VALUES( ?1" ;

		for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
			insert_data_sql_columns << ", `" << m_key_entry_names[i] << "_id`" ;
			insert_data_sql_values << boost::format( ", ?%d") % (i+2) ;
		}
		//insert_data_sql_columns << ")" ;
		//insert_data_sql_values << ")" ;

		VariableMap::right_const_iterator
			var_i = m_variables.right.begin(),
			end_var_i = m_variables.right.end() ;

		for( std::size_t bind_i = m_key_entry_names.size()+2; var_i != end_var_i; ++var_i, ++bind_i ) {
			schema_sql << ", " ;
			insert_data_sql_columns << ", " ;
			insert_data_sql_values << ", " ;
			schema_sql
				<< '`'
				<< var_i->second
				<< '`'
				<< " NULL" ;

			insert_data_sql_columns << '`' << var_i->second << '`' ;
			insert_data_sql_values << "?" << to_string( bind_i ) ;
		}

		if( m_without_rowid ) {
			schema_sql << ", PRIMARY KEY( `analysis_id`" ;
			for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
				schema_sql  << ", `" << m_key_entry_names[i] << "_id`" ;
			}
			schema_sql << " ) ) WITHOUT ROWID ;" ;
		} else {
			schema_sql << " ) ; " ;
		}

		insert_data_sql_columns << ") " ;
		insert_data_sql_values << ") ; " ;

#if DEBUG_FLATTABLEDBOUTPUTTER
		std::cerr << "Creating table " << table_name << " using this SQL:\n"
			<< schema_sql.str()
			<< "\n" ;
#endif
		m_outputter.connection().run_statement( schema_sql.str() ) ;

		std::ostringstream view_sql ;
		view_sql
			<< "CREATE VIEW IF NOT EXISTS `" << table_name << "View` AS SELECT A.name AS `analysis`, "
		;
		for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
			view_sql << boost::format( " V%d.rsid AS `%s_rsid`, V%d.chromosome AS `%s_chromosome`, V%d.position AS `%s_position`," )
				% (i+1)
				% m_key_entry_names[i]
				% (i+1)
				% m_key_entry_names[i]
				% (i+1)
				% m_key_entry_names[i]
			;
		}
		view_sql  << "T.* FROM `"
			<< table_name << "` T " ;
		for( std::size_t i = 0; i < m_key_entry_names.size(); ++i ) {
			view_sql << (boost::format( "INNER JOIN Variant V%d ON V%d.id = " + m_key_entry_names[i] + "_id " )
				% (i+1) % (i+1) ) ;
		}
		view_sql << "INNER JOIN Analysis A ON A.id = T.analysis_id" ;

#if DEBUG_FLATTABLEDBOUTPUTTER
		std::cerr << "Creating view using SQL:\n" << view_sql.str() << "\n" ;
#endif
		m_outputter.connection().run_statement( view_sql.str() ) ;

		insert_data_sql << " " << insert_data_sql_columns.str() << " " << insert_data_sql_values.str() ;
#if DEBUG_FLATTABLEDBOUTPUTTER
		std::cerr << "Inserts will use this SQL:\n"
			<< insert_data_sql.str()
			<< "\n" ;
#endif

		m_insert_data_sql = m_outputter.connection().get_statement(
			insert_data_sql.str()
		) ;
	}

	void FlatTableMultiVariantDBOutputter::store_data_for_variants(
		std::size_t const key_i,
		genfile::db::Connection::RowId const analysis_id,
		std::vector< genfile::db::Connection::RowId > const& variant_ids
	) {
		m_insert_data_sql->bind( 1, analysis_id ) ;
		std::size_t i = 0 ;
		for( ; i < variant_ids.size(); ++i ) {
			m_insert_data_sql->bind( i+2, variant_ids[i] ) ;
		}

		VariableMap::right_const_iterator
			var_i = m_variables.right.begin(),
			end_var_i = m_variables.right.end() ;

		for( std::size_t bind_i = i+2; var_i != end_var_i; ++var_i, ++bind_i ) {
			ValueMap::const_iterator where = m_values.find( std::make_pair( key_i, var_i->first )) ;
			if( where == m_values.end() ) {
				m_insert_data_sql->bind( bind_i, genfile::MissingValue() ) ;
			} else {
#if DEBUG_FLATTABLEDBOUTPUTTER
				std::cerr << "FlatTableMultiVariantDBOutputter::store_data_for_variants(): key " << key_i << ", bind( " << bind_i << ", " << where->second << " )\n" ;
#endif
				m_insert_data_sql->bind( bind_i, where->second ) ;
			}
		}

		m_insert_data_sql->step() ;
		m_insert_data_sql->reset() ;
	}
}

max_stars_repo_name: gavinband/qctool

max_issues_repo_name: gavinband/qctool

max_forks_repo_name: gavinband/qctool


text:
// Copyright (C) 2005-2006 Douglas Gregor <doug.gregor -at- gmail.com>.

// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// Message Passing Interface 1.1 -- Section 4.7. Gather-to-all
#ifndef BOOST_MPI_ALL_GATHER_HPP
#define BOOST_MPI_ALL_GATHER_HPP

#include <boost/mpi/exception.hpp>
#include <boost/mpi/datatype.hpp>
#include <vector>
#include <boost/serialization/vector.hpp>

// all_gather falls back to gather+broadcast in some cases
#include <boost/mpi/collectives/broadcast.hpp>
#include <boost/mpi/collectives/gather.hpp>

namespace boost { namespace mpi {

namespace detail {
  // We're all-gathering for a type that has an associated MPI
  // datatype, so we'll use MPI_Gather to do all of the work.
  template<typename T>
  void
  all_gather_impl(const communicator& comm, const T* in_values, int n,
                  T* out_values, mpl::true_)
  {
    MPI_Datatype type = boost::mpi::get_mpi_datatype<T>(*in_values);
    BOOST_MPI_CHECK_RESULT(MPI_Allgather,
                           (const_cast<T*>(in_values), n, type,
                            out_values, n, type, comm));
  }

  // We're all-gathering for a type that has no associated MPI
  // type. So, we'll do a manual gather followed by a broadcast.
  template<typename T>
  void
  all_gather_impl(const communicator& comm, const T* in_values, int n,
                  T* out_values, mpl::false_)
  {
    std::cerr << comm.rank() << ": gathering " << n << " values to root 0..." << std::endl;
    std::cerr << in_values << ", " << out_values << std::endl;
    gather(comm, in_values, n, out_values, 0);
    std::cerr << comm.rank() << ":  broadcasting from root 0..." << std::endl;
    broadcast(comm, out_values, comm.size() * n, 0);
  }
} // end namespace detail

template<typename T>
inline void
all_gather(const communicator& comm, const T& in_value, T* out_values)
{
  detail::all_gather_impl(comm, &in_value, 1, out_values, is_mpi_datatype<T>());
}

template<typename T>
void
all_gather(const communicator& comm, const T& in_value,
           std::vector<T>& out_values)
{
  out_values.resize(comm.size());
  ::boost::mpi::all_gather(comm, &in_value, 1, &out_values[0]);
}

template<typename T>
inline void
all_gather(const communicator& comm, const T* in_values, int n, T* out_values)
{
  detail::all_gather_impl(comm, in_values, n, out_values, is_mpi_datatype<T>());
}

template<typename T>
void
all_gather(const communicator& comm, const T* in_values, int n,
           std::vector<T>& out_values)
{
  out_values.resize(comm.size() * n);
  ::boost::mpi::all_gather(comm, in_values, n, &out_values[0]);
}

} } // end namespace boost::mpi

#endif // BOOST_MPI_ALL_GATHER_HPP

ext: hpp

max_stars_repo_name: jaredhoberock/gotham

max_issues_repo_name: foxostro/CheeseTesseract

max_forks_repo_name: foxostro/CheeseTesseract


text:
#pragma once

#include <memory>
#include <boost/filesystem.hpp>

class PathManager {
public:
	PathManager();
	PathManager(PathManager const &) = delete;
	~PathManager() = default;
	static PathManager &Get();

	/// \brief Add Path
	/// \details Store a Path linked to a name
	///				like "resources" linked to the resources path
	/// \param Name path
	/// \param Path
	void addPath(std::string const &name, boost::filesystem::path path);

	/// \brief Get Path linked to a name
	/// \details Shearch in map, the Path associated to this name
	///             If the name are not found, throw an exception
	/// \param Name of path
	/// \return Path
	/// \throw It throws out_of_range if "Name" is not the key of an element in the Path map.
	boost::filesystem::path &getPath(std::string const &name);

private:
	std::map< std::string, boost::filesystem::path > mapPaths_;

	static std::unique_ptr<PathManager> instance_;
};


text:
/* Copyright (c) AIRBUS and its affiliates.
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */
#ifndef SKDECIDE_IW_HH
#define SKDECIDE_IW_HH

#include <functional>
#include <memory>
#include <unordered_set>
#include <vector>
#include <queue>
#include <list>
#include <chrono>

#include <boost/container_hash/hash.hpp>

#include "spdlog/spdlog.h"
#include "spdlog/sinks/stdout_color_sinks.h"

#include "utils/associative_container_deducer.hh"
#include "utils/string_converter.hh"
#include "utils/execution.hh"

namespace skdecide {

/** Use default hasher provided with domain's states */
template <typename Tdomain, typename Tfeature_vector>
struct DomainStateHash {
    typedef typename Tdomain::State Key;

    template <typename Tnode>
    static const Key& get_key(const Tnode& n) {
        return n.state;
    }

    struct Hash {
        std::size_t operator()(const Key& k) const {
            return typename Tdomain::State::Hash()(k);
        }
    };

    struct Equal {
        bool operator()(const Key& k1, const Key& k2) const {
            return typename Tdomain::State::Equal()(k1, k2);
        }
    };
};

/** Use state binary feature vector to hash states */
template <typename Tdomain, typename Tfeature_vector>
struct StateFeatureHash {
    typedef Tfeature_vector Key;

    template <typename Tnode>
    static const Key& get_key(const Tnode& n) {
        return *n.features;
    }

    struct Hash {
        std::size_t operator()(const Key& k) const {
            std::size_t seed = 0;
            for (std::size_t i = 0 ; i < k.size() ; i++) {
                boost::hash_combine(seed, k[i]);
            }
            return seed;
        }
    };

    struct Equal {
        bool operator()(const Key& k1, const Key& k2) const {
            std::size_t size = k1.size();
            if (size != k2.size()) {
                return false;
            }
            for (std::size_t i = 0 ; i < size ; i++) {
                if (!(k1[i] == k2[i])) {
                    return false;
                }
            }
            return true;
        }
    };
};

template <typename Tdomain,
          typename Tfeature_vector,
          template <typename...> class Thashing_policy = DomainStateHash,
          typename Texecution_policy = SequentialExecution>
class IWSolver {
public :
    typedef Tdomain Domain;
    typedef typename Domain::State State;
    typedef typename Domain::Event Action;
    typedef Tfeature_vector FeatureVector;
    typedef Thashing_policy<Domain, FeatureVector> HashingPolicy;
    typedef Texecution_policy ExecutionPolicy;

    IWSolver(Domain& domain,
             const std::function<std::unique_ptr<FeatureVector> (Domain& d, const State& s)>& state_features,
             const std::function<bool (const double&, const std::size_t&, const std::size_t&,
                                       const double&, const std::size_t&, const std::size_t&)>& node_ordering = nullptr,
             std::size_t time_budget = 0,  // time budget to continue searching for better plans after a goal has been reached
             bool debug_logs = false)
    : _domain(domain), _state_features(state_features), _time_budget(time_budget), _debug_logs(debug_logs) {
        if (!node_ordering) {
            _node_ordering = [](const double& a_gscore, const std::size_t& a_novelty, const std::size_t&  a_depth,
                                const double& b_gscore, const std::size_t& b_novelty, const std::size_t& b_depth) -> bool {
                                    return a_gscore > b_gscore;
                                };
        } else {
            _node_ordering = node_ordering;
        }
        if (debug_logs) {
            spdlog::set_level(spdlog::level::debug);
        } else {
            spdlog::set_level(spdlog::level::info);
        }
    }

    // clears the solver (clears the search graph, thus preventing from reusing
    // previous search results)
    void clear() {
        _graph.clear();
    }

    // solves from state s
    void solve(const State& s) {
        try {
            spdlog::info("Running " + ExecutionPolicy::print_type() + " IW solver from state " + s.print());
            auto start_time = std::chrono::high_resolution_clock::now();
            std::size_t nb_of_binary_features = _state_features(_domain, s)->size();
            bool found_goal = false;
            _intermediate_scores.clear();

            for (std::size_t w = 1 ; w <= nb_of_binary_features ; w++) {
                std::pair<bool, bool> res = WidthSolver(_domain, _state_features, _node_ordering, w,
                                                        _graph, _time_budget, _intermediate_scores,
                                                        _debug_logs).solve(s, start_time, found_goal);
                if (res.first) { // solution found with width w
                    auto now_time = std::chrono::high_resolution_clock::now();
                    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now_time - start_time).count();

                    if (static_cast<std::size_t>(duration) < _time_budget) {
                        if (_debug_logs) spdlog::debug("Remaining time budget, trying to improve the solution by augmenting the search width");
                    } else {
                        auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now_time - start_time).count();
                        spdlog::info("IW finished to solve from state " + s.print() + " in " + StringConverter::from((double) duration / (double) 1e9) + " seconds.");
                        return;
                    }
                } else if (found_goal) {
                    auto now_time = std::chrono::high_resolution_clock::now();
                    auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now_time - start_time).count();
                    spdlog::info("IW finished to solve from state " + s.print() + " in " + StringConverter::from((double) duration / (double) 1e9) + " seconds.");
                    return;
                } else if (!res.second) { // no states pruned => problem is unsolvable
                    break;
                }
            }

            spdlog::info("IW could not find a solution from state " + s.print());
        } catch (const std::exception& e) {
            spdlog::error("IW failed solving from state " + s.print() + ". Reason: " + e.what());
            throw;
        }
    }

    bool is_solution_defined_for(const State& s) const {
        auto si = _graph.find(Node(s, _domain, _state_features));
        if ((si == _graph.end()) || (si->best_action == nullptr) || (si->solved == false)) {
            return false;
        } else {
            return true;
        }
    }

    const Action& get_best_action(const State& s) const {
        auto si = _graph.find(Node(s, _domain, _state_features));
        if ((si == _graph.end()) || (si->best_action == nullptr)) {
            spdlog::error("SKDECIDE exception: no best action found in state " + s.print());
            throw std::runtime_error("SKDECIDE exception: no best action found in state " + s.print());
        }
        return *(si->best_action);
    }

    const double& get_best_value(const State& s) const {
        auto si = _graph.find(Node(s, _domain, _state_features));
        if (si == _graph.end()) {
            spdlog::error("SKDECIDE exception: no best action found in state " + s.print());
            throw std::runtime_error("SKDECIDE exception: no best action found in state " + s.print());
        }
        return si->fscore;
    }

    std::size_t get_nb_of_explored_states() const {
        return _graph.size();
    }

    std::size_t get_nb_of_pruned_states() const {
        std::size_t cnt = 0;
        for (const auto&  n : _graph)  {
            if (n.pruned) {
                cnt++;
            }
        }
        return cnt;
    }

    const std::list<std::tuple<std::size_t, std::size_t, double>>& get_intermediate_scores() const {
        return _intermediate_scores;
    }

private :

    Domain& _domain;
    std::function<std::unique_ptr<FeatureVector> (Domain& d, const State& s)> _state_features;
    std::function<bool (const double&, const std::size_t&, const std::size_t&,
                                       const double&, const std::size_t&, const std::size_t&)> _node_ordering;
    std::size_t _time_budget;
    std::list<std::tuple<std::size_t, std::size_t, double>> _intermediate_scores;
    bool _debug_logs;

    struct Node {
        State state;
        std::unique_ptr<FeatureVector> features;
        std::tuple<Node*, Action, double> best_parent;
        double gscore;
        double fscore; // not in A*'s meaning but rather to store cost-to-go once a solution is found
        std::size_t novelty;
        std::size_t depth;
        Action* best_action; // computed only when constructing the solution path backward from the goal state
        bool pruned; // true if pruned by the novelty test (used only to report nb of pruned states)
        bool solved; // set to true if on the solution path constructed backward from the goal state

        Node(const State& s, Domain& d,
             const std::function<std::unique_ptr<FeatureVector> (Domain& d, const State& s)>& state_features)
            : state(s),
              gscore(std::numeric_limits<double>::infinity()),
              fscore(std::numeric_limits<double>::infinity()),
              novelty(std::numeric_limits<std::size_t>::max()),
              depth(std::numeric_limits<std::size_t>::max()),
              best_action(nullptr),
              pruned(false),
              solved(false) {
            features = state_features(d, s);
        }

        struct Key {
            const typename HashingPolicy::Key& operator()(const Node& n) const {
                return HashingPolicy::get_key(n);
            }
        };
    };

    typedef typename SetTypeDeducer<Node, HashingPolicy>::Set Graph;
    Graph _graph;

    class WidthSolver { // known as IW(i), i.e. the fixed-width solver sequentially run by IW
    public :
        typedef Tdomain Domain;
        typedef typename Domain::State State;
        typedef typename Domain::Event Action;
        typedef Tfeature_vector FeatureVector;
        typedef Thashing_policy<Domain, FeatureVector> HashingPolicy;
        typedef Texecution_policy ExecutionPolicy;

        WidthSolver(Domain& domain,
                    const std::function<std::unique_ptr<FeatureVector> (Domain& d, const State& s)>& state_features,
                    const std::function<bool (const double&, const std::size_t&, const std::size_t&,
                                              const double&, const std::size_t&, const std::size_t&)>& node_ordering,
                    std::size_t width,
                    Graph& graph,
                    std::size_t time_budget,
                    std::list<std::tuple<std::size_t, std::size_t, double>>& intermediate_scores,
                    bool debug_logs)
            : _domain(domain), _state_features(state_features), _node_ordering(node_ordering),
              _width(width), _graph(graph), _time_budget(time_budget),
              _intermediate_scores(intermediate_scores), _debug_logs(debug_logs) {}

        // solves from state s
        // returned pair p: p.first==true iff solvable, p.second==true iff states have been pruned
        std::pair<bool, bool> solve(const State& s, const std::chrono::time_point<std::chrono::high_resolution_clock>& start_time, bool& found_goal) {
            try {
                spdlog::info("Running " + ExecutionPolicy::print_type() + " IW(" + StringConverter::from(_width) + ") solver from state " + s.print());
                auto local_start_time = std::chrono::high_resolution_clock::now();

                // Create the root node containing the given state s
                auto si = _graph.emplace(Node(s, _domain, _state_features));
                if (si.first->solved || _domain.is_goal(s)) { // problem already solved from this state (was present in _graph and already solved)
                    return std::make_pair(true, false);
                } else if (_domain.is_terminal(s)) { // dead-end state
                    return std::make_pair(false, false);
                }
                Node& root_node = const_cast<Node&>(*(si.first)); // we won't change the real key (Node::state) so we are safe
                root_node.depth = 0;
                root_node.gscore = 0;
                bool states_pruned = false;

                auto node_ordering = [this](const auto& a, const auto& b) -> bool {
                    return _node_ordering(a->gscore, a->novelty, a->depth,
                                          b->gscore, b->novelty, b->depth);
                };

                // Priority queue used to sort non-goal unsolved tip nodes by increasing cost-to-go values (so-called OPEN container)
                std::priority_queue<Node*, std::vector<Node*>, decltype(node_ordering)> open_queue(node_ordering);
                open_queue.push(&root_node);

                // Set of states that have already been explored
                std::unordered_set<Node*> closed_set;

                // Vector of sets of state feature tuples generated so far, for each w <= _width
                TupleVector feature_tuples(_width);
                novelty(feature_tuples, root_node); // initialize feature_tuples with the root node's bits

                while (!open_queue.empty()) {
                    auto best_tip_node = open_queue.top();
                    open_queue.pop();

                    // Check that the best tip node has not already been closed before
                    // (since this implementation's open_queue does not check for element uniqueness,
                    // it can contain many copies of the same node pointer that could have been closed earlier)
                    if (closed_set.find(best_tip_node) != closed_set.end()) { // this implementation's open_queue can contain several
                        continue;
                    }

                    if (_debug_logs) spdlog::debug("Current best tip node: " + best_tip_node->state.print() +
                                                   ", gscore=" + StringConverter::from(best_tip_node->gscore) +
                                                   ", novelty=" + StringConverter::from(best_tip_node->novelty) +
                                                   ", depth=" + StringConverter::from(best_tip_node->depth));

                    closed_set.insert(best_tip_node);

                    if (_domain.is_goal(best_tip_node->state) || best_tip_node->solved) {
                        auto current_node = best_tip_node;
                        double tentative_fscore = 0;

                        while (current_node != &root_node) {
                            Node* parent_node = std::get<0>(current_node->best_parent);
                            tentative_fscore += std::get<2>(current_node->best_parent);
                            current_node = parent_node;
                        }

                        if (!found_goal || root_node.fscore > tentative_fscore) {
                            current_node = best_tip_node;
                            if (!(best_tip_node->solved)) { current_node->fscore = 0; } // goal state
                            best_tip_node->solved = true;

                            while (current_node != &root_node) {
                                Node* parent_node = std::get<0>(current_node->best_parent);
                                parent_node->best_action = &std::get<1>(current_node->best_parent);
                                parent_node->fscore = std::get<2>(current_node->best_parent) + current_node->fscore;
                                parent_node->solved = true;
                                current_node = parent_node;
                            }
                        }

                        spdlog::info("Found a goal state: " + best_tip_node->state.print() +
                                     " (cost=" + StringConverter::from(tentative_fscore) +
                                     "; best=" + StringConverter::from(root_node.fscore) + ")");
                        found_goal = true;
                        auto now_time = std::chrono::high_resolution_clock::now();
                        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now_time - start_time).count();
                        _intermediate_scores.push_back(std::make_tuple(static_cast<std::size_t>(duration), _width, root_node.fscore));

                        if (static_cast<std::size_t>(duration) < _time_budget) {
                            if (_debug_logs) spdlog::debug("Remaining time budget, continuing searching for better plans with current width...");
                            continue;
                        } else {
                            auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now_time - local_start_time).count();
                            spdlog::info("IW(" + StringConverter::from(_width) + ") finished to solve from state " + s.print() + " in " + StringConverter::from((double) duration / (double) 1e9) + " seconds.");
                            return std::make_pair(true, states_pruned);
                        }
                    }

                    if (_domain.is_terminal(best_tip_node->state)) { // dead-end state
                        if (_debug_logs) spdlog::debug("Found a dead-end state: " + best_tip_node->state.print());
                        continue;
                    }

                    // Expand best tip node
                    auto applicable_actions = _domain.get_applicable_actions(best_tip_node->state).get_elements();
                    std::for_each(ExecutionPolicy::policy, applicable_actions.begin(), applicable_actions.end(), [this, &best_tip_node, &open_queue, &feature_tuples, &states_pruned](auto a){
                        if (_debug_logs) spdlog::debug("Current expanded action: " + a.print() + ExecutionPolicy::print_thread());
                        auto next_state = _domain.get_next_state(best_tip_node->state, a);
                        std::pair<typename Graph::iterator, bool> i;
                        _execution_policy.protect([this, &i, &next_state]{
                            i = _graph.emplace(Node(next_state, _domain, _state_features));
                        });
                        Node& neighbor = const_cast<Node&>(*(i.first)); // we won't change the real key (StateNode::state) so we are safe
                        if (_debug_logs) spdlog::debug("Exploring next state (among " +
                                                       StringConverter::from(_graph.size()) + ")" +
                                                       ExecutionPolicy::print_thread());

                        double transition_cost = _domain.get_transition_cost(best_tip_node->state, a, neighbor.state);
                        double tentative_gscore = best_tip_node->gscore + transition_cost;
                        std::size_t tentative_depth = best_tip_node->depth + 1;

                        if ((i.second) || (tentative_gscore < neighbor.gscore)) {
                            if (_debug_logs) spdlog::debug("New gscore: " +
                                                           StringConverter::from(best_tip_node->gscore) + "+" +
                                                           StringConverter::from(transition_cost) + "=" +
                                                           StringConverter::from(tentative_gscore) +
                                                           ExecutionPolicy::print_thread());
                            neighbor.gscore = tentative_gscore;
                            neighbor.best_parent = std::make_tuple(best_tip_node, a, transition_cost);
                        }

                        if ((i.second) || (tentative_depth < neighbor.depth)) {
                            if (_debug_logs) spdlog::debug("New depth: " +
                                                           StringConverter::from(best_tip_node->depth) + "+" +
                                                           StringConverter::from(1) + "=" +
                                                           StringConverter::from(tentative_depth) +
                                                           ExecutionPolicy::print_thread());
                            neighbor.depth = tentative_depth;
                        }

                        _execution_policy.protect([this, &feature_tuples, &open_queue, &neighbor, &states_pruned]{
                            if (this->novelty(feature_tuples, neighbor) > _width) {
                                if (_debug_logs) spdlog::debug("Pruning state " + neighbor.state.print() + ExecutionPolicy::print_thread());
                                states_pruned = true;
                                neighbor.pruned = true;
                            } else {
                                if (_debug_logs) spdlog::debug("Adding state to open queue (among " +
                                                               StringConverter::from(open_queue.size()) + ")" +
                                                               ExecutionPolicy::print_thread());
                                open_queue.push(&neighbor);
                            }
                        });
                    });
                }

                if (found_goal) {
                    auto now_time = std::chrono::high_resolution_clock::now();
                    auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(now_time - local_start_time).count();
                    spdlog::info("IW(" + StringConverter::from(_width) + ") finished to solve from state " + s.print() + " in " + StringConverter::from((double) duration / (double) 1e9) + " seconds.");
                    return std::make_pair(true, states_pruned);
                } else {
                    spdlog::info("IW(" + StringConverter::from(_width) + ") could not find a solution from state " + s.print());
                    return std::make_pair(false, states_pruned);
                }
            } catch (const std::exception& e) {
                spdlog::error("IW(" + StringConverter::from(_width) + ") failed solving from state " + s.print() + ". Reason: " + e.what());
                throw;
            }
        }

    private :
        Domain& _domain;
        const std::function<std::unique_ptr<FeatureVector> (Domain& d, const State& s)>& _state_features;
        const std::function<bool (const double&, const std::size_t&, const std::size_t&,
                                  const double&, const std::size_t&, const std::size_t&)>& _node_ordering;
        std::size_t _width;
        Graph& _graph;
        std::size_t _time_budget;
        std::list<std::tuple<std::size_t, std::size_t, double>>& _intermediate_scores;
        bool _debug_logs;
        ExecutionPolicy _execution_policy;

        typedef std::vector<std::pair<std::size_t, typename FeatureVector::value_type>> TupleType;
        typedef std::vector<std::unordered_set<TupleType, boost::hash<TupleType>>> TupleVector;

        std::size_t novelty(TupleVector& feature_tuples, Node& n) const {
            // feature_tuples is a set of state variable combinations of size _width
            std::size_t nov = n.features->size() + 1;
            const FeatureVector& state_features = *n.features;

            for (std::size_t k = 1 ; k <= std::min(_width, state_features.size()) ; k++) {
                // we must recompute combinations from previous width values just in case
                // this state would be visited for the first time across width iterations
                generate_tuples(k, state_features.size(),
                                [&state_features, &feature_tuples, &k, &nov](TupleType& cv){
                    for (auto& e : cv) {
                        e.second = state_features[e.first];
                    }
                    if(feature_tuples[k-1].insert(cv).second) {
                        nov = std::min(nov, k);
                    }
                });
            }
            if (_debug_logs) spdlog::debug("Novelty: " + StringConverter::from(nov));
            n.novelty = nov;
            return nov;
        }

        // Generates all combinations of size k from [0 ... (n-1)]
        void generate_tuples(const std::size_t& k,
                             const std::size_t& n,
                             const std::function<void (TupleType&)>& f) const {
            TupleType cv(k); // one combination (the first one)
            for (std::size_t i = 0 ; i < k ; i++) {
                cv[i].first = i;
            }
            f(cv);
            bool more_combinations = true;
            while (more_combinations) {
                more_combinations = false;
                // find the rightmost element that has not yet reached its highest possible value
                for (std::size_t i = k; i > 0; i--) {
                    if (cv[i-1].first < n - k + i - 1) {
                        // once finding this element, we increment it by 1,
                        // and assign the lowest valid value to all subsequent elements
                        cv[i-1].first++;
                        for (std::size_t j = i; j < k; j++) {
                            cv[j].first = cv[j-1].first + 1;
                        }
                        f(cv);
                        more_combinations = true;
                        break;
                    }
                }
            }
        }
    };
};

} // namespace skdecide

#endif // SKDECIDE_IW_HH

ext: hh

max_stars_repo_name: jeromerobert/scikit-decide

max_issues_repo_name: jeromerobert/scikit-decide

max_forks_repo_name: jeromerobert/scikit-decide


text:
#include <gtest/gtest.h>

#include "scheme/numeric/euler_angles.hh"

#include <Eigen/Geometry>

#include <random>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>

#include "scheme/util/Timer.hh"

namespace scheme { namespace numeric { namespace test {

using std::cout;
using std::endl;

TEST(euler_angles,test){
	using namespace Eigen;
	std::mt19937 rng((unsigned int)time(0));
	std::normal_distribution<> gauss;
	std::uniform_real_distribution<> uniform;

	for(int i = 0; i < 10000; ++i){
		Quaterniond q( fabs(gauss(rng)), gauss(rng), gauss(rng), gauss(rng) );
		q.normalize();
		Matrix3d m = q.matrix();
		Vector3d euler;
		euler_angles(m,euler);
		// cout << euler.transpose() << endl;
		// cout << m << endl;
		// cout << endl;
		Matrix3d m2;
		from_euler_angles(euler,m2);
		double thresh = 0.0000001;
		if( euler[2] < 0.000001 || euler[2] > M_PI-0.000001 ) thresh = 0.0002;
		if( !m2.isApprox(m,thresh) ){
			cout << euler << endl;
			cout << m << endl;
			cout << m2 << endl;
		}
		ASSERT_TRUE( m2.isApprox(m,thresh) );

	}
}

TEST( euler_angles, performance ){
	using namespace Eigen;
	std::mt19937 rng((unsigned int)time(0));
	std::normal_distribution<> gauss;
	std::uniform_real_distribution<> uniform;

	int NSAMP = 1*1000*1000;

	std::vector<Matrix3d> samp(NSAMP);
	std::vector<Vector3d> euler(NSAMP);

	for(int i = 0; i < NSAMP; ++i){
		Quaterniond q( fabs(gauss(rng)), gauss(rng), gauss(rng), gauss(rng) );
		q.normalize();
		Matrix3d m = q.matrix();
		samp[i] = m;
	}

	util::Timer<> t;
	for(int i = 0; i < NSAMP; ++i){
		euler_angles(samp[i],euler[i]);
	}
	cout << "rate " << NSAMP/(double)t.elapsed() << endl;

	for(int i = 0; i < NSAMP; ++i){
		// cout << euler.transpose() << endl;
		// cout << m << endl;
		// cout << endl;
		Matrix3d m2;
		from_euler_angles(euler[i],m2);
		double thresh = 0.0000001;
		if( euler[i][2] < 0.000001 || euler[i][2] > M_PI-0.000001 ) thresh = 0.0002;
		if( !m2.isApprox(samp[i],thresh) ){
			cout << euler[i] << endl;
			cout << samp[i] << endl;
			cout << m2 << endl;
		}
		ASSERT_TRUE( m2.isApprox(samp[i],thresh) );

	}
}

}}}

ext: cc

max_stars_repo_name: YaoYinYing/rifdock

max_issues_repo_name: YaoYinYing/rifdock

max_forks_repo_name: YaoYinYing/rifdock


text:
#define BOOST_TEST_MODULE "test_read_worm_like_chain_offset_potential"

#ifdef BOOST_TEST_DYN_LINK
#include <boost/test/unit_test.hpp>
#else
#include <boost/test/included/unit_test.hpp>
#endif

#include <mjolnir/input/read_local_potential.hpp>
#include <tuple>

using test_types = std::tuple<double, float>;

constexpr inline float  tolerance_value(float)  noexcept {return 1e-4;}
constexpr inline double tolerance_value(double) noexcept {return 1e-8;}

template<typename Real>
decltype(boost::test_tools::tolerance(std::declval<Real>()))
tolerance() {return boost::test_tools::tolerance(tolerance_value(Real()));}

BOOST_AUTO_TEST_CASE_TEMPLATE(read_worm_like_chain_offset_noenv, T, test_types)
{
    mjolnir::LoggerManager::set_default_logger("test_read_worm_like_chain_offset.log");

    using namespace toml::literals;
    using real_type = T;
    const toml::value env;
    const auto param = u8R"(
        indices = [1, 2]
        p       = 3.9
        lc      = 19.0
        l0      = 5.0
    )"_toml;

    const auto w = mjolnir::read_worm_like_chain_offset_potential<real_type>(param, env);
    BOOST_TEST(w.p()    == real_type(3.9),  tolerance<real_type>());
    BOOST_TEST(w.lc()   == real_type(19.0), tolerance<real_type>());
    BOOST_TEST(w.l0()   == real_type(5.0),  tolerance<real_type>());
}

BOOST_AUTO_TEST_CASE_TEMPLATE(read_worm_like_chain_offset_env, T, test_types)
{
    mjolnir::LoggerManager::set_default_logger("test_read_worm_like_chain_offset.log");

    using real_type = T;
    {
        using namespace toml::literals;
        const auto env = u8R"(
            indices = [1, 2]
            p       = 3.9
            lc      = 19.0
            l0      = 5.0
        )"_toml;
        const auto param = u8R"(
            indices = "indices"
            p       = "p"
            lc      = "lc"
            l0      = "l0"
        )"_toml;

        const auto w = mjolnir::read_worm_like_chain_offset_potential<real_type>(param, env);
        BOOST_TEST(w.p()     == real_type(3.9),  tolerance<real_type>());
        BOOST_TEST(w.lc()    == real_type(19.0), tolerance<real_type>());
        BOOST_TEST(w.l0()    == real_type(5.0),  tolerance<real_type>());
    }
}

ext: cpp

max_stars_repo_name: ToruNiina/Mjolnir

max_issues_repo_name: Mjolnir-MD/Mjolnir

max_forks_repo_name: Mjolnir-MD/Mjolnir


text:
/***************************************************************************
 *                                  _   _ ____  _
 * Copyright (C) 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at http://curl.haxx.se/docs/copyright.html.
 *
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ***************************************************************************/

/*
 * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
 */

#include "http_client.h"
#include "http_curl.h"
#include <curl/curl.h>
#include <boost/asio.hpp>
#include <boost/bind.hpp>
#include <boost/intrusive_ptr.hpp>
#include <tbb/mutex.h>

#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */

using tbb::mutex;

/* boost::asio related objects
 * using global variables for simplicity
 */
std::map<curl_socket_t, HttpClientSession *> socket_map;

/* Update the event timer after curl_multi library calls */
static int multi_timer_cb(CURLM *multi, long timeout_ms, HttpClient *client)
{

  if ( timeout_ms > 0 )
  {
    client->StartTimer(timeout_ms);
  }
  else
  {
    client->CancelTimer();
    timer_cb(client->GlobalInfo());
  }

  return 0;
}

/* Die if we get a bad CURLMcode somewhere */
static void mcode_or_die(const char *where, CURLMcode code)
{
  if ( CURLM_OK != code )
  {
    const char *s;
    switch ( code )
    {
    case CURLM_CALL_MULTI_PERFORM: s="CURLM_CALL_MULTI_PERFORM"; break;
    case CURLM_BAD_HANDLE:         s="CURLM_BAD_HANDLE";         break;
    case CURLM_BAD_EASY_HANDLE:    s="CURLM_BAD_EASY_HANDLE";    break;
    case CURLM_OUT_OF_MEMORY:      s="CURLM_OUT_OF_MEMORY";      break;
    case CURLM_INTERNAL_ERROR:     s="CURLM_INTERNAL_ERROR";     break;
    case CURLM_UNKNOWN_OPTION:     s="CURLM_UNKNOWN_OPTION";     break;
    case CURLM_LAST:               s="CURLM_LAST";               break;
    case CURLM_BAD_SOCKET:         s="CURLM_BAD_SOCKET";         break;
    default:                       s="CURLM_unknown";            break;
    }

    fprintf(MSG_OUT, "\nERROR: %s returns %s", where, s);
  }
}

/* Check for completed transfers, and remove their easy handles */
static void check_multi_info(GlobalInfo *g)
{
  char *eff_url;
  CURLMsg *msg;
  int msgs_left;
  ConnInfo *conn;
  CURL *easy;
  CURLcode res;

  while ((msg = curl_multi_info_read(g->multi, &msgs_left)))
  {
    if (msg->msg == CURLMSG_DONE)
    {
      easy = msg->easy_handle;
      res = msg->data.result;
      curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
      curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);

      boost::system::error_code error(res, boost::system::system_category());
      std::string empty_str("");
      conn->connection->HttpClientCb()(empty_str, error);
    }
  }
}

typedef boost::intrusive_ptr<HttpClientSession> TcpSessionPtr;

static void event_cb_impl(GlobalInfo *g, TcpSessionPtr session, int action,
                          const boost::system::error_code &error,
                          std::size_t bytes_transferred)
{

  if (session->IsClosed()) return;

  if (g->client->IsErrorHard(error)) return;

  CURLMcode rc;
  rc = curl_multi_socket_action(g->multi, session->socket()->native_handle(), action, &g->still_running);

  mcode_or_die("event_cb: curl_multi_socket_action", rc);
  check_multi_info(g);

  if ( g->still_running <= 0 )
  {
    g->client->CancelTimer();
  }
}

/* Called by asio when there is an action on a socket */
static void event_cb(GlobalInfo *g, TcpSessionPtr session, int action,
                     const boost::system::error_code &error, std::size_t bytes_transferred)
{
  tbb::mutex::scoped_lock lock(session->mutex());

  // Ignore if the connection is already deleted.
  if (!session->Connection()) return;

  HttpClient *client = session->Connection()->client();
  client->ProcessEvent(boost::bind(&event_cb_impl, g, session, action, error,
                                   bytes_transferred));
}

/* Called by asio when our timeout expires */
void timer_cb(GlobalInfo *g)
{
    CURLMcode rc;
    rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);

    mcode_or_die("timer_cb: curl_multi_socket_action", rc);
    check_multi_info(g);
}

/* Clean up any data */
static void remsock(int *f, GlobalInfo *g)
{
  if ( f )
  {
    free(f);
  }
}

static void setsock(int *fdp, curl_socket_t s, CURL*e, int act, GlobalInfo *g)
{
  std::map<curl_socket_t, HttpClientSession *>::iterator it = socket_map.find(s);

  if ( it == socket_map.end() )
  {
    return;
  }

  HttpClientSession *session = it->second;
  if (session->IsClosed()) return;

  boost::asio::ip::tcp::socket * tcp_socket = session->socket();

  *fdp = act;

  if ( act == CURL_POLL_IN )
  {
    tcp_socket->async_read_some(boost::asio::null_buffers(),
                                boost::bind(&event_cb, g, TcpSessionPtr(session),
                                act, _1, _2));
  }
  else if ( act == CURL_POLL_OUT )
  {
    tcp_socket->async_write_some(boost::asio::null_buffers(),
                                 boost::bind(&event_cb, g, TcpSessionPtr(session),
                                 act, _1, _2));
  }
  else if ( act == CURL_POLL_INOUT )
  {
    tcp_socket->async_read_some(boost::asio::null_buffers(),
                                boost::bind(&event_cb, g, TcpSessionPtr(session),
                                act, _1, _2));
    tcp_socket->async_write_some(boost::asio::null_buffers(),
                                 boost::bind(&event_cb, g, TcpSessionPtr(session),
                                 act, _1, _2));
  }
}

static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo *g)
{
  int *fdp = (int *)calloc(sizeof(int), 1); /* fdp is used to store current action */

  setsock(fdp, s, easy, action, g);
  curl_multi_assign(g->multi, s, fdp);
}

/* CURLMOPT_SOCKETFUNCTION */
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp)
{
  GlobalInfo *g = (GlobalInfo*) cbp;
  int *actionp = (int*) sockp;

  if ( what == CURL_POLL_REMOVE )
  {
    remsock(actionp, g);
  }
  else
  {
    if ( !actionp )
    {
      addsock(s, e, what, g);
    }
    else
    {
      setsock(actionp, s, e, what, g);
    }
  }
  return 0;
}

/* CURLOPT_WRITEFUNCTION */
static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data)
{
  size_t written = size * nmemb;

  HttpConnection *conn = static_cast<HttpConnection *>(data);
  conn->AssignData((char *)ptr, written);
  return written;
}

static size_t read_cb(void *ptr, size_t size, size_t nmemb, void *data)
{
  HttpConnection *conn = static_cast<HttpConnection *>(data);
  const std::string str = conn->GetData();

  size_t maxb = size*nmemb;
  size_t offset = conn->GetOffset();
  size_t datasize = str.length() - offset;

  if (maxb >= datasize) {
      memcpy(ptr, str.data() + offset, datasize);
      conn->UpdateOffset(datasize);
      return str.length();
  } else {
      memcpy(ptr, str.data() + offset, maxb);
      conn->UpdateOffset(maxb);
      return maxb;
  }

  return 0;
}

/* CURLOPT_PROGRESSFUNCTION */
static int prog_cb (void *p, double dltotal, double dlnow, double ult,
                    double uln)
{
  (void)ult;
  (void)uln;

  return 0;
}

/* CURLOPT_OPENSOCKETFUNCTION */
static curl_socket_t opensocket(void *data,
                                curlsocktype purpose,
                                struct curl_sockaddr *address)
{
  HttpConnection *conn = static_cast<HttpConnection *>(data);

  curl_socket_t sockfd = CURL_SOCKET_BAD;

  /* restrict to ipv4 */
  if (purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET)
  {
      HttpClientSession *session = conn->CreateSession();
      sockfd = session->socket()->native_handle();
      socket_map.insert(std::pair<curl_socket_t, HttpClientSession *>(sockfd,
                  static_cast<HttpClientSession *>(session)));
      conn->set_session(session);
  }

  return sockfd;
}

/* CURLOPT_CLOSESOCKETFUNCTION */
static int closesocket(void *clientp, curl_socket_t item)
{
  std::map<curl_socket_t, HttpClientSession *>::iterator it = socket_map.find(item);
  if ( it != socket_map.end() ) {
      socket_map.erase(it);
  }

  return 0;
}

void del_conn(HttpConnection *connection, GlobalInfo *g) {

    if (connection->session()) {
        tbb::mutex::scoped_lock lock(connection->session()->mutex());
        closesocket(NULL, connection->session()->socket()->native_handle());
        connection->session()->SetConnection(NULL);
    }

    struct _ConnInfo *curl_handle = connection->curl_handle();
    if (curl_handle) {
        curl_multi_remove_handle(g->multi, curl_handle->easy);
        curl_slist_free_all(curl_handle->headers);
        free(curl_handle->post);
        free(curl_handle->url);
        curl_easy_cleanup(curl_handle->easy);
        connection->set_curl_handle(NULL);
        free(curl_handle);
    }
}

/* Create a new easy handle, and add it to the global curl_multi */
ConnInfo *new_conn(HttpConnection *connection, GlobalInfo *g,
                   bool header, bool timeout)
{
  ConnInfo *conn = (ConnInfo *)calloc(1, sizeof(ConnInfo));
  memset(conn, 0, sizeof(ConnInfo));
  conn->error[CURL_ERROR_SIZE]='\0';

  conn->easy = curl_easy_init();

  if ( !conn->easy )
  {
    exit(2);
  }
  conn->global = g;
  curl_easy_setopt(conn->easy, CURLOPT_FOLLOWLOCATION, 1L);
  curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb);
  curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, connection);
  curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION, read_cb);
  curl_easy_setopt(conn->easy, CURLOPT_READDATA, connection);
  curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
  curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
  curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L);
  curl_easy_setopt(conn->easy, CURLOPT_PROGRESSFUNCTION, prog_cb);
  curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn);
  curl_easy_setopt(conn->easy, CURLOPT_CONNECTTIMEOUT, 4L); // in secs
  if (timeout) {
      /* set the timeout limits to abort the connection */
      curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 3L);
      curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 10L);
  }
  curl_easy_setopt(conn->easy, CURLOPT_FORBID_REUSE, 1L);

  /* to include the header in the body */
  if (header)
      curl_easy_setopt(conn->easy, CURLOPT_HEADER, 1);

  /* call this function to get a socket */
  curl_easy_setopt(conn->easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
  curl_easy_setopt(conn->easy, CURLOPT_OPENSOCKETDATA, connection);

  /* call this function to close a socket */
  curl_easy_setopt(conn->easy, CURLOPT_CLOSESOCKETFUNCTION, closesocket);

  return conn;
}

void set_url(ConnInfo *conn, const char *url) {
  conn->url = strdup(url);
  curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url);
}

void set_header_options(ConnInfo *conn, const char *options) {
    conn->headers = curl_slist_append(conn->headers, options);
    curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->headers);
}

void set_put_string(ConnInfo *conn, const char *post) {
    conn->headers = curl_slist_append(conn->headers, "Content-Type: application/xml");
    curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->headers);

    conn->post = strdup(post);
    curl_easy_setopt(conn->easy, CURLOPT_POSTFIELDS, conn->post);
}

int http_get(ConnInfo *conn, GlobalInfo *g) {
    CURLMcode rc = curl_multi_add_handle(g->multi, conn->easy);
    return (int)rc;
}

int http_put(ConnInfo *conn, GlobalInfo *g) {
    CURLMcode rc = curl_multi_add_handle(g->multi, conn->easy);
    return (int)rc;
}

int curl_init(HttpClient *client)
{

  struct _GlobalInfo *g = client->GlobalInfo();

  memset(g, 0, sizeof(GlobalInfo));
  g->multi = curl_multi_init();
  g->client = client;

  curl_multi_setopt(g->multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
  curl_multi_setopt(g->multi, CURLMOPT_SOCKETDATA, g);
  curl_multi_setopt(g->multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
  curl_multi_setopt(g->multi, CURLMOPT_TIMERDATA, client);

  return 0;
}

max_stars_repo_name: safchain/contrail-controller

max_issues_repo_name: safchain/contrail-controller

max_forks_repo_name: safchain/contrail-controller


text:
/*
 * Copyright (c) 2016 Shanghai Jiao Tong University.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://ipads.se.sjtu.edu.cn/projects/wukong
 *
 */

#pragma once

#include <pthread.h>
#include <stdint.h>  // uint64_t
#include <atomic>
#include <iostream>
#include <limits>
#include <queue>
#include <vector>

#include <boost/unordered_set.hpp>

#include <tbb/concurrent_hash_map.h>       // NOLINT
#include <tbb/concurrent_unordered_set.h>  // NOLINT
#include <tbb/concurrent_vector.h>         // NOLINT

#include "core/common/global.hpp"
#include "core/common/mem.hpp"
#include "core/common/rdma.hpp"
#include "core/common/type.hpp"

#include "core/store/rdma_cache.hpp"
#include "core/store/segment_meta.hpp"
#include "core/store/vertex.hpp"

#include "utils/math.hpp"
#include "utils/timer.hpp"
#include "utils/unit.hpp"

namespace wukong {

/**
 * @brief A general RDMA-based KV store
 *
 * KVStore: key (main-header and indirect-header region) | value (entry region)
 * head region is a cluster chaining hash-table (with associativity)
 * entry region is a varying-size array
 *
 * @tparam KeyType
 * @tparam PtrType
 * @tparam ValueType
 */
template <class KeyType, class PtrType, class ValueType>
class KVStore {
    friend class GChecker;
    friend class DGraph;
    friend class SegmentRDFGraph;

public:
    // the number of locks to protect buckets
    static const int NUM_LOCKS = 1024;
    // the associativity of slots in each bucket
    static const int ASSOCIATIVITY = 8;

    /**
     * Memory Usage (estimation):
     *   header region: |vertex| = 128-bit;
     *   #verts = (#S + #O) * AVG(#P) ～= #T
     *
     *   entry region:  |edge| = 32-bit;
     *   #edges = #T * 2 + (#S + #O) * AVG(#P) ～= #T * 3
     *
     *   (+VERSATILE)
     *   #verts += #S + #O
     *   #edges += (#S + #O) * AVG(#P) ~= #T
     */
    // main-header / (main-header + indirect-header)
    static const int MHD_RATIO = 80;
    // header * 100 / (header + entry)
    static const int HD_RATIO = (128 * 100 / (128 + 3 * std::numeric_limits<sid_t>::digits));

protected:
    // slot = [key|pointer]
    struct slot_t {
        KeyType key;
        PtrType ptr;
    };

    int sid;

    Mem* mem;

    slot_t* slots;
    ValueType* values;

    uint64_t num_slots;        // 1 bucket = ASSOCIATIVITY * slots
    uint64_t num_buckets;      // main-header region (static)
    uint64_t num_buckets_ext;  // indirect-header region (dynamical)
    uint64_t last_ext;         // allocation offset of ext bucket

    // lock virtualization (see paper: vLock CGO'13)
    pthread_spinlock_t bucket_locks[NUM_LOCKS];
    pthread_spinlock_t bucket_ext_lock;

    uint64_t num_entries;  // value region

    /**
     * Cache remote vertex(location) of the given key, eleminating one RDMA read.
     * This only works when RDMA enabled.
     */
    RDMA_Cache<KeyType, slot_t> rdma_cache;

    // allocate space to store values of given size. Return offset of allocated space.
    virtual uint64_t alloc_entries(uint64_t num_values, int tid = 0) = 0;

    // check the validation of given values according to given key.
    virtual bool value_is_valid(slot_t& slot, ValueType* value_ptr) = 0;

    // get the values size of given key
    virtual uint64_t get_value_sz(const slot_t& slot) = 0;

    /**
     * @brief insert a given key to store,
     *
     * The values of this key are allocated before this function call
     *
     * @param key given key
     * @param ptr value pointer
     * @return uint64_t slot id
     */
    uint64_t insert_key(KeyType key, PtrType ptr) {
        uint64_t bucket_id = bucket_local(key);
        uint64_t slot_id = bucket_id * ASSOCIATIVITY;
        uint64_t lock_id = bucket_id % NUM_LOCKS;

        bool found = false;
        pthread_spin_lock(&bucket_locks[lock_id]);
        while (slot_id < num_slots) {
            // the last slot of each bucket is always reserved for pointer to indirect header
            /// TODO: add type info to slot and reuse the last slot to store key
            /// TODO: key.vid is reused to store the bucket_id of indirect header rather than ptr.off,
            ///       since the is_empty() is not robust.
            for (int i = 0; i < ASSOCIATIVITY - 1; i++, slot_id++) {
                if (this->slots[slot_id].key == key) {
                    logstream(LOG_ERROR) << "Conflict at slot["
                                         << slot_id << "] of bucket["
                                         << bucket_id << "], "
                                         << key.to_string() << ", "
                                         << this->slots[slot_id].key.to_string()
                                         << LOG_endl;
                    ASSERT(false);
                }

                // insert to an empty slot
                if (this->slots[slot_id].key.is_empty()) {
                    this->slots[slot_id].key = key;
                    this->slots[slot_id].ptr = ptr;
                    goto done;
                }
            }

            // whether the bucket_ext (indirect-header region) is used
            if (!this->slots[slot_id].key.is_empty()) {
                slot_id = this->slots[slot_id].key.vid * ASSOCIATIVITY;
                continue;  // continue and jump to next bucket
            }

            // allocate and link a new indirect header
            this->slots[slot_id].key.vid = alloc_ext_buckets(1);

            // move to a new bucket_ext
            slot_id = this->slots[slot_id].key.vid * ASSOCIATIVITY;
            // insert to the first slot
            this->slots[slot_id].key = key;
            this->slots[slot_id].ptr = ptr;
            goto done;
        }
    done:
        pthread_spin_unlock(&bucket_locks[lock_id]);
        ASSERT(slot_id < num_slots);
        return slot_id;
    }

    /**
     * @brief search slot id for given key
     *
     * if not found, return the new slot id to insert
     *
     * @param key given key
     * @return uint64_t slot id
     */
    uint64_t search_key(KeyType key) {
        uint64_t bucket_id = bucket_local(key);
        uint64_t slot_id = bucket_id * ASSOCIATIVITY;
        uint64_t lock_id = bucket_id % NUM_LOCKS;

        bool found = false;
        pthread_spin_lock(&bucket_locks[lock_id]);
        while (slot_id < num_slots) {
            // the last slot of each bucket is always reserved for pointer to indirect header
            /// TODO: add type info to slot and reuse the last slot to store key
            /// TODO: key.vid is reused to store the bucket_id of indirect header rather than ptr.off,
            ///       since the is_empty() is not robust.
            for (int i = 0; i < ASSOCIATIVITY - 1; i++, slot_id++) {
                if (this->slots[slot_id].key == key) {
                    goto done;
                }

                // insert to an empty slot
                if (this->slots[slot_id].key.is_empty()) {
                    goto done;
                }
            }

            // whether the bucket_ext (indirect-header region) is used
            if (!this->slots[slot_id].key.is_empty()) {
                slot_id = this->slots[slot_id].key.vid * ASSOCIATIVITY;
                continue;  // continue and jump to next bucket
            }

            // allocate and link a new indirect header
            this->slots[slot_id].key.vid = alloc_ext_buckets(1);
            // move to a new bucket_ext
            slot_id = this->slots[slot_id].key.vid * ASSOCIATIVITY;
            goto done;
        }
    done:
        pthread_spin_unlock(&bucket_locks[lock_id]);
        ASSERT(slot_id < num_slots);
        return slot_id;
    }

    /**
     * @brief calculate the bucket of given key
     *
     * @param key given key
     * @param seg segment-based search
     * @return uint64_t bucket id
     */
    uint64_t bucket_local(KeyType key, rdf_seg_meta_t* seg = nullptr) {
        uint64_t bucket_id;
        if (seg != nullptr) {
            bucket_id = seg->bucket_start + key.hash() % seg->num_buckets;
        } else {
            bucket_id = key.hash() % this->num_buckets;
        }
        return bucket_id;
    }

    /**
     * @brief calculate the bucket of given key
     *
     * NOTICE: we assume every machine has the same memory layout
     *
     * @param key
     * @param dst_sid target server
     * @param seg segment-based search
     * @return uint64_t bucket id
     */
    uint64_t bucket_remote(KeyType key, int dst_sid, rdf_seg_meta_t* seg = nullptr) {
        return bucket_local(key, seg);
    }

    /**
     * @brief Get the local slot of given key
     *
     * @param tid caller thread id
     * @param key given key
     * @param seg segment-based search
     * @return slot_t
     */
    slot_t get_slot_local(int tid, KeyType key, rdf_seg_meta_t* seg = nullptr) {
        uint64_t bucket_id = bucket_local(key, seg);
        while (true) {
            for (int i = 0; i < ASSOCIATIVITY; i++) {
                uint64_t slot_id = bucket_id * ASSOCIATIVITY + i;
                if (i < ASSOCIATIVITY - 1) {
                    if (this->slots[slot_id].key == key) {
                        // we found it
                        return this->slots[slot_id];
                    }
                } else {
                    if (this->slots[slot_id].key.is_empty())
                        // not found, return empty slot
                        return slot_t();

                    // move to next bucket
                    bucket_id = this->slots[slot_id].key.vid;
                    // break for-loop
                    break;
                }
            }
        }
    }

    /**
     * @brief Get remote slot of given key
     *
     * This func will fail if RDMA is disabled.
     *
     * @param tid caller thread id
     * @param dst_sid target server
     * @param key given key
     * @param seg segment-based search
     * @return slot_t
     */
    slot_t get_slot_remote(int tid, int dst_sid, KeyType key, rdf_seg_meta_t* seg = nullptr) {
        uint64_t bucket_id = bucket_remote(key, dst_sid, seg);
        slot_t slot;

        // NOTICE: wukong doesn't support to
        // directly get remote key/value without RDMA
        ASSERT(Global::use_rdma);

        // check cache
        if (rdma_cache.lookup(key, slot))
            return slot;

        // get remote bucket by RDMA-read
        char* buf = mem->buffer(tid);
        uint64_t buf_sz = mem->buffer_size();
        while (true) {
            uint64_t off = bucket_id * ASSOCIATIVITY * sizeof(slot_t);
            uint64_t sz = ASSOCIATIVITY * sizeof(slot_t);
            ASSERT(sz < buf_sz);  // enough space to host the slot

            RDMA& rdma = RDMA::get_rdma();
            rdma.dev->RdmaRead(tid, dst_sid, buf, sz, off);
            slot_t* slots = reinterpret_cast<slot_t*>(buf);
            for (int i = 0; i < ASSOCIATIVITY; i++) {
                if (i < ASSOCIATIVITY - 1) {
                    if (slots[i].key == key) {
                        rdma_cache.insert(slots[i]);
                        return slots[i];  // found
                    }
                } else {
                    if (slots[i].key.is_empty())
                        return slot_t();  // not found

                    // move to next bucket
                    bucket_id = slots[i].key.vid;
                    // break for-loop
                    break;
                }
            }
        }
    }

    // Get values of given key from dst_sid by RDMA read.
    ValueType* rdma_get_values(int tid, int dst_sid, slot_t& slot) {
        ASSERT(Global::use_rdma);

        char* buf = mem->buffer(tid);
        uint64_t r_off = num_slots * sizeof(slot_t) + slot.ptr.off * sizeof(ValueType);
        // the size of values
        uint64_t r_sz = get_value_sz(slot);
        uint64_t buf_sz = mem->buffer_size();
        ASSERT(r_sz < buf_sz);  // enough space to host the values

        RDMA& rdma = RDMA::get_rdma();
        rdma.dev->RdmaRead(tid, dst_sid, buf, r_sz, r_off);
        return reinterpret_cast<ValueType*>(buf);
    }

    /**
     * @brief Get local values according to given key
     *
     * @param tid caller thread id
     * @param key given key
     * @param sz for return (value size)
     * @param seg for segment-based search
     * @return ValueType*
     */
    ValueType* get_values_local(int tid, KeyType key, uint64_t& sz, rdf_seg_meta_t* seg = nullptr) {
        slot_t slot = get_slot_local(tid, key, seg);

        if (slot.key.is_empty()) {
            sz = 0;
            return nullptr;  // not found
        }

        // local values
        ValueType* value_ptr = &(this->values[slot.ptr.off]);

        sz = slot.ptr.size;
        return value_ptr;
    }

    /**
     * @brief Get remote values according to given key.
     *
     * @param tid caller thread id
     * @param dst_sid target server id
     * @param key given key
     * @param sz for return (value size)
     * @param seg for segment-based search
     * @return ValueType*
     */
    ValueType* get_values_remote(int tid, int dst_sid, KeyType key, uint64_t& sz, rdf_seg_meta_t* seg = nullptr) {
        slot_t slot = get_slot_remote(tid, dst_sid, key, seg);

        if (slot.key.is_empty()) {
            sz = 0;
            return nullptr;  // not found
        }

        // remote values
        ValueType* value_ptr = rdma_get_values(tid, dst_sid, slot);
        // check cache validation
        while (!value_is_valid(slot, value_ptr)) {
            // invalidate cache and try again
            rdma_cache.invalidate(key);
            slot = get_slot_remote(tid, dst_sid, key, seg);
            value_ptr = rdma_get_values(tid, dst_sid, slot);
        }

        sz = slot.ptr.size;
        return value_ptr;
    }

    /**
     * @brief Allocate extended buckets
     *
     * @param n number of extended buckets to allocate
     * @return uint64_t start offset of allocated extended buckets
     */
    uint64_t alloc_ext_buckets(uint64_t n) {
        uint64_t orig;
        pthread_spin_lock(&bucket_ext_lock);
        orig = this->last_ext;
        this->last_ext += n;
        if (this->last_ext >= this->num_buckets_ext) {
            logstream(LOG_ERROR) << "out of indirect-header region." << LOG_endl;
            ASSERT(false);
        }
        pthread_spin_unlock(&bucket_ext_lock);
        return this->num_buckets + orig;
    }

public:
    /**
     * @brief Construct a new KVStore object
     *
     * @param sid server id
     * @param mem main memory
     */
    KVStore(int sid, Mem* mem) : sid(sid), mem(mem) {
        uint64_t header_region = mem->kvstore_size() * HD_RATIO / 100;
        uint64_t entry_region = mem->kvstore_size() - header_region;

        // header region
        this->num_slots = header_region / sizeof(slot_t);
        this->num_buckets = wukong::math::hash_prime_u64((num_slots / ASSOCIATIVITY) * MHD_RATIO / 100);
        this->num_buckets_ext = (num_slots / ASSOCIATIVITY) - num_buckets;
        // entry region
        this->num_entries = entry_region / sizeof(ValueType);

        this->slots = reinterpret_cast<slot_t*>(mem->kvstore());
        this->values = reinterpret_cast<ValueType*>(mem->kvstore() + this->num_slots * sizeof(slot_t));

        pthread_spin_init(&this->bucket_ext_lock, 0);
        for (int i = 0; i < NUM_LOCKS; i++) {
            pthread_spin_init(&this->bucket_locks[i], 0);
        }

        // print kvstore usage
        logstream(LOG_INFO) << "[KV] kvstore = ";
        logstream(LOG_INFO) << mem->kvstore_size() << " bytes " << LOG_endl;
        logstream(LOG_INFO) << "  header region: " << this->num_slots << " slots"
                            << " (main = " << this->num_buckets
                            << ", indirect = " << this->num_buckets_ext << ")" << LOG_endl;
        logstream(LOG_INFO) << "  entry region: " << this->num_entries << " entries" << LOG_endl;
    }

    virtual ~KVStore() {}

    /**
     * @brief clean kv store
     */
    virtual void refresh() {
        #pragma omp parallel for num_threads(Global::num_engines)
        for (uint64_t i = 0; i < num_slots; i++) {
            this->slots[i].key = KeyType();
            this->slots[i].ptr = PtrType();
        }
        this->last_ext = 0;
    }

    inline void* get_slot_addr() { return reinterpret_cast<void*>(this->slots); }
    inline void* get_value_addr() { return reinterpret_cast<void*>(this->values); }

    /**
     * @brief Get the values for given key
     *
     * @param tid caller thread id
     * @param dst_sid target server id
     * @param key given key
     * @param sz value size(return value)
     * @param seg for segment-based
     * @return ValueType* value address
     */
    ValueType* get_values(int tid, int dst_sid, KeyType key, uint64_t& sz, rdf_seg_meta_t* seg = nullptr) {
        if (dst_sid == this->sid)
            return get_values_local(tid, key, sz, seg);
        else
            return get_values_remote(tid, dst_sid, key, sz, seg);
    }

    /**
     * @brief Check if the given key exists
     *
     * @param key
     * @return true the key exist
     */
    bool check_key_exist(KeyType key) {
        uint64_t bucket_id = bucket_local(key);
        uint64_t slot_id = bucket_id * ASSOCIATIVITY;
        uint64_t lock_id = bucket_id % NUM_LOCKS;

        pthread_spin_lock(&bucket_locks[lock_id]);
        while (slot_id < num_slots) {
            // the last slot of each bucket is always reserved for pointer to indirect header
            /// TODO: add type info to slot and reuse the last slot to store key
            /// TODO: key.vid is reused to store the bucket_id of indirect header rather than ptr.off,
            ///       since the is_empty() is not robust.
            for (int i = 0; i < ASSOCIATIVITY - 1; i++, slot_id++) {
                if (this->slots[slot_id].key == key) {
                    pthread_spin_unlock(&bucket_locks[lock_id]);
                    return true;
                }

                // insert to an empty slot
                if (this->slots[slot_id].key.is_empty()) {
                    pthread_spin_unlock(&bucket_locks[lock_id]);
                    return false;
                }
            }
            // whether the bucket_ext (indirect-header region) is used
            if (!this->slots[slot_id].key.is_empty()) {
                slot_id = this->slots[slot_id].key.vid * ASSOCIATIVITY;
                continue;  // continue and jump to next bucket
            }
            pthread_spin_unlock(&bucket_locks[lock_id]);
            return false;
        }
    }

    /**
     * @brief insert a new key-value pair (dynamically)
     *
     * @param key
     * @param value
     * @param dedup_or_isdup param:if dedup; return value:is duplicate
     * @param tid caller
     * @return true insert succeed
     * @return false insert failed
     */
    virtual bool insert_key_value(KeyType key, ValueType value, bool& dedup_or_isdup, int tid) = 0;

    /**
     * @brief print the memory usage of KV
     */
    virtual void print_mem_usage() {
        uint64_t used_slots = 0;
        uint64_t used_entries = 0;
        for (uint64_t x = 0; x < this->num_buckets; x++) {
            uint64_t slot_id = x * ASSOCIATIVITY;
            for (int y = 0; y < ASSOCIATIVITY - 1; y++, slot_id++) {
                if (this->slots[slot_id].key.is_empty())
                    continue;
                used_slots++;
                used_entries += this->slots[slot_id].ptr.size;
            }
        }

        logstream(LOG_INFO) << "[KV] main header: " << B2MiB(this->num_buckets * ASSOCIATIVITY * sizeof(slot_t))
                            << " MB (" << this->num_buckets * ASSOCIATIVITY << " slots)" << LOG_endl;
        logstream(LOG_INFO) << "\tused: " << 100.0 * used_slots / (this->num_buckets * ASSOCIATIVITY)
                            << " % (" << used_slots << " slots)" << LOG_endl;
        logstream(LOG_INFO) << "\tchain: " << 100.0 * this->num_buckets / (this->num_buckets * ASSOCIATIVITY)
                            << " % (" << this->num_buckets << " slots)" << LOG_endl;

        used_slots = 0;
        for (uint64_t x = this->num_buckets; x < this->num_buckets + this->last_ext; x++) {
            uint64_t slot_id = x * ASSOCIATIVITY;
            for (int y = 0; y < ASSOCIATIVITY - 1; y++, slot_id++) {
                if (this->slots[slot_id].key.is_empty())
                    continue;
                used_slots++;
                used_entries += this->slots[slot_id].ptr.size;
            }
        }

        logstream(LOG_INFO) << "[KV] indirect header: " << B2MiB(this->num_buckets_ext * ASSOCIATIVITY * sizeof(slot_t))
                            << " MB (" << this->num_buckets_ext * ASSOCIATIVITY << " slots)" << LOG_endl;
        logstream(LOG_INFO) << "\talloced: " << 100.0 * this->last_ext / this->num_buckets_ext
                            << " % (" << this->last_ext << " buckets)" << LOG_endl;
        logstream(LOG_INFO) << "\tused: " << 100.0 * used_slots / (num_buckets_ext * ASSOCIATIVITY)
                            << " % (" << used_slots << " slots)" << LOG_endl;

        logstream(LOG_INFO) << "[KV] entry: " << B2MiB(this->num_entries * sizeof(ValueType))
                            << " MB (" << num_entries << " entries)" << LOG_endl;
        logstream(LOG_INFO) << "\tused entries: " << B2MiB(used_entries * sizeof(ValueType))
                            << " MB (" << used_entries << " entries)" << LOG_endl;
    }
};

}  // end of namespace wukong

max_stars_repo_name: SJTU-IPADS/wukong-cube

max_issues_repo_name: SJTU-IPADS/wukong-cube

max_forks_repo_name: SJTU-IPADS/wukong-cube


text:
// Copyright (C) 2016-2018 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#include <boost/yap/expression.hpp>

#include <boost/test/minimal.hpp>

#include <sstream>

template<typename T>
using term = boost::yap::terminal<boost::yap::expression, T>;

template<typename T>
using term_ref = boost::yap::expression_ref<boost::yap::expression, term<T> &>;

template<typename T>
using term_cref =
    boost::yap::expression_ref<boost::yap::expression, term<T> const &>;

namespace yap = boost::yap;
namespace bh = boost::hana;

int test_main(int, char * [])
{
    {
        term<uint32_t> x{{2u}};
        term<uint32_t> const cx{{3u}};

        {
            yap::expression<
                yap::expr_kind::unary_plus,
                bh::tuple<term<uint32_t>>>
                term_expr = +term<uint32_t>{{1u}};
            yap::expression<
                yap::expr_kind::unary_plus,
                bh::tuple<term_ref<uint32_t>>>
                term_ref_expr = +x;
            yap::expression<
                yap::expr_kind::unary_plus,
                bh::tuple<term_cref<uint32_t>>>
                term_const_ref_expr = +cx;

            BOOST_CHECK(evaluate(term_expr) == 1u);
            BOOST_CHECK(evaluate(term_ref_expr) == 2u);
            BOOST_CHECK(evaluate(term_const_ref_expr) == 3u);
        }

        {
            yap::expression<yap::expr_kind::negate, bh::tuple<term<uint32_t>>>
                term_expr = -term<uint32_t>{{1u}};
            yap::expression<
                yap::expr_kind::negate,
                bh::tuple<term_ref<uint32_t>>>
                term_ref_expr = -x;
            yap::expression<
                yap::expr_kind::negate,
                bh::tuple<term_cref<uint32_t>>>
                term_const_ref_expr = -cx;

            BOOST_CHECK(evaluate(term_expr) == 0u - 1u);
            BOOST_CHECK(evaluate(term_ref_expr) == 0u - 2u);
            BOOST_CHECK(evaluate(term_const_ref_expr) == 0u - 3u);
        }

        {
            yap::expression<
                yap::expr_kind::complement,
                bh::tuple<term<uint32_t>>>
                term_expr = ~term<uint32_t>{{1u}};
            yap::expression<
                yap::expr_kind::complement,
                bh::tuple<term_ref<uint32_t>>>
                term_ref_expr = ~x;
            yap::expression<
                yap::expr_kind::complement,
                bh::tuple<term_cref<uint32_t>>>
                term_const_ref_expr = ~cx;

            BOOST_CHECK(evaluate(term_expr) == 0xfffffffe);
            BOOST_CHECK(evaluate(term_ref_expr) == 0xfffffffd);
            BOOST_CHECK(evaluate(term_const_ref_expr) == 0xfffffffc);
        }

        {
            yap::expression<
                yap::expr_kind::logical_not,
                bh::tuple<term<uint32_t>>>
                term_expr = !term<uint32_t>{{1u}};
            yap::expression<
                yap::expr_kind::logical_not,
                bh::tuple<term_ref<uint32_t>>>
                term_ref_expr = !x;
            yap::expression<
                yap::expr_kind::logical_not,
                bh::tuple<term_cref<uint32_t>>>
                term_const_ref_expr = !cx;

            BOOST_CHECK(evaluate(term_expr) == false);
            BOOST_CHECK(evaluate(term_ref_expr) == false);
            BOOST_CHECK(evaluate(term_const_ref_expr) == false);
        }

        {
            yap::expression<yap::expr_kind::pre_inc, bh::tuple<term<uint32_t>>>
                term_expr = ++term<uint32_t>{{1u}};
            yap::expression<
                yap::expr_kind::pre_inc,
                bh::tuple<term_ref<uint32_t>>>
                term_ref_expr = ++x;
            yap::expression<
                yap::expr_kind::pre_inc,
                bh::tuple<term_cref<uint32_t>>>
                term_const_ref_expr = ++cx;
            (void)term_const_ref_expr;

            BOOST_CHECK(evaluate(term_expr) == 2u);
            BOOST_CHECK(evaluate(term_ref_expr) == 3u);
        }
    }

    {
        {
            uint32_t i = 1, j = 2, k = 3;
            term<uint32_t &> x{{j}};
            term<uint32_t &> const cx{{k}};

            yap::expression<
                yap::expr_kind::address_of,
                bh::tuple<term<uint32_t &>>>
                term_expr = &term<uint32_t &>{{i}};
            yap::expression<
                yap::expr_kind::address_of,
                bh::tuple<term_ref<uint32_t &>>>
                term_ref_expr = &x;
            yap::expression<
                yap::expr_kind::address_of,
                bh::tuple<term_cref<uint32_t &>>>
                term_const_ref_expr = &cx;

            BOOST_CHECK(evaluate(term_expr) == &i);
            BOOST_CHECK(evaluate(term_ref_expr) == &j);
            BOOST_CHECK(evaluate(term_const_ref_expr) == &k);
        }

        {
            uint32_t i = 1, j = 2, k = 3;
            term<uint32_t *> x{{&j}};
            term<uint32_t *> const cx{{&k}};

            yap::expression<
                yap::expr_kind::dereference,
                bh::tuple<term<uint32_t *>>>
                term_expr = *term<uint32_t *>{{&i}};
            yap::expression<
                yap::expr_kind::dereference,
                bh::tuple<term_ref<uint32_t *>>>
                term_ref_expr = *x;
            yap::expression<
                yap::expr_kind::dereference,
                bh::tuple<term_cref<uint32_t *>>>
                term_const_ref_expr = *cx;

            BOOST_CHECK(evaluate(term_expr) == i);
            BOOST_CHECK(evaluate(term_ref_expr) == j);
            BOOST_CHECK(evaluate(term_const_ref_expr) == k);
        }
    }

    return 0;
}

max_issues_repo_name: avplayer/cxxrpc

max_forks_repo_name: avplayer/cxxrpc


text:
//
// Created by Lu WJ on 5/7/2017 AD.
//

#ifndef CRYPTCONV_MATRIX_HPP
#define CRYPTCONV_MATRIX_HPP
#include <NTL/matrix.h>
typedef long val_t;
typedef NTL::Mat<val_t> Matrix;
typedef NTL::Vec<val_t> PlainVec;
void randomize(Matrix *mat);
void zeros(Matrix *mat, long N);
void add(Matrix *mat, const Matrix &b);
PlainVec mul(const Matrix &a, const PlainVec &b);
Matrix mul(const Matrix &a, const Matrix &b);
void add(PlainVec *a, const PlainVec &b);
void transpose(Matrix *T, const Matrix &mat);
bool is_same(const Matrix &a, const Matrix &b);
bool is_same(const Matrix &a, const Matrix &b, long modulus);
bool load_matrix(Matrix *mat, const std::string &file);
bool load_matrix(std::istream& in, Matrix *mat);
bool save_matrix(const Matrix &mat, const std::string &file);
bool save_matrix(std::ostream &out, const Matrix &mat);
bool load_vector(PlainVec *vec, const std::string &file);
bool load_vector(std::istream &in, PlainVec *vec);
bool save_vector(const PlainVec &vec, const std::string &file);
bool save_vector(std::ostream &out, const PlainVec &vec);
#endif //CRYPTCONV_MATRIX_HPP

max_stars_repo_name: Vampsj/SMP

max_issues_repo_name: Vampsj/SMP

max_forks_repo_name: Vampsj/SMP


text:
//           Copyright Maarten L. Hekkelman 2021
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#pragma once

#include <pinch/pinch.hpp>
#include <pinch/types.hpp>

#include <boost/asio/dispatch.hpp>

#include <functional>
#include <future>
#include <string>
#include <type_traits>

/// \file This file contains the class known_hosts which is used to keep track of
/// already trusted host keys.

namespace pinch
{

/// \brief The status for a host key's name/alg/key combo.
enum class host_key_state
{
	no_match,    ///< The host key is not known at all
	keys_differ, ///< There is a known host key, but the key values differ
	match        ///< The key is known
};

/// \brief The class known_hosts is used to keep track of already trusted host keys.

class known_hosts
{
  public:

	/// \brief internal struct to store known hosts
	struct host_key
	{
		std::string m_host_name;
		std::string m_algorithm;
		blob m_key;

		host_key_state compare(const std::string &host_name, const std::string &algorithm, const pinch::blob &key) const;
	};

	/// \brief known_hosts is a singleton
	static known_hosts &instance()
	{
		static std::unique_ptr<known_hosts> s_instance(new known_hosts);
		return *s_instance;
	}

	/// \brief Read a host file (in openssh format)
	void load_host_file(std::istream &host_file);

	/// \brief Write a host file (in openssh format)
	void save_host_file(std::ostream &host_file);

	/// \brief Add a single host key, \a key is the base64 encoded key
	void add_host_key(const std::string &host, const std::string &algorithm, const std::string &key);

	/// \brief Add a single host key, \a key is the binary, decoded key
	void add_host_key(const std::string &host, const std::string &algorithm, const blob &key);

	/// \brief Return true if the host/algorithm/key pair should be trusted
	host_key_state accept_host_key(const std::string &host, const std::string &algorithm, const blob &key);

	/// \brief making the known_hosts iterable, but const only
	using iterator = std::vector<host_key>::const_iterator;

	/// \brief return iterator to first const host_key
	iterator begin() const { return m_host_keys.cbegin(); }

	/// \brief return iterator past last const host_key
	iterator end() const { return m_host_keys.cend(); }

	/// \brief return true if list is empty
	bool empty() const { return m_host_keys.empty(); }

	/// \brief return number of keys in list
	std::size_t size() const { return m_host_keys.size(); }

  private:
	known_hosts() {}

	known_hosts(const known_hosts &) = delete;
	known_hosts &operator=(const known_hosts &) = delete;

	std::vector<host_key> m_host_keys;
	std::mutex m_mutex;

	static std::unique_ptr<known_hosts> s_instance;
};

} // namespace pinch

max_stars_repo_name: mhekkel/pinch

max_issues_repo_name: mhekkel/pinch

max_forks_repo_name: mhekkel/pinch