text:
/*
 *	GlobalConfiguration.cpp
 *	!CHOAS
 *	Created by Bisegni Claudio.
 *
 *    	Copyright 2012 INFN, National Institute of Nuclear Physics
 *
 *    	Licensed under the Apache License, Version 2.0 (the "License");
 *    	you may not use this file except in compliance with the License.
 *    	You may obtain a copy of the License at
 *
 *    	http://www.apache.org/licenses/LICENSE-2.0
 *
 *    	Unless required by applicable law or agreed to in writing, software
 *    	distributed under the License is distributed on an "AS IS" BASIS,
 *    	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    	See the License for the specific language governing permissions and
 *    	limitations under the License.
 */
#include <fstream>
#include <iostream>

#include <chaos/common/log/LogManager.h>
#include <boost/interprocess/sync/file_lock.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>

#include "GlobalConfiguration.h"

using namespace chaos;
namespace po = boost::program_options;

#define _RPC_PORT					8888
#define _DIRECT_IO_PRIORITY_PORT	1672
#define _DIRECT_IO_SERVICE_PORT		30175

/*

 */
void GlobalConfiguration::preParseStartupParameters() throw (CException){

    try{
        addOption(InitOption::OPT_HELP, "Produce help message");
		//cache parameter
		addOption<std::string>(InitOption::OPT_CONF_FILE,"File configuration path");
		addOption(InitOption::OPT_DATA_IO_IMPL, po::value< string >()->default_value("IODirect"), "Specify the data io implementation");
		addOption(InitOption::OPT_DIRECT_IO_IMPLEMENTATION, po::value< string >()->default_value("ZMQ"), "Specify the direct io implementation");
		addOption(InitOption::OPT_DIRECT_IO_PRIORITY_SERVER_PORT, po::value<int>()->default_value(_DIRECT_IO_PRIORITY_PORT), "DirectIO priority server port");
		addOption(InitOption::OPT_DIRECT_IO_SERVICE_SERVER_PORT, po::value<int>()->default_value(_DIRECT_IO_SERVICE_PORT), "DirectIO service server port");
        addOption(InitOption::OPT_DIRECT_IO_SERVER_THREAD_NUMBER, po::value<int>()->default_value(2),"DirectIO server thread number");
        addOption(InitOption::OPT_RPC_IMPLEMENTATION, po::value< string >()->default_value("ZMQ"), "Specify the rpc implementation");
		addOption(InitOption::OPT_RPC_SERVER_PORT, po::value<int>()->default_value(_RPC_PORT), "RPC server port");
        addOption(InitOption::OPT_RPC_SERVER_THREAD_NUMBER, po::value<int>()->default_value(2),"RPC server thread number");
        addOption(InitOption::OPT_LIVE_DATA_SERVER_ADDRESS, po::value< vector<string> >()->multitoken(), "Live server:port address");
        addOption(InitOption::OPT_METADATASERVER_ADDRESS, po::value< string >()->default_value("localhost:5000"), "Metadataserver server:port address");
        addOption(InitOption::OPT_LOG_ON_CONSOLE, po::value< bool >()->zero_tokens(), "Specify when the log must be forwarded on console");
        addOption(InitOption::OPT_LOG_ON_FILE, po::value< bool >()->zero_tokens(), "Specify when the log must be forwarded on file");
        addOption(InitOption::OPT_LOG_FILE, po::value< string >()->default_value("chaos_frameowrk.log"), "Specify when the file path of the log");
        addOption(InitOption::OPT_LOG_LEVEL, po::value< string >()->default_value("info"), "Specify the level of the log using the value [debug, info, notice, warning, fatal]"),
        addOption(InitOption::OPT_PUBLISHING_IP, po::value< string >(), "Specify the ip address where to publish the framework rpc system");
    }catch (po::error &e) {
        throw CException(0, e.what(), "GlobalConfiguration::preParseStartupParameters");
    }
}

/*!
 Specialized option for startup c and cpp program main options parameter
 */
void GlobalConfiguration::parseStartupParameters(int argc, char* argv[]) throw (CException) {
    parseParameter(po::parse_command_line(argc, argv, desc));
}
//!stringbuffer parser
/*
 specialized option for string stream buffer with boost semantics
 */
void GlobalConfiguration::parseStringStream(std::istream &sStreamOptions) throw (CException) {
    parseParameter(po::parse_config_file(sStreamOptions, desc));
}

//!conver user input into enumeration
/*
 Conver the string into enumeration for the log level
 */
int32_t GlobalConfiguration::filterLogLevel(string& levelStr) throw (CException) {
    chaos::log::level::LogSeverityLevel level = chaos::log::level::LSLInfo;

    if (levelStr == "info")
        level = chaos::log::level::LSLInfo;
    else if (levelStr == "debug")
        level = chaos::log::level::LSLDebug;
    else if (levelStr == "notice")
        level = chaos::log::level::LSLNotice;
    else if (levelStr == "warning")
        level = chaos::log::level::LSLWarning;
    else if (levelStr == "fatal")
        level = chaos::log::level::LSLFatal;
    else
        throw chaos::CException(1, "Invalid log level", "GlobalConfiguration::filterLogLevel");

    return static_cast< int32_t >(level);
}

void GlobalConfiguration::loadStartupParameter(int argc, char* argv[]) throw (CException) {
	try{
        //
        po::store(po::parse_command_line(argc, argv, desc), vm);
    }catch (po::error &e) {
        //write error also on cerr
        std::cerr << e.what();
        throw CException(0, e.what(), __PRETTY_FUNCTION__);

    }
}

void GlobalConfiguration::loadStreamParameter(std::istream &config_file)  throw (CException) {
	try{
        //
        po::store(po::parse_config_file(config_file, desc), vm);
    }catch (po::error &e) {
        //write error also on cerr
        std::cerr << e.what();
        throw CException(0, e.what(), __PRETTY_FUNCTION__);

    }
}

void GlobalConfiguration::scanOption()  throw (CException) {
	try{
        po::notify(vm);
    }catch (po::error &e) {
        //write error also on cerr
        std::cerr << e.what();
        throw CException(0, e.what(), __PRETTY_FUNCTION__);

    }

	if (hasOption(InitOption::OPT_HELP)) {
        std::cout << desc;
        exit(0);
        return;

    }
}

/*
 parse the tandard startup parameters
 */
void GlobalConfiguration::parseParameter(const po::basic_parsed_options<char>& optionsParser) throw (CException){
    //int rpcServerPort;
    //int rpcServerThreadNumber;
    //string metadataServerAddress;
    //vector<string> liveDataServer;
    try{
        //
        po::store(optionsParser, vm);
    }catch (po::error &e) {
        //write error also on cerr
        std::cerr << e.what();
        throw CException(0, e.what(), "GlobalConfiguration::preParseStartupParameters");

    }

	//scan option
	scanOption();

	//check the default option
	checkDefaultOption();
}

void GlobalConfiguration::checkDefaultOption() throw (CException) {

	//lock file for permit to choose different tcp port for services
	std::fstream domain_file_lock_stream("/tmp/chaos_init.lock",
										 std::ios_base::out |
										 std::ios_base::binary);

	//check if we have got the lock
	boost::interprocess::file_lock flock("/tmp/chaos_init.lock");
	boost::interprocess::scoped_lock<boost::interprocess::file_lock> e_lock(flock);

    //now we can fill the gloabl configuration
    //start with getting log configuration
    CHECK_AND_DEFINE_BOOL_ZERO_TOKEN_OPTION(logOnConsole, InitOption::OPT_LOG_ON_CONSOLE)
    configuration.addBoolValue(InitOption::OPT_LOG_ON_CONSOLE, logOnConsole);

    CHECK_AND_DEFINE_BOOL_ZERO_TOKEN_OPTION(logOnFile, InitOption::OPT_LOG_ON_FILE)
    configuration.addBoolValue(InitOption::OPT_LOG_ON_FILE, logOnFile);

    CHECK_AND_DEFINE_OPTION(string, logFilePath, InitOption::OPT_LOG_FILE)
    configuration.addStringValue(InitOption::OPT_LOG_FILE, logFilePath);

    CHECK_AND_DEFINE_OPTION(string, logLevel, InitOption::OPT_LOG_LEVEL)
    configuration.addInt32Value(InitOption::OPT_LOG_LEVEL, filterLogLevel(logLevel));

    CHECK_AND_DEFINE_OPTION(string, publishingIp, InitOption::OPT_PUBLISHING_IP)
    bool isIp = regex_match(publishingIp, ServerIPRegExp);
    if(isIp) configuration.addStringValue(InitOption::OPT_PUBLISHING_IP, publishingIp);

    //configure rpc
    CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(int, rpcServerPort, InitOption::OPT_RPC_SERVER_PORT, 8888)
    int32_t freeFoundPort = InetUtility::scanForLocalFreePort(rpcServerPort);
    addLocalServerBasePort(freeFoundPort);
    configuration.addInt32Value(RpcConfigurationKey::CS_CMDM_RPC_ADAPTER_TCP_UDP_PORT, freeFoundPort);

    CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(int, rpcServerThreadNumber, InitOption::OPT_RPC_SERVER_THREAD_NUMBER, 1)
    configuration.addInt32Value(RpcConfigurationKey::CS_CMDM_RPC_ADAPTER_THREAD_NUMBER, rpcServerThreadNumber);

    CHECK_AND_DEFINE_OPTION(string, rpcImpl, InitOption::OPT_RPC_IMPLEMENTATION)
    configuration.addStringValue(RpcConfigurationKey::CS_CMDM_RPC_ADAPTER_TYPE, rpcImpl);

	//direct io
	CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(int, direct_io_server_thread_number, InitOption::OPT_DIRECT_IO_SERVER_THREAD_NUMBER, 2)
	configuration.addInt32Value(common::direct_io::DirectIOConfigurationKey::DIRECT_IO_SERVER_THREAD_NUMBER, direct_io_server_thread_number);

	CHECK_AND_DEFINE_OPTION(string, direct_io_server_impl, InitOption::OPT_DIRECT_IO_IMPLEMENTATION)
	configuration.addStringValue(common::direct_io::DirectIOConfigurationKey::DIRECT_IO_IMPL_TYPE, direct_io_server_impl);

	CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(int, direct_io_priority_port, InitOption::OPT_DIRECT_IO_PRIORITY_SERVER_PORT, _DIRECT_IO_PRIORITY_PORT)
    freeFoundPort = InetUtility::scanForLocalFreePort(direct_io_priority_port);
    configuration.addInt32Value(common::direct_io::DirectIOConfigurationKey::DIRECT_IO_PRIORITY_PORT, (uint32_t)freeFoundPort);

	CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(int, direct_io_service_port, InitOption::OPT_DIRECT_IO_SERVICE_SERVER_PORT, _DIRECT_IO_SERVICE_PORT)
    freeFoundPort = InetUtility::scanForLocalFreePort(direct_io_service_port);
    configuration.addInt32Value(common::direct_io::DirectIOConfigurationKey::DIRECT_IO_SERVICE_PORT, (uint32_t)freeFoundPort);

    //cevent
    configuration.addStringValue(event::EventConfiguration::OPTION_KEY_EVENT_ADAPTER_IMPLEMENTATION, "AsioImpl");

    //configure the live data
    CHECK_AND_DEFINE_OPTION(vector<string>, liveDataServer, InitOption::OPT_LIVE_DATA_SERVER_ADDRESS)
    if(liveDataServer.size()==0){
        configuration.appendStringToArray("localhost:11211");
    }else{
        for (int idx = 0; idx < liveDataServer.size(); idx++) {
            configuration.appendStringToArray(liveDataServer[idx]);
        }
    }
    configuration.finalizeArrayForKey(DataProxyConfigurationKey::CS_DM_LD_SERVER_ADDRESS);

    //configure metadataserver
    CHECK_AND_DEFINE_OPTION_WITH_DEFAULT(string, metadataServerAddress, InitOption::OPT_METADATASERVER_ADDRESS, "localhost:5000")
    if (metadataServerAddress.size()>0) {
        addMetadataServerAddress(metadataServerAddress);
    }
}

/*
 Add a custom option
 */
void GlobalConfiguration::addOption(const char* name,
                                    const char* description) throw (CException){
    try{
        desc.add_options()(name, description);
    }catch (po::error &e) {
        throw CException(0, e.what(), "GlobalConfiguration::addOption");
    }
}

/*
 Add a custom option
 */
void GlobalConfiguration::addOption(const char* name,
                                    const po::value_semantic* s,
                                    const char* description) throw (CException){
    try {
        desc.add_options()(name, s, description);
    }catch (po::error &e) {
        throw CException(0, e.what(), "GlobalConfiguration::addOption");
    }
}


text:
#ifndef _BIN_FILE_HPP
#define _BIN_FILE_HPP 1

#include <boost/shared_ptr.hpp>
#include <stddef.h>
#include <stdint.h>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>

class BinFile
{
public:
  BinFile(const std::string &filename);
  virtual ~BinFile();
  // intentionally no copy constructor or assignment operator

  std::size_t size_in_bytes();

  /// The number of data items that must be in the file for the supplied data size.
  std::size_t num_items(const std::size_t data_size, const std::size_t items = 0);

  /// Seek to a position in the file relative to the beginning.
  /// \param bytes The position in bytes.
  void seek(const std::size_t bytes);

  /// Turn on/off byteswapping
  void setByteSwap(const bool);

  /// Read a chunk of data from the current offset position
  /// \param items the number of elements to read. Zero means read all.
  template <typename NumT>
  void read(std::vector<NumT> &data, const std::size_t items = 0);
  void read(std::stringstream & data, const size_t items = 0);

private:
  template <typename NumT>
  void read_block(NumT *buffer, const std::size_t buffer_size);

  bool byteSwap;
  std::size_t size_bytes;
  std::ifstream *handle;
  std::string filename;
};

#endif

ext: hpp

max_stars_repo_name: quantumsteve/morebin

max_issues_repo_name: quantumsteve/morebin

max_forks_repo_name: quantumsteve/morebin


text:
//==================================================================================================
/**
  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
**/
//==================================================================================================

#ifndef BOOST_SIMD_FUNCTION_SIMD_LOGICAL_OR_HPP_INCLUDED
#define BOOST_SIMD_FUNCTION_SIMD_LOGICAL_OR_HPP_INCLUDED

#include <boost/simd/function/scalar/logical_or.hpp>
#include <boost/simd/arch/common/generic/function/autodispatcher.hpp>
#include <boost/simd/arch/common/simd/function/logical_or.hpp>

#endif

max_stars_repo_name: xmar/pythran

max_issues_repo_name: xmar/pythran

max_forks_repo_name: xmar/pythran


text:
/*

Copyright (c) 2005-2016, University of Oxford.
All rights reserved.

University of Oxford means the Chancellor, Masters and Scholars of the
University of Oxford, having an administrative office at Wellington
Square, Oxford OX1 2JD, UK.

This file is part of Chaste.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
 * Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.
 * Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.
 * Neither the name of the University of Oxford nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#ifndef TESTCABASEDDIVISIONRULE_HPP_
#define TESTCABASEDDIVISIONRULE_HPP_

#include <cxxtest/TestSuite.h>

#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>

#include "ArchiveOpener.hpp"
#include "CellsGenerator.hpp"
#include "CaBasedCellPopulation.hpp"
#include "FixedDurationGenerationBasedCellCycleModel.hpp"
#include "AbstractCellBasedTestSuite.hpp"
#include "AbstractCaBasedDivisionRule.hpp"
#include "ExclusionCaBasedDivisionRule.hpp"
#include "PottsMeshGenerator.hpp"
#include "SmartPointers.hpp"

//This test is always run sequentially (never in parallel)
#include "FakePetscSetup.hpp"

class TestCaBasedDivisionRules : public AbstractCellBasedTestSuite
{
public:

    void TestAddCellwithExclusionBasedDivisionRule()
    {
        /**
         * In this test we basically test that the AbstractCaBasedDivisionRule is implemented and joined with the population
         * correctly. We make a new ExclusionCaBasedDivisionRule, divide a cell with it and check that the new cells
         * are in the correct locations.
         */

        // Make a simple Potts mesh
        PottsMeshGenerator<2> generator(3, 0, 0, 3, 0, 0);
        PottsMesh<2>* p_mesh = generator.GetMesh();

        // Create 6 cells in the bottom 2 rows
        std::vector<unsigned> location_indices;
        for (unsigned index=0; index<6; index++)
        {
            location_indices.push_back(index);
        }

        std::vector<CellPtr> cells;
        CellsGenerator<FixedDurationGenerationBasedCellCycleModel, 1> cells_generator;
        cells_generator.GenerateBasic(cells, location_indices.size());

        // Create cell population
        CaBasedCellPopulation<2> cell_population(*p_mesh, cells, location_indices);

        CellPtr p_cell_0 = cell_population.GetCellUsingLocationIndex(0);
        MAKE_PTR(WildTypeCellMutationState, p_state);
        MAKE_PTR(StemCellProliferativeType, p_stem_type);

        FixedDurationGenerationBasedCellCycleModel* p_model = new FixedDurationGenerationBasedCellCycleModel();
        CellPtr p_temp_cell(new Cell(p_state, p_model));
        p_temp_cell->SetCellProliferativeType(p_stem_type);
        p_temp_cell->SetBirthTime(-1);

        // Set the division rule for our population to be the exclusion division rule (note that this is the default

        boost::shared_ptr<AbstractCaBasedDivisionRule<2> > p_division_rule_to_set(new ExclusionCaBasedDivisionRule<2>());
        cell_population.SetCaBasedDivisionRule(p_division_rule_to_set);

        // Get the division rule back from the population and try to add new cell by dividing cell at site 0;
        boost::shared_ptr<AbstractCaBasedDivisionRule<2> > p_division_rule = cell_population.GetCaBasedDivisionRule();

        CellPtr p_parent_cell = cell_population.GetCellUsingLocationIndex(0);
        TS_ASSERT(!(p_division_rule->IsRoomToDivide(p_parent_cell,cell_population)));

        // Test adding the new cell in the population (note this calls CalculateDaughterNodeIndex)
        TS_ASSERT_THROWS_THIS(cell_population.AddCell(p_cell_0, zero_vector<double>(2), p_parent_cell),
                              "Trying to divide when there is no room to divide, check your division rule");

        // Test adding it in a free space
        p_parent_cell = cell_population.GetCellUsingLocationIndex(4);
        TS_ASSERT(p_division_rule->IsRoomToDivide(p_parent_cell,cell_population));

        TS_ASSERT_EQUALS(p_division_rule->CalculateDaughterNodeIndex(p_cell_0,p_parent_cell,cell_population), 7u);

        // Test adding the new cell in the population (note this calls CalculateDaughterNodeIndex)
        cell_population.AddCell(p_cell_0, zero_vector<double>(2), p_parent_cell);

        // Now check the cells are in the correct place
        TS_ASSERT_EQUALS(cell_population.GetNumRealCells(), 7u);
    }
};

#endif /*TESTCABASEDDIVISIONRULE_HPP_*/

max_stars_repo_name: uofs-simlab/ChasteOS

max_issues_repo_name: uofs-simlab/ChasteOS

max_forks_repo_name: uofs-simlab/ChasteOS


text:
#include <stdio.h>
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/stream.hpp>
#include <fstream>
#include <iostream>
#include <string.h>

#include "fbyfilefind.h"
#include "fbyregex.h"

using namespace std;
using namespace boost::iostreams ;
namespace fs = boost::filesystem;

Regex regexJheadResolution("JHead Resolution",
                           "Resolution\\s+\\:\\s+(\\d+)\\s*x\\s*(\\d+)");

void FindImageSize(const string &filename,
                   int & /* width */, int & /* height */)
{
    string command("jhead '" + filename + "'");

    cout << command << endl;

    FILE *f = popen(command.c_str(), "r");
    if (NULL == f)
    {
        string errstr("Unable to popen: " + command);
        perror(errstr.c_str());
        return;
    }

    char buffer[256];
    RegexMatch match;
    while (NULL != fgets(buffer, sizeof(buffer) - 1, f))
    {
        string s(buffer);
        if (regexJheadResolution.Match(s, match))
        {
            cout << "Image " << filename << " is " << match.Match(1) << " by " << match.Match(2) << endl;
        }
// << s << endl;
    }

    pclose(f);
    return;
}

int main(int argc, char **argv)
{
    for (int arg = 1; arg < argc; ++arg)
    {
        FileFind(argv[arg],
                 [](fs::directory_iterator dir_iter)
                 {
                     string path(dir_iter->path().filename().string());
                     size_t pos = path.rfind("/");
                     if (pos != string::npos)
                     {
                         path = path.erase(0, pos);
                     }
                     pos = path.rfind('.');
                     if (pos != string::npos
                         && !strcasecmp(path.c_str() + pos, ".jpg"))
                     {
                         int width = -1, height = -1;
                         FindImageSize(dir_iter->path().filename().string(),
                                       width, height);
                     }
                 }
            );
    }
}

ext: cpp

max_stars_repo_name: danlyke/libfby

max_issues_repo_name: danlyke/libfby

max_forks_repo_name: danlyke/libfby


text:
/*
 * A C++ memory pool that is Boost-friendly and performance oriented.
 *
 * Inspired by:
 *  - https://thinkingeek.com/2017/11/19/simple-memory-pool/: a C++ memory pool for raw pointers
 *    (no smart pointers) using free lists and perfect forwarding to handle non-default ctors.
 *  - https://github.com/steinwurf/recycle: a C++ memory pool for std::shared_ptr<>, which however
 *    produces 1 malloc operation for each allocation/recycle operation
 *
 * Author: fmontorsi
 * Created: Feb 2019
 * License: BSD license
 *
 */

#pragma once

//------------------------------------------------------------------------------
// Includes
//------------------------------------------------------------------------------

// #include <boost/intrusive/slist.hpp> // not really used finally
#include <boost/smart_ptr/intrusive_ptr.hpp>
#include <boost/smart_ptr/intrusive_ref_counter.hpp>

namespace memorypool {
//------------------------------------------------------------------------------
// Constants
//------------------------------------------------------------------------------

#define BOOST_INTRUSIVE_POOL_DEFAULT_POOL_SIZE (64)
#define BOOST_INTRUSIVE_POOL_INCREASE_STEP (64)
#define BOOST_INTRUSIVE_POOL_NO_MAX_SIZE (0)

#ifndef BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
// if you define BOOST_INTRUSIVE_POOL_DEBUG_CHECKS=1 before including this header file,
// you will activate a lot more debug checks on the memory pool to verify its integrity;
// this is useful during e.g. debug builds
#define BOOST_INTRUSIVE_POOL_DEBUG_CHECKS (0)
#endif

#ifndef BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
// if you define BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS=1 before including this header file,
// you will activate a pthread-specific check about correct thread access to the memory pool
#define BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS (0)
#else
// checks are active: we use pthread_self() API:
#include <pthread.h>
#endif

#ifndef BOOST_INTRUSIVE_POOL_DEBUG_MAX_REFCOUNT
// completely-arbitrary threshold about what range of refcounts can be considered
// sane and valid and which range cannot be considered valid!
#define BOOST_INTRUSIVE_POOL_DEBUG_MAX_REFCOUNT (1024)
#endif

typedef enum {
    RECYCLE_METHOD_NONE,
    RECYCLE_METHOD_DESTROY_FUNCTION,
    RECYCLE_METHOD_CUSTOM_FUNCTION,
    // RECYCLE_METHOD_DTOR,
} recycle_method_e;

//------------------------------------------------------------------------------
// Forward declarations
//------------------------------------------------------------------------------

class boost_intrusive_pool_item;

//------------------------------------------------------------------------------
// boost_intrusive_pool_iface
//------------------------------------------------------------------------------

class boost_intrusive_pool_iface
    : public boost::intrusive_ref_counter<boost_intrusive_pool_iface, boost::thread_unsafe_counter> {
public:
    virtual ~boost_intrusive_pool_iface() = default;

    virtual void recycle(boost_intrusive_pool_item* item) = 0;

    virtual bool is_bounded() const = 0;
    virtual bool is_memory_exhausted() const = 0;
};

//------------------------------------------------------------------------------
// boost_intrusive_pool_item
// Base class for any C++ class that will be used inside a boost_intrusive_pool
//------------------------------------------------------------------------------

class boost_intrusive_pool_item {
    friend void intrusive_ptr_add_ref(boost_intrusive_pool_item* x);
    friend void intrusive_ptr_release(boost_intrusive_pool_item* x);

public:
    boost_intrusive_pool_item()
    {
        m_boost_intrusive_pool_next = nullptr;
        m_boost_intrusive_pool_refcount = 0;
        m_boost_intrusive_pool_owner = nullptr;
    }
    boost_intrusive_pool_item(const boost_intrusive_pool_item& other)
    {
        // IMPORTANT:
        //   do not copy the members of this class around:
        //   whether "this" instance is inside a memory pool or not, that has to stay that way,
        //   regardless of whether "other" is inside a memory pool or not.

        m_boost_intrusive_pool_next = nullptr;
        m_boost_intrusive_pool_refcount = 0;
        m_boost_intrusive_pool_owner = nullptr;
    }
    boost_intrusive_pool_item(const boost_intrusive_pool_item&& other)
    {
        // IMPORTANT:
        //   do not copy the members of this class around:
        //   whether "this" instance is inside a memory pool or not, that has to stay that way,
        //   regardless of whether "other" is inside a memory pool or not.

        m_boost_intrusive_pool_next = nullptr;
        m_boost_intrusive_pool_refcount = 0;
        m_boost_intrusive_pool_owner = nullptr;
    }
    virtual ~boost_intrusive_pool_item() {}

    //------------------------------------------------------------------------------
    // emulate the boost::intrusive_ref_counter class implementation:
    //------------------------------------------------------------------------------

    unsigned int use_count() const noexcept { return m_boost_intrusive_pool_refcount; }

    boost_intrusive_pool_item& operator=(const boost_intrusive_pool_item& other)
    {
        // IMPORTANT:
        //   do not copy the members of this class around:
        //   whether "this" instance is inside a memory pool or not, that has to stay that way,
        //   regardless of whether "other" is inside a memory pool or not.
        //   This is also what boost::intrusive_ref_counter implementation does.
        return *this;
    }
    boost_intrusive_pool_item& operator=(const boost_intrusive_pool_item&& other)
    {
        // IMPORTANT:
        //   do not copy the members of this class around:
        //   whether "this" instance is inside a memory pool or not, that has to stay that way,
        //   regardless of whether "other" is inside a memory pool or not.
        //   This is also what boost::intrusive_ref_counter implementation does.
        return *this;
    }

    //------------------------------------------------------------------------------
    // memorypool::boost_intrusive_pool private functions
    //------------------------------------------------------------------------------

    boost_intrusive_pool_item* _refcounted_item_get_next() { return m_boost_intrusive_pool_next; }
    void _refcounted_item_set_next(boost_intrusive_pool_item* p) { m_boost_intrusive_pool_next = p; }

    boost::intrusive_ptr<boost_intrusive_pool_iface> _refcounted_item_get_pool()
    {
        return m_boost_intrusive_pool_owner;
    }
    void _refcounted_item_set_pool(boost::intrusive_ptr<boost_intrusive_pool_iface> p)
    {
        m_boost_intrusive_pool_owner = p;
    }

    //------------------------------------------------------------------------------
    // default init-after-recycle, destroy-before-recycle methods:
    //------------------------------------------------------------------------------

    virtual void destroy() {}

    //------------------------------------------------------------------------------
    // memorypool utility functions
    //------------------------------------------------------------------------------

    // is_in_memory_pool() returns true both in case
    //  - this item is currently marked as "in use" in some memory pool
    //  - this item lies unused in some memory pool
    // This function returns false if e.g. this item has been allocated on the heap
    // bypassing any memory pool mechanism.
    bool is_in_memory_pool() const { return m_boost_intrusive_pool_owner != nullptr; }

    // sanity checks for this item. Useful for debug only.
    void check() const
    {
        if (is_in_memory_pool()) {
            assert(m_boost_intrusive_pool_refcount < BOOST_INTRUSIVE_POOL_DEBUG_MAX_REFCOUNT);

            if (m_boost_intrusive_pool_refcount == 0) {
                // this item is apparently inside the free list of the memory pool:
                // in such case it should be always linked to the list; the only case where
                // the "next" pointer can be NULL is in the case the memory pool is memory-bounded
                // and the free items are exhausted or the memory pool is infinite but the memory is over
                assert(m_boost_intrusive_pool_next != nullptr || m_boost_intrusive_pool_owner->is_bounded()
                    || m_boost_intrusive_pool_owner->is_memory_exhausted());
            } else {
                // this item is in use and thus must be UNLINKED from the free list of the memory pool:
                assert(m_boost_intrusive_pool_next == nullptr);
            }
        }
    }

private:
    size_t m_boost_intrusive_pool_refcount; // intrusive refcount
    boost_intrusive_pool_item* m_boost_intrusive_pool_next; // we use a free-list-based memory pool algorithm
    boost::intrusive_ptr<boost_intrusive_pool_iface>
        m_boost_intrusive_pool_owner; // used for auto-return to the memory pool
};

inline void intrusive_ptr_add_ref(boost_intrusive_pool_item* x)
{
#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
    assert(x->m_boost_intrusive_pool_refcount < BOOST_INTRUSIVE_POOL_DEBUG_MAX_REFCOUNT - 1);
#endif
    ++x->m_boost_intrusive_pool_refcount;
}

inline void intrusive_ptr_release(boost_intrusive_pool_item* x)
{
#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
    x->check();
#endif
    if (--x->m_boost_intrusive_pool_refcount == 0) {
        if (x->m_boost_intrusive_pool_owner)
            x->m_boost_intrusive_pool_owner->recycle(x);
        else
            // assume the item has been allocated out of the pool:
            delete x;
    }
}

//------------------------------------------------------------------------------
// boost_intrusive_pool_arena
// Internal helper class for a boost_intrusive_pool.
//------------------------------------------------------------------------------

// Arena of items. This is just an array of items and a pointer
// to another arena. All arenas are singly linked between them.
template <typename Item> class boost_intrusive_pool_arena {
public:
    // Creates an arena with arena_size items.
    boost_intrusive_pool_arena(size_t arena_size, boost::intrusive_ptr<boost_intrusive_pool_iface> p)
    {
        assert(arena_size > 0 && p);
        m_storage_size = arena_size;
        m_storage = new Item[arena_size];
        if (m_storage) {
            for (size_t i = 1; i < arena_size; i++) {
                m_storage[i - 1]._refcounted_item_set_next(&m_storage[i]);
                m_storage[i - 1]._refcounted_item_set_pool(p);
            }
            m_storage[arena_size - 1]._refcounted_item_set_next(nullptr);
            m_storage[arena_size - 1]._refcounted_item_set_pool(p);
        }
        // else: malloc failed!

        m_next_arena = nullptr;
    }

    boost_intrusive_pool_arena(const boost_intrusive_pool_arena& other) = delete;
    boost_intrusive_pool_arena(const boost_intrusive_pool_arena&& other) = delete;

    ~boost_intrusive_pool_arena()
    {
        // m_storage is automatically cleaned up
        if (m_storage) {
            delete[] m_storage;
            m_storage = nullptr;
        }
    }

    // Returns a pointer to the array of items. This is used by the arena
    // itself. This is only used to update free_list during initialization
    // or when creating a new arena when the current one is full.
    boost_intrusive_pool_item* get_first_item() const
    {
        return dynamic_cast<boost_intrusive_pool_item*>(&m_storage[0]);
    }

    size_t get_stored_item_count() const { return m_storage_size; }

    // Sets the next arena. Used when the current arena is full and
    // we have created this one to get more storage.
    void set_next_arena(boost_intrusive_pool_arena* p)
    {
        assert(!m_next_arena && p);
        m_next_arena = p;
        m_storage[m_storage_size - 1]._refcounted_item_set_next(p->get_first_item());
    }
    boost_intrusive_pool_arena* get_next_arena() { return m_next_arena; }
    const boost_intrusive_pool_arena* get_next_arena() const { return m_next_arena; }

    boost_intrusive_pool_arena operator=(const boost_intrusive_pool_arena& other) = delete;
    boost_intrusive_pool_arena operator=(const boost_intrusive_pool_arena&& other) = delete;

private:
    // Pointer to the next arena.
    boost_intrusive_pool_arena* m_next_arena;

    // Storage of this arena.
    // std::unique_ptr<Item[]> m_storage;
    size_t m_storage_size;
    Item* m_storage;
};

//------------------------------------------------------------------------------
// boost_intrusive_pool
// The actual memory pool implementation.
//------------------------------------------------------------------------------

template <class Item> class boost_intrusive_pool {
public:
    // using dummy = typename std::enable_if<std::is_base_of<boost_intrusive_pool_item, Item>::value>::type;

    // The type of pointers provided by this memory pool implementation
    using item_ptr = boost::intrusive_ptr<Item>;

    // The allocate function type
    using allocate_function = std::function<void(Item&)>;

    // The recycle function type
    // If specified the recycle function will be called every time a resource gets recycled into the pool. This allows
    // temporary resources, e.g., file handles to be closed when an object is longer used.
    using recycle_function = std::function<void(Item&)>;

public:
    // Default constructor
    // Constructs a memory pool quite small which increases its size by rather small steps.
    // Tuning of these steps is critical for performances.
    // The ctor also allows you to specify which function should be run on items returning to the pool.
    boost_intrusive_pool(size_t init_size = BOOST_INTRUSIVE_POOL_DEFAULT_POOL_SIZE,
        size_t enlarge_size = BOOST_INTRUSIVE_POOL_INCREASE_STEP, size_t max_size = BOOST_INTRUSIVE_POOL_NO_MAX_SIZE,
        recycle_method_e method = RECYCLE_METHOD_NONE, recycle_function recycle = nullptr)
    {
        assert(init_size > 0);
        assert((max_size == BOOST_INTRUSIVE_POOL_NO_MAX_SIZE) || (max_size >= init_size && enlarge_size > 0));

        m_pool = boost::intrusive_ptr<impl>(new impl(enlarge_size, max_size, method, recycle));

        // do initial malloc
        m_pool->enlarge(init_size);
    }
    virtual ~boost_intrusive_pool() { m_pool->trigger_self_destruction(); }

    // Copy constructor
    boost_intrusive_pool(const boost_intrusive_pool& other) = delete;

    // Move constructor
    boost_intrusive_pool(boost_intrusive_pool&& other) = delete;

    // Copy assignment
    boost_intrusive_pool& operator=(const boost_intrusive_pool& other) = delete;

    // Move assignment
    boost_intrusive_pool& operator=(boost_intrusive_pool&& other) = delete;

    //------------------------------------------------------------------------------
    // configuration methods
    //------------------------------------------------------------------------------

    void set_recycle_method(recycle_method_e method, recycle_function recycle_fn = nullptr)
    {
        m_pool->set_recycle_method(method, recycle_fn);
    }

    //------------------------------------------------------------------------------
    // allocate method variants
    //------------------------------------------------------------------------------

    item_ptr allocate()
    {
        Item* recycled_item = m_pool->allocate_safe_get_recycled_item();
        if (!recycled_item)
            return nullptr;

        // in this case we don't need to call ANY function

        item_ptr ret_ptr(recycled_item);
#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
        ret_ptr->check();
#endif
        return ret_ptr;
    }

    // Returns first available free item or, if necessary and the memory pool is unbounded and has not reached the
    // maximum size, allocates a new item. Uses C++11 perfect forwarding to the init() function of the memory pooled
    // item.
    template <typename... Args> item_ptr allocate_through_init(Args&&... args)
    {
        Item* recycled_item = m_pool->allocate_safe_get_recycled_item();
        if (!recycled_item)
            return nullptr;

        // Construct the object in the obtained storage
        // using the init() method of the item itself:
        recycled_item->init(std::forward<Args>(args)...);

        // AFTER the init() call, run the check() function
        item_ptr ret_ptr(recycled_item);
#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
        ret_ptr->check();
#endif
        return ret_ptr;
    }

    // Returns first available free item or, if necessary and the memory pool is unbounded,
    // allocates a new item.
    template <typename... Args> item_ptr allocate_through_function(allocate_function fn)
    {
        Item* recycled_item = m_pool->allocate_safe_get_recycled_item();
        if (!recycled_item)
            return nullptr;

        // Construct the object in the obtained storage
        // uses perfect forwarding to the class ctor:
        /// new (recycled_item) Item(std::forward<Args>(args)...);
        fn(*recycled_item);

        // relinking the item to the pool is instead a critical step: we just executed
        // the ctor of the recycled item; that resulted in a call to
        // boost_intrusive_pool_item::boost_intrusive_pool_item()!
        recycled_item->_refcounted_item_set_pool(m_pool);

        // AFTER the ctor call, run the check() function
        item_ptr ret_ptr(recycled_item);
#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
        ret_ptr->check();
#endif
        return ret_ptr;
    }

    //------------------------------------------------------------------------------
    // other functions operating on items
    //------------------------------------------------------------------------------

    void clear()
    {
        // VERY IMPORTANT: this function is very tricky: to do this correctly we cannot
        // simply call m_pool->clear(): that would remove all arenas that are the memory
        // support of memory pool items.
        // At this point we don't know yet if there are boost::intrusive_ptr<> out there
        // still alive... so we must play safe:
        size_t init_size = m_pool->m_enlarge_step;
        size_t enlarge_size = m_pool->m_enlarge_step;
        size_t max_size = m_pool->m_max_size;
        recycle_method_e method = m_pool->m_recycle_method;
        recycle_function recycle = m_pool->m_recycle_fn;
        m_pool->trigger_self_destruction();
        m_pool = nullptr; // release old pool
        m_pool = boost::intrusive_ptr<impl>(new impl(enlarge_size, max_size, method, recycle));
    }

    void check() { m_pool->check(); }

    //------------------------------------------------------------------------------
    // getters
    //------------------------------------------------------------------------------

    // returns true if there are no elements in use.
    // Note that if empty()==true, it does not mean that capacity()==0 as well!
    bool empty() const { return m_pool->empty(); }

    bool is_bounded() const { return m_pool->is_bounded(); }

    bool is_limited() const { return m_pool->is_limited(); }

    bool is_memory_exhausted() const { return m_pool->is_memory_exhausted(); }

    // returns the current (=maximum) capacity of the object pool
    size_t capacity() const { return m_pool->capacity(); }

    size_t max_size() const { return m_pool->max_size(); }

    // returns the number of free entries of the pool
    size_t unused_count() const { return m_pool->unused_count(); }

    // returns the number of items currently malloc()ed from this pool
    size_t inuse_count() const { return m_pool->inuse_count(); }

    // returns the number of mallocs done so far
    size_t enlarge_steps_done() const { return m_pool->enlarge_steps_done(); }

private:
    /// The actual pool implementation. We use the
    /// enable_shared_from_this helper to make sure we can pass a
    /// "back-pointer" to the pooled objects. The idea behind this
    /// is that we need objects to be able to add themselves back
    /// into the pool once they go out of scope.
    class impl : public boost_intrusive_pool_iface {
    public:
        impl(size_t enlarge_size, size_t max_size, recycle_method_e method, recycle_function recycle)
        {
            // assert(enlarge_size > 0); // NOTE: enlarge_size can be zero to create a limited-size memory pool

            // configurations
            m_recycle_method = method;
            m_recycle_fn = recycle;
            m_enlarge_step = enlarge_size;
            m_max_size = max_size;

            // status
            m_first_free_item = nullptr;
            m_first_arena = nullptr;
            m_last_arena = nullptr;
            m_memory_exhausted = false;
            m_trigger_self_destruction = false;

            // stats
            m_free_count = 0;
            m_inuse_count = 0;
            m_total_count = 0;

#if BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
            m_allowed_thread = 0;
#endif
        }

        ~impl()
        {
            // if this dtor is called, it means that all memory pooled items have been destroyed:
            // they are holding a shared_ptr<> back to us, so if one of them was alive, this dtor would not be called!
            clear();
        }

        void set_recycle_method(recycle_method_e method, recycle_function recycle = nullptr)
        {
            m_recycle_method = method;
            m_recycle_fn = recycle;
        }

        void trigger_self_destruction()
        {
            m_trigger_self_destruction = true;

            // walk over the free list and reduce our own refcount by removing the link between the items and ourselves:
            // this is important because it allows the last item that will return to this pool to trigger the pool
            // dtor: see recycle() implementation
            boost_intrusive_pool_item* pcurr = m_first_free_item;
            while (pcurr) {
                pcurr->_refcounted_item_set_pool(nullptr);
                pcurr = pcurr->_refcounted_item_get_next();
            }
        }

        size_t get_effective_enlarge_step() const
        {
            size_t enlarge_step = m_enlarge_step;
            if (m_enlarge_step > 0 && (m_max_size > 0 && (m_total_count + m_enlarge_step > m_max_size)))
                enlarge_step = m_max_size - m_total_count; // enlarge_step can be zero if we reach the max_size
            return enlarge_step;
        }

        Item* allocate_safe_get_recycled_item()
        {
#if BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
            if (m_allowed_thread == 0)
                m_allowed_thread = pthread_self();
            else
                assert(m_allowed_thread == pthread_self());
#endif

            if (m_free_count == 0) {
                assert(m_first_free_item == nullptr);
                size_t enlarge_step = get_effective_enlarge_step();
                if (enlarge_step == 0 || !enlarge(enlarge_step)) {
                    m_memory_exhausted = true;
                    return nullptr; // allocation by enlarge() failed or this is a fixed-size memory pool!
                }
            }

            // get first item from free list
            assert(m_first_free_item != nullptr);
            Item* recycled_item = dynamic_cast<Item*>(m_first_free_item); // downcast (base class -> derived class)
            assert(recycled_item != nullptr); // we always allocate all items of the same type,
                                              // so the dynamic cast cannot fail
            assert(
                recycled_item->_refcounted_item_get_pool().get() == this); // this was set during arena initialization
                                                                           // and must be valid at all times

            // update stats
            m_free_count--;
            m_inuse_count++;

            // update the pointer to the next free item available
            m_first_free_item = m_first_free_item->_refcounted_item_get_next();
            if (m_first_free_item == nullptr && m_enlarge_step > 0) {
                size_t enlarge_step = get_effective_enlarge_step();
                if (enlarge_step == 0) { // enlarge_step can be zero if we reach the max_size
                    m_memory_exhausted = true;
                } else {
                    // this is a memory pool which can be still enlarged:
                    // exit the function leaving the m_first_free_item as a valid pointer to a free item!
                    // this is just to simplify debugging and make more effective the check() function implementation!

                    assert(m_free_count == 0);
                    if (!enlarge(enlarge_step)) {
                        m_memory_exhausted = true;
                        // We tried to fetch memory from the O.S. but we failed. However we succeeded in getting the
                        // last available item. So fallback and provide that last item to the caller.
                    }
                }
            }

            // unlink the item to return
            recycled_item->_refcounted_item_set_next(nullptr);
            return recycled_item;
        }

        bool enlarge(size_t arena_size)
        {
#if BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
            assert(m_allowed_thread == 0 || m_allowed_thread == pthread_self());
#endif
            // If the current arena is full, create a new one.
            boost_intrusive_pool_arena<Item>* new_arena = new boost_intrusive_pool_arena<Item>(arena_size, this);
            if (!new_arena)
                return false; // malloc failed... memory finished... very likely this is a game over

            // Link the new arena to the last one.
            if (m_last_arena)
                m_last_arena->set_next_arena(new_arena);
            if (m_first_arena == nullptr)
                m_first_arena = new_arena; // apparently we are initializing the memory pool for the very first time

            // Seek pointer to last arena
            m_last_arena = new_arena;

            // Update the free_list with the storage of the just created arena.
            if (m_first_free_item == nullptr)
                m_first_free_item = m_last_arena->get_first_item();

            m_free_count += arena_size;
            m_total_count += arena_size;

            return true;
        }

        virtual void recycle(boost_intrusive_pool_item* pitem_base) override
        {
#if BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
            assert(m_allowed_thread != 0);
            assert(m_allowed_thread == pthread_self());
#endif
            assert(pitem_base
                && pitem_base->_refcounted_item_get_next()
                    == nullptr); // Recycling an item that has been already recycled?
            assert(pitem_base->_refcounted_item_get_pool().get() == this);

            Item* pitem = dynamic_cast<Item*>(pitem_base); // downcast (base class -> derived class)
            assert(pitem != nullptr); // we always allocate all items of the same type,
                                      // so the dynamic cast cannot fail
            switch (m_recycle_method) {
            case RECYCLE_METHOD_NONE:
                break;

            case RECYCLE_METHOD_DESTROY_FUNCTION:
                pitem->destroy();
                break;

            case RECYCLE_METHOD_CUSTOM_FUNCTION:
                m_recycle_fn(*pitem);
                break;

                // the big problem with using the class destructor is that the virtual table of the item will
                // be destroyed; attempting to dynamic_cast<> the item later will fail (NULL returned).
                // so this recycling option is disabled for now
                // case RECYCLE_METHOD_DTOR:
                // Destroy the object using its dtor:
                // pitem->Item::~Item();
                // break;
            }

            // sanity check:
            if (!is_bounded()) {
                assert(m_first_free_item != nullptr || m_memory_exhausted);
            }

            // Add the item at the beginning of the free list.
            pitem_base->_refcounted_item_set_next(m_first_free_item);
            m_first_free_item = pitem_base;
            m_free_count++;

            assert(m_inuse_count > 0);
            m_inuse_count--;

#if BOOST_INTRUSIVE_POOL_DEBUG_CHECKS
            pitem_base->check();
#endif

            // test for self-destruction:
            // is this an orphan pool (i.e. a pool without any boost_intrusive_pool<> associated to it anymore)?
            if (m_trigger_self_destruction) {
                // in such case break the link between the items being recycled and this pool:
                // in this way the very last memory-pooled item returning to this pool will lower the refcount
                // to this pool to zero, and impl::~impl() will get called, freeing all arenas!
                pitem->_refcounted_item_set_pool(nullptr);
            }
        }

        //------------------------------------------------------------------------------
        // other functions operating on items
        //------------------------------------------------------------------------------

        // THIS IS A SUPER DANGEROUS FUNCTION: IT JUST REMOVES ALL ARENAS OF THIS MEMORY POOL WITHOUT ANY
        // CHECK WHETHER THERE ARE boost::intrusive_ptr<> OUT THERE STILL POINTING TO ITEMS INSIDE THOSE
        // ARENAS. USE
        void clear()
        {
            if (m_first_arena) {
                size_t init_size = m_first_arena->get_stored_item_count();
                boost_intrusive_pool_arena<Item>* pcurr = m_first_arena;
                while (pcurr) {
                    boost_intrusive_pool_arena<Item>* pnext = pcurr->get_next_arena();
                    delete pcurr;
                    pcurr = pnext;
                }
            } else {
                // this memory pool has just been clear()ed... the last arena pointer should be null as well:
                assert(m_last_arena == nullptr);
            }

            // status
            m_first_free_item = nullptr;
            m_first_arena = nullptr;
            m_last_arena = nullptr;
            m_memory_exhausted = false;

            // stats
            m_free_count = 0;
            m_inuse_count = 0;
            m_total_count = 0;
        }

        void check()
        {
            if (m_first_arena) {
                // this memory pool has been correctly initialized
                assert(m_last_arena);
                assert(m_total_count > 0);

                // this condition should hold at any time:
                assert(m_free_count + m_inuse_count == m_total_count);
                if (is_bounded()) {
                    // when the memory pool is bounded it contains only 1 arena of a fixed size:
                    assert(m_first_arena == m_last_arena);
                } else {
                    // infinite or max size memory pool: either we have a valid free element or the last malloc() must
                    // have failed or the maximum size has been reached:
                    assert(m_first_free_item != nullptr || m_memory_exhausted);
                }
            } else {
                // this memory pool has just been cleared with clear() apparently:
                assert(!m_last_arena);
                assert(!m_first_free_item);
                assert(m_free_count == 0);
                assert(m_inuse_count == 0);
                assert(m_total_count == 0);
            }
        }

        //------------------------------------------------------------------------------
        // getters
        //------------------------------------------------------------------------------

        // returns true if there are no elements in use.
        // Note that if empty()==true, it does not mean that capacity()==0 as well!
        bool empty() const { return m_free_count == m_total_count; }

        bool is_bounded() const { return m_enlarge_step == 0; }

        bool is_limited() const { return (m_enlarge_step == 0 || m_max_size != 0); }

        bool can_be_enlarged() const { return m_enlarge_step > 0 && (m_max_size == 0 || m_total_count < m_max_size); }

        bool is_memory_exhausted() const { return m_memory_exhausted; }

        // returns the current (=maximum) capacity of the object pool
        size_t capacity() const { return m_total_count; }

        size_t max_size() const { return (m_enlarge_step != 0) ? m_max_size : m_total_count; }

        // returns the number of free entries of the pool
        size_t unused_count() const { return m_free_count; }

        // returns the number of items currently malloc()ed from this pool
        size_t inuse_count() const { return m_inuse_count; }

        // returns the number of mallocs done so far
        size_t enlarge_steps_done() const
        {
            size_t num_arenas_allocated = 0;

            const boost_intrusive_pool_arena<Item>* pcurr = m_first_arena;
            while (pcurr) {
                pcurr = pcurr->get_next_arena();
                num_arenas_allocated++;
            }

            return num_arenas_allocated;
        }

    public:
        // The recycle strategy & function
        recycle_method_e m_recycle_method;
        recycle_function m_recycle_fn;

        // How many new items to add each time the pool becomes full?
        // If this is zero, then this is a bounded pool, which cannot grow beyond
        // the initial size provided at construction time.
        size_t m_enlarge_step;
        // Maximum pool size.
        // If this is zero, then it is ignored.
        // If this is greater then zero, then no more items will be added to the pull if resulting size would exceed
        // this value. If enlarge_step is zero, the max_size parameter become meaningless.
        size_t m_max_size;

        // Pointers to first and last arenas.
        // First arena is changed only at
        //  - construction time
        //  - in clear()
        //  - in reset(size_t)
        // Pointer to last arena is instead updated on every enlarge step.
        boost_intrusive_pool_arena<Item>* m_first_arena;
        boost_intrusive_pool_arena<Item>* m_last_arena;

        // List of free elements. The list can be threaded between different arenas
        // depending on the deallocation pattern.
        // This pointer can be NULL only whether:
        // - an infinite memory pool has exhausted memory (malloc returned NULL);
        // - a bounded memory pool has exhausted all its items
        // - a maximum size memory pool has exhausted all its items and reached the limit
        // In such cases m_memory_exhausted==true
        boost_intrusive_pool_item* m_first_free_item;
        // This flag can be true if allocation by enlarge() failed or this is a fixed-size memory pool or this is a
        // maximum size memory pool!
        bool m_memory_exhausted;

        // stats
        // This should hold always:
        //         m_free_count+m_inuse_count == m_total_count
        size_t m_free_count;
        size_t m_inuse_count;
        size_t m_total_count;

        bool m_trigger_self_destruction;

#if BOOST_INTRUSIVE_POOL_DEBUG_THREAD_ACCESS
        pthread_t m_allowed_thread;
#endif
    };

private:
    // The pool impl
    boost::intrusive_ptr<impl> m_pool;
};

} // namespace memorypool


text:
// This file is part of comma, a generic and flexible library
// Copyright (c) 2011 The University of Sydney
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. Neither the name of the University of Sydney nor the
//    names of its contributors may be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
// GRANTED BY THIS LICENSE.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
// HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
// OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

/// @authors dewey nguyen, vsevolod vlaskine

#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <iostream>
#include <deque>

#ifndef WIN32
#include <sysexits.h>
#endif

#include <boost/unordered_map.hpp>
#include "../../application/command_line_options.h"
#include "../../base/types.h"
#include "../../csv/stream.h"
#include "../../csv/impl/unstructured.h"
#include "../../string/string.h"
#include "../../visiting/traits.h"

static const char* name() { return "csv-blocks: "; }

struct input_t
{
    comma::csv::impl::unstructured key;
    input_t()  {}
    input_t( const comma::csv::impl::unstructured& key ): key( key ) {}
};

struct input_with_block
{
    comma::uint32 block;
    input_with_block() : block( 0 ) {}
};

struct input_with_index
{
    comma::uint32 index;
    input_with_index() : index( 0 ) {}
};

struct appended_column
{
    comma::uint32 value;
    appended_column( comma::uint32 val = 0 ) : value( val ) {}
};

struct accumulate_input
{
    comma::uint32 block;
    comma::csv::impl::unstructured key;
    accumulate_input() : block( 0 ) {}
    typedef boost::unordered_map< comma::csv::impl::unstructured, std::string, comma::csv::impl::unstructured::hash > unordered_map;
};

namespace comma { namespace visiting {

template <> struct traits< input_t >
{
    template < typename K, typename V > static void visit( const K&, const input_t& p, V& v ) { v.apply( "key", p.key ); }
    template < typename K, typename V > static void visit( const K&, input_t& p, V& v ) { v.apply( "key", p.key ); }
};

template <> struct traits< input_with_block >
{
    template < typename K, typename V > static void visit( const K&, const input_with_block& p, V& v ) { v.apply( "block", p.block ); }
    template < typename K, typename V > static void visit( const K&, input_with_block& p, V& v ) { v.apply( "block", p.block ); }
};

template <> struct traits< input_with_index >
{
    template < typename K, typename V > static void visit( const K&, const input_with_index& p, V& v ) { v.apply( "index", p.index ); }
    template < typename K, typename V > static void visit( const K&, input_with_index& p, V& v ) { v.apply( "index", p.index ); }
};

template <> struct traits< appended_column >
{
    template < typename K, typename V > static void visit( const K&, const appended_column& p, V& v ) { v.apply( "value", p.value ); }
    template < typename K, typename V > static void visit( const K&, appended_column& p, V& v ) { v.apply( "value", p.value ); }
};

template <> struct traits< accumulate_input >
{
    template < typename K, typename V > static void visit( const K&, const accumulate_input& p, V& v )
    {
        v.apply( "block", p.block );
        v.apply( "key", p.key );
    }

    template < typename K, typename V > static void visit( const K&, accumulate_input& p, V& v )
    {
        v.apply( "block", p.block );
        v.apply( "key", p.key );
    }
};

} } // namespace comma { namespace visiting {

static void usage( bool more )
{
    std::cerr << std::endl;
    std::cerr << "operations contiguous blocks of csv data" << std::endl;
    std::cerr << std::endl;
    std::cerr << "usage: cat some-data.csv | csv-blocks <operation> [<options>]" << std::endl;
    std::cerr << std::endl;
    std::cerr << "operations" << std::endl;
    std::cerr << "    accumulate" << std::endl;
    std::cerr << "        accumulate records from block to block, keeping the last seen record for each id" << std::endl;
    std::cerr << "        attention: output does not preserve input order, since there is no reasonable tradeof there" << std::endl;
    std::cerr << "                   use csv-sort for post-processing, if required" << std::endl;
    std::cerr << "    group|make-blocks" << std::endl;
    std::cerr << "        usage: cat something.csv | csv-blocks group --fields=,id, " << std::endl;
    std::cerr << "            appends group's block field based on specified id key or keys" << std::endl;
    std::cerr << "        options" << std::endl;
    std::cerr << "            --fields=<fields>" << std::endl;
    std::cerr << "                id: any number of id fields to group by" << std::endl;
    std::cerr << "                scalar: group by scalar, which can be integer, floating point number, or time" << std::endl;
    std::cerr << "                        options" << std::endl;
    std::cerr << "                            --block-gap,--gap=<value>; minimum gap in values between block" << std::endl;
    std::cerr << "                                                       double (for time: seconds as double), see examples" << std::endl;
    std::cerr << "                            --block-span,--span=<value>; maximum block span, double (for time: seconds as" << std::endl;
    std::cerr << "                                                         double), see examples" << std::endl;
    std::cerr << "                            --discard-out-of-range; discard input records with scalar out of range defined by --min and --max" << std::endl;
    std::cerr << "                            --min=[<value>]; convenience option: min value for the scalar range, see examples" << std::endl;
    std::cerr << "                            --max=[<value>]; convenience option: max value for the scalar range, see examples" << std::endl;
    std::cerr << "    head" << std::endl;
    std::cerr << "        reads records from first block to stdout, if --num-of-blocks=<num> specified, read more than one blocks" << std::endl;
    std::cerr << "        requires the index from 'index' mode in the inputs" << std::endl;
    std::cerr << "            cat something.csv | csv-blocks index --fields=,index " << std::endl;
    std::cerr << "    increment" << std::endl;
    std::cerr << "        increments specified field value, must be uint32 type - any such field can be used as a block:" << std::endl;
    std::cerr << "            cat something.csv | csv-blocks increment --fields=,,block" << std::endl;
    std::cerr << "            cat something.csv | csv-blocks increment --fields=,,block --step 2" << std::endl;
    std::cerr << "            cat something.csv | csv-blocks increment --fields=,,block --step -1" << std::endl;
    std::cerr << "    index" << std::endl;
    std::cerr << "        appends an index field counting the number of records for each block. Use --reverse for count down." << std::endl;
    std::cerr << "            cat something.csv | csv-blocks index --fields=,block " << std::endl;
    std::cerr << "    read-until" << std::endl;
    std::cerr << "        reads records until a given index value, e.g:" << std::endl;
    std::cerr << "            cat something.csv | csv-blocks read-until --fields=,index --index=5" << std::endl;
    std::cerr << std::endl;
    std::cerr << "options" << std::endl;
    std::cerr << "    --help,-h: help; --help --verbose: more help" << std::endl;
    if( more ) { std::cerr << comma::csv::options::usage() << std::endl; } else { std::cerr << std::endl; }
    std::cerr << "  head" << std::endl;
    std::cerr << "    --lines,--num-of-blocks,-n; outputs only the first specified number of blocks, default is 1" << std::endl;
    std::cerr << "    --strict; exits with an error if a full block (as indicated by the index number) is not found" << std::endl;
    std::cerr << std::endl;
    std::cerr << "  increment" << std::endl;
    std::cerr << "    --step; the number of increment/decrement for specified field, default is 1" << std::endl;
    std::cerr << std::endl;
    std::cerr << "  index" << std::endl;
    std::cerr << "    --reverse; output the indices in descending order instead of ascending" << std::endl;
    std::cerr << std::endl;
    std::cerr << "  group" << std::endl;
    std::cerr << "    --from,--starting-block; the starting block number to use, default is 1" << std::endl;
    std::cerr << std::endl;
    std::cerr << "  read-until" << std::endl;
    std::cerr << "    --index=<index>" << std::endl;
    std::cerr << std::endl;
    std::cerr << "examples" << std::endl;
    std::cerr << "    block_csv=block.csv" << std::endl;
    std::cerr << "    ( echo \"a,1,2,3\"; echo \"a,4,2,3\"; echo \"b,5,5,6\"; echo \"c,7,5,6\"; echo \"c,7,8,9\"; echo \"c,7,8,9\" ) >$block_csv" << std::endl;
    std::cerr << std::endl;
    std::cerr << "    group|make-blocks" << std::endl;
    std::cerr << "        unique ascending block number are assigned based on one id field" << std::endl;
    std::cerr << "            cat $block_csv | csv-blocks group --fields=id" << std::endl;
    std::cerr << "        unique ascending block number are assigned based on two id fields" << std::endl;
    std::cerr << "            cat $block_csv | csv-blocks group --fields=id,,id" << std::endl;
    std::cerr << "        group by scalar span - try it" << std::endl;
    std::cerr << "            seq 20 | csv-blocks group --fields=scalar --span 5" << std::endl;
    std::cerr << "            seq 1 3 20 | csv-blocks group --fields scalar --span 4" << std::endl;
    std::cerr << "        group by scalar gap - try it" << std::endl;
    std::cerr << "            seq 20 | csv-blocks group --fields=scalar --gap 1" << std::endl;
    std::cerr << "            seq 20 | csv-blocks group --fields=scalar --gap 2" << std::endl;
    std::cerr << std::endl;
    std::cerr << "    index" << std::endl;
    std::cerr << "        cat $block_csv | csv-blocks group --fields=id | csv-blocks index --fields=,,,,block" << std::endl;
    std::cerr << "            Append will add block field at the end, 'index' will append how many lines/records are left in the block" << std::endl;
    std::cerr << "            See 'head operation' below" << std::endl;
    std::cerr << std::endl;
    std::cerr << "    increment" << std::endl;
    std::cerr << "        cat $block_csv | csv-blocks increment --fields=,block " << std::endl;
    std::cerr << "            Given an integer field (any field), mark it as a block field so the field value is incremented." << std::endl;
    std::cerr << "            This increments the second field" << std::endl;
    std::cerr << std::endl;
    std::cerr << "    head" << std::endl;
    std::cerr << "        cat $block_csv | csv-blocks group --fields=id | csv-blocks index --fields=,,,,block | csv-blocks head --fields=,,,,,index " << std::endl;
    std::cerr << "            After appending the block field, then the block reverse index field, reading a single block from the input is possible" << std::endl;
    std::cerr << std::endl;
    std::cerr << "    index and head together" << std::endl;
    std::cerr << "        echo -e \"0\\n0\\n1\\n1\\n1\\n0\\n0\\n0\\n0\" | csv-blocks index --fields=block --reverse | while true ; do" << std::endl;
    std::cerr << "            block=$( csv-blocks head --fields block,index ) || break ; echo \"---\" ; echo \"$block\"" << std::endl;
    std::cerr << "        done" << std::endl;
    std::cerr << "            After indexing the input in reverse order, the stream is read block-by-block, with the first line given the maximal" << std::endl;
    std::cerr << "            element id in the block, essentially, the block size" << std::endl;
    std::cerr << std::endl;
    std::cerr << std::endl;
    exit(0);
}

static bool verbose;
static bool strict = false;
static comma::csv::options csv;
static bool reverse_index = false;
// All the data for this block
static std::deque< std::string > block_records;
static comma::int32 increment_step = 1;

static void output_record_and_index( const std::string& input, comma::uint32 index, bool is_binary, char delimiter )
{
    if( is_binary )
    {
        std::cout.write( &input[0], input.size() );
        std::cout.write( (const char *) &index, sizeof(comma::uint32));
        if( csv.flush ) { std::cout.flush(); }
    }
    else
    {
        std::cout << input << delimiter << index << std::endl;
        if( csv.flush ) { std::cout.flush(); }
    }
}

static void output_reverse_indexing( std::deque< std::string >& block_records, bool is_binary, char delimiter )
{
    std::size_t size = block_records.size();
    for( std::size_t i=0; i<size; ++i )
    {
        const std::string& input = block_records[i];
        comma::uint32 index = size - (i+1);
        output_record_and_index( input, index, is_binary, delimiter );
    }
    block_records.clear();
}

static bool empty_( const std::string& s ) // quick and dirty
{
    for( unsigned int i = 0; i < s.size(); ++i ) { if( s[i] != ' ' ) { return false; } }
    return true;
}

static double to_double( const input_t& lhs ) // quick and dirty
{
    if( lhs.key.longs.size() == 1 ) { return lhs.key.longs[0]; }
    if( lhs.key.doubles.size() == 1 ) { return lhs.key.doubles[0]; }
    if( lhs.key.time.size() == 1 ) { COMMA_THROW( comma::exception, "cannot convert time to double" ); }
    if( lhs.key.strings.size() == 1 ) { COMMA_THROW( comma::exception, "cannot convert strings to double" ); }
    COMMA_THROW( comma::exception, "never here" );
}

static double diff( const input_t& lhs, const input_t& rhs ) // quick and dirty
{
    if( lhs.key.longs.size() == 1 ) { return std::abs( double( lhs.key.longs[0] ) - rhs.key.longs[0] ); }
    if( lhs.key.doubles.size() == 1 ) { return std::abs( lhs.key.doubles[0] - rhs.key.doubles[0] ); }
    if( lhs.key.time.size() == 1 ) { return std::abs( double( ( lhs.key.time[0] - rhs.key.time[0] ).total_microseconds() ) / 1000000 ); }
    if( lhs.key.strings.size() == 1 ) { COMMA_THROW( comma::exception, "difference for strings: not implemented" ); }
    COMMA_THROW( comma::exception, "never here" );
}

struct how_t { enum values { none, by_id, by_scalar }; };

template < typename T > static how_t::values set_fields( const comma::command_line_options& options, std::string& first_line, T& default_input )
{
    std::vector< std::string > v = comma::split( csv.fields, ',' );
    comma::csv::format f;
    if( csv.binary() ) { f = csv.format(); }
    else if( options.exists( "--format,--binary" ) ) { f = comma::csv::format( options.value< std::string >( "--format,--binary" ) ); }
    else
    {
        while( std::cin.good() && empty_( first_line ) ) { std::getline( std::cin, first_line ); }
        if( !std::cin.good() ) { exit( 0 ); }
        f = comma::csv::impl::unstructured::guess_format( first_line, csv.delimiter );
        if( verbose ) { std::cerr << name() << "guessed format: " << f.string() << std::endl; }
    }
    // This is to load the keys into input_t structure
    unsigned int size = f.count();
    how_t::values how = how_t::none;
    for( std::size_t i = 0; i < size; ++i )
    {
        if( i < v.size() )
        {
            if( v[i] == "id" )
            {
                if( how == how_t::by_scalar ) { COMMA_THROW( comma::exception, "expected either id or scalar in --fields; got both in: \"" << csv.fields << "\"" ); }
                how = how_t::by_id;
                v[i] = "key/" + default_input.key.append( f.offset( i ).type );
            }
            else if( v[i] == "scalar" )
            {
                if( how == how_t::by_id ) { COMMA_THROW( comma::exception, "expected either id or scalar in --fields; got both in: \"" << csv.fields << "\"" ); }
                if( how == how_t::by_scalar ) { COMMA_THROW( comma::exception, "expected not more than one scalar in --fields; got: \"" << csv.fields << "\"" ); }
                how = how_t::by_scalar;
                v[i] = "key/" + default_input.key.append( f.offset( i ).type );
            }
        }
    }
    if( how == how_t::none ) { COMMA_THROW( comma::exception, "please specify at least one id or scalar in --fields; got: \"" << csv.fields << "\"" ); }
    csv.fields = comma::join( v, ',' );
    return how;
}

#ifndef WIN32

struct memory_buffer
{
    char* buffer;
    size_t size;
    memory_buffer();
    memory_buffer( size_t size ) { allocate( size ); }
    ~memory_buffer();

    void allocate( size_t size );

    // if strict is true, fails when less data then expected is received
    comma::uint32 read_binary_records( comma::uint32 num_record=1, bool strict=false );

};
memory_buffer::memory_buffer() : buffer(NULL), size(0) {}
memory_buffer::~memory_buffer()
{
    if( buffer != NULL ) {
//         std::cerr  << "freeing head ascii memory, size: " << size << std::endl;
        free( (void*) buffer );
    }
}

void memory_buffer::allocate(size_t size)
{
    this->size = size;
    buffer = (char*) ::malloc( size );
    if( buffer == NULL ) { std::cerr << name() << "failed to allocate memory buffer of size: " << size << std::endl; exit(1); }
}

comma::uint32 memory_buffer::read_binary_records( comma::uint32 num_record, bool strict )
{
    static comma::uint32 one_record_size = csv.format().size();
    comma::uint32 bytes_to_read = one_record_size * num_record;
    size_t bytes_read = 0;
    while ( bytes_read < bytes_to_read && !::feof( stdin ) && !::ferror(stdin) )
    {
        bytes_read +=  ::fread( &buffer[bytes_read], 1, bytes_to_read - bytes_read, stdin );
    }

    if ( bytes_read == 0 ) { exit( EX_NOINPUT ); } // signals end of stream, not an error

    if( bytes_read % one_record_size != 0 ) { std::cerr << name() << "expected " << one_record_size << " bytes, got only read: " << ( bytes_read % one_record_size ) << std::endl; exit( 1 ); }
    if ( strict && (bytes_read < bytes_to_read) ) { std::cerr << name() << "expected " << bytes_to_read << " bytes (" << ( bytes_to_read / one_record_size  ) << " records); got only: " << bytes_read << " bytes (" << ( bytes_read / one_record_size ) << " records)" << std::endl; exit( 1 ); }

    return bytes_read;
}

static comma::uint32 extended_buffer_size = 65536; // 64Kb, default, overwritten on demand if a record is longer

static comma::uint32 read_and_write_binary_record()
{
    static memory_buffer memory( csv.format().size() );
    static comma::csv::binary< input_with_index > binary( csv );
    // Reads from stdin, the exact number of bytes to be memory.size
    //  It should blocks and keep reading until we have the entire message,
    //  This is to reassemble the message if it is broken into pieces (e.g. TCP input piped into stdin)
    comma::uint32 num_of_bytes = memory.read_binary_records(1, strict);
    std::cout.write( memory.buffer, num_of_bytes ); // send data to stdout, expect it to read one record
    if( csv.flush ) { std::cout.flush(); }

    static input_with_index record;
    // fill 'record' param
    binary.get( record, memory.buffer );
    return record.index;
}

/// Read a record and  fills out param 'record', the binary data is immediately send to stdout
static void read_and_write_binary_block()
{
    comma::uint32 records_to_read = read_and_write_binary_record();
    if( records_to_read == 0 ) { return; }
    static const comma::uint32 one_record_size = csv.format().size();
    extended_buffer_size = std::max( extended_buffer_size, one_record_size );
    static memory_buffer extended_buffer( extended_buffer_size );
    // maximum number that will fit in the extened_buffer
    comma::uint32 records_in_buffer = comma::uint32(extended_buffer_size / one_record_size) ;
    // only read up to the number of bytes we can hold
    comma::uint32 read_per_loop = (records_to_read <= records_in_buffer) ? records_to_read : records_in_buffer;
    comma::uint32 records_read = 0;
    while( records_read < records_to_read  && !::feof( stdin ) && !::ferror(stdin) )
    {
        if( read_per_loop > records_to_read  - records_read ) { read_per_loop = records_to_read  - records_read; }
        comma::uint32 read_chunk_size = extended_buffer.read_binary_records( read_per_loop, strict);
        std::cout.write( extended_buffer.buffer, read_chunk_size );
        if( csv.flush ) { std::cout.flush(); }
        records_read += read_per_loop;
    }
}

/// Read a record and  fills out param 'record', the binary data is immediately send to stdout
// when return value is false, 'record' is not filled because line read is empty
static boost::optional< comma::uint32 > read_and_write_ascii_record()
{
    static memory_buffer memory( 1024 );
    static comma::csv::ascii< input_with_index > ascii( csv );
    while( true )
    {
        input_with_index record;
        std::stringstream sstream;
        bool has_end_of_line = false;
        while( !has_end_of_line && !::feof(stdin) )
        {
            // getline may actually changes the buffer with realloc and extend the size
            ssize_t bytes_read = ::getline( &memory.buffer, &memory.size, stdin );
            if ( bytes_read <= 0 ) { return boost::none; }
#ifndef WIN32
            has_end_of_line = ( memory.buffer[bytes_read-1] == '\n' );
            sstream.write( memory.buffer, has_end_of_line ? bytes_read-1 : bytes_read );
#else
            has_end_of_line = ( bytes_read > 1
                                && memory.buffer[bytes_read-2] == '\r'
                                && memory.buffer[bytes_read-1] == '\n' );
            sstream.write( memory.buffer, has_end_of_line ? bytes_read-2 : bytes_read );
#endif
        }
        std::string line = sstream.str();
        if( line.empty() ) { continue; }
        // filling 'record' param
        ascii.get( record, line );
        // send line to stdout
        std::cout << line << std::endl;
        if( csv.flush ) { std::cout.flush(); }
        return record.index; // 'record' filled
    }
}

#endif // #ifndef WIN32

int main( int ac, char** av )
{
    try
    {
        comma::command_line_options options( ac, av, usage );
        verbose = options.exists( "--verbose,-v" );
        strict = options.exists( "--strict" );
        csv = comma::csv::options( options );
        csv.quote.reset();
        comma::csv::options csv_out;
        csv_out.full_xpath = false;
        if( csv.binary() ) { csv_out.format( comma::csv::format("ui") ); }
        std::vector< std::string > unnamed = options.unnamed( "--help,-h,--reverse,--verbose,-v,--discard-out-of-range", "-.*" );
        if( unnamed.empty() ) { std::cerr << name() << "please specify operation" << std::endl; return 1; }
        const std::string  operation = unnamed.front();
        if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << std::endl; }
        if( operation == "accumulate" )
        {
            std::string first_line;
            accumulate_input default_input;
            set_fields( options, first_line, default_input );
            if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << std::endl; }
            if ( default_input.key.empty() ) { std::cerr << name() << "please specify at least one id field" << std::endl; return 1; }
            accumulate_input::unordered_map map;
            comma::csv::input_stream< accumulate_input > istream( std::cin, csv, default_input );
            comma::uint32 block = 0;
            if( !first_line.empty() )
            {
                accumulate_input p = comma::csv::ascii< accumulate_input >( csv, default_input ).get( first_line );
                block = p.block;
                map[ p.key ] = first_line;
            }
            std::vector< std::string > fields = comma::split( csv.fields, ',' );
            for( unsigned int i = 0; i < fields.size(); ++i ) { if( fields[i] != "block" ) { fields[i] = ""; } }
            comma::csv::options block_csv = csv;
            block_csv.fields = comma::join( fields, ',' );
            boost::scoped_ptr< comma::csv::ascii< accumulate_input > > ascii;
            boost::scoped_ptr< comma::csv::binary< accumulate_input > > binary;
            if( csv.binary() ) { binary.reset( new comma::csv::binary< accumulate_input >( block_csv, default_input ) ); }
            else { ascii.reset( new comma::csv::ascii< accumulate_input >( block_csv, default_input ) ); }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const accumulate_input* p = istream.read();
                if( !p || block != p->block )
                {
                    for( accumulate_input::unordered_map::iterator it = map.begin(); it != map.end(); ++it )
                    {
                        std::cout.write( &( it->second[0] ), it->second.size() );
                        if( !csv.binary() ) { std::cout << std::endl; }
                    }
                    if( csv.flush ) { std::cout.flush(); }
                    if( p )
                    {
                        block = p->block;
                        for( accumulate_input::unordered_map::iterator it = map.begin(); it != map.end(); ++it ) { if( binary ) { binary->put( *p, &it->second[0] ); } else { ascii->put( *p, it->second ); } }
                    }
                }
                if( !p ) { break; }
                map[ p->key ] = istream.last();
            }
            return 0;
        }
        if( operation == "group" || operation == "make-blocks" )
        {
            comma::uint32 current_block = options.value< comma::uint32 >( "--starting-block,--from", 0 );
            std::string first_line;
            input_t default_input;
            auto how = set_fields( options, first_line, default_input );
            if( verbose ) { std::cerr << name() << "csv fields: " << csv.fields << "; making blocks by " << ( how == how_t::by_id ? "id" : "scalar" ) << std::endl; }
            boost::optional< double > gap;
            boost::optional< double > span;
            boost::optional< double > min;
            boost::optional< double > max;
            if( how == how_t::by_scalar )
            {
                options.assert_mutually_exclusive( "--gap,--span", "--min,--max" ); // for now
                gap = options.optional< double >( "--block-gap,--gap" );
                span = options.optional< double >( "--block-span,--span" );
                min = options.optional< double >( "--min" );
                max = options.optional< double >( "--max" );
            }
            comma::csv::input_stream< input_t > istream( std::cin, csv, default_input );
            comma::csv::output_stream< appended_column > ostream( std::cout, csv_out );
            comma::csv::tied< input_t, appended_column > tied( istream, ostream );
            auto update_block = [&]( const input_t& p )->bool
            {
                static input_t last = p;
                switch( how )
                {
                    case how_t::by_id:
                        if( !( last.key == p.key ) ) { ++current_block; }
                        last = p;
                        return true;
                    case how_t::by_scalar:
                    {
                        static input_t first = p;
                        if( gap || span )
                        {
                            if( ( gap && diff( last, p ) >= *gap ) || ( span && diff( first, p ) >= *span ) ) { ++current_block; first = p; }
                            last = p;
                            return true;
                        }
                        else
                        {
                            static bool last_in_range = false;
                            static bool discard_output_out_of_range = options.exists( "--discard-out-of-range" );
                            double v = to_double( p );
                            bool in_range = ( !min || !comma::math::less( v, *min ) ) && ( !max || !comma::math::less( *max, v ) );
                            static bool first_record = true;
                            if( !first_record ) // quick and dirty
                            {
                                if( discard_output_out_of_range )
                                {
                                    if( last_in_range && !in_range ) { ++current_block; }
                                }
                                else
                                {
                                    if( last_in_range != in_range ) { ++current_block; } // quick and dirty
                                }
                            }
                            last = p;
                            last_in_range = in_range;
                            first_record = false;
                            return in_range || !discard_output_out_of_range;
                        }
                    }
                    case how_t::none: // never here
                        return true;
                }
                return true; // never here
            };
            if( !first_line.empty() )
            {
                input_t p = comma::csv::ascii< input_t >( csv, default_input ).get( first_line );
                if( update_block( p ) )
                {
                    if( istream.is_binary() ) { std::cout.write( (char*)&p, istream.binary().size() ); }
                    else { std::cout << first_line << istream.ascii().ascii().delimiter(); }
                    ostream.write( appended_column( current_block ) );
                    if( csv.flush ) { std::cout.flush(); }
                }
            }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const input_t* p = istream.read();
                if( !p ) { break; }
                if( update_block( *p ) )
                {
                    tied.append( appended_column( current_block ) );
                    if( csv.flush ) { std::cout.flush(); }
                }
            }
            return 0;
        }
        else if( operation == "head" )
        {
#ifdef WIN32
            std::cerr << "csv-blocks: head: not implemented on windows" << std::endl; return 1;
#else // #ifdef WIN32
            ::setvbuf( stdin, (char *)NULL, _IONBF, 0 );
            #ifdef WIN32
                if( csv.binary() ) { _setmode( _fileno( stdin ), _O_BINARY ); } // for the time, when windows actually will be supported... maybe...
            #endif
            comma::uint32 num_of_blocks = options.value< comma::uint32 >( "--lines,--num-of-blocks,-n", 1 );

            if( !csv.binary() )
            {
                comma::uint32 index = 0;    // set as 0 for previous_index at start
                while( num_of_blocks > 0 )
                {
                    boost::optional< comma::uint32 > i = read_and_write_ascii_record();
                    if( !i )
                    {
                        if( index != 0 )
                        {
                            std::cerr << name() << "failed to read a full block, last index read is:" << index <<  ", hence " << index << " more record/s expected" << std::endl;
                            if( strict ) { return 1; }
                        }
                        return EX_NOINPUT; // we are done, end of stream
                    }
                    index = *i;
                    if( index == 0 ) { --num_of_blocks; }
                }
            }
            else
            {
                for( std::size_t n=0; n <num_of_blocks; ++n ) { read_and_write_binary_block(); }
            }
            return 0;
#endif // #ifdef WIN32
        }
        else if( operation == "index" )
        {
            reverse_index = options.exists("--reverse");
            comma::csv::input_stream< input_with_block > istream( std::cin, csv );
            char delimiter = istream.is_binary() ? ',' : istream.ascii().ascii().delimiter();
            comma::uint32 block = 0;
            comma::uint32 index = 0;
            std::string buffer;
            if( istream.is_binary() ) { buffer.resize( istream.binary().size() ); }
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const input_with_block* p = istream.read();
                if( !p ) { break; }
                if( block != p->block )
                {
                    if ( reverse_index ) { output_reverse_indexing( block_records, istream.is_binary(), delimiter  ); }
                    else { index = 0; }
                }
                block = p->block;
                if ( reverse_index )
                {
                    if( istream.is_binary() )
                    {
                        ::memcpy( &buffer[0], istream.binary().last(),  istream.binary().size() );
                        block_records.push_back( buffer );
                    }
                    else
                    {
                        block_records.push_back( comma::join( istream.ascii().last(), delimiter ) );
                    }
                }
                else
                {
                    if( istream.is_binary() ) { ::memcpy( &buffer[0], istream.binary().last(),  istream.binary().size() ); }
                    else { buffer = comma::join( istream.ascii().last(), delimiter ); }
                    output_record_and_index( buffer, index, istream.is_binary(), delimiter );
                    index++;
                }
            }
            if ( reverse_index ) { output_reverse_indexing( block_records, istream.is_binary(), delimiter  ); }
            return 0;
        }
        else if( operation == "increment" )    // operation is head
        {
            increment_step = options.value< comma::int32 >( "--step", 1 );
            comma::csv::input_stream< input_with_block > istream( std::cin, csv );
            comma::csv::output_stream< appended_column > ostream( std::cout, csv_out );
            comma::csv::tied< input_with_block, appended_column > tied( istream, ostream );
            appended_column incremented;
            while( istream.ready() || ( std::cin.good() && !std::cin.eof() ) )
            {
                const input_with_block* p = istream.read();
                if( !p ) { break; }
                incremented.value = p->block + increment_step;
                tied.append( incremented );
                if( csv.flush ) { std::cout.flush(); }
            }
            return 0;
        }
        else if( operation == "read-until" )
        {
#ifdef WIN32
            std::cerr << "csv-blocks: read-until: not implemented on windows" << std::endl; return 1;
#else // #ifdef WIN32
            ::setvbuf( stdin, (char *)NULL, _IONBF, 0 );
            #ifdef WIN32
                if( csv.binary() ) { _setmode( _fileno( stdin ), _O_BINARY ); } // for the time, when windows actually will be supported... maybe...
            #endif
            comma::uint32 index = options.value< comma::uint32 >( "--index" );
            while( true )
            {
                boost::optional< comma::uint32 > i = csv.binary() ? read_and_write_binary_record() : read_and_write_ascii_record();
                if( !i || *i == index ) { break; }
            }
            return 0;
#endif // #ifdef WIN32
        }
        else { std::cerr << name() << "unrecognised operation '" << operation << "'" << std::endl; }

        return 1;
    }
    catch( std::exception& ex ) { std::cerr << name() << ex.what() << std::endl; }
    catch( ... ) { std::cerr << name() << "unknown exception" << std::endl; }
    return 1;
}

max_stars_repo_name: mission-systems-pty-ltd/comma

max_issues_repo_name: mission-systems-pty-ltd/comma

max_forks_repo_name: mission-systems-pty-ltd/comma