text:
/**********************************************************************
*  Copyright (c) 2008-2014, Alliance for Sustainable Energy.
*  All rights reserved.
*
*  This library is free software; you can redistribute it and/or
*  modify it under the terms of the GNU Lesser General Public
*  License as published by the Free Software Foundation; either
*  version 2.1 of the License, or (at your option) any later version.
*
*  This library is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
*  Lesser General Public License for more details.
*
*  You should have received a copy of the GNU Lesser General Public
*  License along with this library; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
**********************************************************************/

#include <gtest/gtest.h>
#include "AnalysisDriverFixture.hpp"
#include "StopWatcher.hpp"

#include "../AnalysisDriver.hpp"
#include "../CurrentAnalysis.hpp"
#include "../AnalysisRunOptions.hpp"

#include "../../project/ProjectDatabase.hpp"
#include "../../project/AnalysisRecord.hpp"

#include "../../analysis/Analysis.hpp"
#include "../../analysis/Problem.hpp"
#include "../../analysis/DataPoint.hpp"
#include "../../analysis/DesignOfExperiments.hpp"
#include "../../analysis/DesignOfExperiments_Impl.hpp"
#include "../../analysis/DesignOfExperimentsOptions.hpp"
#include "../../analysis/DesignOfExperimentsOptions_Impl.hpp"
#include "../../analysis/SamplingAlgorithmOptions.hpp"
#include "../../analysis/SamplingAlgorithmOptions_Impl.hpp"
#include "../../analysis/SamplingAlgorithm.hpp"
#include "../../analysis/ContinuousVariable.hpp"
#include "../../analysis/ContinuousVariable_Impl.hpp"

#include <runmanager/Test/ToolBin.hxx>
#include "../../runmanager/lib/JobErrors.hpp"
#include "../../runmanager/lib/RunManager.hpp"
#include "../../runmanager/lib/Job.hpp"

#include "../../model/Model.hpp"

#include "../../utilities/core/FileReference.hpp"

#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_real.hpp>
#include <boost/random/variate_generator.hpp>

#include <OpenStudio.hxx>

#include <resources.hxx>

using namespace openstudio;
using namespace openstudio::model;
using namespace openstudio::runmanager;
using namespace openstudio::project;
using namespace openstudio::analysis;
using namespace openstudio::analysisdriver;

TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopDakotaAnalysis) {
  // Tests for stopping time < 20s.

  // RETRIEVE PROBLEM
  Problem problem = retrieveProblem("SimpleHistogramBinUQ",true,false);

  // DEFINE SEED
  Model model = model::exampleModel();
  openstudio::path p = toPath("./example.osm");
  model.save(p,true);
  FileReference seedModel(p);

  // CREATE ANALYSIS
  SamplingAlgorithmOptions algOptions;
  algOptions.setSamples(100);
  Analysis analysis("Stop Dakota Analysis",
                    problem,
                    SamplingAlgorithm(algOptions),
                    seedModel);

  // RUN ANALYSIS
  ProjectDatabase database = getCleanDatabase("StopDakotaAnalysis");
  AnalysisDriver analysisDriver(database);
  AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
  StopWatcher watcher(analysisDriver);
  watcher.watch(analysis.uuid());
  CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
  analysisDriver.waitForFinished();
  EXPECT_FALSE(analysisDriver.isRunning());
  EXPECT_GE(watcher.nComplete(),watcher.stopNum());
  EXPECT_LE(watcher.stoppingTime(),openstudio::Time(0,0,0,20));

  // check conditions afterward
  boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
  ASSERT_TRUE(jobErrors);
  EXPECT_FALSE(jobErrors->errors().empty());
  EXPECT_FALSE(currentAnalysis.analysis().dataPoints().empty());
  EXPECT_FALSE(currentAnalysis.analysis().algorithm()->isComplete());
  EXPECT_FALSE(currentAnalysis.analysis().algorithm()->failed());
  EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
}

TEST_F(AnalysisDriverFixture,RuntimeBehavior_StopAndRestartDakotaAnalysis) {

  // RETRIEVE PROBLEM
  Problem problem = retrieveProblem("SimpleHistogramBinUQ",true,false);

  // DEFINE SEED
  Model model = model::exampleModel();
  openstudio::path p = toPath("./example.osm");
  model.save(p,true);
  FileReference seedModel(p);

  // CREATE ANALYSIS
  SamplingAlgorithmOptions algOptions;
  algOptions.setSamples(10);
  Analysis analysis("Stop and Restart Dakota Analysis",
                    problem,
                    SamplingAlgorithm(algOptions),
                    seedModel);

  // RUN ANALYSIS
  ProjectDatabase database = getCleanDatabase("StopAndRestartDakotaAnalysis");
  AnalysisDriver analysisDriver(database);
  AnalysisRunOptions runOptions = standardRunOptions(analysisDriver.database().path().parent_path());
  StopWatcher watcher(analysisDriver);
  watcher.watch(analysis.uuid());
  CurrentAnalysis currentAnalysis = analysisDriver.run(analysis,runOptions);
  analysisDriver.waitForFinished();
  EXPECT_FALSE(analysisDriver.isRunning());

  // check conditions afterward
  boost::optional<runmanager::JobErrors> jobErrors = currentAnalysis.dakotaJobErrors();
  ASSERT_TRUE(jobErrors);
  EXPECT_FALSE(jobErrors->errors().empty());
  EXPECT_FALSE(currentAnalysis.analysis().dataPoints().empty());
  EXPECT_FALSE(currentAnalysis.analysis().dataPointsToQueue().empty());
  EXPECT_FALSE(currentAnalysis.analysis().completeDataPoints().empty());
  EXPECT_FALSE(currentAnalysis.analysis().successfulDataPoints().empty());
  EXPECT_TRUE(currentAnalysis.analysis().failedDataPoints().empty());
  EXPECT_FALSE(currentAnalysis.analysis().algorithm()->isComplete());
  EXPECT_FALSE(currentAnalysis.analysis().algorithm()->failed());
  EXPECT_EQ(0u,analysisDriver.currentAnalyses().size());
  LOG(Debug,"After initial stop, there are " << currentAnalysis.analysis().dataPoints().size()
      << " data points, of which " << currentAnalysis.analysis().completeDataPoints().size()
      << " are complete.");

  // try to restart from database contents
  analysis = AnalysisRecord::getAnalysisRecords(database)[0].analysis();
  ASSERT_TRUE(analysis.algorithm());
  EXPECT_FALSE(analysis.algorithm()->isComplete());
  EXPECT_FALSE(analysis.algorithm()->failed());
  currentAnalysis = analysisDriver.run(analysis,runOptions);
  analysisDriver.waitForFinished();
  EXPECT_EQ(10u,analysis.dataPoints().size());
  EXPECT_EQ(0u,analysis.dataPointsToQueue().size());
  EXPECT_EQ(10u,analysis.completeDataPoints().size());
  EXPECT_EQ(10u,analysis.successfulDataPoints().size());
  EXPECT_EQ(0u,analysis.failedDataPoints().size());
}

TEST_F(AnalysisDriverFixture,RuntimeBehavior_ClearAndRerunDakotaAnalysis) {
}


text:
#include "generator/raw_generator.hpp"

#include "generator/osm_source.hpp"
#include "generator/processor_factory.hpp"
#include "generator/raw_generator_writer.hpp"
#include "generator/translator_factory.hpp"

#include "base/thread_pool_computational.hpp"

#include <future>
#include <string>
#include <vector>

#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>

#include <sys/mman.h>

using namespace std;

namespace generator
{
RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t chunkSize)
  : m_genInfo(genInfo)
  , m_chunkSize(chunkSize)
  , m_cache(std::make_shared<generator::cache::IntermediateData>(genInfo))
  , m_queue(std::make_shared<FeatureProcessorQueue>())
  , m_translators(std::make_shared<TranslatorCollection>())
{
}

std::shared_ptr<FeatureProcessorQueue> RawGenerator::GetQueue()
{
  return m_queue;
}

void RawGenerator::GenerateRegionFeatures(
    string const & regionsFeaturesPath, std::string const & regionsInfoPath)
{
  auto processor = CreateProcessor(ProcessorType::Simple, m_queue, regionsFeaturesPath);
  m_translators->Append(
      CreateTranslator(TranslatorType::Regions, processor, m_cache, regionsInfoPath));
}

void RawGenerator::GenerateStreetsFeatures(string const & filename)
{
  auto processor = CreateProcessor(ProcessorType::Simple, m_queue, filename);
  m_translators->Append(CreateTranslator(TranslatorType::Streets, processor, m_cache));
}

void RawGenerator::GenerateGeoObjectsFeatures(string const & filename)
{
  auto processor = CreateProcessor(ProcessorType::Simple, m_queue, filename);
  m_translators->Append(CreateTranslator(TranslatorType::GeoObjects, processor, m_cache));
}

void RawGenerator::GenerateCustom(std::shared_ptr<TranslatorInterface> const & translator)
{
  m_translators->Append(translator);
}

void RawGenerator::GenerateCustom(std::shared_ptr<TranslatorInterface> const & translator,
                                  std::shared_ptr<FinalProcessorIntermediateMwmInterface> const & finalProcessor)
{
  m_translators->Append(translator);
  m_finalProcessors.emplace(finalProcessor);
}

bool RawGenerator::Execute()
{
  if (!GenerateFilteredFeatures())
    return false;

  while (!m_finalProcessors.empty())
  {
    base::thread_pool::computational::ThreadPool threadPool(m_genInfo.m_threadsCount);
    while (true)
    {
      auto const finalProcessor = m_finalProcessors.top();
      m_finalProcessors.pop();
      threadPool.SubmitWork([finalProcessor{finalProcessor}]() {
        finalProcessor->Process();
      });
      if (m_finalProcessors.empty() || *finalProcessor != *m_finalProcessors.top())
        break;
    }
  }

  LOG(LINFO, ("Final processing is finished."));
  return true;
}

std::vector<std::string> const & RawGenerator::GetNames() const
{
  return m_names;
}

bool RawGenerator::GenerateFilteredFeatures()
{
  RawGeneratorWriter rawGeneratorWriter(m_queue);
  rawGeneratorWriter.Run();

  auto processorThreadsCount = std::max(m_genInfo.m_threadsCount, 2u) - 1 /* writer */;
  if (m_genInfo.m_osmFileName.empty())  // stdin
    processorThreadsCount = 1;

  if (!GenerateFeatures(processorThreadsCount, rawGeneratorWriter))
    return false;

  rawGeneratorWriter.ShutdownAndJoin();
  m_names = rawGeneratorWriter.GetNames();
  LOG(LINFO, ("Names:", m_names));
  return true;
}

bool RawGenerator::GenerateFeatures(
    unsigned int threadsCount, RawGeneratorWriter & /* rawGeneratorWriter */)
{
  auto translators = std::vector<std::shared_ptr<TranslatorInterface>>{};
  auto sourceMap = boost::optional<boost::iostreams::mapped_file_source>{};
  if (!m_genInfo.m_osmFileName.empty())
  {
    sourceMap = MakeFileMap(m_genInfo.m_osmFileName);
    LOG_SHORT(LINFO, ("Reading OSM data from", m_genInfo.m_osmFileName));
  }

  std::vector<std::thread> threads;
  for (unsigned int i = 0; i < threadsCount; ++i)
  {
    auto translator = m_translators->Clone();
    translators.push_back(translator);

    constexpr size_t chunkSize = 10'000;
    auto processorMaker =
        [osmFileType = m_genInfo.m_osmFileType, threadsCount, i, chunkSize] (auto & reader)
            -> std::unique_ptr<ProcessorOsmElementsInterface>
    {
      switch (osmFileType)
      {
      case feature::GenerateInfo::OsmSourceType::O5M:
        return std::make_unique<ProcessorOsmElementsFromO5M>(reader, threadsCount, i, chunkSize);
      case feature::GenerateInfo::OsmSourceType::XML:
        return std::make_unique<ProcessorOsmElementsFromXml>(reader);
      }
      UNREACHABLE();
    };

    threads.emplace_back([translator, processorMaker, &sourceMap] {
      if (!sourceMap)
      {
        auto reader = SourceReader{};
        auto processor = processorMaker(reader);
        TranslateToFeatures(*processor, *translator);
        return;
      }

      namespace io = boost::iostreams;
      auto && sourceArray = io::array_source{sourceMap->data(), sourceMap->size()};
      auto && stream = io::stream<io::array_source>{sourceArray, std::ios::binary};
      auto && reader = SourceReader(stream);
      auto processor = processorMaker(reader);
      TranslateToFeatures(*processor, *translator);
    });
  }
  for (auto & thread : threads)
    thread.join();
  LOG(LINFO, ("Input was processed."));

  return FinishTranslation(translators);
}

// static
void RawGenerator::TranslateToFeatures(ProcessorOsmElementsInterface & sourceProcessor,
                                       TranslatorInterface & translator)
{
  OsmElement osmElement{};
  while (sourceProcessor.TryRead(osmElement))
    translator.Emit(osmElement);
}

bool RawGenerator::FinishTranslation(
    std::vector<std::shared_ptr<TranslatorInterface>> & translators)
{
  using TranslatorPtr = std::shared_ptr<TranslatorInterface>;

  base::threads::ThreadSafeQueue<std::future<TranslatorPtr>> queue;
  for (auto const & translator : translators)
  {
    std::promise<TranslatorPtr> p;
    p.set_value(translator);
    queue.Push(p.get_future());
  }
  CHECK_GREATER_OR_EQUAL(queue.Size(), 1, ());

  base::thread_pool::computational::ThreadPool pool(queue.Size() / 2 + 1);
  while (queue.Size() != 1)
  {
    std::future<TranslatorPtr> left;
    std::future<TranslatorPtr> right;
    queue.WaitAndPop(left);
    queue.WaitAndPop(right);
    queue.Push(pool.Submit([left{move(left)}, right{move(right)}]() mutable {
      auto leftTranslator = left.get();
      auto rigthTranslator = right.get();
      rigthTranslator->Finish();
      leftTranslator->Finish();
      leftTranslator->Merge(*rigthTranslator);
      return leftTranslator;
    }));
  }

  std::future<TranslatorPtr> translatorFuture;
  queue.WaitAndPop(translatorFuture);
  auto translator = translatorFuture.get();
  translator->Finish();
  return translator->Save();
}

boost::iostreams::mapped_file_source RawGenerator::MakeFileMap(std::string const & filename)
{
  CHECK(!filename.empty(), ());
  auto fileMap = boost::iostreams::mapped_file_source{filename};
  if (!fileMap.is_open())
    MYTHROW(Writer::OpenException, ("Failed to open", filename));

  // Try aggressively (MADV_WILLNEED) and asynchronously read ahead the o5m-file.
  auto readaheadTask = std::thread([data = fileMap.data(), size = fileMap.size()] {
    ::madvise(const_cast<char*>(data), size, MADV_WILLNEED);
  });
  readaheadTask.detach();

  return fileMap;
}

}  // namespace generator

ext: cpp

max_stars_repo_name: mapsme/geocore

max_issues_repo_name: mapsme/geocore

max_forks_repo_name: mapsme/geocore


text:
/**
 *  @file datastream.hpp
 *  @copyright defined in oac/LICENSE.txt
 */
#pragma once
#include <oaclib/system.h>
#include <oaclib/memory.h>
#include <oaclib/vector.hpp>
#include <boost/container/flat_set.hpp>
#include <boost/container/flat_map.hpp>
#include <oaclib/varint.hpp>
#include <array>
#include <set>
#include <map>
#include <string>

#include <boost/fusion/algorithm/iteration/for_each.hpp>
#include <boost/fusion/include/for_each.hpp>

#include <boost/pfr.hpp>

namespace oac {

/**
 *  @brief A data stream for reading and writing data in the form of bytes
 */
template<typename T>
class datastream {
   public:
      datastream( T start, size_t s )
      :_start(start),_pos(start),_end(start+s){}

     /**
      *  Skips a specified number of bytes from this stream
      *  @brief Skips a specific number of bytes from this stream
      *  @param s The number of bytes to skip
      */
      inline void skip( size_t s ){ _pos += s; }

     /**
      *  Reads a specified number of bytes from the stream into a buffer
      *  @brief Reads a specified number of bytes from this stream into a buffer
      *  @param d pointer to the destination buffer
      *  @param s the number of bytes to read
      */
      inline bool read( char* d, size_t s ) {
        oac_assert( size_t(_end - _pos) >= (size_t)s, "read" );
        memcpy( d, _pos, s );
        _pos += s;
        return true;
      }

     /**
      *  Writes a specified number of bytes into the stream from a buffer
      *  @brief Writes a specified number of bytes into the stream from a buffer
      *  @param d pointer to the source buffer
      *  @param s The number of bytes to write
      */
      inline bool write( const char* d, size_t s ) {
        oac_assert( _end - _pos >= (int32_t)s, "write" );
        memcpy( (void*)_pos, d, s );
        _pos += s;
        return true;
      }

     /**
      *  Writes a byte into the stream
      *  @brief Writes a byte into the stream
      *  @param c byte to write
      */
      inline bool put(char c) {
        oac_assert( _pos < _end, "put" );
        *_pos = c;
        ++_pos;
        return true;
      }

     /**
      *  Reads a byte from the stream
      *  @brief Reads a byte from the stream
      *  @param c reference to destination byte
      */
      inline bool get( unsigned char& c ) { return get( *(char*)&c ); }
      inline bool get( char& c )
      {
        oac_assert( _pos < _end, "get" );
        c = *_pos;
        ++_pos;
        return true;
      }

     /**
      *  Retrieves the current position of the stream
      *  @brief Retrieves the current position of the stream
      *  @return the current position of the stream
      */
      T pos()const { return _pos; }
      inline bool valid()const { return _pos <= _end && _pos >= _start;  }

     /**
      *  Sets the position within the current stream
      *  @brief Sets the position within the current stream
      *  @param p offset relative to the origin
      */
      inline bool seekp(size_t p) { _pos = _start + p; return _pos <= _end; }

     /**
      *  Gets the position within the current stream
      *  @brief Gets the position within the current stream
      *  @return p the position within the current stream
      */
      inline size_t tellp()const      { return size_t(_pos - _start); }

     /**
      *  Returns the number of remaining bytes that can be read/skipped
      *  @brief Returns the number of remaining bytes that can be read/skipped
      *  @return number of remaining bytes
      */
      inline size_t remaining()const  { return _end - _pos; }
    private:
      T _start;
      T _pos;
      T _end;
};

/**
 *  @brief Specialization of datastream used to help determine the final size of a serialized value
 */
template<>
class datastream<size_t> {
   public:
     datastream( size_t init_size = 0):_size(init_size){}
     inline bool     skip( size_t s )                 { _size += s; return true;  }
     inline bool     write( const char* ,size_t s )  { _size += s; return true;  }
     inline bool     put(char )                      { ++_size; return  true;    }
     inline bool     valid()const                     { return true;              }
     inline bool     seekp(size_t p)                  { _size = p;  return true;  }
     inline size_t   tellp()const                     { return _size;             }
     inline size_t   remaining()const                 { return 0;                 }
  private:
     size_t _size;
};

/**
 *  Serialize a public_key into a stream
 *  @brief Serialize a public_key
 *  @param ds stream to write
 *  @param pubkey value to serialize
 */
template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const public_key pubkey) {
  ds.write( (const char*)&pubkey, sizeof(pubkey));
  return ds;
}
/**
 *  Deserialize a public_key from a stream
 *  @brief Deserialize a public_key
 *  @param ds stream to read
 *  @param pubkey destination for deserialized value
 */
template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, public_key& pubkey) {
  ds.read((char*)&pubkey, sizeof(pubkey));
  return ds;
}

/**
 *  Serialize a key256 into a stream
 *  @brief Serialize a key256
 *  @param ds stream to write
 *  @param d value to serialize
 */
template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const key256& d) {
  ds.write( (const char*)d.data(), d.size() );
  return ds;
}
/**
 *  Deserialize a key256 from a stream
 *  @brief Deserialize a key256
 *  @param ds stream to read
 *  @param d destination for deserialized value
 */
template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, key256& d) {
  ds.read((char*)d.data(), d.size() );
  return ds;
}

template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const bool& d) {
  return ds << uint8_t(d);
}
template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, bool& d) {
  uint8_t t;
  ds >> t;
  d = t;
  return ds;
}

/**
 *  Serialize a checksum256 into a stream
 *  @brief Serialize a checksum256
 *  @param ds stream to write
 *  @param d value to serialize
 */
template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const checksum256& d) {
   ds.write( (const char*)&d.hash[0], sizeof(d.hash) );
   return ds;
}
/**
 *  Deserialize a checksum256 from a stream
 *  @brief Deserialize a checksum256
 *  @param ds stream to read
 *  @param d destination for deserialized value
 */
template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, checksum256& d) {
   ds.read((char*)&d.hash[0], sizeof(d.hash) );
   return ds;
}

template<typename DataStream>
DataStream& operator << ( DataStream& ds, const std::string& v ) {
   ds << unsigned_int( v.size() );
   if (v.size())
      ds.write(v.data(), v.size());
   return ds;
}

template<typename DataStream>
DataStream& operator >> ( DataStream& ds, std::string& v ) {
   std::vector<char> tmp;
   ds >> tmp;
   if( tmp.size() )
      v = std::string(tmp.data(),tmp.data()+tmp.size());
   else
      v = std::string();
   return ds;
}

template<typename DataStream, typename T, std::size_t N>
DataStream& operator << ( DataStream& ds, const std::array<T,N>& v ) {
   for( const auto& i : v )
      ds << i;
   return ds;
}

template<typename DataStream, typename T, std::size_t N>
DataStream& operator >> ( DataStream& ds, std::array<T,N>& v ) {
   for( auto& i : v )
      ds >> i;
   return ds;
}

namespace _datastream_detail {
   template<typename T>
   constexpr bool is_pointer() {
      return std::is_pointer<T>::value ||
             std::is_null_pointer<T>::value ||
             std::is_member_pointer<T>::value;
   }

   template<typename T>
   constexpr bool is_primitive() {
      return std::is_arithmetic<T>::value ||
             std::is_enum<T>::value;
   }
}

template<typename DataStream, typename T, std::enable_if_t<_datastream_detail::is_pointer<T>()>* = nullptr>
DataStream& operator >> ( DataStream& ds, T ) {
   static_assert(!_datastream_detail::is_pointer<T>(), "Pointers should not be serialized" );
   return ds;
}

template<typename DataStream, typename T, std::size_t N,
         std::enable_if_t<!_datastream_detail::is_primitive<T>() &&
                          !_datastream_detail::is_pointer<T>()>* = nullptr>
DataStream& operator << ( DataStream& ds, const T (&v)[N] ) {
   ds << unsigned_int( N );
   for( uint32_t i = 0; i < N; ++i )
      ds << v[i];
   return ds;
}

template<typename DataStream, typename T, std::size_t N,
         std::enable_if_t<_datastream_detail::is_primitive<T>()>* = nullptr>
DataStream& operator << ( DataStream& ds, const T (&v)[N] ) {
   ds << unsigned_int( N );
   ds.write((char*)&v[0], sizeof(v));
   return ds;
}

template<typename DataStream, typename T, std::size_t N,
         std::enable_if_t<!_datastream_detail::is_primitive<T>() &&
                          !_datastream_detail::is_pointer<T>()>* = nullptr>
DataStream& operator >> ( DataStream& ds, T (&v)[N] ) {
   unsigned_int s;
   ds >> s;
   oac_assert( N == s.value, "T[] size and unpacked size don't match");
   for( uint32_t i = 0; i < N; ++i )
      ds >> v[i];
   return ds;
}

template<typename DataStream, typename T, std::size_t N,
         std::enable_if_t<_datastream_detail::is_primitive<T>()>* = nullptr>
DataStream& operator >> ( DataStream& ds, T (&v)[N] ) {
   unsigned_int s;
   ds >> s;
   oac_assert( N == s.value, "T[] size and unpacked size don't match");
   ds.read((char*)&v[0], sizeof(v));
   return ds;
}

template<typename DataStream>
DataStream& operator << ( DataStream& ds, const vector<char>& v ) {
   ds << unsigned_int( v.size() );
   ds.write( v.data(), v.size() );
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator << ( DataStream& ds, const vector<T>& v ) {
   ds << unsigned_int( v.size() );
   for( const auto& i : v )
      ds << i;
   return ds;
}

template<typename DataStream>
DataStream& operator >> ( DataStream& ds, vector<char>& v ) {
   unsigned_int s;
   ds >> s;
   v.resize( s.value );
   ds.read( v.data(), v.size() );
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator >> ( DataStream& ds, vector<T>& v ) {
   unsigned_int s;
   ds >> s;
   v.resize(s.value);
   for( auto& i : v )
      ds >> i;
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator << ( DataStream& ds, const std::set<T>& s ) {
   ds << unsigned_int( s.size() );
   for( const auto& i : s ) {
      ds << i;
   }
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator >> ( DataStream& ds, std::set<T>& s ) {
   s.clear();
   unsigned_int sz; ds >> sz;

   for( uint32_t i = 0; i < sz.value; ++i ) {
      T v;
      ds >> v;
      s.emplace( std::move(v) );
   }
   return ds;
}

template<typename DataStream, typename K, typename V>
DataStream& operator << ( DataStream& ds, const std::map<K,V>& m ) {
   ds << unsigned_int( m.size() );
   for( const auto& i : m ) {
      ds << i.first << i.second;
   }
   return ds;
}

template<typename DataStream, typename K, typename V>
DataStream& operator >> ( DataStream& ds, std::map<K,V>& m ) {
   m.clear();
   unsigned_int s; ds >> s;

   for (uint32_t i = 0; i < s.value; ++i) {
      K k; V v;
      ds >> k >> v;
      m.emplace( std::move(k), std::move(v) );
   }
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator << ( DataStream& ds, const boost::container::flat_set<T>& s ) {
   ds << unsigned_int( s.size() );
   for( const auto& i : s ) {
      ds << i;
   }
   return ds;
}

template<typename DataStream, typename T>
DataStream& operator >> ( DataStream& ds, boost::container::flat_set<T>& s ) {
   s.clear();
   unsigned_int sz; ds >> sz;

   for( uint32_t i = 0; i < sz.value; ++i ) {
      T v;
      ds >> v;
      s.emplace( std::move(v) );
   }
   return ds;
}

template<typename DataStream, typename K, typename V>
DataStream& operator<<( DataStream& ds, const boost::container::flat_map<K,V>& m ) {
   ds << unsigned_int( m.size() );
   for( const auto& i : m )
      ds << i.first << i.second;
   return ds;
}

template<typename DataStream, typename K, typename V>
DataStream& operator>>( DataStream& ds, boost::container::flat_map<K,V>& m ) {
   m.clear();
   unsigned_int s; ds >> s;

   for( uint32_t i = 0; i < s.value; ++i ) {
      K k; V v;
      ds >> k >> v;
      m.emplace( std::move(k), std::move(v) );
   }
   return ds;
}

template<typename DataStream, typename... Args>
DataStream& operator<<( DataStream& ds, const std::tuple<Args...>& t ) {
   boost::fusion::for_each( t, [&]( const auto& i ) {
       ds << i;
   });
   return ds;
}

template<typename DataStream, typename... Args>
DataStream& operator>>( DataStream& ds, std::tuple<Args...>& t ) {
   boost::fusion::for_each( t, [&]( auto& i ) {
       ds >> i;
   });
   return ds;
}

template<typename DataStream, typename T, std::enable_if_t<std::is_class<T>::value>* = nullptr>
DataStream& operator<<( DataStream& ds, const T& v ) {
   boost::pfr::for_each_field(v, [&](const auto& field) {
      ds << field;
   });
   return ds;
}
template<typename DataStream, typename T, std::enable_if_t<std::is_class<T>::value>* = nullptr>
DataStream& operator>>( DataStream& ds, T& v ) {
   boost::pfr::for_each_field(v, [&](auto& field) {
      ds >> field;
   });
   return ds;
}

template<typename DataStream, typename T, std::enable_if_t<_datastream_detail::is_primitive<T>()>* = nullptr>
DataStream& operator<<( DataStream& ds, const T& v ) {
   ds.write( (const char*)&v, sizeof(T) );
   return ds;
}

template<typename DataStream, typename T, std::enable_if_t<_datastream_detail::is_primitive<T>()>* = nullptr>
DataStream& operator>>( DataStream& ds, T& v ) {
   ds.read( (char*)&v, sizeof(T) );
   return ds;
}

template<typename T>
T unpack( const char* buffer, size_t len ) {
   T result;
   datastream<const char*> ds(buffer,len);
   ds >> result;
   return result;
}

template<typename T>
T unpack( const vector<char>& bytes ) {
   return unpack<T>( bytes.data(), bytes.size() );
}

template<typename T>
size_t pack_size( const T& value ) {
  datastream<size_t> ps;
  ps << value;
  return ps.tellp();
}

template<typename T>
bytes pack( const T& value ) {
  bytes result;
  result.resize(pack_size(value));

  datastream<char*> ds( result.data(), result.size() );
  ds << value;
  return result;
}

template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const checksum160& cs) {
   ds.write((const char*)&cs.hash[0], sizeof(cs.hash));
   return ds;
}

template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, checksum160& cs) {
   ds.read((char*)&cs.hash[0], sizeof(cs.hash));
   return ds;
}

template<typename Stream>
inline datastream<Stream>& operator<<(datastream<Stream>& ds, const checksum512& cs) {
   ds.write((const char*)&cs.hash[0], sizeof(cs.hash));
   return ds;
}

template<typename Stream>
inline datastream<Stream>& operator>>(datastream<Stream>& ds, checksum512& cs) {
   ds.read((char*)&cs.hash[0], sizeof(cs.hash));
   return ds;
}

}

ext: hpp

max_stars_repo_name: openaichain/openaichain

max_issues_repo_name: openaichain/openaichain

max_forks_repo_name: openaichain/openaichain