text:
/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/
#include "hphp/hhbbc/stats.h"

#include <atomic>
#include <array>
#include <string>
#include <cinttypes>
#include <cstdlib>
#include <iostream>
#include <memory>
#include <tuple>
#include <type_traits>

#include <boost/variant.hpp>

#include <tbb/concurrent_hash_map.h>

#include <folly/Conv.h>
#include <folly/String.h>
#include <folly/Format.h>
#include <folly/ScopeGuard.h>
#include <folly/portability/Stdlib.h>

#include "hphp/runtime/vm/hhbc.h"

#include "hphp/util/trace.h"

#include "hphp/hhbbc/analyze.h"
#include "hphp/hhbbc/func-util.h"
#include "hphp/hhbbc/index.h"
#include "hphp/hhbbc/interp-internal.h"
#include "hphp/hhbbc/interp.h"
#include "hphp/hhbbc/context.h"
#include "hphp/hhbbc/misc.h"
#include "hphp/hhbbc/parallel.h"
#include "hphp/hhbbc/representation.h"

namespace HPHP { namespace HHBBC {

//////////////////////////////////////////////////////////////////////

namespace {

TRACE_SET_MOD(hhbbc_stats);

//////////////////////////////////////////////////////////////////////

#define STAT_TYPES                              \
  X(Gen)                                        \
  X(InitGen)                                    \
  X(Ref)                                        \
  X(Cell)                                       \
  X(InitCell)                                   \
  X(Unc)                                        \
  X(InitUnc)                                    \
  X(Obj)                                        \
  X(OptObj)                                     \
  X(Arr)                                        \
  X(SArr)                                       \
  X(ArrE)                                       \
  X(ArrN)                                       \
  X(SArrN)                                      \
  X(SArrE)                                      \
  X(Null)                                       \
  X(Bottom)

struct TypeStat {
#define X(x) std::atomic<uint64_t> sub_##x; \
             std::atomic<uint64_t> eq_##x;
  STAT_TYPES
#undef X
};

struct ISameCmp {
  bool equal(SString s1, SString s2) const {
    return s1->isame(s2);
  }

  size_t hash(SString s) const {
    return s->hash();
  }
};

/*
 * Information about builtins usage.
 *
 * The tuple contains the known return type for the builtin, the total
 * number of calls seen and the total number of calls that could be
 * reduced.  A builtin call is considered reducible if its output is a
 * constant and all its inputs are constants. That is not a guaranteed
 * condition but it gives us an idea of what's possible.
 */
using BuiltinInfo = tbb::concurrent_hash_map<
  SString,
  std::tuple<Type,uint64_t,uint64_t>,
  ISameCmp
>;

struct Builtins {
  std::atomic<uint64_t> totalBuiltins;
  std::atomic<uint64_t> reducibleBuiltins;
  BuiltinInfo builtinsInfo{};
};

#define TAG(x) 1 +
constexpr uint32_t kNumRATTags = REPO_AUTH_TYPE_TAGS 0 ;
#undef TAG

}

struct Stats {
  std::array<std::atomic<uint64_t>,Op_count> op_counts;
  std::array<std::atomic<uint64_t>,kNumRATTags> ratL_tags;
  std::array<std::atomic<uint64_t>,kNumRATTags> ratStk_tags;
  std::atomic<uint64_t> ratL_specialized_array;
  std::atomic<uint64_t> ratStk_specialized_array;
  std::atomic<uint64_t> persistentClasses;
  std::atomic<uint64_t> persistentFunctions;
  std::atomic<uint64_t> uniqueClasses;
  std::atomic<uint64_t> uniqueFunctions;
  std::atomic<uint64_t> totalClasses;
  std::atomic<uint64_t> totalFunctions;
  std::atomic<uint64_t> totalPseudoMains;
  std::atomic<uint64_t> totalMethods;
  std::atomic<uint64_t> persistentSPropsPub;
  std::atomic<uint64_t> persistentSPropsProt;
  std::atomic<uint64_t> persistentSPropsPriv;
  std::atomic<uint64_t> totalSProps;
  TypeStat returns;
  TypeStat privateProps;
  TypeStat privateStatics;
  TypeStat cgetmBase;
  TypeStat iterInitBase;
  TypeStat iterInitKBase;
  Builtins builtins;
};

namespace {

void type_stat_string(std::string& ret,
                      const std::string& prefix,
                      const TypeStat& st) {
#define X(x)                                    \
  folly::format(&ret, "  {}_=_{: <9} {: >8}\n",  \
    prefix, #x ":", st.eq_##x.load());          \
  folly::format(&ret, "  {}_<_{: <9} {: >8}\n",  \
    prefix, #x ":", st.sub_##x.load());
  STAT_TYPES
#undef X
  ret += "\n";
}

std::string show(const Builtins& builtins) {
  auto ret = std::string{};

  if (builtins.builtinsInfo.begin() != builtins.builtinsInfo.end()) {
    folly::format(&ret, "Total number of builtin calls: {: >15}\n",
                  builtins.totalBuiltins.load());
    folly::format(&ret, "Possible reducible builtins: {: >15}\n",
                  builtins.reducibleBuiltins.load());

    ret += "Builtins Info:\n";
    for (auto it = builtins.builtinsInfo.begin();
         it != builtins.builtinsInfo.end(); ++it) {
      folly::format(
        &ret,
        "  {: >30} [tot:{: >8}, red:{: >8}]\t\ttype: {}\n",
        it->first,
        std::get<1>(it->second),
        std::get<2>(it->second),
        show(std::get<0>(it->second))
      );
    }
    ret += "\n";
  }
  return ret;
}

std::string show(const Stats& stats) {
  auto ret = std::string{};

  for (auto i = uint32_t{}; i < stats.op_counts.size(); ++i) {
    folly::format(
      &ret,
      "  {: >20}:  {: >15}\n",
      opcodeToName(static_cast<Op>(i)),
      stats.op_counts[i].load()
    );
  }
  ret += "\n";

  type_stat_string(ret, "ret",            stats.returns);
  type_stat_string(ret, "priv_prop",      stats.privateProps);
  type_stat_string(ret, "priv_static",    stats.privateStatics);
  type_stat_string(ret, "cgetm_base",     stats.cgetmBase);
  type_stat_string(ret, "iterInit_base",  stats.iterInitBase);
  type_stat_string(ret, "iterInitK_base", stats.iterInitKBase);

  folly::format(
    &ret,
    "       total_methods:  {: >8}\n"
    "   total_pseudomains:  {: >8}\n"
    "         total_funcs:  {: >8}\n"
    "        unique_funcs:  {: >8}\n"
    "    persistent_funcs:  {: >8}\n"
    "       total_classes:  {: >8}\n"
    "      unique_classes:  {: >8}\n"
    "  persistent_classes:  {: >8}\n"
    "\n"
    "        total_sprops:      {: >8}\n"
    "   persistent_sprops_pub:  {: >8}\n"
    "   persistent_sprops_prot: {: >8}\n"
    "   persistent_sprops_priv: {: >8}\n",
    stats.totalMethods.load(),
    stats.totalPseudoMains.load(),
    stats.totalFunctions.load(),
    stats.uniqueFunctions.load(),
    stats.persistentFunctions.load(),
    stats.totalClasses.load(),
    stats.uniqueClasses.load(),
    stats.persistentClasses.load(),
    stats.totalSProps.load(),
    stats.persistentSPropsPub.load(),
    stats.persistentSPropsProt.load(),
    stats.persistentSPropsPriv.load()
  );

  ret += "\n";
  ret += show(stats.builtins);

  ret += "\n";
  using T = RepoAuthType::Tag;
  using U = std::underlying_type<T>::type;
#define TAG(x)                                                          \
  folly::format(&ret, "  {: >24}:  {: >8}\n"                            \
                      "  {: >24}:  {: >8}\n",                           \
                      "RATL_" #x,                                       \
                      stats.ratL_tags[static_cast<U>(T::x)].load(),     \
                      "RATStk_" #x,                                     \
                      stats.ratStk_tags[static_cast<U>(T::x)].load());
  REPO_AUTH_TYPE_TAGS
#undef TAG

  folly::format(&ret, "  {: >24}:  {: >8}\n"
                      "  {: >24}:  {: >8}\n",
                      "RATL_Arr_Special",
                      stats.ratL_specialized_array.load(),
                      "RATStk_Arr_Special",
                      stats.ratStk_specialized_array.load());

  return ret;
}

//////////////////////////////////////////////////////////////////////

void add_type(TypeStat& stat, const Type& t) {
#define X(x)                                    \
  if (t.strictSubtypeOf(T##x)) ++stat.sub_##x;  \
  if (t == T##x) ++stat.eq_##x;
  STAT_TYPES
#undef X
}

//////////////////////////////////////////////////////////////////////

struct StatsSS : ISS {
  explicit StatsSS(ISS& env, Stats& stats)
    : ISS(env)
    , stats(stats)
  {}

  Stats& stats;
};

//////////////////////////////////////////////////////////////////////

template <class OpCode>
bool in(StatsSS& /*env*/, const OpCode&) {
  return false;
}

bool in(StatsSS& env, const bc::IterInit& /*op*/) {
  add_type(env.stats.iterInitBase, topC(env));
  return false;
}

bool in(StatsSS& env, const bc::IterInitK& /*op*/) {
  add_type(env.stats.iterInitKBase, topC(env));
  return false;
}

bool in(StatsSS& env, const bc::FCallBuiltin& op) {
  ++env.stats.builtins.totalBuiltins;

  bool reducible = op.arg1 > 0;
  for (auto i = uint32_t{0}; i < op.arg1; ++i) {
    auto t = topT(env, i);
    auto const v = tv(t);
    if (!v || v->m_type == KindOfUninit) {
      reducible = false;
      break;
    }
  }

  default_dispatch(env, op);

  auto builtin = op.str3;
  {
    BuiltinInfo::accessor acc;
    auto inserted = env.stats.builtins.builtinsInfo.insert(acc, builtin);
    if (inserted) {
      auto f = env.index.resolve_func(env.ctx, builtin);
      auto t = env.index.lookup_return_type(env.ctx, f);
      acc->second = std::make_tuple(t, 1, 0);
    } else {
      ++std::get<1>(acc->second);
      if (reducible) ++std::get<2>(acc->second);
    }
  }

  return true;
}

//////////////////////////////////////////////////////////////////////

// Run the interpreter. The "bool in(StatsSS&, const bc::XX)" functions
// can call the default dispatch on their own or return false to let
// the main loop call the default dispatch. Returning true stops
// the call to the default interpreter which implies the handler for
// that opcode must perform all the right steps wrt the state.
void dispatch(StatsSS& env, const Bytecode& op) {
#define O(opcode, ...)                                   \
  case Op::opcode:                                       \
    if (!in(env, op.opcode)) default_dispatch(env, op);  \
    return;

  switch (op.op) { OPCODES }
#undef O
  not_reached();
}

//////////////////////////////////////////////////////////////////////

// Simple stats about opcodes (that don't require full type
// information---those cases are only enabled when extendedStats is
// on).
void collect_simple(Stats& stats, const Bytecode& bc) {
  ++stats.op_counts[static_cast<uint64_t>(bc.op)];

  RepoAuthType rat;
  switch (bc.op) {
  case Op::AssertRATL:
    rat = bc.AssertRATL.rat;
    break;
  case Op::AssertRATStk:
    rat = bc.AssertRATStk.rat;
    break;
  default:
    return;
  }

  using U = std::underlying_type<RepoAuthType::Tag>::type;
  auto const tagInt = static_cast<U>(rat.tag());
  assert(tagInt < stats.ratL_tags.size());
  if (bc.op == Op::AssertRATL) {
    ++stats.ratL_tags[tagInt];
  } else {
    ++stats.ratStk_tags[tagInt];
  }

  if (rat.mayHaveArrData()) {
    if (rat.hasArrData()) {
      if (bc.op == Op::AssertRATL) {
        ++stats.ratL_specialized_array;
      } else {
        ++stats.ratStk_specialized_array;
      }
    }
  }
}

void collect_func(Stats& stats, const Index& index, php::Func& func) {
  if (!func.cls) {
    if (is_pseudomain(&func)) {
      ++stats.totalPseudoMains;
    } else {
      ++stats.totalFunctions;
      if (func.attrs & AttrPersistent) {
        ++stats.persistentFunctions;
      }
      if (func.attrs & AttrUnique) {
        if (!(func.attrs & AttrPersistent)) {
          FTRACE(1, "Func unique but not persistent: {} : {}\n",
                 func.name, func.unit->filename);
        }
        ++stats.uniqueFunctions;
      } else {
        FTRACE(1, "Func not unique: {} : {}\n",
               func.name, func.unit->filename);
      }
    }
  }

  auto const ty = index.lookup_return_type_raw(&func);

  add_type(stats.returns, ty);

  for (auto const bid : func.blockRange()) {
    auto const blk = func.blocks[bid].get();
    if (blk->dead) continue;
    for (auto& bc : blk->hhbcs) {
      collect_simple(stats, bc);
    }
  }

  if (!options.extendedStats) return;

  auto const ctx = Context { func.unit, &func, func.cls };
  auto const fa  = analyze_func(index, ctx, CollectionOpts{});
  {
    Trace::Bump bumper{Trace::hhbbc, kStatsBump};
    for (auto const bid : func.blockRange()) {
      auto const blk = func.blocks[bid].get();
      auto state = fa.bdata[bid].stateIn;
      if (!state.initialized) continue;

      CollectedInfo collect {
        index, ctx, nullptr, CollectionOpts {}, &fa
      };
      Interp interp { index, ctx, collect, bid, blk, state };
      for (auto& bc : blk->hhbcs) {
        auto noop    = [] (BlockId, const State*) {};
        ISS env { interp, noop };
        StatsSS sss { env, stats };
        dispatch(sss, bc);
        if (state.unreachable) break;
      }
    }
  }
}

void collect_class(Stats& stats, const Index& index, const php::Class& cls) {
  ++stats.totalClasses;
  if (cls.attrs & AttrPersistent) {
    ++stats.persistentClasses;
  }
  if (cls.attrs & AttrUnique) {
    if (!(cls.attrs & AttrPersistent)) {
      FTRACE(1, "Class unique but not persistent: {} : {}\n",
             cls.name, cls.unit->filename);
    }
    ++stats.uniqueClasses;
  } else {
    FTRACE(1, "Class not unique: {} : {}\n",
           cls.name, cls.unit->filename);
  }
  stats.totalMethods += cls.methods.size();

  for (auto& kv : index.lookup_private_props(&cls)) {
    add_type(stats.privateProps, kv.second.ty);
  }
  for (auto& kv : index.lookup_private_statics(&cls)) {
    add_type(stats.privateStatics, kv.second.ty);
  }

  for (auto& prop : cls.properties) {
    if (prop.attrs & AttrStatic) {
      ++stats.totalSProps;
      if (prop.attrs & AttrPersistent) {
        if (prop.attrs & AttrPublic)    ++stats.persistentSPropsPub;
        if (prop.attrs & AttrProtected) ++stats.persistentSPropsProt;
        if (prop.attrs & AttrPrivate)   ++stats.persistentSPropsPriv;
      }
    }
  }
}

void collect_stats(Stats& stats,
                   const Index& index,
                   const php::Program& program) {
  parallel::for_each(
    program.units,
    [&] (const std::unique_ptr<php::Unit>& unit) {
      for (auto& c : unit->classes) {
        collect_class(stats, index, *c);
        for (auto& m : c->methods) {
          collect_func(stats, index, *m);
        }
      }
      for (auto& x : unit->funcs) {
        collect_func(stats, index, *x);
      }
      collect_func(stats, index, *unit->pseudomain);
    }
  );
}

//////////////////////////////////////////////////////////////////////

}

//////////////////////////////////////////////////////////////////////

StatsHolder::StatsHolder() {
  if (!Trace::moduleEnabledRelease(Trace::hhbbc_time, 1)) return;
  stats = new Stats{};
}

StatsHolder::~StatsHolder() {
  delete stats;
}

StatsHolder allocate_stats() {
  return StatsHolder();
}

void collect_stats(const StatsHolder& stats,
                   const Index& index,
                   const php::Unit* unit) {
  if (!stats) return;
  for (auto& c : unit->classes) {
    collect_class(*stats.stats, index, *c);
    for (auto& m : c->methods) {
      collect_func(*stats.stats, index, *m);
    }
  }
  for (auto& x : unit->funcs) {
    collect_func(*stats.stats, index, *x);
  }
  collect_func(*stats.stats, index, *unit->pseudomain);
}

void print_stats(const StatsHolder& stats) {
  if (!stats) return;

  auto const str = show(*stats.stats);
  if (Trace::moduleEnabledRelease(Trace::hhbbc_time, 2)) {
    std::cout << str;
  }

  auto stats_file = options.stats_file;

  auto const file = [&] () -> std::FILE* {
    if (!stats_file.empty()) {
      return fopen(stats_file.c_str(), "w");
    }

    char fileBuf[] = "/tmp/hhbbcXXXXXX";
    auto const fd = mkstemp(fileBuf);
    stats_file = fileBuf;
    if (fd == -1) return nullptr;

    if (auto const fp = fdopen(fd, "w")) return fp;
    close(fd);
    return nullptr;
  }();

  if (file == nullptr) {
    std::cerr << "couldn't open file for stats: "
              << folly::errnoStr(errno) << '\n';
    return;
  }

  SCOPE_EXIT { fclose(file); };
  std::cout << "stats saved to " << stats_file << '\n';
  std::fwrite(str.c_str(), sizeof(char), str.size(), file);
  std::fflush(file);
}

//////////////////////////////////////////////////////////////////////

}}

max_stars_repo_name: jeffomatic/hhvm

max_issues_repo_name: jeffomatic/hhvm

max_forks_repo_name: jeffomatic/hhvm


text:
//          Copyright 2003-2013 Christopher M. Kohlhoff
//          Copyright Oliver Kowalke, Nat Goodspeed 2015.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_FIBERS_ASIO_YIELD_HPP
#define BOOST_FIBERS_ASIO_YIELD_HPP

#include <boost/asio/async_result.hpp>
#include <boost/asio/detail/config.hpp>
#include <boost/asio/handler_type.hpp>
#include <boost/assert.hpp>
#include <boost/config.hpp>
#include <boost/system/error_code.hpp>
#include <boost/system/system_error.hpp>
#include <boost/throw_exception.hpp>

#include <boost/fiber/all.hpp>

#include <mutex> // std::unique_lock

#ifdef BOOST_HAS_ABI_HEADERS
#include BOOST_ABI_PREFIX
#endif

namespace boost {
    namespace fibers {
        namespace asio {
            //[fibers_asio_yield_t
            class yield_t {
                public:
                    yield_t() = default;

                    /**
                     * @code
                     * static yield_t yield;
                     * boost::system::error_code myec;
                     * func(yield[myec]);
                     * @endcode
                     * @c yield[myec] returns an instance of @c yield_t whose @c ec_ points
                     * to @c myec. The expression @c yield[myec] "binds" @c myec to that
                     * (anonymous) @c yield_t instance, instructing @c func() to store any
                     * @c error_code it might produce into @c myec rather than throwing @c
                     * boost::system::system_error.
                     */
                    yield_t operator[](boost::system::error_code &ec) const {
                        yield_t tmp;
                        tmp.ec_ = &ec;
                        return tmp;
                    }

                    // private:
                    // ptr to bound error_code instance if any
                    boost::system::error_code *ec_{
                        nullptr
                    };
            };
            //]

            //[fibers_asio_yield
            // canonical instance
            thread_local yield_t yield{};
            //]
        }
    }
}

namespace boost {
    namespace fibers {
        namespace asio {
            namespace detail {
                //[fibers_asio_yield_completion
                // Bundle a completion bool flag with a spinlock to protect it.
                struct yield_completion {
                    typedef fibers::detail::spinlock mutex_t;
                    typedef std::unique_lock<mutex_t> lock_t;

                    mutex_t mtx_{};
                    bool completed_{false};

                    void wait() {
                        // yield_handler_base::operator()() will set completed_ true and
                        // attempt to wake a suspended fiber. It would be Bad if that call
                        // happened between our detecting (! completed_) and suspending.
                        lock_t lk{mtx_};
                        // If completed_ is already set, we're done here: don't suspend.
                        if(!completed_) {
                            // suspend(unique_lock<spinlock>) unlocks the lock in the act of
                            // resuming another fiber
                            fibers::context::active()->suspend(lk);
                        }
                    }
                };
                //]

                //[fibers_asio_yield_handler_base
                // This class encapsulates common elements between yield_handler<T> (capturing
                // a value to return from asio async function) and yield_handler<void> (no
                // such value). See yield_handler<T> and its <void> specialization below. Both
                // yield_handler<T> and yield_handler<void> are passed by value through
                // various layers of asio functions. In other words, they're potentially
                // copied multiple times. So key data such as the yield_completion instance
                // must be stored in our async_result<yield_handler<>> specialization, which
                // should be instantiated only once.
                class yield_handler_base {
                    public:
                        yield_handler_base(yield_t const &y)
                        // capture the context* associated with the running fiber
                        ctx_{
                            boost::fibers::context::active()
                        }
                        // capture the passed yield_t
                        yt_(y) {

                        }

                        // completion callback passing only (error_code)
                        void operator()(boost::system::error_code const &ec) {
                            BOOST_ASSERT_MSG(ycomp_,
                                             "Must inject yield_completion* "
                                             "before calling yield_handler_base::operator()()");
                            BOOST_ASSERT_MSG(yt_.ec_,
                                             "Must inject boost::system::error_code* "
                                             "before calling yield_handler_base::operator()()");
                            // If originating fiber is busy testing completed_ flag, wait until it
                            // has observed (! completed_).
                            yield_completion::lock_t lk {
                                ycomp_->mtx_
                            };
                            // Notify a subsequent yield_completion::wait() call that it need not suspend.
                            ycomp_->completed_ = true;
                            // set the error_code bound by yield_t
                            *yt_.ec_ = ec;
                            // If ctx_ is still active, e.g. because the async operation
                            // immediately called its callback (this method!) before the asio
                            // async function called async_result_base::get(), we must not set it ready.
                            if(fibers::context::active() != ctx_) {
                                // wake the fiber
                                fibers::context::active()->set_ready(ctx_);
                            }
                        }

                        // private:
                        boost::fibers::context *ctx_;
                        yield_t yt_;
                        // We depend on this pointer to yield_completion, which will be injected by async_result.
                        yield_completion *ycomp_{nullptr};
                };
                //]

                //[fibers_asio_yield_handler_T
                // asio uses handler_type<completion token type, signature>::type to decide
                // what to instantiate as the actual handler. Below, we specialize
                // handler_type< yield_t, ... > to indicate yield_handler<>. So when you pass
                // an instance of yield_t as an asio completion token, asio selects
                // yield_handler<> as the actual handler class.
                template <typename T>
                class yield_handler : public yield_handler_base {
                    public:
                        // asio passes the completion token to the handler constructor
                        explicit yield_handler(yield_t const &y) : yield_handler_base{y} {

                        }

                        // completion callback passing only value (T)
                        void operator()(T t) {
                            // just like callback passing success error_code
                            (*this)(boost::system::error_code(), std::move(t));
                        }

                        // completion callback passing (error_code, T)
                        void operator()(boost::system::error_code const &ec, T t) {
                            BOOST_ASSERT_MSG(value_,
                                             "Must inject value ptr "
                                             "before caling yield_handler<T>::operator()()");
                            // move the value to async_result<> instance BEFORE waking up a suspended fiber
                            *value_ = std::move(t);
                            // forward the call to base-class completion handler
                            yield_handler_base::operator()(ec);
                        }

                        // private:
                        // pointer to destination for eventual value
                        // this must be injected by async_result before operator()() is called
                        T *value_{
                            nullptr
                        }
                };
                //]

                //[fibers_asio_yield_handler_void
                // yield_handler<void> is like yield_handler<T> without value_. In fact it's
                // just like yield_handler_base.
                template <>
                class yield_handler<void> : public yield_handler_base {
                    public:
                        explicit yield_handler(yield_t const &y) : yield_handler_base{y} {

                        }

                        // nullary completion callback
                        void operator()() {
                            (*this)(boost::system::error_code());
                        }

                        // inherit operator()(error_code) overload from base class
                        using yield_handler_base::operator();
                };
                //]

                // Specialize asio_handler_invoke hook to ensure that any exceptions thrown
                // from the handler are propagated back to the caller
                template <typename Fn, typename T>
                void asio_handler_invoke(Fn fn, yield_handler<T> *h) {
                    fn();
                }

                //[fibers_asio_async_result_base
                // Factor out commonality between async_result<yield_handler<T>> and
                // async_result<yield_handler<void>>
                class async_result_base {
                    public:
                        explicit async_result_base(yield_handler_base &h) {
                            // Inject ptr to our yield_completion instance into this
                            // yield_handler<>.
                            h.ycomp_ = &this->ycomp_;
                            // if yield_t didn't bind an error_code, make yield_handler_base's
                            // error_code* point to an error_code local to this object so
                            // yield_handler_base::operator() can unconditionally store through
                            // its error_code*
                            if(!h.yt_.ec_) {
                                h.yt_.ec_ = &ec_;
                            }
                        }

                        void get() {
                            // Unless yield_handler_base::operator() has already been called,
                            // suspend the calling fiber until that call.
                            ycomp_.wait();
                            // The only way our own ec_ member could have a non-default value is
                            // if our yield_handler did not have a bound error_code AND the
                            // completion callback passed a non-default error_code.
                            if(ec_) {
                                throw_exception(boost::system::system_error{ec_});
                            }
                        }

                    private:
                        // If yield_t does not bind an error_code instance, store into here.
                        boost::system::error_code ec_{

                        }
                        // async_result_base owns the yield_completion because, unlike
                        // yield_handler<>, async_result<> is only instantiated once.
                        yield_completion ycomp_{};
                };
                //]
            }
        }
    }
}

namespace boost {
    namespace asio {
        //[fibers_asio_async_result_T
        // asio constructs an async_result<> instance from the yield_handler specified
        // by handler_type<>::type. A particular asio async method constructs the
        // yield_handler, constructs this async_result specialization from it, then
        // returns the result of calling its get() method.
        template <typename T>
        class async_result<boost::fibers::asio::detail::yield_handler<T>>
        public boost::fibers::asio::detail::async_result_base {
            public:
                // type returned by get()
                typedef T type;

                explicit async_result(boost::fibers::asio::detail::yield_handler<T> &h)
                boost::fibers::asio::detail::async_result_base{h} {
                    // Inject ptr to our value_ member into yield_handler<>: result will
                    // be stored here.
                    h.value_ = &value_;
                }

                // asio async method returns result of calling get()
                type get() {
                    boost::fibers::asio::detail::async_result_base::get();
                    return std::move(value_);
                }

            private:
                type value_{

                }
        };
        //]

        //[fibers_asio_async_result_void
        // Without the need to handle a passed value, our yield_handler<void>
        // specialization is just like async_result_base.
        template <>
        class async_result<boost::fibers::asio::detail::yield_handler<void>>
        public boost::fibers::asio::detail::async_result_base {
            public:
                typedef void type;

                explicit async_result(boost::fibers::asio::detail::yield_handler<void> &h)
                boost::fibers::asio::detail::async_result_base{h} {

                }
        };
        //]

        // Handler type specialisation for fibers::asio::yield.
        // When 'yield' is passed as a completion handler which accepts no parameters,
        // use yield_handler<void>.
        template <typename ReturnType>
        struct handler_type<fibers::asio::yield_t, ReturnType()> {
            typedef fibers::asio::detail::yield_handler<void> type;
        };

        // Handler type specialisation for fibers::asio::yield.
        // When 'yield' is passed as a completion handler which accepts a data
        // parameter, use yield_handler<parameter type> to return that parameter to
        // the caller.
        template <typename ReturnType, typename Arg1>
        struct handler_type<fibers::asio::yield_t, ReturnType(Arg1)> {
            typedef fibers::asio::detail::yield_handler<Arg1> type;
        };

        //[asio_handler_type
        // Handler type specialisation for fibers::asio::yield.
        // When 'yield' is passed as a completion handler which accepts only
        // error_code, use yield_handler<void>. yield_handler will take care of the
        // error_code one way or another.
        template <typename ReturnType>
        struct handler_type<fibers::asio::yield_t, ReturnType(boost::system::error_code)> {
            typedef fibers::asio::detail::yield_handler<void> type;
        }
        //]

        // Handler type specialisation for fibers::asio::yield.
        // When 'yield' is passed as a completion handler which accepts a data
        // parameter and an error_code, use yield_handler<parameter type> to return
        // just the parameter to the caller. yield_handler will take care of the
        // error_code one way or another.
        template <typename ReturnType, typename Arg2>
        struct handler_type<fibers::asio::yield_t, ReturnType(boost::system::error_code, Arg2)> {
            typedef fibers::asio::detail::yield_handler<Arg2> type;
        };
    }
}

#ifdef BOOST_HAS_ABI_HEADERS
#include BOOST_ABI_SUFFIX
#endif

#endif // BOOST_FIBERS_ASIO_YIELD_HPP

ext: hpp

max_stars_repo_name: FrostMineMC/Frost

max_issues_repo_name: FrostMineMC/Frost

max_forks_repo_name: FrostMineMC/Frost


text:
#include<iostream>
//#define EIGEN_USE_MKL_ALL
#include"basis.hpp"
#include"operators.hpp"
#include"diag.h"
#include"tpoperators.hpp"
#include"files.hpp"
#include"timeev.hpp"
#include"ETH.hpp"
#include<iomanip>
#include <boost/program_options.hpp>

using namespace boost::program_options;
int main(int argc, char *argv[])
{
  using namespace Eigen;
using namespace std;
using namespace Many_Body;
  using Fermi_HubbardBasis= TensorProduct<ElectronBasis, ElectronBasis>;
   using HolsteinBasis= TensorProduct<ElectronBasis, PhononBasis>;
  using Mat= Operators::Mat;
   using boost::program_options::value;
   int M{};
  int L{};
    double T{};
  double t0{};
  double omega{};
  double gamma{};
  double cutoff{};
  bool PB{};

  std::string sM{};
  std::string sL{};
  std::string star{};
  std::string sT{};
  std::string st0{};
  std::string somega{};
  std::string sgamma{};
  std::string starget{};
  std::string scutoff{};
  std::string sPB{};
  std::string filename="exT";
  try
  {
    boost::program_options::options_description desc{"Options"};
    desc.add_options()
      ("help,h", "Help screen")
      ("L", value(&L)->default_value(4), "L")
      ("T", value(&T)->default_value(0.1), "T")
      ("M,m", value(&M)->default_value(2), "M")
      ("t0", value(&t0)->default_value(1.), "t0")
      ("gam", value(&gamma)->default_value(std::sqrt(2)), "gamma")
      ("omg", value(&omega)->default_value(1.), "omega")
      ("pb", value(&PB)->default_value(0), "pb");
  boost::program_options::variables_map vm;
  boost::program_options::store(parse_command_line(argc, argv, desc), vm);
  boost::program_options::notify(vm);

  if (vm.count("help"))
      {std::cout << desc << '\n'; return 0;}
    else{
      if (vm.count("L"))
      {      std::cout << "L: " << vm["L"].as<int>() << '\n';
      	sL="L"+std::to_string(vm["L"].as<int>());
	filename+=sL;
      }
         if (vm.count("M"))
      {      std::cout << "M: " << vm["M"].as<int>() << '\n';
      	sM="M"+std::to_string(vm["M"].as<int>());
	filename+=sM;
      }
      	 if (vm.count("t0"))
      {      std::cout << "t0: " << vm["t0"].as<double>() << '\n';
      	st0="t0"+std::to_string(vm["t0"].as<double>()).substr(0, 3);
      	filename+=st0;
      }
      	 	 if (vm.count("omg"))
      {      std::cout << "omega: " << vm["omg"].as<double>() << '\n';
      	somega="omg"+std::to_string(vm["omg"].as<double>()).substr(0, 3);
      		filename+=somega;
      }

      		 if (vm.count("gam"))
      {      std::cout << "gamma: " << vm["gam"].as<double>() << '\n';
      	sgamma="gam"+std::to_string(vm["gam"].as<double>()).substr(0, 3);
      		filename+=sgamma;
      }
      // 		 if (vm.count("T"))
      // {      std::cout << "T: " << vm["T"].as<double>() << '\n';
      // 	sT="T"+std::to_string(vm["T"].as<double>()).substr(0, 3);
      // 	filename+=sT;
      // }
		 if (vm.count("pb"))
      {      std::cout << "PB: " << vm["pb"].as<bool>() << '\n';
      	sPB="PB"+std::to_string(vm["pb"].as<bool>());
      		filename+=sPB;
      }
      }
    }
  catch (const boost::program_options::error &ex)
  {
    std::cerr << ex.what() << '\n';
    return 0;
  }
  filename+=".bin";
  double mean=0.5*L*M*omega;
  double dt=0.1;
  ElectronState estate1(L, 1);
  ElectronBasis e( L, 1);
  std::cout<< e<<std::endl;

  PhononBasis ph(L, M);
  //                 std::cout<< ph<<std::endl;
      HolsteinBasis TP(e, ph);
      std::cout<< TP.dim << std::endl;
         Eigen::VectorXcd inistate(TP.dim);

   	 Mat O=Operators::NumberOperator(TP, ph, 1,  PB);
   inistate.setZero();

   inistate[0]=1;
   	          Eigen::VectorXcd i0=inistate;

   		  std::cout<< TP.dim << std::endl;
      Mat E1=Operators::EKinOperatorL(TP, e, t0,PB);
       Mat Ebdag=Operators::NBosonCOperator(TP, ph, gamma, PB);
       Mat Eb=Operators::NBosonDOperator(TP, ph, gamma, PB);
      Mat Eph=Operators::NumberOperator(TP, ph, omega,  PB);

      Eigen::VectorXd eigenVals(TP.dim);
      Mat H=E1+Eb+Eph+Ebdag;
   auto J=Operators::CurOperatorL(TP, e, t0, PB);
Eigen::MatrixXcd Jcp=J*std::complex<double>(0,1);

    Eigen::MatrixXd HH=Eigen::MatrixXd(H);

    //Eigen::MatrixXcd HHcp=HH;
    Eigen::MatrixXcd HHcp=Eigen::MatrixXd(HH);
             Eigen::MatrixXcd M2=HHcp*Jcp;
             Eigen::MatrixXcd M1=Jcp*HHcp;
	     Eigen::MatrixXcd COMM=M1-M2;
	     //                  std::cout<< M1<<std::endl;
	     //           std::cout<< COMM<<std::endl;
	     //   std::cout<< M2<<std::endl;
	       COMM*=Jcp;

		  //           std::cout<< HHcp<<std::endl;
		  //           std::cout<< Jcp<<std::endl;
        Eigen::MatrixXd N=Eigen::MatrixXd(Eph);
	  Many_Body::diagMat(HH, eigenVals);
     std::cout<<"MIN E "<<std::setprecision(15)<< eigenVals(0)<< std::endl;
     std::cout<< "MAX "<<eigenVals(eigenVals.size()-1)<< std::endl;
    		      std::cout<<endl<<eigenVals.mean()-mean<<std::endl;
         Eigen::VectorXd energy(TP.dim);
    	   std::vector<double> obs(TP.dim);
    	   std::vector<double> obs2(TP.dim);
	   std::vector<double> obs3(TP.dim);
	   std::vector<double> obs4(TP.dim);
	   std::vector<double> obs5(TP.dim);
	   std::vector<double> obs6(TP.dim);
	   //std::vector<double> obs(TP.dim);
    	   std::vector<double> ensvec(TP.dim);

	   Eigen::MatrixXd PHD2=HH.adjoint()*N*HH;
	   Eigen::MatrixXd EKIN=HH.adjoint()*E1*HH;
	   Eigen::MatrixXd EC=HH.adjoint()*(Ebdag+Eb)*HH;
	   Eigen::MatrixXcd Jmat=HH.adjoint()*(Jcp*HH);
	   Eigen::MatrixXcd JJmat=Jmat*Jmat;
	   Eigen::MatrixXcd COMMmat=HH.adjoint()*(COMM*HH);
	   //	   std::complex<double> J_in=(newIn.adjoint()*(Jcp*newIn))(0);
	   // std::complex<double> JJsq_in=(newIn.adjoint()*(Jcp*Jcp*newIn))(0);
	    Eigen::VectorXd v=PHD2.diagonal();
	    Eigen::VectorXd v2=EKIN.diagonal();
	    Eigen::VectorXd v3=EC.diagonal();
	    Eigen::VectorXd v4=(JJmat.diagonal().real());
	     Eigen::VectorXd v5=(Jmat.diagonal().real());
	     Eigen::VectorXd v6=(COMMmat.diagonal().real());
	    std::cout<< "GS "<<eigenVals(0)<<std::endl;
    	   for(int i=0; i<energy.size(); i++)
    {
      //obs[i]=real(obstot[i]);

      ensvec[i]=eigenVals(i);
      obs[i]=v(i);
      obs2[i]=v2(i);
      obs3[i]=v3(i);
      obs4[i]=v4(i);
      obs5[i]=v5(i);
      obs6[i]=v6(i);
      std::cout<< COMMmat(i,i)<<std::endl;
      if(gamma!=0)
	{	obs3[i]/=gamma;}

      //std::cout<< ensvec[i]<<'\n';
    }// 	   	    std::cout<< "gS "<<   eigenVals[0] <<std::endl;
    		    std::cout<< "mean "<<   eigenVals.mean() <<std::endl;
    	   std::vector<double> Ovec;
	    std::vector<double> Ovec2;
	     std::vector<double> Ovec3;
	     std::vector<double> Ovec4;
	     	     std::vector<double> Ovec5;

   	    	   std::vector<double> Evec;
   	    	   //		   std::vector<double> Tr={0.01, 0.05, 0.1, 0.15, 0.2, 0.25};
   	    	   		   std::vector<double> Tr;
   	    	   for(int i=1; i<22; i++)
   	    	     {
   	    	       Tr.push_back(i*0.05);

   	    	     }

   	    	  	  for(auto t: Tr){

   	    	 // //	      std::cout<< l<<std::endl;
   	         // bin_write(filename, l);
   	    	 // n++;
			    double e1=expvalCan(ensvec, ensvec,  t);
			    double o=expvalCan(ensvec, obs,  t);
			    double o2=expvalCan(ensvec, obs2,  t);
			    double o3=expvalCan(ensvec, obs3,  t);
			    double o4=expvalCan(ensvec, obs4,  t);
			     double o5=expvalCan(ensvec, obs5,  t);
			     double o6=expvalCan(ensvec, obs6,  t);
   	    		    std::cout<< " at T "<< t<<std::endl;
			    std::cout<<std::setprecision(15)<<e1 <<"  "<< o+o2+o3*gamma <<" Nph "<<o << " JJ " <<o4 << "o 6 2 "<< o6<< std::endl;
			    Ovec.push_back(o);
			    Ovec2.push_back(o2);
			    Ovec3.push_back(o3);
			    Ovec4.push_back(o4);
			    			    Ovec5.push_back(o5);
			    Evec.push_back(e1);

   	    }

   			  int pb=PB;
   		  	  std::string Hs="H";
			  std::string Ts="T";
			  std::string phds="PHD";

		      bin_write("E"+filename, Evec);
		      bin_write("Nph"+filename, Ovec);
		      bin_write("EK"+filename, Ovec2);
		      bin_write("nX"+filename, Ovec3);
		      bin_write("JJ"+filename, Ovec4);
		      bin_write("J"+filename, Ovec5);
		      bin_write("temp"+filename, Tr);
  return 0;
}

ext: cpp

max_stars_repo_name: jansendavid/many-body-lib

max_issues_repo_name: jansendavid/many-body-lib

max_forks_repo_name: jansendavid/many-body-lib


text:
#ifndef S3_TRANSPORT_CALLBACKS_HPP
#define S3_TRANSPORT_CALLBACKS_HPP

// stdlib and misc includes
#include <string>
#include <thread>
#include <vector>
#include <cstdio>
#include <iostream>
#include <mutex>
#include <condition_variable>
#include <new>
#include <ctime>
#include <fstream>
#include <cstring>

// boost includes
#include <boost/algorithm/string/predicate.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/map.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/containers/list.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/interprocess/sync/named_mutex.hpp>
#include <boost/container/scoped_allocator.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem.hpp>

// local includes
#include "circular_buffer.hpp"
#include "managed_shared_memory_object.hpp"
#include "s3_multipart_shared_data.hpp"
#include "s3_transport_types.hpp"
#include "s3_transport.hpp"

namespace irods::experimental::io::s3_transport
{
    template <typename CharT>
    class s3_transport;

    class callback_for_read_from_s3_base
    {
        public:

            callback_for_read_from_s3_base(libs3_types::bucket_context& _saved_bucket_context)
                : saved_bucket_context{_saved_bucket_context}
                , offset{0}
                , content_length{0}
                , thread_identifier{0}
                , bytes_read_from_s3{0}
                , shmem_key{}
                , shared_memory_timeout_in_seconds{constants::DEFAULT_SHARED_MEMORY_TIMEOUT_IN_SECONDS}
                , callback_counter{0}
            {}

            virtual libs3_types::status callback_implementation(int libs3_buffer_size,
                                                                const libs3_types::char_type *libs3_buffer) = 0;

            static libs3_types::status invoke_callback(int libs3_buffer_size,
                                                       const libs3_types::char_type *libs3_buffer,
                                                       void *callback_data)
            {
                using named_shared_memory_object =
                    irods::experimental::interprocess::shared_memory::named_shared_memory_object
                    <shared_data::multipart_shared_data>;

                callback_for_read_from_s3_base *data =
                    static_cast<callback_for_read_from_s3_base*>(callback_data);

                // just touch shmem so we know we are active
                if (data->callback_counter++ % 10000 == 0) {
                    auto shmem_key =  data->shmem_key;
                    auto shared_memory_timeout_in_seconds = data->shared_memory_timeout_in_seconds;

                    named_shared_memory_object shm_obj{shmem_key,
                        shared_memory_timeout_in_seconds,
                        constants::MAX_S3_SHMEM_SIZE};
                }

                return data->callback_implementation(libs3_buffer_size, libs3_buffer);
            }

            static libs3_types::status on_response_properties(const libs3_types::response_properties *properties,
                                                              void *callback_data)
            {
                // Don't need to do anything here
                return libs3_types::status_ok;
            }

            static void on_response_completion (libs3_types::status status,
                                                const libs3_types::error_details *error,
                                                void *callback_data)
            {
                callback_for_read_from_s3_base *data = static_cast<callback_for_read_from_s3_base*>(callback_data);
                store_and_log_status( status, error, "callback_for_read_from_s3_base::on_response_completion", data->saved_bucket_context, data->status);
                // Don't change the global error, we may want to retry at a higher level.
                // The WorkerThread will note that status!=OK and act appropriately (retry or fail)
            }

            virtual ~callback_for_read_from_s3_base() {};

            libs3_types::bucket_context& saved_bucket_context; /* To enable more detailed error messages */
            int64_t                      offset;       /* For multiple upload */
            int64_t                      content_length;
            uint64_t                     thread_identifier;
            int64_t                      bytes_read_from_s3;
            std::string                  shmem_key;
            time_t                       shared_memory_timeout_in_seconds;

            // Counter incremented each data callback.  Every Nth iteration touch shared memory
            // so that we know the process didn't die and leave shared memory corrupted
            int                          callback_counter;

            libs3_types::status          status;
    };

    class callback_for_read_from_s3_to_cache : public callback_for_read_from_s3_base
    {

        public:

            explicit callback_for_read_from_s3_to_cache(libs3_types::bucket_context& _saved_bucket_context)
                : callback_for_read_from_s3_base{_saved_bucket_context}
            {}

            libs3_types::status callback_implementation(int libs3_buffer_size,
                                                        const libs3_types::char_type *libs3_buffer)
            {
                assert(libs3_buffer_size >= 0);

                if (!cache_fstream.is_open()) {
                    cache_fstream.open(filename.c_str(), std::ios_base::out);
                }

                if (!cache_fstream) {
                    rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                            __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                    return S3StatusAbortedByCallback;
                }

                // writing output to cache file
                cache_fstream.seekp(this->offset);
                cache_fstream.write(libs3_buffer, libs3_buffer_size);

                auto wrote = static_cast<int64_t>(cache_fstream.tellp()) - this->offset;
                if (wrote>0) {
                    this->offset += wrote;
                    this->bytes_read_from_s3 += wrote;
                }

                return ((wrote < static_cast<decltype(wrote)>(libs3_buffer_size)) ?
                        S3StatusAbortedByCallback : libs3_types::status_ok);

            }

            ~callback_for_read_from_s3_to_cache() {
                if (cache_fstream.is_open()) {
                    cache_fstream.close();
                }
            };

            void set_and_open_cache_file(std::string& f)
            {
                filename = f;
                cache_fstream.open(filename.c_str(), std::ios_base::out);
                if (!cache_fstream) {
                    rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                            __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                }
            }

        private:

            std::string   filename;
            std::ofstream cache_fstream;

    };

    class callback_for_read_from_s3_to_buffer : public callback_for_read_from_s3_base
    {

        public:

            explicit callback_for_read_from_s3_to_buffer(libs3_types::bucket_context& _saved_bucket_context)
                : callback_for_read_from_s3_base{_saved_bucket_context}
                , output_buffer{nullptr}
                , output_buffer_size{0}
            {}

            libs3_types::status callback_implementation(int libs3_buffer_size,
                                                        const libs3_types::char_type *libs3_buffer)
            {
                assert(libs3_buffer_size >= 0);

                // writing to buffer

                int64_t bytes_to_write = this->offset + libs3_buffer_size > output_buffer_size
                    ? output_buffer_size - this->offset
                    : libs3_buffer_size;

                memcpy(output_buffer + this->offset, libs3_buffer, bytes_to_write);

                this->offset += bytes_to_write;
                this->bytes_read_from_s3 += bytes_to_write;

                return ((bytes_to_write < static_cast<int64_t>(libs3_buffer_size)) ?
                        S3StatusAbortedByCallback : libs3_types::status_ok);

            }

            ~callback_for_read_from_s3_to_buffer() {};

            void set_output_buffer_size(int64_t size)
            {
                output_buffer_size = size;
            }

            void set_output_buffer(libs3_types::char_type *buffer)
            {
                output_buffer = buffer;
            }

        private:

            libs3_types::char_type *output_buffer;
            int64_t                output_buffer_size;

    };

    namespace s3_head_object_callback
    {
        libs3_types::status on_response_properties (const libs3_types::response_properties *properties,
                                                    void *callback_data);

        void on_response_complete (libs3_types::status status,
                                       const libs3_types::error_details *error,
                                       void *callback_data);
    }

    namespace s3_upload
    {

        template <typename CharT>
        class callback_for_write_to_s3_base
        {

            public:

                callback_for_write_to_s3_base(libs3_types::bucket_context& _saved_bucket_context,
                                              upload_manager& _manager)
                    : enable_md5{false}
                    , thread_identifier{0}
                    , object_key{}
                    , shmem_key{}
                    , shared_memory_timeout_in_seconds{constants::DEFAULT_SHARED_MEMORY_TIMEOUT_IN_SECONDS}
                    , content_length{0}
                    , saved_bucket_context{_saved_bucket_context}
                    , manager{_manager}
                    , bytes_written{0}
                    , callback_counter{0}
                    , offset{0}
                    , transport_object_ptr{nullptr}
                {}

                virtual int callback_implementation(int libs3_buffer_size,
                                                    libs3_types::char_type *libs3_buffer) = 0;

                static int invoke_callback(int libs3_buffer_size,
                                           libs3_types::char_type *libs3_buffer,
                                           void *callback_data)
                {

                    using named_shared_memory_object =
                        irods::experimental::interprocess::shared_memory::named_shared_memory_object
                        <shared_data::multipart_shared_data>;

                    callback_for_write_to_s3_base *data =
                        static_cast<callback_for_write_to_s3_base*>(callback_data);

                    // just touch shmem so we know we are active
                    if (data->callback_counter++ % 10000 == 0) {
                        auto shmem_key =  data->shmem_key;
                        auto shared_memory_timeout_in_seconds = data->shared_memory_timeout_in_seconds;

                        named_shared_memory_object shm_obj{shmem_key,
                            shared_memory_timeout_in_seconds,
                            constants::MAX_S3_SHMEM_SIZE};
                    }

                    return data->callback_implementation(libs3_buffer_size, libs3_buffer);
                }

                static libs3_types::status on_response_properties(const libs3_types::response_properties *properties,
                                                                  void *callback_data)
                {
                    return libs3_types::status_ok;
                }

                static void on_response_completion (libs3_types::status status,
                                                    const libs3_types::error_details *error,
                                                    void *callback_data)
                {
                    callback_for_write_to_s3_base *data =
                        static_cast<callback_for_write_to_s3_base*>(callback_data);
                    store_and_log_status( status, error, "s3_upload::callback_for_write_to_s3_base::on_response_completion", data->saved_bucket_context,
                            data->status);

                    if (libs3_types::status_ok == status) {
                        // do cleanup
                        callback_for_write_to_s3_base *data =
                            static_cast<callback_for_write_to_s3_base*>(callback_data);
                        data->post_success_cleanup();
                    }

                }

                virtual void post_success_cleanup() = 0;

                virtual ~callback_for_write_to_s3_base() {};

                libs3_types::status          status;
                bool                         enable_md5;
                uint64_t                     thread_identifier;
                std::string                  object_key;
                std::string                  shmem_key;
                time_t                       shared_memory_timeout_in_seconds;

                int64_t                      content_length;
                libs3_types::bucket_context& saved_bucket_context; // To enable more detailed error messages
                upload_manager&              manager;
                int64_t                      bytes_written;

                // Counter incremented each data callback.  Every Nth iteration touch shared memory
                // so that we know the process didn't die and leave shared memory corrupted
                int                          callback_counter;
                int64_t                      offset;       /* For multiple upload */
                s3_transport<CharT>*         transport_object_ptr;

        };

        template <typename CharT>
        class callback_for_write_from_cache_to_s3 : public callback_for_write_to_s3_base<CharT>
        {

            public:

                callback_for_write_from_cache_to_s3(libs3_types::bucket_context& _saved_bucket_context,
                                                    upload_manager& _manager)
                    : callback_for_write_to_s3_base<CharT>{_saved_bucket_context, _manager}
                {}

                int callback_implementation(int libs3_buffer_size,
                                            libs3_types::buffer_type libs3_buffer)
                {

                    assert(libs3_buffer_size >= 0);

                    if (!cache_fstream.is_open()) {
                        cache_fstream.open(filename.c_str(), std::ios_base::in);
                    }

                    if (!cache_fstream) {
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                        return S3StatusAbortedByCallback;
                    }

                    // writing cache file to s3 buffer
                    int64_t length_to_read_from_cache = this->content_length - this->bytes_written
                        > static_cast<int64_t>(libs3_buffer_size)
                        ? static_cast<int64_t>(libs3_buffer_size)
                        : this->content_length - this->bytes_written;

                    cache_fstream.seekg(this->offset);
                    cache_fstream.read(libs3_buffer, length_to_read_from_cache);

                    auto bytes_read_from_cache = static_cast<int64_t>(cache_fstream.tellg()) - this->offset;
                    if (bytes_read_from_cache > 0) {
                        this->offset += bytes_read_from_cache;
                        this->bytes_written += bytes_read_from_cache;
                    }

                    return bytes_read_from_cache;

                }

                void post_success_cleanup() {}

                ~callback_for_write_from_cache_to_s3() {
                    if (cache_fstream.is_open()) {
                        cache_fstream.close();
                    }
                };

                void set_and_open_cache_file(std::string& f)
                {
                    filename = f;
                    cache_fstream.open(filename.c_str(), std::ios_base::in);
                    if (!cache_fstream) {
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                    }
                }

            private:

                std::string   filename;
                std::ifstream cache_fstream;

        };

        template <typename CharT>
        class callback_for_write_from_buffer_to_s3 : public callback_for_write_to_s3_base<CharT>
        {

            public:

                using circular_char_type = irods::experimental::circular_buffer<libs3_types::char_type>;

                callback_for_write_from_buffer_to_s3(libs3_types::bucket_context& _saved_bucket_context,
                                                     upload_manager& _manager,
                                                     circular_char_type& _circular_buffer)
                    : callback_for_write_to_s3_base<CharT>{_saved_bucket_context, _manager}
                    , circular_buffer{_circular_buffer}
                {}

                int callback_implementation(int libs3_buffer_size,
                                            libs3_types::buffer_type libs3_buffer)
                {
                    using named_shared_memory_object =
                        irods::experimental::interprocess::shared_memory::named_shared_memory_object
                        <shared_data::multipart_shared_data>;

                    assert(libs3_buffer_size >= 0);

                    // if a critical error occurred in the transport, the writer to the buffer
                    // may have hung up, so as not to get in a deadlock, just return
                    if (this->transport_object_ptr && !this->transport_object_ptr->get_error().ok()) {
                        return 0;
                    }

                    // if we've already written the expected number of bytes, just return 0 which will
                    // trigger the completion
                    if (this->content_length <= this->bytes_written) {
                        return 0;
                    }

                    auto bytes_to_return =
                        libs3_buffer_size < this->content_length - this->bytes_written
                        ? libs3_buffer_size
                        : this->content_length - this->bytes_written;

                    try {
                        circular_buffer.peek(this->bytes_written, bytes_to_return, libs3_buffer);
                    } catch (timeout_exception& e) {

                        // timeout reading from circular buffer

                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] "
                                "Timed out waiting to read from circular buffer.  "
                                "Remote likely hung up...\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);

                        // save that we got a timeout so that we don't keep retrying

                        auto shmem_key =  this->shmem_key;
                        auto shared_memory_timeout_in_seconds = this->shared_memory_timeout_in_seconds;

                        named_shared_memory_object shm_obj{shmem_key,
                            shared_memory_timeout_in_seconds,
                            constants::MAX_S3_SHMEM_SIZE};

                        shm_obj.atomic_exec([](auto& data) {

                            data.circular_buffer_read_timeout = true;

                        });

                        return 0;
                    }

                    this->bytes_written += bytes_to_return;

                    return bytes_to_return;

                }

                void post_success_cleanup() {

                    // had a success, remove all processed bytes from buffer
                    try {
                        circular_buffer.pop_front(this->bytes_written);
                    } catch (timeout_exception& e) {
                        // this should never happen but catch and log just in case
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] "
                                "Unexpected timeout when removing entries from circular buffer."
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                    }
                }

                ~callback_for_write_from_buffer_to_s3() {};

                irods::experimental::circular_buffer<libs3_types::char_type>& circular_buffer;

        };

    } // end namespace s3_upload

    namespace s3_multipart_upload
    {

        namespace initialization_callback
        {

            libs3_types::status on_response (const libs3_types::char_type* upload_id,
                                             void *callback_data );

            libs3_types::status on_response_properties (const libs3_types::response_properties *properties,
                                                        void *callback_data);

            void on_response_complete (libs3_types::status status,
                                       const libs3_types::error_details *error,
                                       void *callback_data);
        } // end namespace initialization_callback

        /* Uploading the multipart completion XML from our buffer */
        namespace commit_callback
        {
            int on_response (int buffer_size,
                          libs3_types::buffer_type buffer,
                          void *callback_data);

            libs3_types::status on_response_properties (const libs3_types::response_properties *properties,
                                                        void *callback_data);

            void on_response_completion (libs3_types::status status,
                                         const libs3_types::error_details *error,
                                         void *callback_data);

        } // end namespace commit_callback

        template <typename CharT>
        class callback_for_write_to_s3_base
        {

            public:

                callback_for_write_to_s3_base(libs3_types::bucket_context& _saved_bucket_context,
                                              upload_manager& _manager)
                    : enable_md5{false}
                    , thread_identifier{0}
                    , shared_memory_timeout_in_seconds{constants::DEFAULT_SHARED_MEMORY_TIMEOUT_IN_SECONDS}
                    , object_key{}
                    , shmem_key{}
                    , sequence{0}
                    , content_length{0}
                    , saved_bucket_context{_saved_bucket_context}
                    , manager{_manager}
                    , bytes_written{0}
                    , callback_counter{0}
                    , offset{0}
                    , transport_object_ptr{nullptr}
                {}

                virtual int callback_implementation(int libs3_buffer_size,
                                                    libs3_types::char_type *libs3_buffer) = 0;

                static int invoke_callback(int libs3_buffer_size,
                                           libs3_types::char_type *libs3_buffer,
                                           void *callback_data)
                {

                    using named_shared_memory_object =
                        irods::experimental::interprocess::shared_memory::named_shared_memory_object
                        <shared_data::multipart_shared_data>;

                    callback_for_write_to_s3_base *data = static_cast<callback_for_write_to_s3_base*>(callback_data);

                    // just touch shmem so we know we are active
                    if (data->callback_counter++ % 10000 == 0) {
                        auto shmem_key =  data->shmem_key;
                        auto shared_memory_timeout_in_seconds = data->shared_memory_timeout_in_seconds;

                        named_shared_memory_object shm_obj{shmem_key,
                            shared_memory_timeout_in_seconds,
                            constants::MAX_S3_SHMEM_SIZE};
                    }

                    return data->callback_implementation(libs3_buffer_size, libs3_buffer);
                }

                static libs3_types::status on_response_properties(const libs3_types::response_properties *properties,
                                                                  void *callback_data)
                {
                    namespace bi = boost::interprocess;
                    namespace types = shared_data::interprocess_types;

                    callback_for_write_to_s3_base *callback_for_write_to_s3_base_data
                        = static_cast<callback_for_write_to_s3_base*>(callback_data);

                    using named_shared_memory_object =
                        irods::experimental::interprocess::shared_memory::named_shared_memory_object
                        <shared_data::multipart_shared_data>;

                    auto shmem_key =  callback_for_write_to_s3_base_data->shmem_key;
                    auto shared_memory_timeout_in_seconds =
                        callback_for_write_to_s3_base_data->shared_memory_timeout_in_seconds;

                    named_shared_memory_object shm_obj{shmem_key,
                        shared_memory_timeout_in_seconds,
                        constants::MAX_S3_SHMEM_SIZE};

                    return shm_obj.atomic_exec([properties,
                            &callback_for_write_to_s3_base_data,
                            &shm_obj](auto& data) {

                        const char *etag = properties->eTag;

                        // Update the etags vector.  It should be sized large enough
                        // to not require a resize but resize if necessary.
                        if (callback_for_write_to_s3_base_data->sequence > data.etags.size()) {
                            try {
                                data.etags.resize(callback_for_write_to_s3_base_data->sequence,
                                        types::shm_char_string("", shm_obj.get_allocator()));
                            } catch (std::bad_alloc& ba) {
                                return S3StatusOutOfMemory;
                            }
                        }

                        if (etag) {
                            data.etags[callback_for_write_to_s3_base_data->sequence - 1] = etag;
                        } else {
                            data.etags[callback_for_write_to_s3_base_data->sequence - 1] = "";
                        }

                        return libs3_types::status_ok;
                    });
                }

                static void on_response_completion (libs3_types::status status,
                                                    const libs3_types::error_details *error,
                                                    void *callback_data)
                {
                    callback_for_write_to_s3_base *data =
                        static_cast<callback_for_write_to_s3_base*>(callback_data);
                    store_and_log_status( status, error, "s3_multipart_upload::callback_for_write_to_s3_base::on_response_completion", data->saved_bucket_context,
                            data->status);

                    if (libs3_types::status_ok == status) {
                        // do cleanup
                        callback_for_write_to_s3_base *data =
                            static_cast<callback_for_write_to_s3_base*>(callback_data);
                        data->post_success_cleanup();
                    }

                }

                virtual void post_success_cleanup() = 0;

                virtual ~callback_for_write_to_s3_base() {};

                libs3_types::status          status;
                bool                         enable_md5;
                uint64_t                     thread_identifier;
                time_t                       shared_memory_timeout_in_seconds;
                std::string                  object_key;
                std::string                  shmem_key;

                unsigned long                sequence;
                int64_t                      content_length;
                libs3_types::bucket_context& saved_bucket_context; // To enable more detailed error messages

                upload_manager&              manager;

                int64_t                      bytes_written;

                // Counter incremented each data callback.  Every Nth iteration touch shared memory
                // so that we know the process didn't die and leave shared memory corrupted
                int                          callback_counter;
                int64_t                      offset;
                s3_transport<CharT>*         transport_object_ptr;

        };

        template <typename CharT>
        class callback_for_write_from_cache_to_s3 : public callback_for_write_to_s3_base<CharT>
        {

            public:

                callback_for_write_from_cache_to_s3(libs3_types::bucket_context& _saved_bucket_context,
                                                    upload_manager& _manager)
                    : callback_for_write_to_s3_base<CharT>{_saved_bucket_context, _manager}
                {}

                int callback_implementation(int libs3_buffer_size,
                                            libs3_types::buffer_type libs3_buffer)
                {

                    assert(libs3_buffer_size >= 0);

                    if (!cache_fstream.is_open()) {
                        cache_fstream.open(filename.c_str(), std::ios_base::in);
                    }

                    if (!cache_fstream) {
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                        return 0;
                    }

                    // writing cache file to s3 buffer
                    int64_t length_to_read_from_cache = this->content_length - this->bytes_written
                        > static_cast<int64_t>(libs3_buffer_size)
                        ? static_cast<int64_t>(libs3_buffer_size)
                        : this->content_length - this->bytes_written;

                    cache_fstream.seekg(this->offset);
                    cache_fstream.read(libs3_buffer, length_to_read_from_cache);

                    auto bytes_read_from_cache = static_cast<int64_t>(cache_fstream.tellg()) - this->offset;
                    if (bytes_read_from_cache > 0) {
                        this->offset += bytes_read_from_cache;
                        this->bytes_written += bytes_read_from_cache;
                    }

                    return bytes_read_from_cache;

                }

                ~callback_for_write_from_cache_to_s3() {
                    if (cache_fstream.is_open()) {
                        cache_fstream.close();
                    }
                };

                void set_and_open_cache_file(std::string& f)
                {
                    filename = f;
                    cache_fstream.open(filename.c_str(), std::ios_base::in);
                    if (!cache_fstream) {
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] could not open cache file\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                    }
                }

                void post_success_cleanup() {}

            private:

                std::string   filename;
                std::ifstream cache_fstream;

        };

        template <typename CharT>
        class callback_for_write_from_buffer_to_s3 : public callback_for_write_to_s3_base<CharT>
        {

            public:

                using circular_char_type = irods::experimental::circular_buffer<libs3_types::char_type>;

                callback_for_write_from_buffer_to_s3(libs3_types::bucket_context& _saved_bucket_context,
                                                     upload_manager& _manager,
                                                     circular_char_type& _circular_buffer)
                    : callback_for_write_to_s3_base<CharT>{_saved_bucket_context, _manager}
                    , circular_buffer{_circular_buffer}
                {}

                int callback_implementation(int libs3_buffer_size,
                                            libs3_types::buffer_type libs3_buffer)
                {
                    using named_shared_memory_object =
                        irods::experimental::interprocess::shared_memory::named_shared_memory_object
                        <shared_data::multipart_shared_data>;

                    assert(libs3_buffer_size >= 0);

                    // if a critical error occurred in the transport, the writer to the buffer
                    // may have hung up, so as not to get in a deadlock, just return
                    if (this->transport_object_ptr && !this->transport_object_ptr->get_error().ok()) {
                        return 0;
                    }

                    // if we've already written the expected number of bytes, just return 0 which will
                    // trigger the completion
                    if (this->content_length <= this->bytes_written) {
                        return 0;
                    }

                    auto bytes_to_return =
                        libs3_buffer_size < this->content_length - this->bytes_written
                        ? libs3_buffer_size
                        : this->content_length - this->bytes_written;

                    try {
                        circular_buffer.peek(this->bytes_written, bytes_to_return, libs3_buffer);
                    } catch (timeout_exception& e) {

                        // timeout reading from circular buffer

                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] "
                                "Timed out waiting to read from circular buffer.  "
                                "Remote likely hung up...\n",
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);

                        // save that we got a timeout so that we don't keep retrying
                        auto shmem_key =  this->shmem_key;
                        auto shared_memory_timeout_in_seconds = this->shared_memory_timeout_in_seconds;

                        named_shared_memory_object shm_obj{shmem_key,
                            shared_memory_timeout_in_seconds,
                            constants::MAX_S3_SHMEM_SIZE};

                        shm_obj.atomic_exec([](auto& data) {

                            data.circular_buffer_read_timeout = true;

                        });

                        return 0;

                    }

                    this->bytes_written += bytes_to_return;

                    return bytes_to_return;

                }

                void post_success_cleanup() {
                    // had a success, remove all processed bytes from buffer
                    try {
                        circular_buffer.pop_front(this->bytes_written);
                    } catch (timeout_exception& e) {
                        // this should never happen but catch and log just in case
                        rodsLog(LOG_ERROR, "%s:%d (%s) [[%lu]] "
                                "Unexpected timeout when removing entries from circular buffer."
                                __FILE__, __LINE__, __FUNCTION__, this->thread_identifier);
                    }
                }

                ~callback_for_write_from_buffer_to_s3() {};

                irods::experimental::circular_buffer<libs3_types::char_type>& circular_buffer;

        };

        namespace cancel_callback
        {
            libs3_types::status on_response_properties (const libs3_types::response_properties *properties,
                                                        void *callback_data);

            // S3_abort_multipart_upload() does not allow a callback_data parameter, so pass the
            // final operation status using this global.

            extern libs3_types::status g_response_completion_status;
            extern libs3_types::bucket_context *g_response_completion_saved_bucket_context;

            void on_response_completion (libs3_types::status status,
                                      const libs3_types::error_details *error,
                                      void *callback_data);
        } // end namespace cancel_callback

    } // end namespace s3_multipart_upload

    namespace restore_object_callback
    {
        int on_response (int buffer_size,
                      libs3_types::buffer_type buffer,
                      void *callback_data);

        libs3_types::status on_response_properties (const libs3_types::response_properties *properties,
                                      void *callback_data);

        void on_response_completion (libs3_types::status status,
                                  const libs3_types::error_details *error,
                                  void *callback_data);

    } // end namespace restore_object_callback

} // irods::experimental::io::s3_transport

#endif // S3_TRANSPORT_CALLBACKS_HPP


text:
/*
 * GridTools
 *
 * Copyright (c) 2014-2019, ETH Zurich
 * All rights reserved.
 *
 * Please, refer to the LICENSE file in the root directory.
 * SPDX-License-Identifier: BSD-3-Clause
 */
#pragma once

#include <boost/mpl/copy.hpp>
#include <boost/mpl/inserter.hpp>

#include "../defs.hpp"

namespace gridtools {

    namespace _impl {
        struct variadic_push_back {
            template <class, class>
            struct apply;
            template <template <class...> class L, class... Ts, class T>
            struct apply<L<Ts...>, T> {
                using type = L<Ts..., T>;
            };
        };
    } // namespace _impl

    /// Helper to copy MPL sequence to a variadic typelist.
    //
    //  \tparam Src  - a type thst models MPL sequence concept
    //  \tparam Dst  - a type that is instantiation of the template with the variadic template class parameter
    //
    //  Example:
    //     copy_into_variadic<boost::mpl::vector<int, double>, std::tuple> is the same as std::tuple<int, double>
    //
    template <class Src, class Dst>
    struct lazy_copy_into_variadic : boost::mpl::copy<Src, boost::mpl::inserter<Dst, _impl::variadic_push_back>> {};

    template <class Src, class Dst>
    using copy_into_variadic =
        typename boost::mpl::copy<Src, boost::mpl::inserter<Dst, _impl::variadic_push_back>>::type;
} // namespace gridtools

max_stars_repo_name: mbianco/gridtools

max_issues_repo_name: mbianco/gridtools

max_forks_repo_name: mbianco/gridtools


text:
#include "generator/osm_source.hpp"

#include "generator/intermediate_data.hpp"
#include "generator/intermediate_elements.hpp"
#include "generator/osm_element.hpp"
#include "generator/towns_dumper.hpp"
#include "generator/translator_factory.hpp"

#include "platform/platform.hpp"

#include "geometry/mercator.hpp"

#include "base/assert.hpp"
#include "base/stl_helpers.hpp"
#include "base/file_name_utils.hpp"

#include <fstream>
#include <memory>
#include <set>
#include <thread>
#include <vector>

#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/iostreams/stream.hpp>

#include <sys/mman.h>

#include "defines.hpp"

using namespace std;

namespace generator
{
// SourceReader ------------------------------------------------------------------------------------
SourceReader::SourceReader() : m_file(unique_ptr<istream, Deleter>(&cin, Deleter(false)))
{
  LOG_SHORT(LINFO, ("Reading OSM data from stdin"));
}

SourceReader::SourceReader(string const & filename)
  : m_file(unique_ptr<istream, Deleter>(new ifstream(filename), Deleter()))
{
  CHECK(static_cast<ifstream *>(m_file.get())->is_open(), ("Can't open file:", filename));
  LOG_SHORT(LINFO, ("Reading OSM data from", filename));
}

SourceReader::SourceReader(std::istream & stream)
  : m_file(unique_ptr<istream, Deleter>(&stream, Deleter(false)))
{ }

uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize)
{
  m_file->read(buffer, bufferSize);
  return m_file->gcount();
}

// Functions ---------------------------------------------------------------------------------------

void BuildIntermediateNode(OsmElement && element, NodeElement & node)
{
  auto position = MercatorBounds::FromLatLon(element.m_lat, element.m_lon);
  node = {element.m_id, position.y, position.x};
}

bool BuildIntermediateWay(OsmElement && element, WayElement & way)
{
  way.m_wayOsmId = element.m_id;
  way.nodes = std::move(element.Nodes());
  return way.IsValid();
}

bool BuildIntermediateRelation(OsmElement && element, RelationElement & relation)
{
  for (auto & member : element.Members())
  {
    switch (member.m_type) {
    case OsmElement::EntityType::Node:
      relation.nodes.emplace_back(member.m_ref, std::move(member.m_role));
      break;
    case OsmElement::EntityType::Way:
      relation.ways.emplace_back(member.m_ref, std::move(member.m_role));
      break;
    case OsmElement::EntityType::Relation:
      // we just ignore type == "relation"
      break;
    default:
      break;
    }
  }

  for (auto & tag : element.Tags())
    relation.tags.emplace(std::move(tag.m_key), std::move(tag.m_value));

  return relation.IsValid();
}

void AddElementToCache(cache::IntermediateDataWriter & cache, OsmElement && element)
{
  switch (element.m_type)
  {
  case OsmElement::EntityType::Node:
  {
    NodeElement node;
    BuildIntermediateNode(std::move(element), node);
    cache.AddNode(node.m_nodeOsmId, node.m_lat, node.m_lon);
    break;
  }
  case OsmElement::EntityType::Way:
  {
    // Store way.
    auto const id = element.m_id;
    WayElement way(id);
    if (BuildIntermediateWay(std::move(element), way))
      cache.AddWay(id, way);
    break;
  }
  case OsmElement::EntityType::Relation:
  {
    // store relation
    auto const id = element.m_id;
    RelationElement relation;
    if (BuildIntermediateRelation(std::move(element), relation))
      cache.AddRelation(id, relation);
    break;
  }
  default:
    break;
  }
}

void BuildIntermediateDataFromXML(SourceReader & stream, cache::IntermediateDataWriter & cache,
                                  TownsDumper & towns)
{
  ProcessorOsmElementsFromXml processorOsmElementsFromXml(stream);
  OsmElement element;
  while (processorOsmElementsFromXml.TryRead(element))
  {
    towns.CheckElement(element, false /* concurrent */);
    AddElementToCache(cache, std::move(element));
  }
}

void BuildIntermediateDataFromXML(
    std::string const & filename, cache::IntermediateDataWriter & cache, TownsDumper & towns)
{
  SourceReader reader = filename.empty() ? SourceReader() : SourceReader(filename);
  BuildIntermediateDataFromXML(reader, cache, towns);
}

void ProcessOsmElementsFromXML(SourceReader & stream, function<void(OsmElement &&)> processor)
{
  ProcessorOsmElementsFromXml processorOsmElementsFromXml(stream);
  OsmElement element;
  while (processorOsmElementsFromXml.TryRead(element))
    processor(std::move(element));
}

void BuildIntermediateData(std::vector<OsmElement> && elements,
                           cache::IntermediateDataWriter & cache, TownsDumper & towns,
                           bool concurrent)
{
  if (elements.empty())
    return;

  auto const firstElementType = elements.front().m_type;
  auto nodes = cache::IntermediateDataWriter::Nodes{};
  if (firstElementType == OsmElement::EntityType::Node)
    nodes.reserve(elements.size());

  auto ways = cache::IntermediateDataWriter::Ways{};
  if (firstElementType == OsmElement::EntityType::Way)
    ways.reserve(elements.size());

  auto relations = cache::IntermediateDataWriter::Relations{};
  if (firstElementType == OsmElement::EntityType::Relation)
    relations.reserve(elements.size());

  for (auto & osmElement : elements)
  {
    towns.CheckElement(osmElement, concurrent);

    auto const id = osmElement.m_id;
    switch (osmElement.m_type)
    {
    case OsmElement::EntityType::Node:
    {
      nodes.emplace_back(id, NodeElement{});
      BuildIntermediateNode(std::move(osmElement), nodes.back().second);
      break;
    }
    case OsmElement::EntityType::Way:
    {
      WayElement way{id};
      if (BuildIntermediateWay(std::move(osmElement), way))
        ways.emplace_back(id, std::move(way));
      break;
    }
    case OsmElement::EntityType::Relation:
    {
      RelationElement relation;
      if (BuildIntermediateRelation(std::move(osmElement), relation))
        relations.emplace_back(id, std::move(relation));
      break;
    }
    default:
      break;
    }
  }

  if (!nodes.empty())
    cache.AddNodes(std::move(nodes), concurrent);
  if (!ways.empty())
    cache.AddWays(std::move(ways), concurrent);
  if (!relations.empty())
    cache.AddRelations(std::move(relations), concurrent);
}

void BuildIntermediateDataFromO5M(
    ProcessorOsmElementsFromO5M & o5mReader, cache::IntermediateDataWriter & cache,
    TownsDumper & towns, bool concurrent)
{
  std::vector<OsmElement> elements(o5mReader.ChunkSize());
  size_t elementsCount = 0;
  while (o5mReader.TryRead(elements[elementsCount]))
  {
    ++elementsCount;
    if (elementsCount < o5mReader.ChunkSize())
      continue;

    BuildIntermediateData(std::move(elements), cache, towns, concurrent);
    elements.resize(o5mReader.ChunkSize());  // restore capacity after std::move(elements)
    elementsCount = 0;
  }

  elements.resize(elementsCount);
  BuildIntermediateData(std::move(elements), cache, towns, concurrent);
}

void BuildIntermediateDataFromO5M(
    std::string const & filename, cache::IntermediateDataWriter & cache, TownsDumper & towns,
    unsigned int threadsCount)
{
  if (filename.empty())
  {
    // Read form stdin.
    auto && reader = SourceReader{};
    auto && o5mReader = ProcessorOsmElementsFromO5M(reader);
    return BuildIntermediateDataFromO5M(o5mReader, cache, towns, false /* concurrent */);
  }

  LOG_SHORT(LINFO, ("Reading OSM data from", filename));

  auto sourceMap = boost::iostreams::mapped_file_source{filename};
  if (!sourceMap.is_open())
    MYTHROW(Writer::OpenException, ("Failed to open", filename));
  // Try aggressively (MADV_WILLNEED) and asynchronously read ahead the o5m-file.
  auto readaheadTask = std::thread([data = sourceMap.data(), size = sourceMap.size()] {
    ::madvise(const_cast<char*>(data), size, MADV_WILLNEED);
  });
  readaheadTask.detach();

  constexpr size_t chunkSize = 10'000;
  std::vector<std::thread> threads;
  for (unsigned int i = 0; i < std::max(threadsCount, 1u); ++i)
  {
    threads.emplace_back([&sourceMap, &cache, &towns, threadsCount, i] {
      namespace io = boost::iostreams;
      auto && sourceArray = io::array_source{sourceMap.data(), sourceMap.size()};
      auto && stream = io::stream<io::array_source>{sourceArray, std::ios::binary};
      auto && reader = SourceReader(stream);
      auto && o5mReader = ProcessorOsmElementsFromO5M(reader, threadsCount, i, chunkSize);
      BuildIntermediateDataFromO5M(o5mReader, cache, towns, threadsCount > 1);
    });
  }

  for (auto & thread : threads)
    thread.join();
}

void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement &&)> processor)
{
  ProcessorOsmElementsFromO5M processorOsmElementsFromO5M(stream);

  OsmElement element;
  while (processorOsmElementsFromO5M.TryRead(element))
    processor(std::move(element));
}

ProcessorOsmElementsFromO5M::ProcessorOsmElementsFromO5M(
    SourceReader & stream, size_t taskCount, size_t taskId, size_t chunkSize)
  : m_stream(stream)
  , m_dataset([&](uint8_t * buffer, size_t size) {
      return m_stream.Read(reinterpret_cast<char *>(buffer), size);
    }, 1024 * 1024)
  , m_taskCount{taskCount}
  , m_taskId{taskId}
  , m_chunkSize{chunkSize}
  , m_pos(m_dataset.begin())
{
}

bool ProcessorOsmElementsFromO5M::TryRead(OsmElement & element)
{
  while (m_pos != m_dataset.end())
  {
    auto const chunkId = m_elementCounter / m_chunkSize;
    auto const chunkTaskId = chunkId % m_taskCount;
    if (chunkTaskId == m_taskId)
      return Read(element);

    ++m_pos;
    ++m_elementCounter;
  }

  return false;
}

bool ProcessorOsmElementsFromO5M::Read(OsmElement & element)
{
  using Type = osm::O5MSource::EntityType;

  element.Clear();

  auto const translate = [](Type t) -> OsmElement::EntityType {
    switch (t)
    {
    case Type::Node: return OsmElement::EntityType::Node;
    case Type::Way: return OsmElement::EntityType::Way;
    case Type::Relation: return OsmElement::EntityType::Relation;
    default: return OsmElement::EntityType::Unknown;
    }
  };

  // Be careful, we could call Nodes(), Members(), Tags() from O5MSource::Entity
  // only once (!). Because these functions read data from file simultaneously with
  // iterating in loop. Furthermore, into Tags() method calls Nodes.Skip() and Members.Skip(),
  // thus first call of Nodes (Members) after Tags() will not return any results.
  // So don not reorder the "for" loops (!).
  auto const & entity = *m_pos;
  element.m_id = entity.id;
  switch (entity.type)
  {
  case Type::Node:
  {
    element.m_type = OsmElement::EntityType::Node;
    element.m_lat = entity.lat;
    element.m_lon = entity.lon;
    break;
  }
  case Type::Way:
  {
    element.m_type = OsmElement::EntityType::Way;
    for (uint64_t nd : entity.Nodes())
      element.AddNd(nd);
    break;
  }
  case Type::Relation:
  {
    element.m_type = OsmElement::EntityType::Relation;
    for (auto const & member : entity.Members())
      element.AddMember(member.ref, translate(member.type), member.role);
    break;
  }
  default: break;
  }

  for (auto const & tag : entity.Tags())
    element.AddTag(tag.key, tag.value);

  ++m_pos;
  ++m_elementCounter;
  return true;
}

ProcessorOsmElementsFromXml::ProcessorOsmElementsFromXml(SourceReader & stream)
  : m_xmlSource([&, this](auto * element) { m_queue.emplace(*element); })
  , m_parser(stream, m_xmlSource)
{
}

bool ProcessorOsmElementsFromXml::TryReadFromQueue(OsmElement & element)
{
  if (m_queue.empty())
    return false;

  element = m_queue.front();
  m_queue.pop();
  return true;
}

bool ProcessorOsmElementsFromXml::TryRead(OsmElement & element)
{
  do {
    if (TryReadFromQueue(element))
      return true;
  } while (m_parser.Read());

  return TryReadFromQueue(element);
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// Generate functions implementations.
///////////////////////////////////////////////////////////////////////////////////////////////////

bool GenerateIntermediateData(feature::GenerateInfo const & info)
{
  auto nodes = cache::CreatePointStorageWriter(info.m_nodeStorageType,
                                               info.GetIntermediateFileName(NODES_FILE));
  cache::IntermediateDataWriter cache(*nodes, info);
  TownsDumper towns;

  LOG(LINFO, ("Data source:", info.m_osmFileName));

  switch (info.m_osmFileType)
  {
  case feature::GenerateInfo::OsmSourceType::XML:
    BuildIntermediateDataFromXML(info.m_osmFileName, cache, towns);
    break;
  case feature::GenerateInfo::OsmSourceType::O5M:
    BuildIntermediateDataFromO5M(info.m_osmFileName, cache, towns, info.m_threadsCount);
    break;
  }

  cache.SaveIndex();
  towns.Dump(info.GetIntermediateFileName(TOWNS_FILE));
  LOG(LINFO, ("Added points count =", nodes->GetNumProcessedPoints()));
  return true;
}
}  // namespace generator

max_stars_repo_name: mapsme/geocore

max_issues_repo_name: mapsme/geocore

max_forks_repo_name: mapsme/geocore


text:
//=======================================================================
// Copyright 2001 Jeremy G. Siek, Andrew Lumsdaine, Lie-Quan Lee,
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================
#include <boost/config.hpp>
#include <iostream>
#include <fstream>
#include <vector>
#include <boost/lexical_cast.hpp>
#include <boost/graph/graphviz.hpp>
#include <boost/graph/prim_minimum_spanning_tree.hpp>
int
main()
{
  using namespace boost;
  GraphvizGraph g_dot;
  read_graphviz("figs/telephone-network.dot", g_dot);

  typedef adjacency_list < vecS, vecS, undirectedS, no_property,
    property < edge_weight_t, int > > Graph;
  Graph g(num_vertices(g_dot));
  property_map < GraphvizGraph, edge_attribute_t >::type
    edge_attr_map = get(edge_attribute, g_dot);
  graph_traits < GraphvizGraph >::edge_iterator ei, ei_end;
  for (boost::tie(ei, ei_end) = edges(g_dot); ei != ei_end; ++ei) {
    int weight = lexical_cast < int >(edge_attr_map[*ei]["label"]);
    property < edge_weight_t, int >edge_property(weight);
    add_edge(source(*ei, g_dot), target(*ei, g_dot), edge_property, g);
  }

  typedef graph_traits < Graph >::vertex_descriptor Vertex;
  std::vector < Vertex > parent(num_vertices(g));
  property_map < Graph, edge_weight_t >::type weight = get(edge_weight, g);
#if defined(BOOST_MSVC) && BOOST_MSVC <= 1300
  property_map<Graph, vertex_index_t>::type indexmap = get(vertex_index, g);
  std::vector<std::size_t> distance(num_vertices(g));
  prim_minimum_spanning_tree(g, *vertices(g).first, &parent[0], &distance[0],
                             weight, indexmap, default_dijkstra_visitor());
#else
  prim_minimum_spanning_tree(g, &parent[0]);
#endif

  int total_weight = 0;
  for (int v = 0; v < num_vertices(g); ++v)
    if (parent[v] != v)
      total_weight += get(weight, edge(parent[v], v, g).first);
  std::cout << "total weight: " << total_weight << std::endl;

  for (int u = 0; u < num_vertices(g); ++u)
    if (parent[u] != u)
      edge_attr_map[edge(parent[u], u, g_dot).first]["color"] = "black";
  std::ofstream out("figs/telephone-mst-prim.dot");
  graph_property < GraphvizGraph, graph_edge_attribute_t >::type &
    graph_edge_attr_map = get_property(g_dot, graph_edge_attribute);
  graph_edge_attr_map["color"] = "gray";
  write_graphviz(out, g_dot);

  return EXIT_SUCCESS;
}

max_stars_repo_name: zyiacas/boost-doc-zh

max_issues_repo_name: sdfict/boost-doc-zh

max_forks_repo_name: sdfict/boost-doc-zh