text:
// Copyright 2015-2018 RWTH Aachen University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <boost/program_options.hpp>

#include <iostream>
#include <fstream>
#include <string>

#include "ast.hh"
#include "ast_print_visitor.hh"
#include "ast_preprocessor_visitor.hh"
#include "ast_policy_compressor_visitor.hh"
#include "binary.hh"
#include "policy_header.hh"
#include "policy_stack.hh"
#include "relation_set.hh"
#include "variable_set.hh"

#include "equation_driver.hh"

#include "debug.hh"

#include "performance.h"

namespace  {
    const size_t SUCCESS = 0;
    const size_t ERROR_IN_COMMAND_LINE = 1;
    const size_t ERROR_UNHANDLED_EXCEPTION = 2;
    const size_t POLICY_ERROR = 3;
}

string getFile(string filename)  {
    ifstream file(filename);
    if(!file.is_open())
        throw runtime_error("Can't open file "+ filename);
    //read the file into string
    stringstream strStream;
    strStream << file.rdbuf();
    file.close();
    return strStream.str();
}

int main (int argc, char *argv[])  {
    try  {
        //default options that can be overwriten by the program options
        bool traceParsingEnabled = false;
        bool traceScanningEnabled = false;
        string inputFile;
        string policyDefinitionFile;
        string outputFile;

        //Define and parse the program options
        namespace po = boost::program_options;
        po::options_description desc("Options");
        desc.add_options()
            ("help", "Print help message")
            ("input-file", po::value<string>(), "input file")
            ("policy-definition-file", po::value<string>(), "JSON file that represents the policy definition")
            ("output-file,o", po::value<string>(), "Writes the compressed policy to this file (example: $(basename <input-file>).ccppl)")
            ("trace-parsing,p", "Print the parser trace (for debugging)")
            ("trace-scanning,s", "Print the scanning trace (for debugging)");

        //all positional options should be translated into "input-file" options
        po::positional_options_description p;
        p.add("input-file", -1);

        po::variables_map vm;
        try  {
            po::store(po::command_line_parser(argc, argv).
                              options(desc).positional(p).run(), vm);

            //--help option
            if(vm.count("help"))  {
                std::cout << "Compact Privacy Policy Language Generator" << std::endl
                    << desc << std::endl;
                return SUCCESS;
            }

            //trace parsing options
            if(vm.count("trace-parsing"))  {
                traceParsingEnabled = true;
            }

            //trace scanning options
            if(vm.count("trace-scanning"))  {
                traceScanningEnabled = true;
            }

            //input-file
            if(vm.count("input-file"))  {
                inputFile = vm["input-file"].as<string>();
            } else  {
                std::cerr << "No input file!" << std::endl;
                return ERROR_IN_COMMAND_LINE;
            }

            //policy-definition-file
            if(vm.count("policy-definition-file"))  {
                policyDefinitionFile = vm["policy-definition-file"].as<string>();
            } else  {
                std::cerr << "No policy definition file parameter!" << std::endl;
                return ERROR_IN_COMMAND_LINE;
            }

            //output-file
            if(vm.count("output-file"))  {
                outputFile = vm["output-file"].as<string>();
            } else  {
                std::cerr << "No output file!" << std::endl;
                return ERROR_IN_COMMAND_LINE;
            }

            po::notify(vm); // throws on error
        }
        catch(po::error& e)  {
            std::cerr << "ERROR: " << e.what() << std::endl << std::endl;
            std::cerr << desc << std::endl;
            return ERROR_IN_COMMAND_LINE;
        }

        //application code

        //load the necessary files before measuring to compensate file caching
        //get the policy definition from a json file
        string policyDefinitionJsonString = getFile(policyDefinitionFile);

        #if EVALUATION_OUTPUT
            performance_start_benchmark();
        #endif

        //contains all information of the policy definition
        PolicyDefinition policyDefinition(policyDefinitionJsonString);
        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Policy Definition time");
            performance_start_benchmark();
        #endif

        //generate the function handler that evaluates the user defined function
        FunctionHandler functionHandler(policyDefinition);

        Binary genBinary(NULL);
        VariableSet variableSet(policyDefinition, functionHandler, genBinary);
        RelationSet relationSet(policyDefinition, variableSet, genBinary);
        PolicyStack policyStack(relationSet, genBinary, 0);

        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Policy Class init time");
            performance_start_benchmark();
        #endif

        EquationDriver eqDriver;

        eqDriver.trace_parsing = traceParsingEnabled;
        eqDriver.trace_scanning = traceScanningEnabled;

        eqDriver.parse(inputFile);

        //show an error if the policy is empty
        if(eqDriver.ast == NULL)  {
            cerr << "Policy is empty" << endl;
            return POLICY_ERROR;
        }

        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Policy Parsing time");
            performance_start_benchmark();
        #endif

        //preprocess the AST
        // - check that the types inside the AST are used correctly
        // - sort the children s.t. a correct PN is created by a DFS
        AstPreprocessorVisitor preprocessor = AstPreprocessorVisitor(policyDefinition);
        preprocessor.visit(*(eqDriver.ast));

        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen AST Preprocessing time");
            performance_start_benchmark();
        #endif

        //print the AST
        #if DEBUG_AST_PRINT
            AstPrintVisitor printVisitor = AstPrintVisitor();
            printVisitor.visit(*(eqDriver.ast));
        #endif

        //generate the policy header
        PolicyHeader policyHeader(genBinary, policyDefinition.getVersion());

        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Policy Header time");
            performance_start_benchmark();
        #endif

        //compress the AST as RPN stack with equation set
        AstPolicyCompressorVisitor policyCompressor = AstPolicyCompressorVisitor(policyDefinition, policyStack, relationSet);
        #if DEBUG_POLICY_GENERATION
            cout << "Compressing the policy: " << endl;
        #endif
        policyCompressor.visit(*(eqDriver.ast));
        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Policy Compressor time");
            performance_start_benchmark();
        #endif
        #if DEBUG_POLICY_GENERATION
            cout << endl;
        #endif

        //add the policy parts in the right order to the binary
        policyHeader.store();
        policyStack.store();
        relationSet.store();
        variableSet.store();
        #if EVALUATION_OUTPUT
            performance_stop_benchmark("Gen Binary Creation time");
        #endif

        //DEBUG output for PolicyStack sizes
        #if EVALUATION_OUTPUT
            policyHeader.printSize();
            policyStack.printSize();
            relationSet.printSize();
            variableSet.printSize();
        #endif

        #if DEBUG_POLICY_GENERATION
            //debug output for the complete policy parts
            cout << "PolicyHeader: " << policyHeader.legend << endl << policyHeader << endl;
            cout << "PolicyStack: "  <<  policyStack.legend << endl << policyStack << endl;
            cout << "RelationSet: "  <<  relationSet.legend << relationSet << endl;
            cout << "VariableSet: "  <<  variableSet.legend << variableSet << endl;
        #endif

        #if DEBUG_POLICY_GENERATION
            cout << "Policy: "<< endl;
            genBinary.print();
        #endif
        #if EVALUATION_OUTPUT
            cout << "Policy length: " << genBinary.size() << endl;
        #endif

        genBinary.write(outputFile);
    }
    catch(char const *msg)  {
        std::cerr << "Policy Error: " << msg << std::endl;
        return POLICY_ERROR;
    }
    catch(std::exception& e)  {
        std::cerr << "Unhandled Exception reached the top of main: "
            << e.what() << ", application will now exit" << std::endl;
        return ERROR_UNHANDLED_EXCEPTION;

    }

    return SUCCESS;
}

max_stars_repo_name: SSICLOPS/cppl

max_issues_repo_name: SSICLOPS/cppl

max_forks_repo_name: SSICLOPS/cppl


text:
/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/

#include "hphp/util/extern-worker.h"

#include "hphp/util/assertions.h"
#include "hphp/util/current-executable.h"
#include "hphp/util/hash-map.h"
#include "hphp/util/logger.h"
#include "hphp/util/match.h"
#include "hphp/util/trace.h"

#include <folly/FileUtil.h>
#include <folly/Subprocess.h>
#include <folly/gen/Base.h>

#include <boost/filesystem.hpp>

#include <mutex>

namespace HPHP::extern_worker {

//////////////////////////////////////////////////////////////////////

using namespace detail;
using namespace folly::gen;

//////////////////////////////////////////////////////////////////////

const char* const s_option = "--extern-worker";

std::atomic<uint64_t> RequestId::s_next{0};
std::atomic<uint64_t> RequestId::s_active{0};

const std::array<OutputType, 1> Client::s_valOutputType{OutputType::Val};
const std::array<OutputType, 1> Client::s_vecOutputType{OutputType::Vec};
const std::array<OutputType, 1> Client::s_optOutputType{OutputType::Opt};

ImplHook g_impl_hook{nullptr};

//////////////////////////////////////////////////////////////////////

namespace {

//////////////////////////////////////////////////////////////////////

TRACE_SET_MOD(extern_worker);

//////////////////////////////////////////////////////////////////////

struct Registry {
  hphp_fast_string_map<JobBase*> registry;
  std::mutex lock;
};

Registry& registry() {
  static Registry registry;
  return registry;
}

//////////////////////////////////////////////////////////////////////

// folly::writeFile expects a container-like input, so this makes a
// ptr/length pair behave like one (without copying).
struct Adaptor {
  size_t size() const { return m_size; }
  bool empty() const { return !size(); }
  const char& operator[](size_t idx) const { return m_ptr[idx]; }
  const char* m_ptr;
  size_t m_size;
};

//////////////////////////////////////////////////////////////////////

}

//////////////////////////////////////////////////////////////////////

namespace detail {

//////////////////////////////////////////////////////////////////////

// Wrappers around the folly functions with error handling

std::string readFile(const folly::fs::path& path) {
  std::string s;
  if (!folly::readFile(path.c_str(), s)) {
    throw Error{
      folly::sformat(
        "Unable to read input from {} [{}]",
        path.c_str(), folly::errnoStr(errno)
      )
    };
  }
  return s;
}

void writeFile(const folly::fs::path& path,
               const char* ptr, size_t size) {
  if (!folly::writeFile(Adaptor{ptr, size}, path.c_str())) {
    throw Error{
      folly::sformat(
        "Unable to write output to {} [{}]",
        path.c_str(), folly::errnoStr(errno)
      )
    };
  }
}

//////////////////////////////////////////////////////////////////////

JobBase::JobBase(const std::string& name)
  : m_name{name}
{
  FTRACE(4, "registering remote worker command \"{}\"\n", m_name);
  auto& r = registry();
  std::lock_guard<std::mutex> _{r.lock};
  auto const insert = r.registry.emplace(m_name, this);
  always_assert(insert.second);
}

//////////////////////////////////////////////////////////////////////

}

//////////////////////////////////////////////////////////////////////

/*
 * A note on conventions: A worker expects to be invoked with the name
 * of the command, followed by 3 paths to directories. The directories
 * represent the config inputs (for init()), the inputs (for multiple
 * runs()), and the last is the output directory, which will be
 * written to.
 *
 * All three directories use the same format for representing
 * data:
 *
 * - The first level contains numbered directories (from 0 to N). Each
 * numbered directory represents a set of inputs/outputs. For outputs,
 * the number corresponds to the matching number of the inputs. The
 * config directory does not have this level, as there's only ever one
 * of them.
 *
 * - The next level contains numbered files or directories (from 0 to
 * N). Each file/directory specifies a particular input. The exact
 * number depends on the job being executed (both sides need to
 * agree). The format of the file or directory depends on that
 * input/output's type.
 *
 * - If the input/output is a "normal" type, it will be a file
 * containing the serialized data (using BlobEncoder) for that
 * input/output. Strings are a special case. If the type is a
 * std::string, it will not be serialized and the string will be
 * stored directly (this makes it easier to represent files as their
 * contents).
 *
 * - If the input/output is an Opt<T>, it will be represented like a
 * "normal" type of type T, except it is not required to be
 * present. If the file is not present (so a gap in the numbering), it
 * is assumed to be std::nullopt.
 *
 * - If the input/output is a Variadic<T>, it will be a
 * directory. Inside of that directory will be numbered files from
 * 0-N, one for each element of the vector. The vector elements will
 * be encoded as "normal" types.
 *
 * NB: Marker types cannot nest, so there aren't any possible deeper
 * levels than this.
 *
 * All implementations must follow this layout when setting up
 * execution. The actual execution on the worker side is completely
 * agnostic to the implementation.
 */

int main(int argc, char** argv) {
  try {
    always_assert(argc > 1);
    always_assert(!strcmp(argv[1], s_option));

    if (argc != 6) {
      std::cerr << "Usage: "
                << argv[0]
                << " " << s_option << " <command name>"
                << " <config dir>"
                << " <output dir>"
                << " <input dir>"
                << std::endl;
      return EXIT_FAILURE;
    }

    std::string name{argv[2]};
    folly::fs::path configPath{argv[3]};
    folly::fs::path outputPath{argv[4]};
    folly::fs::path inputPath{argv[5]};

    FTRACE(2, "extern worker run(\"{}\", {}, {}, {})\n",
           name, configPath.native(), outputPath.native(),
           inputPath.native());

    // Lookup the registered job for the requested name.
    auto const worker = [&] {
      auto& r = registry();
      std::lock_guard<std::mutex> _{r.lock};
      auto const it = r.registry.find(name);
      if (it == r.registry.end()) {
        throw Error{folly::sformat("No command named `{}` registered", name)};
      }
      return it->second;
    }();

    // We insist on a clean output directory.
    if (!folly::fs::create_directory(outputPath)) {
      throw Error{
        folly::sformat("Output directory {} already exists", outputPath.native())
      };
    }

    // First do any global initialization.
    time("init", [&] { worker->init(configPath); });
    time("run-all", [&] {
        // Then execute run() for each given set of inputs.
      for (size_t i = 0;; ++i) {
        auto const thisInput = inputPath / folly::to<std::string>(i);
        if (!folly::fs::exists(thisInput)) break;

        auto const thisOutput = outputPath / folly::to<std::string>(i);
        FTRACE(4, "executing #{} ({} -> {})\n",
               i, thisInput.native(), thisOutput.native());
        folly::fs::create_directory(thisOutput, outputPath);

        time(
          [&]{ return folly::sformat("run {}", i); },
          [&] { worker->run(thisInput, thisOutput); }
        );
      }
    });
    // Do any cleanup
    time("fini", [&] { worker->fini(); });

    return 0;
  } catch (const std::exception& exn) {
    std::cerr << "Error: " << exn.what() << std::endl;
    return EXIT_FAILURE;
  }
}

//////////////////////////////////////////////////////////////////////

namespace {

//////////////////////////////////////////////////////////////////////

/*
 * The builtin implementation which uses fork+exec (and stores data on
 * disk). It is always available, reasonably efficient, and assumed to
 * be "reliable" (IE, never needs a fallback).
 *
 * All data is stored under the "root" (which is the working-dir
 * specified in Options). The two sub-directories are "blobs" and
 * "execs".
 *
 * Data is stored under blobs/, using files with numerically
 * increasing names. Stored files are not stored at all (since they're
 * already on disk). The RefIds are just the path to the file (either
 * under blobs/ or where-ever the stored file was).
 *
 * Executions are stored under execs/. Each entry is a sub-directory
 * with numerically increasing names. Within each sub-directory is the
 * directory layout that the worker expects (see comment above int
 * main()). The data is not actually copied into the input
 * directories. Instead symlinks are created using the path in the
 * RefId. Outputs are not copied either, they just "live" in their
 * output directory.
 */
struct SubprocessImpl : public Client::Impl {
  explicit SubprocessImpl(const Options&);
  ~SubprocessImpl() override;

  bool isSubprocess() const override { return true; }
  bool isDisabled() const override { return false; }

  coro::Task<BlobVec> load(const RequestId&, IdVec) override;
  coro::Task<IdVec> store(const RequestId&, PathVec, BlobVec,
                          size_t*, size_t*) override;
  coro::Task<std::vector<RefValVec>>
  exec(const RequestId&,
       const std::string&,
       RefValVec,
       std::vector<RefValVec>,
       const folly::Range<const OutputType*>&,
       bool*) override;

private:
  folly::fs::path newBlob();
  folly::fs::path newExec();
  static folly::fs::path newRoot(const Options&);

  void doSubprocess(const RequestId&,
                    const std::string&,
                    const folly::fs::path&,
                    const folly::fs::path&,
                    const folly::fs::path&,
                    const folly::fs::path&);

  Options m_options;

  folly::fs::path m_root;
  // SubprocessImpl doesn't need the m_size portion of RefId. So we
  // store an unique integer in it to help verify that the RefId came
  // from this implementation instance.
  size_t m_marker;

  std::atomic<size_t> m_nextBlob;
  std::atomic<size_t> m_nextExec;
};

SubprocessImpl::SubprocessImpl(const Options& options)
  : Impl{"subprocess"}
  , m_options{options}
  , m_root{newRoot(m_options)}
    // Cheap way of generating semi-unique integer:
  , m_marker{std::hash<std::string>{}(m_root.native())}
  , m_nextBlob{0}
  , m_nextExec{0}
{
  FTRACE(2, "Using subprocess extern-worker impl with root at {}\n",
         m_root.native());
  Logger::FInfo("Using subprocess extern-worker at {}", m_root.native());

  folly::fs::create_directory(m_root / "blobs");
  folly::fs::create_directory(m_root / "execs");
}

SubprocessImpl::~SubprocessImpl() {
  // Deleting everything under blobs/ and execs/ can take quite a
  // while...
  if (m_options.m_cleanup) {
    Logger::FInfo(
      "Cleaning up subprocess extern-worker at {}...",
      m_root.native()
    );
    auto const before = std::chrono::steady_clock::now();
    auto const removed = time(
      "subprocess cleanup",
      [&] {
        std::error_code ec; // Suppress exceptions
        return folly::fs::remove_all(m_root, ec);
      }
    );
    auto const elapsed = std::chrono::duration_cast<std::chrono::duration<double>>(
      std::chrono::steady_clock::now() - before
    ).count();
    FTRACE(2, "removed {} files\n", removed);
    Logger::FInfo(
      "Done cleaning up subprocess extern-worker. "
      "({} files removed) (took {})",
      removed, folly::prettyPrint(elapsed, folly::PRETTY_TIME_HMS, false)
    );
  }
}

// Create paths for a new blob or a new execution. I lied a little bit
// above in the description of the directory layout. It's true that
// blobs and execs use numerically increasing names, but I shard them
// into directories of 10000 each, to keep their sizes reasonable.
folly::fs::path SubprocessImpl::newBlob() {
  auto const id = m_nextBlob++;
  auto const blobRoot = m_root / "blobs";
  auto const mid = blobRoot / folly::sformat("{:05}", id / 10000);
  folly::fs::create_directory(mid, blobRoot);
  return mid / folly::sformat("{:04}", id % 10000);
}

folly::fs::path SubprocessImpl::newExec() {
  auto const id = m_nextExec++;
  auto const execRoot = m_root / "execs";
  auto const mid = execRoot / folly::sformat("{:05}", id / 10000);
  folly::fs::create_directory(mid, execRoot);
  auto const full = mid / folly::sformat("{:04}", id % 10000);
  folly::fs::create_directory(full, mid);
  return full;
}

// Ensure we always have an unique root under the working directory.
folly::fs::path SubprocessImpl::newRoot(const Options& opts) {
  auto const base = opts.m_workingDir / "hphp-extern-worker";
  folly::fs::create_directories(base);
  auto const full = base /  boost::filesystem::unique_path(
    "%%%%-%%%%-%%%%-%%%%-%%%%-%%%%"
  ).native();
  folly::fs::create_directory(full, base);
  return folly::fs::canonical(full);
}

coro::Task<BlobVec> SubprocessImpl::load(const RequestId& requestId,
                                         IdVec ids) {
  // Read every file corresponding to the id and return their
  // contents.
  auto out = from(ids)
    | mapped([&] (const RefId& id) {
        assertx(id.m_size == m_marker);
        FTRACE(4, "{} reading blob from {}\n",
               requestId.tracePrefix(), id.m_id);
        auto blob = readFile(id.m_id);
        FTRACE(4, "{} blob is {} bytes\n",
               requestId.tracePrefix(), blob.size());
        return blob;
      })
    | as<std::vector>();
  HPHP_CORO_MOVE_RETURN(out);
}

coro::Task<IdVec> SubprocessImpl::store(const RequestId& requestId,
                                        PathVec paths,
                                        BlobVec blobs,
                                        size_t* read,
                                        size_t* uploaded) {
  // SubprocessImpl always "reads" and "uploads" the data (there's no
  // caching of any kind).
  if (read) *read = paths.size();
  if (uploaded) *uploaded = paths.size() + blobs.size();
  // Create RefIds from the given paths, then write the blobs to disk,
  // then use their paths.
  auto out =
    ((from(paths)
      | mapped([&] (const folly::fs::path& p) {
          return RefId{folly::fs::canonical(p).native(), m_marker};
        })
    ) +
    (from(blobs)
     | mapped([&] (const std::string& b) {
         auto const path = newBlob();
         FTRACE(4, "{} writing size {} blob to {}\n",
                requestId.tracePrefix(), b.size(), path.native());
         writeFile(path, b.data(), b.size());
         return RefId{folly::fs::canonical(path).native(), m_marker};
       })
    ))
    | as<std::vector>();
  HPHP_CORO_MOVE_RETURN(out);
}

coro::Task<std::vector<RefValVec>>
SubprocessImpl::exec(const RequestId& requestId,
                     const std::string& command,
                     RefValVec config,
                     std::vector<RefValVec> inputs,
                     const folly::Range<const OutputType*>& output,
                     bool* cached = nullptr) {
  auto const execPath = newExec();
  auto const configPath = execPath / "config";
  auto const inputsPath = execPath / "input";
  auto const outputsPath = execPath / "output";

  FTRACE(4, "{} executing \"{}\" inside {} ({} runs)\n",
         requestId.tracePrefix(), command,
         execPath.native(), inputs.size());

  // SubprocessImpl never caches
  if (cached) *cached = false;

  // Set up the directory structure that the worker expects:

  auto const symlink = [&] (const RefId& id,
                            const folly::fs::path& path) {
    assertx(id.m_size == m_marker);
    folly::fs::create_symlink(id.m_id, path);
    FTRACE(4, "{} symlinked {} to {}\n",
           requestId.tracePrefix(), id.m_id, path.native());
  };

  auto const prepare = [&] (const RefValVec& params,
                            const folly::fs::path& parent) {
    for (size_t paramIdx = 0; paramIdx < params.size(); ++paramIdx) {
      auto const& p = params[paramIdx];
      auto const path = parent / folly::to<std::string>(paramIdx);
      match<void>(
        p,
        [&] (const RefId& id) { symlink(id, path); },
        [&] (const Optional<RefId>& id) {
          if (!id.has_value()) return;
          symlink(*id, path);
        },
        [&] (const IdVec& ids) {
          folly::fs::create_directory(path, parent);
          for (size_t vecIdx = 0; vecIdx < ids.size(); ++vecIdx) {
            symlink(ids[vecIdx], path / folly::to<std::string>(vecIdx));
          }
        }
      );
    }
  };

  folly::fs::create_directory(configPath, execPath);
  prepare(config, configPath);

  // Each set of inputs should always have the same size.
  if (debug && !inputs.empty()) {
    auto const size = inputs[0].size();
    for (size_t i = 1; i < inputs.size(); ++i) {
      always_assert(inputs[i].size() == size);
    }
  }

  folly::fs::create_directory(inputsPath, execPath);
  for (size_t i = 0; i < inputs.size(); ++i) {
    auto const path = inputsPath / folly::to<std::string>(i);
    folly::fs::create_directory(path, inputsPath);
    prepare(inputs[i], path);
  }

  // Do the actual fork+exec.
  doSubprocess(
    requestId,
    command,
    execPath,
    configPath,
    inputsPath,
    outputsPath
  );

  // Make RefIds corresponding to the outputs.

  auto const makeOutput =
    [&] (OutputType type, folly::fs::path path) -> RefVal {
    switch (type) {
      case OutputType::Val:
        assertx(folly::fs::exists(path));
        return RefId{std::move(path), m_marker};
      case OutputType::Opt:
        if (!folly::fs::exists(path)) return std::nullopt;
        return make_optional(RefId{std::move(path), m_marker});
      case OutputType::Vec: {
        assertx(folly::fs::exists(path));
        IdVec vec;
        size_t i = 0;
        while (true) {
          auto valPath = path / folly::to<std::string>(i);
          if (!folly::fs::exists(valPath)) break;
          vec.emplace_back(valPath.native(), m_marker);
          ++i;
        }
        return vec;
      }
    }
    always_assert(false);
  };

  auto const makeOutputs = [&] (const folly::fs::path& path) {
    RefValVec vec;
    vec.reserve(output.size());
    for (size_t i = 0; i < output.size(); ++i) {
      vec.emplace_back(
        makeOutput(
          output[i],
          path / folly::to<std::string>(i)
        )
      );
    }
    return vec;
  };

  std::vector<RefValVec> out;
  out.reserve(inputs.size());
  for (size_t i = 0; i < inputs.size(); ++i) {
    out.emplace_back(makeOutputs(outputsPath / folly::to<std::string>(i)));
  }
  HPHP_CORO_MOVE_RETURN(out);
}

void SubprocessImpl::doSubprocess(const RequestId& requestId,
                                  const std::string& command,
                                  const folly::fs::path& execPath,
                                  const folly::fs::path& configPath,
                                  const folly::fs::path& inputPath,
                                  const folly::fs::path& outputPath) {
  std::vector<std::string> args{
    current_executable_path(),
    s_option,
    command,
    configPath,
    outputPath,
    inputPath
  };

  // Propagate the TRACE option in the environment. We'll copy the
  // trace output into this process' trace output.
  std::vector<std::string> env;
  Optional<folly::fs::path> traceFile;
  if (auto const trace = getenv("TRACE")) {
    traceFile = execPath / "trace.log";
    env.emplace_back(folly::sformat("TRACE={}", trace));
    env.emplace_back(folly::sformat("HPHP_TRACE_FILE={}", traceFile->c_str()));
  }

  FTRACE(4, "{} executing subprocess\n",
         requestId.tracePrefix());

  auto const DEBUG_ONLY before = std::chrono::steady_clock::now();

  // Do the actual fork+exec.
  folly::Subprocess subprocess{
    args,
    folly::Subprocess::Options{}
      .stdinFd(folly::Subprocess::DEV_NULL)
      .pipeStdout()
      .pipeStderr()
      .closeOtherFds(),
    nullptr,
    &env
  };

  auto const [_, stderr] = subprocess.communicate();
  auto const returnCode = subprocess.wait();

  FTRACE(
    4,
    "{} subprocess finished (took {}). Return code: {}, Stderr: {}\n",
    requestId.tracePrefix(),
    prettyPrint(
      std::chrono::duration_cast<std::chrono::duration<double>>(
        std::chrono::steady_clock::now() - before
      ).count(),
      folly::PRETTY_TIME,
      false
    ),
    returnCode.str(),
    stderr
  );

  // Do this before checking the return code. If the process failed,
  // we want to capture anything it logged before throwing.
  if (traceFile && folly::fs::exists(*traceFile)) {
    auto const contents = readFile(*traceFile);
    if (!contents.empty()) {
      Trace::ftraceRelease(
        "vvvvvvvvvvvvvvvvvv remote-exec ({} \"{}\" {}) vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
        "{}"
        "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
        requestId.toString(),
        command,
        execPath.native(),
        contents
      );
    }
  }

  if (!returnCode.exited() || returnCode.exitStatus() != 0) {
    throw Error{
      folly::sformat(
        "Execution of `{}` failed: {}\nstderr:\n{}",
        command,
        returnCode.str(),
        stderr
      )
    };
  }
}

//////////////////////////////////////////////////////////////////////

}

//////////////////////////////////////////////////////////////////////

Client::Client(folly::Executor::KeepAlive<> executor,
               const Options& options)
  : m_options{options}
  , m_forceFallback{false}
{
  Timer _{"create impl"};
  // Look up which implementation to use. If a hook has been
  // registered, and we're allowed to use it according to the Options,
  // try to use it.
  if (g_impl_hook &&
      m_options.m_useSubprocess != Options::UseSubprocess::Always) {
    m_impl = g_impl_hook(m_options, executor);
  }
  // The hook can return nullptr even if registered. In each case, we
  // have no special implementation to use.
  if (!m_impl) {
    // Use the subprocess implementation (which is always available),
    // unless the Options specifies we shouldn't. If not, it's a fatal
    // error.
    if (m_options.m_useSubprocess == Options::UseSubprocess::Never) {
      throw Error{"No non-subprocess impl available"};
    }
    m_impl = std::make_unique<SubprocessImpl>(m_options);
  }
  FTRACE(2, "created \"{}\" impl\n", m_impl->name());
}

Client::~Client() {
  Timer _{[&] { return folly::sformat("destroy impl {}", m_impl->name()); }};
  m_impl.reset();
  m_fallbackImpl.reset();
}

std::unique_ptr<Client::Impl> Client::makeFallbackImpl() const {
  // This will be called once from within LockFreeLazy, so we only
  // emit this warning once.
  Logger::Warning(
    "Certain operations will use local fallback from this "
    "point on and may run slower."
  );
  return std::make_unique<SubprocessImpl>(m_options);
}

coro::Task<Ref<std::string>> Client::storeFile(folly::fs::path path,
                                               bool* read,
                                               bool* uploaded) {
  RequestId requestId{"store file"};

  FTRACE(2, "{} storing {}\n", requestId.tracePrefix(), path.native());

  size_t readCount;
  size_t uploadedCount;
  auto wasFallback = false;
  auto ids = HPHP_CORO_AWAIT(tryWithFallback<IdVec>(
    [&] (Impl& i, bool) {
      return i.store(requestId, PathVec{path}, {}, &readCount, &uploadedCount);
    },
    wasFallback
  ));
  assertx(ids.size() == 1);
  assertx(readCount <= 1);
  assertx(uploadedCount <= 1);
  if (read) *read = (readCount > 0);
  if (uploaded) *uploaded = (uploadedCount > 0);

  Ref<std::string> ref{std::move(ids[0]), wasFallback};
  HPHP_CORO_MOVE_RETURN(ref);
}

coro::Task<std::vector<Ref<std::string>>>
Client::storeFile(std::vector<folly::fs::path> paths,
                  size_t* read,
                  size_t* uploaded) {
  RequestId requestId{"store files"};

  FTRACE(2, "{} storing {} files\n", requestId.tracePrefix(), paths.size());
  ONTRACE(4, [&] {
    for (auto const& p : paths) {
      FTRACE(4, "{} storing {}\n", requestId.tracePrefix(), p.native());
    }
  }());

  auto const DEBUG_ONLY size = paths.size();
  auto wasFallback = false;
  auto ids = HPHP_CORO_AWAIT(tryWithFallback<IdVec>(
    [&] (Impl& i, bool) {
      return i.store(requestId, paths, {}, read, uploaded);
    },
    wasFallback
  ));
  assertx(ids.size() == size);
  assertx(!read || *read <= ids.size());
  assertx(!uploaded || *uploaded <= ids.size());

  auto out = from(ids)
    | move
    | mapped([&] (auto&& id) {
        return Ref<std::string>{std::move(id), wasFallback};
      })
    | as<std::vector>();
  HPHP_CORO_MOVE_RETURN(out);
}

//////////////////////////////////////////////////////////////////////

}

ext: cpp

max_stars_repo_name: Atry/hhvm

max_issues_repo_name: Atry/hhvm

max_forks_repo_name: Atry/hhvm


text:
/*****************************************************************************
*
* Functions for calculating the gap
*
* Copyright (C) 2018 by Hidemaro Suwa
* e-mail:suwamaro@phys.s.u-tokyo.ac.jp
*
*****************************************************************************/

#include "calc_gap.h"
#include "rpa_util.h"
#include <armadillo>

cx_double calc_intensity_cubic(int L, hoppings const& ts, double mu, double U, double delta, double qx, double qy, double qz, cx_double omega, bool zz){
  double k1 = 2. * M_PI / (double)L;

  cx_double A = 0, B = 0, C = 0, D = 0;
  /* Summing up at all the wavevectors */
  for(int x=-L/2; x < L/2; x++){
    double kx = k1 * x;
    for(int y=-L/2; y < L/2; y++){
      double ky = k1 * y;
      for(int z=-L/2; z < L/2; z++){
	double kz = k1 * z;

	add_to_sus_mat2( ts, mu, A, B, C, D, qx, qy, qz, kx, ky, kz, delta, omega, zz );
	// double e_free = energy_free_electron( t, mu, kx, ky, kz );
	// double e_eps = 1e-12;
	// if ( e_free < e_eps ) {
	//   /* Summing up over all k inside the Brillouin zone. */
	//   double diff_x = wave_vector_in_BZ( kx - qx );
	//   double diff_y = wave_vector_in_BZ( ky - qy );
	//   double diff_z = wave_vector_in_BZ( kz - qz );
	//   double e_free2 = energy_free_electron( t, mu, diff_x, diff_y, diff_z );
	//   add_to_sus_mat( A, B, D, e_free, e_free2, delta, omega );
	// }
      }
    }
  }

  int n_sites = L * L * L;
  A *= 2. / (double)n_sites;
  B *= 2. / (double)n_sites;
  C *= 2. / (double)n_sites;
  D *= 2. / (double)n_sites;

  /* RPA */
  arma::cx_mat chi0_mat(2,2);
  chi0_mat(0,0) = A;   // (A, A) correlation
  chi0_mat(0,1) = B;   // (A, B)
  chi0_mat(1,0) = C;   // (B, A)
  chi0_mat(1,1) = D;   // (B, B)

  /* Transverse = < \sigma^- \sigma^+ >; Longitudinal (zz) = < \sigma^z \sigma^z > */
  /* Note that 2 < \sigma^- \sigma^+ > = < \sigma^z \sigma^z > (U -> 0 for the SU(2) case) */
  double factor_channel = 1.0;
  if ( zz ) { factor_channel = 0.5; }

  arma::cx_mat denom = arma::eye<arma::cx_mat>(2,2) - factor_channel * U * chi0_mat;
  arma::cx_mat chi_mat = chi0_mat * arma::inv(denom);

  // sigma-to-spin factor
  double factor_operator = 0.5;
  cx_double chi = factor_operator * factor_operator * ( chi_mat(0,0) - chi_mat(1,0) - chi_mat(0,1) + chi_mat(1,1) );

  return chi;

  // A *= 2. / (double)n_sites;
  // B *= 2. / (double)n_sites;
  // D *= 2. / (double)n_sites;

  // /* RPA */
  // arma::cx_mat chi0_mat(2,2);
  // chi0_mat(0,0) = A;   // (A, A) correlation
  // chi0_mat(0,1) = B;   // (A, B)
  // chi0_mat(1,0) = B;   // (B, A)
  // chi0_mat(1,1) = D;   // (B, B)
  // arma::cx_mat denom = arma::eye<arma::cx_mat>(2,2) - U * chi0_mat;
  // arma::cx_mat chi_mat = chi0_mat * arma::inv(denom);

  // // Double counting from A and B
  // double factor_sublattice = 0.5;
  // cx_double chi = factor_sublattice * factor_sublattice * ( chi_mat(0,0) - chi_mat(1,0) - chi_mat(0,1) + chi_mat(1,1) );

  // // // for check
  // // std::cout << qx << "  " << qy << "  " << omega << "  " << A << "  " << B << "  " << D << "  " << chi_mat(0,0) << "  " << chi_mat(0,1) << "  " << chi_mat(1,0) << "  " << chi_mat(1,1) << "  " << chi << std::endl;

  // return chi;
}

max_stars_repo_name: suwamaro/rpa

max_issues_repo_name: suwamaro/rpa

max_forks_repo_name: suwamaro/rpa


text:
// Copyright (c) 2020 Bitcoin Association
// Distributed under the Open BSV software license, see the accompanying file
// LICENSE.

#include "mining/journal_change_set.h"
#include "validation.h"
#include "mempooltxdb.h"

#include "mempool_test_access.h"

#include "test/test_bitcoin.h"

#include <boost/test/unit_test.hpp>

#include <vector>

namespace {
    mining::CJournalChangeSetPtr nullChangeSet{nullptr};

    // large enough count to control integer rounding errors in fractions
    constexpr int N_PRIMARY = 50;
    // fixed size transactions so we can correlate sizes and counts
    // large enough transactions so we are not dominated by index ram usage
    constexpr int TX_SIZE = 1000;
    const CFeeRate A_PRIMARY = CFeeRate(Amount(6000));
    const CFeeRate A_BLOCK_MIN_FEE = CFeeRate(Amount(1000));
    const CFeeRate A_SECONDARY = CFeeRate(Amount(100));

    // test representation of mempool entry
    struct Entry {
        TxId txId;
        bool forPrimary;
        size_t size;
        Entry(TxId txId, bool forPrimary, size_t size)
        : txId(txId), forPrimary(forPrimary), size(size) {}
    };

    using Predicate = std::function<bool(CTxMemPoolTestAccess&, const Entry&)>;

    // a collection of entries added to mempool
	struct Entries {
        CTxMemPool &pool;
        std::vector<Entry> entries;
        Entries(CTxMemPool& pool) : pool(pool) {}
        // return entries that satisfy the predicate by consulting the actual mempool entries
        Entries that(Predicate predicate) const {
            return filter(with(predicate));
        }
        // return entries that were submitted to primary mempool
        Entries for_primary() const {
            return filter([](const Entry& entry){return entry.forPrimary;});
        }
        // return entries that were submitted to secondary mempool
        Entries for_secondary() const {
            return filter([](const Entry& entry){return !entry.forPrimary;});
        }
        // number of entries
        size_t count() const { return entries.size(); }
        // number of bytes consumed by transactions of entries
        size_t size() const {
            return std::accumulate(entries.begin(), entries.end(), 0,
                                   [](size_t size, const Entry& entry) {
                return size + entry.size;
                });
            }
	private:
        using Filter = std::function<bool(const Entry&)>;
        Entries filter(Filter cond) const {
            Entries ret(pool);
            std::copy_if(entries.cbegin(), entries.cend(), std::back_inserter(ret.entries), cond);
            return ret;
        }
        Filter with(Predicate predicate) const {
            return [this, predicate](const Entry& entry) {
                CTxMemPoolTestAccess testPoolAccess(pool);
                return predicate(testPoolAccess, entry);
            };
        }
    };

    // Predicates
    bool in_pool(CTxMemPoolTestAccess& pool, const Entry& entry) {
        auto it = pool.mapTx().find(entry.txId);
        return it != pool.mapTx().end();
    }

    bool in_memory(CTxMemPoolTestAccess& pool, const Entry& entry) {
        auto it = pool.mapTx().find(entry.txId);
        return (it != pool.mapTx().end()) && it->IsInMemory();
    }

    bool on_disk(CTxMemPoolTestAccess& pool, const Entry& entry) {
        auto it = pool.mapTx().find(entry.txId);
        return (it != pool.mapTx().end()) && !it->IsInMemory();
    }

    bool in_primary(CTxMemPoolTestAccess& pool, const Entry& entry) {
        auto it = pool.mapTx().find(entry.txId);
        return (it != pool.mapTx().end()) && it->IsInPrimaryMempool();
    }

    bool in_secondary(CTxMemPoolTestAccess& pool, const Entry& entry) {
        auto it = pool.mapTx().find(entry.txId);
        return (it != pool.mapTx().end()) && !it->IsInPrimaryMempool();
    }

    Predicate are(Predicate predicate) {
        return predicate;
    }

	Predicate are_not(Predicate predicate) {
        return [predicate](CTxMemPoolTestAccess& pool, const Entry& entry) {
            return !predicate(pool, entry);
        };
    }

	struct Demand {
        int howMany;
        CFeeRate fee;
        Demand(int howMany, CFeeRate fee) : howMany(howMany), fee(fee) {}
    };

    std::vector<CTxMemPoolEntry> GetABunchOfEntries(Demand demand)
    {
        static int unique = 0;
        std::vector<uint8_t> fluff;
        fluff.resize(TX_SIZE - 71, 42);    // subtract base transaction size
        TestMemPoolEntryHelper entry;
        std::vector<CTxMemPoolEntry> entries;
        for (int i = 0; i < demand.howMany; i++) {
            CMutableTransaction mtx;
            mtx.vin.resize(1);
            mtx.vin[0].prevout = COutPoint(uint256(), unique++);
            mtx.vin[0].scriptSig = CScript() << fluff << OP_DROP << 11 << OP_DROP << OP_11;
            mtx.vout.resize(1);
            mtx.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL;
            mtx.vout[0].nValue = Amount(33000LL);
            // add a bounded oscillating offset to the fee to fight eviction order assumptions
            Amount fee = (demand.fee.GetFee(TX_SIZE)
                          + ((50 * Amount(i%2 ? i : -i)) / demand.howMany));
            entries.emplace_back(entry.Fee(fee).Time(GetTime()).FromTx(mtx));
        }
        return entries;
    }

    // add requested kinds and number of transactions to the mempool
    Entries StuffMempool(CTxMemPool& pool, std::initializer_list<Demand> demand)
    {
        CTxMemPoolTestAccess testPoolAccess(pool);
        testPoolAccess.SetBlockMinTxFee(A_BLOCK_MIN_FEE);

        std::vector<std::vector<CTxMemPoolEntry>> all_txns;
        for (auto d: demand) {
            all_txns.push_back(GetABunchOfEntries(d));
        }
        Entries entries {pool};
        // intersperse transactions to prevent arrival order assumptions by the tests
        bool progress = true;
        while(progress) {
            progress = false;
            for(auto& txns : all_txns) {
                if (txns.empty()) {
                    continue;
                }
                progress = true;
                CTxMemPoolEntry txn = txns.back();
                txns.pop_back();
                const TxId txId = txn.GetTxId();
                pool.AddUnchecked(txId, txn, TxStorage::memory, nullChangeSet);
                bool isForPrimary = txn.GetFee() >= A_BLOCK_MIN_FEE.GetFee(txn.GetTxSize());
                entries.entries.push_back({txId, isForPrimary, txn.GetTxSize()});
            }
        }
        // test the basic assumptions how Entries interacts with the mempool
        BOOST_TEST(entries.for_primary().count() == entries.that(are(in_primary)).count());
        BOOST_TEST(entries.for_secondary().count() == entries.that(are(in_secondary)).count());
        BOOST_TEST(pool.DynamicMemoryUsage() > entries.size());
        BOOST_TEST(pool.GetDiskUsage() == 0);
        BOOST_TEST(pool.SecondaryMempoolUsage() >= entries.for_secondary().size());
        return entries;
    }

	// synchronize with async mempooldb thread
	void sync(CTxMemPool& pool) {
        CTxMemPoolTestAccess testPoolAccess(pool);
        testPoolAccess.SyncWithMempoolTxDB();
    }
}

BOOST_FIXTURE_TEST_SUITE(mempool_limit_tests, TestingSetup)

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitAllInRamSecondaryBelowLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY/11, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = poolTotal;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.count() == entries.that(are(in_memory)).count());
    BOOST_TEST(entries.that(are(on_disk)).count() == 0);
    BOOST_TEST(testPool.DynamicMemoryUsage() == poolTotal);
    BOOST_TEST(testPool.GetDiskUsage() == 0);
    BOOST_TEST(testPool.SecondaryMempoolUsage() == poolSecondary);
}

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitAllInRamSecondaryAboveLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = poolTotal;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() > entries.for_primary().count());
    BOOST_TEST(entries.that(are(in_pool)).count() <= entries.for_primary().count() + entries.for_secondary().count()/3);
    BOOST_TEST(entries.that(are(in_memory)).count() = entries.that(are(in_pool)).count());
    BOOST_TEST(entries.that(are(on_disk)).count() == 0);
    BOOST_TEST(entries.for_secondary().that(are(in_memory)).count() >= N_PRIMARY / 10);
    BOOST_TEST(entries.for_secondary().that(are(in_memory)).count() <= 2 * N_PRIMARY / 10 + 3);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= poolTotal - entries.for_secondary().that(are_not(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() == 0);
    BOOST_TEST(testPool.SecondaryMempoolUsage() >= entries.for_secondary().that(are(in_pool)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() < poolSecondary);
}

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitAllOnDiskSecondaryBelowLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY/11, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = 100;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() == entries.count());
    BOOST_TEST(entries.that(are(in_memory)).count() == 0);
    BOOST_TEST(entries.that(are(on_disk)).count() == entries.count());
    BOOST_TEST(testPool.DynamicMemoryUsage() == poolTotal);
    BOOST_TEST(testPool.GetDiskUsage() >= entries.size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() >= entries.for_secondary().size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() <= poolSecondary);
}

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitAllOnDiskSecondaryAboveLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = 100;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() > entries.for_primary().count());
    BOOST_TEST(entries.that(are(in_pool)).count() <= entries.for_primary().count() + entries.for_secondary().count()/3);
    BOOST_TEST(entries.that(are(in_memory)).count() == 0);
    BOOST_TEST(entries.that(are(on_disk)).count() == entries.that(are(in_pool)).count());
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() >= N_PRIMARY / 10);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() <= 3 * N_PRIMARY / 10);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= poolTotal - entries.that(are_not(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(in_pool)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() < poolSecondary);
    BOOST_TEST(testPool.SecondaryMempoolUsage() >= entries.for_secondary().that(are(in_pool)).size());
}

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitHalfOnDiskSecondaryBelowLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY/11, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = poolTotal / 2;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() == entries.count());
    BOOST_TEST(entries.that(are(in_memory)).count() < entries.count() / 2);
    BOOST_TEST(entries.that(are(on_disk)).count() > entries.count() / 2);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() >= N_PRIMARY / 11);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() <= 2 * N_PRIMARY / 10);
    BOOST_TEST(testPool.DynamicMemoryUsage() == poolTotal);
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(in_pool)).size() / 2);
    BOOST_TEST(testPool.SecondaryMempoolUsage() == poolSecondary);
}

BOOST_AUTO_TEST_CASE(PrimaryBelowLimitHalfOnDiskSecondaryAboveLimit)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {N_PRIMARY, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto poolSecondary = testPool.SecondaryMempoolUsage();
    const auto limitTotal = poolTotal + 10000;
    const auto limitMemory = poolTotal / 3;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() < entries.count());
    BOOST_TEST(entries.that(are(in_pool)).count() >= 11 * entries.for_primary().count() / 10);
    BOOST_TEST(entries.that(are(on_disk)).count() <= entries.that(are(in_pool)).count());
    BOOST_TEST(entries.that(are(on_disk)).count() > entries.that(are(in_pool)).count() / 6);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() >= N_PRIMARY / 10);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() <= 3 * N_PRIMARY / 10);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= poolTotal - entries.that(are_not(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(on_disk)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() < poolSecondary);
    BOOST_TEST(testPool.SecondaryMempoolUsage() >= entries.for_secondary().that(are(in_pool)).size());

}

BOOST_AUTO_TEST_CASE(PrimaryAtLimitAllInRam)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY},});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = poolTotal;
    const auto limitMemory = limitTotal;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_memory)).count() == entries.count());
    BOOST_TEST(testPool.DynamicMemoryUsage() == limitTotal);
    BOOST_TEST(testPool.GetDiskUsage() == 0);
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_CASE(PrimaryAtLimitAllOnDisk)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY},});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = poolTotal;
    const auto limitMemory = 0;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(on_disk)).count() == entries.count());
    BOOST_TEST(testPool.DynamicMemoryUsage() == limitTotal);
    BOOST_TEST(testPool.GetDiskUsage() >= entries.size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_CASE(PrimaryAtLimitThirdOnDiskSecondaryGone)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {2, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = poolTotal - testPool.SecondaryMempoolUsage() + 1000;
    const auto limitMemory = (limitTotal * 2) / 3;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() < entries.count());
    BOOST_TEST(entries.that(are(in_pool)).count() == entries.for_primary().count());
    BOOST_TEST(entries.that(are(in_memory)).count() <= ((N_PRIMARY * 2) / 3));
    BOOST_TEST(entries.that(are(on_disk)).count() >= ((N_PRIMARY * 1) / 3));
    BOOST_TEST(testPool.DynamicMemoryUsage() <= poolTotal - entries.that(are_not(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(on_disk)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_CASE(PrimaryAboveLimitInRamSecondaryGone)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {2, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = (poolTotal * 2) / 3 + 10000;
    const auto limitMemory = limitTotal;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() >= (N_PRIMARY * 2) / 3);
    BOOST_TEST(entries.that(are(in_memory)).count() >= N_PRIMARY / 3);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() == 0);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= limitTotal);
    BOOST_TEST(testPool.DynamicMemoryUsage() > entries.that(are(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() == 0);
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_CASE(PrimaryAboveLimitAllOnDiskSecondaryGone)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {2, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = (poolTotal * 2) / 3 + 10000;
    const auto limitMemory = 0;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() >= (N_PRIMARY * 2) / 3);
    BOOST_TEST(entries.that(are(in_memory)).count() == 0);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() == 0);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= limitTotal);
    BOOST_TEST(testPool.DynamicMemoryUsage() > entries.that(are(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(on_disk)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_CASE(PrimaryAboveLimitThirdOnDiskSecondaryGone)
{
    CTxMemPool testPool;

    const auto entries = StuffMempool(testPool, {{N_PRIMARY, A_PRIMARY}, {2, A_SECONDARY}});

    const auto poolTotal = testPool.DynamicMemoryUsage();
    const auto limitTotal = (poolTotal * 2) / 3 + 10000;
    const auto limitMemory = (limitTotal * 2) / 3;
    const auto limitDisk = limitTotal - limitMemory;

    LimitMempoolSize(testPool, nullChangeSet, MempoolSizeLimits(limitMemory, limitDisk, limitTotal/10, 1000000));
    sync(testPool);

    BOOST_TEST(entries.that(are(in_pool)).count() >= (N_PRIMARY * 2) / 3);
    BOOST_TEST(entries.that(are(in_memory)).count() < (N_PRIMARY * 4) / 9);
    BOOST_TEST(entries.for_secondary().that(are(in_pool)).count() == 0);
    BOOST_TEST(testPool.DynamicMemoryUsage() <= limitTotal);
    BOOST_TEST(testPool.DynamicMemoryUsage() > entries.that(are(in_pool)).size());
    BOOST_TEST(testPool.GetDiskUsage() >= entries.that(are(on_disk)).size());
    BOOST_TEST(testPool.SecondaryMempoolUsage() == 0);
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: shphrd/bitcoin-sv


text: #include <boost/mpl/aux_/preprocessed/bcc/bind_fwd.hpp>

ext: hpp

max_stars_repo_name: miathedev/BoostForArduino

max_issues_repo_name: miathedev/BoostForArduino

max_forks_repo_name: miathedev/BoostForArduino


text:
#include "SpatialAudioSender.hpp"

#include <zmq.hpp>

#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/bind.hpp>
#include <cstdlib>
#include <iostream>
#include <chrono>
#include <thread>

namespace spatialaudiopy{

SpatialAudioSender::SpatialAudioSender(const std::string& server_socket, const unsigned num_slots)
: 	m_running(false),
	m_userdata(num_slots),
	m_mutex(num_slots),
	m_senderthread(0),
	m_serversocket(server_socket)
{
	for(unsigned i = 0; i != num_slots; ++i){
		m_mutex[i] = new boost::mutex;
	}
}

SpatialAudioSender::~SpatialAudioSender(){
	m_running = false;
	m_senderthread->join();
	for(unsigned i = 0; i != m_mutex.size(); ++i){
		delete m_mutex[i];
	}
	delete m_senderthread;
}

void
SpatialAudioSender::start(){
	m_running = true;
	m_senderthread = new boost::thread(boost::bind(&SpatialAudioSender::loop, this));
}

void
SpatialAudioSender::stop(){
	m_running = false;
}

void
SpatialAudioSender::setUserData(const unsigned user_id, const unsigned group_id,
                                         const float pos_x, const float pos_y, const float pos_z,
                                         const float front_x, const float front_y, const float front_z,
                                         const float up_x, const float up_y, const float up_z){

        if(user_id >= m_userdata.size()){
                std::cerr << "ERROR in:  skipping user_id: " << user_id << std::endl;
                return;
        }

        boost::mutex::scoped_lock lock( *(m_mutex[user_id]) );

        m_userdata[user_id].user_id = user_id;
        m_userdata[user_id].group_id = group_id;

        m_userdata[user_id].position[0] = pos_x;
        m_userdata[user_id].position[1] = pos_y;
        m_userdata[user_id].position[2] = pos_z;

        m_userdata[user_id].front[0] = front_x;
        m_userdata[user_id].front[1] = front_y;
        m_userdata[user_id].front[2] = front_z;

        m_userdata[user_id].up[0] = up_x;
        m_userdata[user_id].up[1] = up_y;
        m_userdata[user_id].up[2] = up_z;

}

void
SpatialAudioSender::setUserMatrixA(const unsigned user_id, const float m0, const float m1, const float m2, const float m3){

        if(user_id >= m_userdata.size()){
                std::cerr << "ERROR in:  skipping user_id: " << user_id << std::endl;
                return;
        }

        boost::mutex::scoped_lock lock( *(m_mutex[user_id]) );

        m_userdata[user_id].m_t[0] = m0;
        m_userdata[user_id].m_t[1] = m1;
        m_userdata[user_id].m_t[2] = m2;
        m_userdata[user_id].m_t[3] = m3;

}

void
SpatialAudioSender::setUserMatrixB(const unsigned user_id, const float m0, const float m1, const float m2, const float m3){

        if(user_id >= m_userdata.size()){
                std::cerr << "ERROR in:  skipping user_id: " << user_id << std::endl;
                return;
        }

        boost::mutex::scoped_lock lock( *(m_mutex[user_id]) );

        m_userdata[user_id].m_t[4] = m0;
        m_userdata[user_id].m_t[5] = m1;
        m_userdata[user_id].m_t[6] = m2;
        m_userdata[user_id].m_t[7] = m3;

}

void
SpatialAudioSender::setUserMatrixC(const unsigned user_id, const float m0, const float m1, const float m2, const float m3){

        if(user_id >= m_userdata.size()){
                std::cerr << "ERROR in:  skipping user_id: " << user_id << std::endl;
                return;
        }

        boost::mutex::scoped_lock lock( *(m_mutex[user_id]) );

        m_userdata[user_id].m_t[8] = m0;
        m_userdata[user_id].m_t[9] = m1;
        m_userdata[user_id].m_t[10] = m2;
        m_userdata[user_id].m_t[11] = m3;

}
void
SpatialAudioSender::setUserMatrixD(const unsigned user_id, const float m0, const float m1, const float m2, const float m3){

        if(user_id >= m_userdata.size()){
                std::cerr << "ERROR in:  skipping user_id: " << user_id << std::endl;
                return;
        }

        boost::mutex::scoped_lock lock( *(m_mutex[user_id]) );

        m_userdata[user_id].m_t[12] = m0;
        m_userdata[user_id].m_t[13] = m1;
        m_userdata[user_id].m_t[14] = m2;
        m_userdata[user_id].m_t[15] = m3;

}

void
SpatialAudioSender::loop(){

	// 1. prepare socket
  	zmq::context_t ctx(1); // means single threaded
  	zmq::socket_t  server_socket(ctx, ZMQ_PUB); // means a publisher
  	uint32_t hwm = 1;
  	server_socket.setsockopt(ZMQ_SNDHWM,&hwm, sizeof(hwm));
  	std::string endpoint("tcp://" + m_serversocket);
  	server_socket.bind(endpoint.c_str());

	while(m_running){

		// 2. create and fill message
		const unsigned num_users = m_userdata.size();
		zmq::message_t msg( sizeof(unsigned) + sizeof(user_data) * num_users);

		memcpy( (unsigned char* ) msg.data(), (const unsigned char*) &num_users, sizeof(unsigned));
		for(unsigned i = 0; i != num_users; ++i){
			boost::mutex::scoped_lock lock( *(m_mutex[i]) );
			memcpy( ((unsigned char* ) msg.data() + (sizeof(unsigned) + i * sizeof(user_data))), (const unsigned char*) &(m_userdata[i]), sizeof(user_data));
		}
		// 3. send userdate
		server_socket.send(msg);

		// sleep a tiny littly bit
		std::this_thread::sleep_for(std::chrono::milliseconds(10));
	}

}

}

max_stars_repo_name: steppobeck/spatial-audio-py

max_issues_repo_name: steppobeck/spatial-audio-py

max_forks_repo_name: steppobeck/spatial-audio-py


text:
////////////////////////////////////////////////////////////////////////
//
// This file is part of gmic-8bf, a filter plug-in module that
// interfaces with G'MIC-Qt.
//
// Copyright (c) 2020, 2021 Nicholas Hayes
//
// This file is licensed under the MIT License.
// See LICENSE.txt for complete licensing and attribution information.
//
////////////////////////////////////////////////////////////////////////

#include "GmicQtParameters.h"
#include "GmicPluginTerminology.h"
#include "FileUtil.h"
#include "ScopedBufferSuite.h"
#include "Utilities.h"
#include <boost/predef.h>
#include <array>
#include <codecvt>

namespace
{
    struct GmicQtParametersHeader
    {
        GmicQtParametersHeader()
        {
            // G8FP = GMIC 8BF filter parameters
            signature[0] = 'G';
            signature[1] = '8';
            signature[2] = 'F';
            signature[3] = 'P';
#if BOOST_ENDIAN_BIG_BYTE
            endian[0] = 'B';
            endian[1] = 'E';
            endian[2] = 'D';
            endian[3] = 'N';
#elif BOOST_ENDIAN_LITTLE_BYTE
            endian[0] = 'L';
            endian[1] = 'E';
            endian[2] = 'D';
            endian[3] = 'N';
#else
#error "Unknown endianness on this platform."
#endif
            version = 1;
        }

        GmicQtParametersHeader(FileHandle* fileHandle) : version()
        {
            ReadFile(fileHandle, this, sizeof(*this));

            if (strncmp(signature, "G8FP", 4) != 0)
            {
                throw std::runtime_error("The GmicQt parameters file has an invalid file signature.");
            }

#if BOOST_ENDIAN_BIG_BYTE
            const char* const platformEndian = "BEDN";
#elif BOOST_ENDIAN_LITTLE_BYTE
            const char* const platformEndian = "LEDN";
#else
#error "Unknown endianness on this platform."
#endif

            if (strncmp(endian, platformEndian, 4) != 0)
            {
                throw std::runtime_error("The GmicQt parameters file endianess does not match the current platform.");
            }
        }

        char signature[4];
        char endian[4];
        int32_t version;
    };

    template<class T> class ScopedPICASuite
    {
    public:
        ScopedPICASuite(SPBasicSuite* basicSuite, const char* name, int32 version) noexcept
            : basicSuite(basicSuite), suite(nullptr), name(name), version(version), suiteValid(false)
        {
            SPErr err = basicSuite->AcquireSuite(
                name,
                version,
                const_cast<const void**>(reinterpret_cast<void**>(&suite)));
            suiteValid = err == kSPNoError && suite != nullptr;
        }

        ~ScopedPICASuite()
        {
            if (suiteValid)
            {
                basicSuite->ReleaseSuite(name, version);
                suiteValid = false;
            }
        }

        T* get() const noexcept
        {
            return suite;
        }

        const T* operator->() const noexcept
        {
            return suite;
        }

        explicit operator bool() const noexcept
        {
            return suiteValid;
        }

    private:

        SPBasicSuite* basicSuite;
        T* suite;
        const char* name;
        int32 version;
        bool suiteValid;
    };

    class ScopedActionDescriptor
    {
    public:
        explicit ScopedActionDescriptor(PSActionDescriptorProcs* descriptorProcs)
            : descriptorProcs(descriptorProcs), descriptor(), descriptorValid(false)
        {
           descriptorValid = descriptorProcs->Make(&descriptor) == noErr;
        }

        ScopedActionDescriptor(PSActionDescriptorProcs* descriptorProcs, PIDescriptorHandle handle)
            : descriptorProcs(descriptorProcs), descriptor(), descriptorValid(false)
        {
            descriptorValid = descriptorProcs->HandleToDescriptor(handle, &descriptor) == noErr;
        }

        ~ScopedActionDescriptor()
        {
            if (descriptorValid)
            {
                descriptorProcs->Free(descriptor);
                descriptorValid = false;
            }
        }

        PIActionDescriptor get() const noexcept
        {
            return descriptor;
        }

        explicit operator bool() const noexcept
        {
            return descriptorValid;
        }

    private:

        PSActionDescriptorProcs* descriptorProcs;
        PIActionDescriptor descriptor;
        bool descriptorValid;
    };

    void ThrowIfASError(ASErr err)
    {
        if (err != kASNoError)
        {
            switch (err)
            {
            case kSPOutOfMemoryError:
                throw OSErrException(memFullErr);
            case kSPUnimplementedError:
                throw OSErrException(errPlugInHostInsufficient);
            default:
                throw OSErrException(paramErr);
            }
        }
    }

    class ScopedASZString
    {
    public:

        ScopedASZString(ASZStringSuite* suite, std::vector<ASUnicode> unicodeChars)
            : zstringSuite(suite), zstr(nullptr), zstringValid(false)
        {
            ThrowIfASError(suite->MakeFromUnicode(unicodeChars.data(), unicodeChars.size(), &zstr));
            zstringValid = true;
        }

        ScopedASZString(
            ASZStringSuite* zstringSuite,
            PSActionDescriptorProcs* descriptorProcs,
            PIActionDescriptor descriptor,
            DescriptorKeyID key)
            : zstringSuite(zstringSuite), zstr(nullptr), zstringValid(false)
        {
            OSErrException::ThrowIfError(descriptorProcs->GetZString(descriptor, key, &zstr));
            zstringValid = true;
        }

        ~ScopedASZString()
        {
            if (zstringValid)
            {
                zstringSuite->Release(zstr);
                zstr = nullptr;
                zstringValid = false;
            }
        }

        ASZString get() const
        {
            return zstr;
        }

    private:

        ASZStringSuite* zstringSuite;
        ASZString zstr;
        bool zstringValid;
    };

    void ReadUtf8String(FileHandle* fileHandle, std::string& value)
    {
        int32_t stringLength = 0;

        ReadFile(fileHandle, &stringLength, sizeof(stringLength));

        if (stringLength == 0)
        {
            value = std::string();
        }
        else
        {
            std::vector<char> stringChars(stringLength);

            ReadFile(fileHandle, &stringChars[0], stringLength);

            value.assign(stringChars.begin(), stringChars.end());
        }
    }

    void WriteUtf8String(FileHandle* fileHandle, const std::string& value)
    {
        // Check that the required byte buffer size can fit in a uint32_t.
        if (value.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max()))
        {
            throw std::runtime_error("The string cannot be written to the file because it is too long.");
        }

        int32_t stringLength = static_cast<int32_t>(value.size());

        WriteFile(fileHandle, &stringLength, sizeof(stringLength));

        if (value.size() > 0)
        {
            WriteFile(fileHandle, value.c_str(), stringLength);
        }
    }

    std::vector<ASUnicode> ConvertUtf8StringToASUnicode(const std::string& utf8Str)
    {
        std::codecvt_utf8_utf16<ASUnicode> converter;
        std::codecvt_utf8_utf16<ASUnicode>::state_type state = std::codecvt_utf8_utf16<ASUnicode>::state_type();

        const char* first = utf8Str.data();
        const char* last = first + utf8Str.length();

        const int utf16Length = converter.length(state, first, last, utf8Str.length());
        std::vector<ASUnicode> unicodeChars = std::vector<ASUnicode>(utf16Length);

        ASUnicode* dest = unicodeChars.data();
        ASUnicode* destEnd = dest + unicodeChars.size();
        ASUnicode* outputLocation;

        auto result = converter.in(state, first, last, first, dest, destEnd, outputLocation);

        switch (result)
        {
        case std::codecvt_base::ok:
        case std::codecvt_base::partial:
            if (outputLocation != destEnd)
            {
                throw std::runtime_error("Bad UTF-8 to UTF-16 conversion.");
            }
            break;
        case std::codecvt_base::noconv:
            for (size_t i = 0, length = last - first; i < length; i++, first++)
            {
                unicodeChars[i] = static_cast<ASUnicode>(static_cast<unsigned char>(*first));
            }
            break;
        default:
            throw std::runtime_error("Bad UTF-8 to UTF-16 conversion.");
        }

        return unicodeChars;
    }

    void PutDescriptorString(
        ASZStringSuite* zstringSuite,
        PSActionDescriptorProcs* descriptorProcs,
        PIActionDescriptor descriptor,
        DescriptorKeyID key,
        const std::string& utf8Str)
    {
        std::vector<ASUnicode> unicodeChars = ConvertUtf8StringToASUnicode(utf8Str);

        ScopedASZString zstr(zstringSuite, unicodeChars);

        OSErrException::ThrowIfError(descriptorProcs->PutZString(descriptor, key, zstr.get()));
    }

    bool ActionDescriptiorSuiteSupported(const FilterRecordPtr filterRecord)
    {
        bool result = false;

        if (SPBasicSuiteIsAvailable(filterRecord))
        {
            PSActionDescriptorProcs* suite = nullptr;

            SPErr err = filterRecord->sSPBasic->AcquireSuite(
                kPSActionDescriptorSuite,
                2,
                const_cast<const void**>(reinterpret_cast<void**>(&suite)));

            if (err == kSPNoError)
            {
                // Ensure that the structure function pointer are not null.
                if (suite != nullptr &&
                    suite->Make != nullptr &&
                    suite->Free != nullptr &&
                    suite->HandleToDescriptor != nullptr &&
                    suite->AsHandle != nullptr &&
                    suite->GetZString != nullptr &&
                    suite->PutZString != nullptr &&
                    suite->GetDataLength != nullptr &&
                    suite->GetData != nullptr &&
                    suite->PutData != nullptr)
                {
                    result = true;
                }

                filterRecord->sSPBasic->ReleaseSuite(kPSActionDescriptorSuite, 2);
            }
        }

        return result;
    }

    bool ASZStringSuiteSupported(const FilterRecordPtr filterRecord)
    {
        bool result = false;

        if (SPBasicSuiteIsAvailable(filterRecord))
        {
            ASZStringSuite* suite = nullptr;

            SPErr err = filterRecord->sSPBasic->AcquireSuite(
                kASZStringSuite,
                1,
                const_cast<const void**>(reinterpret_cast<void**>(&suite)));

            if (err == kSPNoError)
            {
                // Ensure that the structure function pointer are not null.
                if (suite != nullptr &&
                    suite->MakeFromUnicode != nullptr &&
                    suite->Release != nullptr &&
                    suite->LengthAsCString != nullptr &&
                    suite->AsCString != nullptr)
                {
                    result = true;
                }

                filterRecord->sSPBasic->ReleaseSuite(kASZStringSuite, 1);
            }
        }

        return result;
    }

    bool HostSupportsActionDescriptiorSuites(const FilterRecordPtr filterRecord)
    {
        bool result = false;

        if (HandleSuiteIsAvailable(filterRecord) && SPBasicSuiteIsAvailable(filterRecord))
        {
            result = ActionDescriptiorSuiteSupported(filterRecord) && ASZStringSuiteSupported(filterRecord);
        }

        return result;
    }

    bool ActionDescriptorSuitesAreAvailable(const FilterRecordPtr filterRecord)
    {
        static bool suitesAvailable = HostSupportsActionDescriptiorSuites(filterRecord);

        return suitesAvailable;
    }

    struct FilterOpaqueDataHeader
    {
        int32_t commandLength;
        int32_t menuPathLength;
    };
}

GmicQtParameters::GmicQtParameters(const FilterRecordPtr filterRecord)
    : command(), filterMenuPath(), inputMode("Active Layer"), filterName()
{
    if (filterRecord->descriptorParameters != nullptr &&
        filterRecord->descriptorParameters->descriptor != nullptr &&
        ActionDescriptorSuitesAreAvailable(filterRecord))
    {
        SPBasicSuite* basicSuite = filterRecord->sSPBasic;

        ScopedPICASuite<PSActionDescriptorProcs> descriptorProcs(basicSuite, kPSActionDescriptorSuite, 2);

        if (descriptorProcs)
        {
            ScopedPICASuite<ASZStringSuite> zstringSuite(basicSuite, kASZStringSuite, 1);

            if (zstringSuite)
            {
                ScopedActionDescriptor descriptor(descriptorProcs.get(), filterRecord->descriptorParameters->descriptor);

                if (descriptor)
                {
                    Boolean hasKey;

                    if (descriptorProcs->HasKey(descriptor.get(), keyFilterInputMode, &hasKey) == noErr && hasKey)
                    {
                        ReadFilterInputMode(zstringSuite.get(), descriptorProcs.get(), descriptor.get());
                    }

                    if (descriptorProcs->HasKey(descriptor.get(), keyFilterOpaqueData, &hasKey) == noErr && hasKey)
                    {
                        ReadFilterOpaqueData(filterRecord, descriptorProcs.get(), descriptor.get());
                    }

                    // G'MIC-Qt does not need the filter name.
                }
            }
        }
    }
}

GmicQtParameters::GmicQtParameters(const boost::filesystem::path& path)
{
    std::unique_ptr<FileHandle> file = OpenFile(path, FileOpenMode::Read);
    GmicQtParametersHeader header(file.get());

    ReadUtf8String(file.get(), command);
    ReadUtf8String(file.get(), filterMenuPath);
    ReadUtf8String(file.get(), inputMode);
    ReadUtf8String(file.get(), filterName);
}

GmicQtParameters::~GmicQtParameters()
{
}

bool GmicQtParameters::IsValid() const
{
    return !command.empty();
}

void GmicQtParameters::SaveToDescriptor(FilterRecordPtr filterRecord) const
{
    if (filterRecord->descriptorParameters != nullptr &&
        ActionDescriptorSuitesAreAvailable(filterRecord))
    {
        SPBasicSuite* basicSuite = filterRecord->sSPBasic;

        ScopedPICASuite<PSActionDescriptorProcs> descriptorProcs(basicSuite, kPSActionDescriptorSuite, 2);

        if (descriptorProcs)
        {
            ScopedPICASuite<ASZStringSuite> zstringSuite(basicSuite, kASZStringSuite, 1);

            if (zstringSuite)
            {
                ScopedActionDescriptor descriptor(descriptorProcs.get());

                if (descriptor)
                {
                    PutDescriptorString(zstringSuite.get(), descriptorProcs.get(), descriptor.get(), keyFilterName, filterName);
                    PutDescriptorString(zstringSuite.get(), descriptorProcs.get(), descriptor.get(), keyFilterInputMode, inputMode);
                    WriteFilterOpaqueData(filterRecord, descriptorProcs.get(), descriptor.get());

                    if (filterRecord->descriptorParameters->descriptor != nullptr)
                    {
                        filterRecord->handleProcs->disposeProc(filterRecord->descriptorParameters->descriptor);
                    }

                    OSErrException::ThrowIfError(descriptorProcs->AsHandle(descriptor.get(), &filterRecord->descriptorParameters->descriptor));
                }
            }
        }
    }
}

void GmicQtParameters::SaveToFile(const boost::filesystem::path& path) const
{
    GmicQtParametersHeader header;

    std::unique_ptr<FileHandle> file = OpenFile(path, FileOpenMode::Write);

    WriteFile(file.get(), &header, sizeof(header));
    WriteUtf8String(file.get(), command);
    WriteUtf8String(file.get(), filterMenuPath);
    WriteUtf8String(file.get(), inputMode);
    // G'MIC-Qt does not need the filter name, so write an empty string.
    WriteUtf8String(file.get(), std::string());
}

OSErr GmicQtParameters::ReadFilterInputMode(
    ASZStringSuite* zstringSuite,
    PSActionDescriptorProcs* descriptorProcs,
    PIActionDescriptor descriptor)
{
    OSErr err = noErr;

    try
    {
        ScopedASZString zstr(zstringSuite, descriptorProcs, descriptor, keyFilterInputMode);

        const ASUInt32 stringLength = zstringSuite->LengthAsCString(zstr.get());

        if (stringLength > 0)
        {
            std::unique_ptr<char[]> buffer = std::make_unique<char[]>(stringLength);

            ThrowIfASError(zstringSuite->AsCString(zstr.get(), buffer.get(), stringLength, true));

            ASUInt32 lengthWithoutTerminator = stringLength;

            // Remove the NUL-terminator from the end of the string.
            if (buffer[stringLength - 1] == 0)
            {
                lengthWithoutTerminator -= 1;
            }

            inputMode = std::string(buffer.get(), buffer.get() + lengthWithoutTerminator);
        }
    }
    catch (const std::bad_alloc&)
    {
        err = memFullErr;
    }
    catch (const OSErrException& e)
    {
        err = e.GetErrorCode();
    }
    catch (...)
    {
        err = paramErr;
    }

    return err;
}

OSErr GmicQtParameters::ReadFilterOpaqueData(
    const FilterRecordPtr filterRecord,
    PSActionDescriptorProcs* suite,
    PIActionDescriptor descriptor)
{
    OSErr err = noErr;

    try
    {
        int32 dataSize;

        OSErrException::ThrowIfError(suite->GetDataLength(descriptor, keyFilterOpaqueData, &dataSize));

        ScopedBufferSuiteBuffer scopedBuffer(filterRecord, dataSize);

        char* data = static_cast<char*>(scopedBuffer.Lock());

        OSErrException::ThrowIfError(suite->GetData(descriptor, keyFilterOpaqueData, data));

        FilterOpaqueDataHeader* header = (FilterOpaqueDataHeader*)data;

        command = std::string(data + sizeof(FilterOpaqueDataHeader), header->commandLength);
        filterMenuPath = std::string(data + sizeof(FilterOpaqueDataHeader) + command.size(), header->menuPathLength);
    }
    catch (const std::bad_alloc&)
    {
        err = memFullErr;
    }
    catch (const OSErrException& e)
    {
        err = e.GetErrorCode();
    }
    catch (...)
    {
        err = paramErr;
    }

    return err;
}

void GmicQtParameters::WriteFilterOpaqueData(
    const FilterRecordPtr filterRecord,
    PSActionDescriptorProcs* suite,
    PIActionDescriptor descriptor) const
{
    uint64 dataSize = static_cast<uint64>(sizeof(FilterOpaqueDataHeader)) + command.size() + filterMenuPath.size();

    if (dataSize > static_cast<uint64>(std::numeric_limits<int32>::max()))
    {
        throw std::runtime_error("The G'MIC-Qt data is larger than 2 GB.");
    }

    ScopedBufferSuiteBuffer scopedBuffer(filterRecord, static_cast<int32>(dataSize));

    uint8* data = static_cast<uint8*>(scopedBuffer.Lock());

    FilterOpaqueDataHeader* header = (FilterOpaqueDataHeader*)data;
    header->commandLength = static_cast<int32>(command.size());
    header->menuPathLength = static_cast<int32>(filterMenuPath.size());

    memcpy(data + sizeof(FilterOpaqueDataHeader), command.data(), command.size());
    memcpy(data + sizeof(FilterOpaqueDataHeader) + command.size(), filterMenuPath.data(), filterMenuPath.size());

    OSErrException::ThrowIfError(suite->PutData(descriptor, keyFilterOpaqueData, static_cast<int32>(dataSize), data));
}


text:
//-----------------------------------------------------------------------------
// Copyright (c) 2012-2018 Benjamin Buch
//
// https://github.com/bebuch/http
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt)
//-----------------------------------------------------------------------------
#ifndef _http__reply__hpp_INCLUDED_
#define _http__reply__hpp_INCLUDED_

#include "header.hpp"

#include <boost/asio.hpp>

#include <string>
#include <vector>

namespace http{

	namespace asio = boost::asio;

	/// \brief A reply to be sent to a client.
	struct reply{
		/// \brief The status of the reply.
		enum status_type{
			// Informational
			continue_ = 100,
			switching_protocols = 101,
			// Successful
			ok = 200,
			created = 201,
			accepted = 202,
			non_authoritative_information = 203,
			no_content = 204,
			reset_content = 205,
			partial_content = 206,
			// Redirection
			multiple_choices = 300,
			moved_permanently = 301,
			found = 302,
			see_other = 303,
			not_modified = 304,
			use_proxy = 305,
			temporary_redirect = 307,
			// Client Error
			bad_request = 400,
			unauthorized = 401,
			payment_required = 402,
			forbidden = 403,
			not_found = 404,
			method_not_allowed = 405,
			not_acceptable = 406,
			proxy_authentication_required = 407,
			request_time_out = 408,
			conflict = 409,
			gone = 410,
			length_required = 411,
			precondition_failed = 412,
			request_entity_too_large = 413,
			request_url_too_long = 414,
			unsupported_media_type = 415,
			requested_range_not_satisfiable = 416,
			expectation_failed = 417,
			// Server Error
			internal_server_error = 500,
			not_implemented = 501,
			bad_gateway = 502,
			service_unavailable = 503,
			gateway_time_out = 504,
			http_version_not_supported = 505
		} status;

		/// \brief The headers to be included in the reply.
		header headers;

		/// \brief The content to be sent in the reply.
		std::string content;

		/// \brief Convert the reply into a vector of buffers.
		///
		/// The buffers do not own the underlying memory blocks,
		/// therefore the reply object must remain valid and
		/// not be changed until the write operation has completed.
		std::vector< asio::const_buffer > to_buffers() const;

		/// \brief Get a stock reply.
		static reply stock_reply(status_type status);
	};

}

#endif

max_stars_repo_name: bebuch/http

max_issues_repo_name: bebuch/http

max_forks_repo_name: bebuch/http