text:
#include <list>

#include "gmock/gmock.h"
#include "gtest/gtest.h"

#include <boost/asio.hpp>

#include "ray/raylet/format/node_manager_generated.h"
#include "ray/raylet/reconstruction_policy.h"

#include "ray/object_manager/object_directory.h"

namespace ray {

namespace raylet {

class MockObjectDirectory : public ObjectDirectoryInterface {
 public:
  MockObjectDirectory() {}

  ray::Status LookupLocations(const ObjectID &object_id,
                              const OnLocationsFound &callback) override {
    callbacks_.push_back({object_id, callback});
    return ray::Status::OK();
  }

  void FlushCallbacks() {
    for (const auto &callback : callbacks_) {
      const ObjectID object_id = callback.first;
      auto it = locations_.find(object_id);
      if (it == locations_.end()) {
        callback.second(object_id, {}, /*created=*/false);
      } else {
        callback.second(object_id, it->second, /*created=*/true);
      }
    }
    callbacks_.clear();
  }

  void SetObjectLocations(const ObjectID &object_id,
                          const std::unordered_set<ClientID> &locations) {
    locations_[object_id] = locations;
  }

  void HandleClientRemoved(const ClientID &client_id) override {
    for (auto &locations : locations_) {
      locations.second.erase(client_id);
    }
  }

  std::string DebugString() const override { return ""; }

  MOCK_METHOD0(RegisterBackend, void(void));
  MOCK_METHOD0(GetLocalClientID, ray::ClientID());
  MOCK_CONST_METHOD1(LookupRemoteConnectionInfo, void(RemoteConnectionInfo &));
  MOCK_CONST_METHOD0(LookupAllRemoteConnections, std::vector<RemoteConnectionInfo>());
  MOCK_METHOD3(SubscribeObjectLocations,
               ray::Status(const ray::UniqueID &, const ObjectID &,
                           const OnLocationsFound &));
  MOCK_METHOD2(UnsubscribeObjectLocations,
               ray::Status(const ray::UniqueID &, const ObjectID &));
  MOCK_METHOD3(ReportObjectAdded,
               ray::Status(const ObjectID &, const ClientID &,
                           const object_manager::protocol::ObjectInfoT &));
  MOCK_METHOD2(ReportObjectRemoved, ray::Status(const ObjectID &, const ClientID &));

 private:
  std::vector<std::pair<ObjectID, OnLocationsFound>> callbacks_;
  std::unordered_map<ObjectID, std::unordered_set<ClientID>> locations_;
};

class MockGcs : public gcs::PubsubInterface<TaskID>,
                public ray::gcs::LogInterface<TaskID, TaskReconstructionData> {
 public:
  MockGcs() : notification_callback_(nullptr), failure_callback_(nullptr){};

  void Subscribe(const gcs::TaskLeaseTable::WriteCallback &notification_callback,
                 const gcs::TaskLeaseTable::FailureCallback &failure_callback) {
    notification_callback_ = notification_callback;
    failure_callback_ = failure_callback;
  }

  void Add(const JobID &job_id, const TaskID &task_id,
           std::shared_ptr<TaskLeaseDataT> &task_lease_data) {
    task_lease_table_[task_id] = task_lease_data;
    if (subscribed_tasks_.count(task_id) == 1) {
      notification_callback_(nullptr, task_id, *task_lease_data);
    }
  }

  Status RequestNotifications(const JobID &job_id, const TaskID &task_id,
                              const ClientID &client_id) {
    subscribed_tasks_.insert(task_id);
    auto entry = task_lease_table_.find(task_id);
    if (entry == task_lease_table_.end()) {
      failure_callback_(nullptr, task_id);
    } else {
      notification_callback_(nullptr, task_id, *entry->second);
    }
    return ray::Status::OK();
  }

  Status CancelNotifications(const JobID &job_id, const TaskID &task_id,
                             const ClientID &client_id) {
    subscribed_tasks_.erase(task_id);
    return ray::Status::OK();
  }

  Status AppendAt(
      const JobID &job_id, const TaskID &task_id,
      std::shared_ptr<TaskReconstructionDataT> &task_data,
      const ray::gcs::LogInterface<TaskID, TaskReconstructionData>::WriteCallback
          &success_callback,
      const ray::gcs::LogInterface<TaskID, TaskReconstructionData>::WriteCallback
          &failure_callback,
      int log_index) {
    if (task_reconstruction_log_[task_id].size() == static_cast<size_t>(log_index)) {
      task_reconstruction_log_[task_id].push_back(*task_data);
      if (success_callback != nullptr) {
        success_callback(nullptr, task_id, *task_data);
      }
    } else {
      if (failure_callback != nullptr) {
        failure_callback(nullptr, task_id, *task_data);
      }
    }
    return Status::OK();
  }

  MOCK_METHOD4(
      Append,
      ray::Status(
          const JobID &, const TaskID &, std::shared_ptr<TaskReconstructionDataT> &,
          const ray::gcs::LogInterface<TaskID, TaskReconstructionData>::WriteCallback &));

 private:
  gcs::TaskLeaseTable::WriteCallback notification_callback_;
  gcs::TaskLeaseTable::FailureCallback failure_callback_;
  std::unordered_map<TaskID, std::shared_ptr<TaskLeaseDataT>> task_lease_table_;
  std::unordered_set<TaskID> subscribed_tasks_;
  std::unordered_map<TaskID, std::vector<TaskReconstructionDataT>>
      task_reconstruction_log_;
};

class ReconstructionPolicyTest : public ::testing::Test {
 public:
  ReconstructionPolicyTest()
      : io_service_(),
        mock_gcs_(),
        mock_object_directory_(std::make_shared<MockObjectDirectory>()),
        reconstruction_timeout_ms_(50),
        reconstruction_policy_(std::make_shared<ReconstructionPolicy>(
            io_service_, [this](const TaskID &task_id,
                                bool created) { TriggerReconstruction(task_id); },
            reconstruction_timeout_ms_, ClientID::from_random(), mock_gcs_,
            mock_object_directory_, mock_gcs_)),
        timer_canceled_(false) {
    mock_gcs_.Subscribe(
        [this](gcs::AsyncGcsClient *client, const TaskID &task_id,
               const TaskLeaseDataT &task_lease) {
          reconstruction_policy_->HandleTaskLeaseNotification(task_id,
                                                              task_lease.timeout);
        },
        [this](gcs::AsyncGcsClient *client, const TaskID &task_id) {
          reconstruction_policy_->HandleTaskLeaseNotification(task_id, 0);
        });
  }

  void TriggerReconstruction(const TaskID &task_id) { reconstructed_tasks_[task_id]++; }

  void Tick(const std::function<void(void)> &handler,
            std::shared_ptr<boost::asio::deadline_timer> timer,
            boost::posix_time::milliseconds timer_period,
            const boost::system::error_code &error) {
    if (timer_canceled_) {
      return;
    }
    ASSERT_FALSE(error);
    handler();
    // Fire the timer again after another period.
    timer->expires_from_now(timer_period);
    timer->async_wait(
        [this, handler, timer, timer_period](const boost::system::error_code &error) {
          Tick(handler, timer, timer_period, error);
        });
  }

  void SetPeriodicTimer(uint64_t period_ms, const std::function<void(void)> &handler) {
    timer_canceled_ = false;
    auto timer_period = boost::posix_time::milliseconds(period_ms);
    auto timer = std::make_shared<boost::asio::deadline_timer>(io_service_, timer_period);
    timer->async_wait(
        [this, handler, timer, timer_period](const boost::system::error_code &error) {
          Tick(handler, timer, timer_period, error);
        });
  }

  void CancelPeriodicTimer() { timer_canceled_ = true; }

  void Run(uint64_t reconstruction_timeout_ms) {
    auto timer_period = boost::posix_time::milliseconds(reconstruction_timeout_ms);
    auto timer = std::make_shared<boost::asio::deadline_timer>(io_service_, timer_period);
    timer->async_wait([this, timer](const boost::system::error_code &error) {
      ASSERT_FALSE(error);
      io_service_.stop();
    });
    io_service_.run();
    io_service_.reset();

    mock_object_directory_->FlushCallbacks();
  }

 protected:
  boost::asio::io_service io_service_;
  MockGcs mock_gcs_;
  std::shared_ptr<MockObjectDirectory> mock_object_directory_;
  uint64_t reconstruction_timeout_ms_;
  std::shared_ptr<ReconstructionPolicy> reconstruction_policy_;
  bool timer_canceled_;
  std::unordered_map<TaskID, int> reconstructed_tasks_;
};

TEST_F(ReconstructionPolicyTest, TestReconstructionSimple) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);

  // Listen for an object.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test for longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was triggered for the task that created the
  // object.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);

  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was triggered again.
  ASSERT_EQ(reconstructed_tasks_[task_id], 2);
}

TEST_F(ReconstructionPolicyTest, TestReconstructionEvicted) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);
  mock_object_directory_->SetObjectLocations(object_id, {ClientID::from_random()});

  // Listen for both objects.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test for longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was not triggered, since the objects still
  // exist on a live node.
  ASSERT_EQ(reconstructed_tasks_[task_id], 0);

  // Simulate evicting one of the objects.
  mock_object_directory_->SetObjectLocations(object_id, {});
  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was triggered, since one of the objects was
  // evicted.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

TEST_F(ReconstructionPolicyTest, TestReconstructionObjectLost) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);
  ClientID client_id = ClientID::from_random();
  mock_object_directory_->SetObjectLocations(object_id, {client_id});

  // Listen for both objects.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test for longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was not triggered, since the objects still
  // exist on a live node.
  ASSERT_EQ(reconstructed_tasks_[task_id], 0);

  // Simulate evicting one of the objects.
  mock_object_directory_->HandleClientRemoved(client_id);
  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction was triggered, since one of the objects was
  // evicted.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

TEST_F(ReconstructionPolicyTest, TestDuplicateReconstruction) {
  // Create two object IDs produced by the same task.
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id1 = ComputeReturnId(task_id, 1);
  ObjectID object_id2 = ComputeReturnId(task_id, 2);

  // Listen for both objects.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id1);
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id2);
  // Run the test for longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction is only triggered once for the task that created
  // both objects.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);

  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction is again only triggered once.
  ASSERT_EQ(reconstructed_tasks_[task_id], 2);
}

TEST_F(ReconstructionPolicyTest, TestReconstructionSuppressed) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);
  // Run the test for much longer than the reconstruction timeout.
  int64_t test_period = 2 * reconstruction_timeout_ms_;

  // Acquire the task lease for a period longer than the test period.
  auto task_lease_data = std::make_shared<TaskLeaseDataT>();
  task_lease_data->node_manager_id = ClientID::from_random().binary();
  task_lease_data->acquired_at = current_sys_time_ms();
  task_lease_data->timeout = 2 * test_period;
  mock_gcs_.Add(DriverID::nil(), task_id, task_lease_data);

  // Listen for an object.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test.
  Run(test_period);
  // Check that reconstruction is suppressed by the active task lease.
  ASSERT_TRUE(reconstructed_tasks_.empty());

  // Run the test again past the expiration time of the lease.
  Run(task_lease_data->timeout * 1.1);
  // Check that this time, reconstruction is triggered.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

TEST_F(ReconstructionPolicyTest, TestReconstructionContinuallySuppressed) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);

  // Listen for an object.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Send the reconstruction manager heartbeats about the object.
  SetPeriodicTimer(reconstruction_timeout_ms_ / 2, [this, task_id]() {
    auto task_lease_data = std::make_shared<TaskLeaseDataT>();
    task_lease_data->node_manager_id = ClientID::from_random().binary();
    task_lease_data->acquired_at = current_sys_time_ms();
    task_lease_data->timeout = reconstruction_timeout_ms_;
    mock_gcs_.Add(DriverID::nil(), task_id, task_lease_data);
  });
  // Run the test for much longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 2);
  // Check that reconstruction is suppressed.
  ASSERT_TRUE(reconstructed_tasks_.empty());

  // Cancel the heartbeats to the reconstruction manager.
  CancelPeriodicTimer();
  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that this time, reconstruction is triggered.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

TEST_F(ReconstructionPolicyTest, TestReconstructionCanceled) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);

  // Listen for an object.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Halfway through the reconstruction timeout, cancel the object
  // reconstruction.
  auto timer_period = boost::posix_time::milliseconds(reconstruction_timeout_ms_);
  auto timer = std::make_shared<boost::asio::deadline_timer>(io_service_, timer_period);
  timer->async_wait([this, timer, object_id](const boost::system::error_code &error) {
    ASSERT_FALSE(error);
    reconstruction_policy_->Cancel(object_id);
  });
  Run(reconstruction_timeout_ms_ * 2);
  // Check that reconstruction is suppressed.
  ASSERT_TRUE(reconstructed_tasks_.empty());

  // Listen for the object again.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that this time, reconstruction is triggered.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

TEST_F(ReconstructionPolicyTest, TestSimultaneousReconstructionSuppressed) {
  TaskID task_id = TaskID::from_random();
  task_id = FinishTaskId(task_id);
  ObjectID object_id = ComputeReturnId(task_id, 1);

  // Log a reconstruction attempt to simulate a different node attempting the
  // reconstruction first. This should suppress this node's first attempt at
  // reconstruction.
  auto task_reconstruction_data = std::make_shared<TaskReconstructionDataT>();
  task_reconstruction_data->node_manager_id = ClientID::from_random().binary();
  task_reconstruction_data->num_reconstructions = 0;
  RAY_CHECK_OK(
      mock_gcs_.AppendAt(DriverID::nil(), task_id, task_reconstruction_data, nullptr,
                         /*failure_callback=*/
                         [](ray::gcs::AsyncGcsClient *client, const TaskID &task_id,
                            const TaskReconstructionDataT &data) { ASSERT_TRUE(false); },
                         /*log_index=*/0));

  // Listen for an object.
  reconstruction_policy_->ListenAndMaybeReconstruct(object_id);
  // Run the test for longer than the reconstruction timeout.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that reconstruction is suppressed by the reconstruction attempt
  // logged by the other node.
  ASSERT_TRUE(reconstructed_tasks_.empty());

  // Run the test for longer than the reconstruction timeout again.
  Run(reconstruction_timeout_ms_ * 1.1);
  // Check that this time, reconstruction is triggered, since we did not
  // receive a task lease notification from the other node yet and our next
  // attempt to reconstruct adds an entry at the next index in the
  // TaskReconstructionLog.
  ASSERT_EQ(reconstructed_tasks_[task_id], 1);
}

}  // namespace raylet

}  // namespace ray

int main(int argc, char **argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}

ext: cc

max_stars_repo_name: opherlieber/ray

max_issues_repo_name: opherlieber/ray

max_forks_repo_name: opherlieber/ray


text:
#include <Eigen/Core>
#include <chrono>
#include <fstream>
#include <iostream>

int main() {
  using namespace std;
  using namespace std::chrono;
  using namespace Eigen;
  int height = 1080;
  int width = 1920;
  int sv_count = 30;

  system_clock::time_point start, stop;
  double elapsed_time;

  int u_buffer_size = 3 * height * sv_count * sizeof(float);
  float *u_buffer = (float *)malloc(u_buffer_size);
  int v_buffer_size = 3 * sv_count * width * sizeof(float);
  float *v_buffer = (float *)malloc(v_buffer_size);
  int sv_buffer_size = 3 * sv_count * sizeof(float);
  float *sv_buffer = (float *)malloc(sv_buffer_size);
  int svd_buffer_size = 3 * width * height * sizeof(float);
  float *svd_buffer = (float *)malloc(svd_buffer_size);

  start = high_resolution_clock::now();
  std::ifstream svd_file("SVD_metadata_10/1.bin", std::ios::binary);
  svd_file.read((char *)sv_buffer, sv_buffer_size);
  svd_file.read((char *)u_buffer, u_buffer_size);
  svd_file.read((char *)v_buffer, v_buffer_size);
  svd_file.close();
  stop = high_resolution_clock::now();
  elapsed_time = duration<double, std::milli>(stop - start).count();
  cout << "Time to read file: " << elapsed_time << endl;

  Map<Matrix<float, Dynamic, Dynamic, RowMajor>> eigen_u_buffer(
      u_buffer, 3 * height, sv_count);
  Map<Matrix<float, Dynamic, Dynamic, RowMajor>> eigen_v_buffer(
      v_buffer, 3 * sv_count, width);
  Map<Matrix<float, Dynamic, 1>> eigen_svd_buffer(svd_buffer, 3 * sv_count);
  start = high_resolution_clock::now();
  for (int color = 0; color < 3; color++) {
    Map<Matrix<float, Dynamic, Dynamic, RowMajor>, 0, InnerStride<3>> eigen_svd(
        svd_buffer + color, height, width);
    eigen_svd =
        eigen_u_buffer.block(color * height, 0, height, sv_count) *
        eigen_svd_buffer.segment(color * sv_count, sv_count).asDiagonal() *
        eigen_v_buffer.block(color * sv_count, 0, sv_count, width);
  }
  stop = high_resolution_clock::now();
  elapsed_time = duration<double, std::milli>(stop - start).count();
  cout << "Time to recover SAT: " << elapsed_time << endl;

  free(u_buffer);
  free(v_buffer);
  free(sv_buffer);
  free(svd_buffer);
}


text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "util/default-path-handlers.h"

#include <sstream>
#include <fstream>
#include <sys/stat.h>
#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <gperftools/malloc_extension.h>
#include <gutil/strings/substitute.h>

#include "common/logging.h"
#include "runtime/mem-tracker.h"
#include "util/debug-util.h"
#include "util/pprof-path-handlers.h"
#include "util/webserver.h"

#include "common/names.h"

using namespace google;
using namespace impala;
using namespace rapidjson;
using namespace strings;

DECLARE_bool(enable_process_lifetime_heap_profiling);
DEFINE_int64(web_log_bytes, 1024 * 1024,
    "The maximum number of bytes to display on the debug webserver's log page");

// Writes the last FLAGS_web_log_bytes of the INFO logfile to a webpage
// Note to get best performance, set GLOG_logbuflevel=-1 to prevent log buffering
void LogsHandler(const Webserver::ArgumentMap& args, Document* document) {
  string logfile;
  impala::GetFullLogFilename(google::INFO, &logfile);
  Value log_path(logfile.c_str(), document->GetAllocator());
  document->AddMember("logfile", log_path, document->GetAllocator());

  struct stat file_stat;
  if (stat(logfile.c_str(), &file_stat) == 0) {
    long size = file_stat.st_size;
    long seekpos = size < FLAGS_web_log_bytes ? 0L : size - FLAGS_web_log_bytes;
    ifstream log(logfile.c_str(), ios::in);
    // Note if the file rolls between stat and seek, this could fail (and we could wind up
    // reading the whole file). But because the file is likely to be small, this is
    // unlikely to be an issue in practice.
    log.seekg(seekpos);
    document->AddMember("num_bytes", FLAGS_web_log_bytes, document->GetAllocator());
    stringstream ss;
    ss << log.rdbuf();
    Value log_json(ss.str().c_str(), document->GetAllocator());
    document->AddMember("log", log_json, document->GetAllocator());
  } else {
    Value error(Substitute("Couldn't open INFO log file: $0", logfile).c_str(),
        document->GetAllocator());
    document->AddMember("error", error, document->GetAllocator());
  }
}

// Registered to handle "/flags", and produces json containing an array of flag metadata
// objects:
//
// "title": "Command-line Flags",
// "flags": [
//     {
//       "name": "catalog_service_port",
//       "type": "int32",
//       "description": "port where the CatalogService is running",
//       "default": "26000",
//       "current": "26000"
//     },
// .. etc
void FlagsHandler(const Webserver::ArgumentMap& args, Document* document) {
  vector<CommandLineFlagInfo> flag_info;
  GetAllFlags(&flag_info);
  Value flag_arr(kArrayType);
  for (const CommandLineFlagInfo& flag: flag_info) {
    Value flag_val(kObjectType);
    Value name(flag.name.c_str(), document->GetAllocator());
    flag_val.AddMember("name", name, document->GetAllocator());

    Value type(flag.type.c_str(), document->GetAllocator());
    flag_val.AddMember("type", type, document->GetAllocator());

    Value description(flag.description.c_str(), document->GetAllocator());
    flag_val.AddMember("description", description, document->GetAllocator());

    Value default_value(flag.default_value.c_str(), document->GetAllocator());
    flag_val.AddMember("default", default_value, document->GetAllocator());

    Value current_value(flag.current_value.c_str(), document->GetAllocator());
    flag_val.AddMember("current", current_value, document->GetAllocator());

    if (!flag.is_default) {
      flag_val.AddMember("value_changed", 1, document->GetAllocator());
    }
    flag_arr.PushBack(flag_val, document->GetAllocator());
  }
  Value title("Command-line Flags", document->GetAllocator());
  document->AddMember("title", title, document->GetAllocator());
  document->AddMember("flags", flag_arr, document->GetAllocator());
}

// Registered to handle "/memz"
void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group,
    const Webserver::ArgumentMap& args, Document* document) {
  DCHECK(mem_tracker != NULL);
  Value mem_limit(PrettyPrinter::Print(mem_tracker->limit(), TUnit::BYTES).c_str(),
      document->GetAllocator());
  document->AddMember("mem_limit", mem_limit, document->GetAllocator());
  Value consumption(
      PrettyPrinter::Print(mem_tracker->consumption(), TUnit::BYTES).c_str(),
      document->GetAllocator());
  document->AddMember("consumption", consumption, document->GetAllocator());

  stringstream ss;
#ifdef ADDRESS_SANITIZER
  ss << "Memory tracking is not available with address sanitizer builds.";
#else
  char buf[2048];
  MallocExtension::instance()->GetStats(buf, 2048);
  ss << string(buf);
#endif

  Value overview(ss.str().c_str(), document->GetAllocator());
  document->AddMember("overview", overview, document->GetAllocator());

  // Dump all mem trackers.
  Value detailed(mem_tracker->LogUsage().c_str(), document->GetAllocator());
  document->AddMember("detailed", detailed, document->GetAllocator());

  if (metric_group != NULL) {
    MetricGroup* jvm_group = metric_group->FindChildGroup("jvm");
    if (jvm_group != NULL) {
      Value jvm(kObjectType);
      jvm_group->ToJson(false, document, &jvm);
      Value heap(kArrayType);
      Value non_heap(kArrayType);
      Value total(kArrayType);
      for (SizeType i = 0; i < jvm["metrics"].Size(); ++i) {
        if (strstr(jvm["metrics"][i]["name"].GetString(), "total") != nullptr) {
          total.PushBack(jvm["metrics"][i], document->GetAllocator());
        } else if (strstr(jvm["metrics"][i]["name"].GetString(), "non-heap") != nullptr) {
          non_heap.PushBack(jvm["metrics"][i], document->GetAllocator());
        } else if (strstr(jvm["metrics"][i]["name"].GetString(), "heap") != nullptr) {
          heap.PushBack(jvm["metrics"][i], document->GetAllocator());
        }
      }
      document->AddMember("jvm_total", total, document->GetAllocator());
      document->AddMember("jvm_heap", heap, document->GetAllocator());
      document->AddMember("jvm_non_heap", non_heap, document->GetAllocator());
    }
  }
}

void impala::AddDefaultUrlCallbacks(
    Webserver* webserver, MemTracker* process_mem_tracker, MetricGroup* metric_group) {
  webserver->RegisterUrlCallback("/logs", "logs.tmpl", LogsHandler);
  webserver->RegisterUrlCallback("/varz", "flags.tmpl", FlagsHandler);
  if (process_mem_tracker != NULL) {
    auto callback = [process_mem_tracker, metric_group]
        (const Webserver::ArgumentMap& args, Document* doc) {
      MemUsageHandler(process_mem_tracker, metric_group, args, doc);
    };
    webserver->RegisterUrlCallback("/memz", "memz.tmpl", callback);
  }

#ifndef ADDRESS_SANITIZER
  // Remote (on-demand) profiling is disabled if the process is already being profiled.
  if (!FLAGS_enable_process_lifetime_heap_profiling) {
    AddPprofUrlCallbacks(webserver);
  }
#endif
}

max_stars_repo_name: michaelhkw/impala

max_issues_repo_name: michaelhkw/impala

max_forks_repo_name: michaelhkw/impala


text:
#include <boost/fiber/fiber.hpp>

#include <iostream>

auto main() -> int
{
    const auto print = [](){
        std::cout << "fiber" << std::endl;
    };

    auto f = boost::fibers::fiber{print};
    f.join();

    return 0;
}

ext: cpp


text:
/*
 * Copyright 2020-present Ksyun
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "cgvhid.h"

#include <atlfile.h>

#include <boost/scope_exit.hpp>

#pragma comment(lib, "hid.lib")
#pragma comment(lib, "SetupAPI.lib")

HANDLE Cgvhid::Open(USAGE usage_page, USAGE usage) noexcept {
  GUID hid_guid;
  HidD_GetHidGuid(&hid_guid);
  HDEVINFO dev_info = SetupDiGetClassDevs(
      &hid_guid, nullptr, nullptr, DIGCF_PRESENT | DIGCF_INTERFACEDEVICE);
  if (INVALID_HANDLE_VALUE == dev_info) {
    ATLTRACE2(atlTraceException, 0, "!SetupDiGetClassDevs() #%u\n",
              GetLastError());
    return dev_info;
  }
  BOOST_SCOPE_EXIT_ALL(&) { SetupDiDestroyDeviceInfoList(dev_info); };

  SP_DEVICE_INTERFACE_DATA device_interface_data{};
  device_interface_data.cbSize = sizeof(device_interface_data);
  for (DWORD i = 0; SetupDiEnumDeviceInterfaces(dev_info, nullptr, &hid_guid, i,
                                                &device_interface_data);
       ++i) {
    HANDLE h =
        OpenInterface(dev_info, &device_interface_data, usage_page, usage);
    if (INVALID_HANDLE_VALUE != h) {
      return h;
    }
  }

  ATLTRACE2(atlTraceException, 0, "Can't find our HID device!\n");
  return INVALID_HANDLE_VALUE;
}

HANDLE Cgvhid::OpenInterface(HDEVINFO dev_info,
                             PSP_DEVICE_INTERFACE_DATA device_interface_data,
                             USAGE usage_page,
                             USAGE usage) noexcept {
  DWORD allocate_length = 0;
  SetupDiGetDeviceInterfaceDetail(dev_info, device_interface_data, nullptr, 0,
                                  &allocate_length, nullptr);
  DWORD error_code = GetLastError();
  if (ERROR_INSUFFICIENT_BUFFER != error_code) {
    ATLTRACE2(atlTraceException, 0, "!SetupDiGetDeviceInterfaceDetail() #%u\n",
              error_code);
    return INVALID_HANDLE_VALUE;
  }

  CHeapPtr<SP_DEVICE_INTERFACE_DETAIL_DATA> data;
  if (!data.AllocateBytes(allocate_length)) {
    error_code = GetLastError();
    ATLTRACE2(atlTraceException, 0, "!AllocateBytes() #%u\n", error_code);
    return INVALID_HANDLE_VALUE;
  }

  data->cbSize = sizeof(SP_DEVICE_INTERFACE_DETAIL_DATA);
  DWORD length = 0;
  if (!SetupDiGetDeviceInterfaceDetail(dev_info, device_interface_data, data,
                                       allocate_length, &length, nullptr)) {
    error_code = GetLastError();
    ATLTRACE2(atlTraceException, 0, "!SetupDiGetDeviceInterfaceDetail() #%u\n",
              error_code);
    return INVALID_HANDLE_VALUE;
  }

  CAtlFile file;
  HRESULT hr = file.Create(data->DevicePath, GENERIC_READ | GENERIC_WRITE,
                           FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING);
  if (FAILED(hr)) {
    ATLTRACE2(atlTraceException, 0, "!Create(%ws) #0x%08x\n", data->DevicePath,
              hr);
    return INVALID_HANDLE_VALUE;
  }

  HIDD_ATTRIBUTES attributes;
  if (!HidD_GetAttributes(file, &attributes)) {
    ATLTRACE2(atlTraceException, 0, "!HidD_GetAttributes()\n");
    return INVALID_HANDLE_VALUE;
  }

  if (attributes.VendorID == VHID_VENDOR_ID &&
      attributes.ProductID == VHID_PRODUCT_ID) {
    PHIDP_PREPARSED_DATA pd = nullptr;
    if (!HidD_GetPreparsedData(file, &pd)) {
      ATLTRACE2(atlTraceException, 0, "!HidD_GetPreparsedData()\n");
      return INVALID_HANDLE_VALUE;
    }
    BOOST_SCOPE_EXIT_ALL(&) { HidD_FreePreparsedData(pd); };

    HIDP_CAPS caps{};
    if (!HidP_GetCaps(pd, &caps)) {
      ATLTRACE2(atlTraceException, 0, "!HidP_GetCaps()\n");
      return INVALID_HANDLE_VALUE;
    }
    if ((caps.UsagePage == usage_page) && (caps.Usage == usage)) {
      ATLTRACE2(atlTraceUtil, 0, "OK: %ws\n", data->DevicePath);
      return file.Detach();
    }
    ATLTRACE2(atlTraceUtil, 0, "Ours, but not the one: %ws\n",
              data->DevicePath);
  }

  return INVALID_HANDLE_VALUE;
}

bool Cgvhid::Init() noexcept {
  if (!control_) {
    HANDLE h = Open(0xff00, 0x0001);
    if (INVALID_HANDLE_VALUE == h) {
      return false;
    }
    control_.Attach(h);
  }

  if (!message_) {
    HANDLE h = Open(0xff00, 0x0002);
    if (INVALID_HANDLE_VALUE == h) {
      return false;
    }
    message_.Attach(h);

    //
    // Set the buffer count to 10 on the message HID
    //
    if (!HidD_SetNumInputBuffers(message_, 10)) {
      Free();
      ATLTRACE2(atlTraceException, 0, "!HidD_SetNumInputBuffers()\n");
      return false;
    }
  }

  return true;
}

void Cgvhid::Free() noexcept {
  control_.Close();
  message_.Close();
}

int Cgvhid::KeyboardUpdate(uint8_t modifiers,
                           uint8_t key_codes[KEYBD_MAX_KEY_COUNT]) noexcept {
  static_assert(CONTROL_REPORT_SIZE >
                    sizeof(VhidControlReport) + sizeof(VhidKeyboardReport),
                "CONTROL_REPORT_SIZE is too small");
  static_assert(sizeof(key_codes) >= KEYBD_MAX_KEY_COUNT,
                "invalid key_codes size");

  auto control_report = GetControlReport();
  control_report->id = REPORT_ID_CONTROL;
  control_report->len = sizeof(VhidKeyboardReport);

  auto keyboard_report = GetKeyboardReport();
  keyboard_report->id = REPORT_ID_KEYBOARD;
  keyboard_report->modifiers = modifiers;
  static_assert(sizeof(keyboard_report->key_codes) == KEYBD_MAX_KEY_COUNT,
                "key_codes size not equal");
  memcpy(keyboard_report->key_codes, key_codes,
         sizeof(keyboard_report->key_codes));

  return WriteControlReport();
}

int Cgvhid::WriteControlReport() noexcept {
  ULONG written = 0;
  if (!WriteFile(control_, control_report_, CONTROL_REPORT_SIZE, &written,
                 nullptr)) {
    int error_code = GetLastError();
    ATLTRACE2(atlTraceException, 0, "!WriteFile(), #%d\n", error_code);
    return error_code;
  }
  ATLTRACE2(atlTraceUtil, 0, "WriteFile() written %d\n", written);
  return 0;
}


text:
#ifndef UUID_3EC30B5B_ABF6_4EFD_94F4_CC95DF4F9A2D
#define UUID_3EC30B5B_ABF6_4EFD_94F4_CC95DF4F9A2D
#pragma once

#include <cstring>
#include <string>
#include <algorithm>
#include <boost/operators.hpp>

namespace Neptools
{

template <size_t N>
class FixedString :
    boost::totally_ordered<FixedString<N>,
        boost::totally_ordered<FixedString<N>, const char*>>
{
public:
    FixedString() = default;
    explicit FixedString(const char* cstr) { *this = cstr; }
    explicit FixedString(const std::string& sstr) { *this = sstr; }

    FixedString& operator=(const char* cstr)
    { strncpy(str, cstr, N-1); str[N-1] = '\0'; return *this; }

    FixedString& operator=(const std::string& sstr)
    {
        auto copied = std::min(sstr.size(), N-1);
        memcpy(str, sstr.data(), copied);
        memset(str+copied, 0, N-copied);
        return *this;
    }

    bool is_valid() const noexcept
    {
        auto len = strnlen(str, N-1);
        return str[len] == '\0';
    }

    char& operator[](size_t i) noexcept { return str[i]; }
    const char& operator[](size_t i) const noexcept { return str[i]; }

    const char* data() const noexcept { return str; }
    const char* c_str() const noexcept { return str; }

    bool empty() const noexcept { return str[0] == 0; }
    size_t size() const noexcept { return strnlen(str, N); }
    size_t length() const noexcept { return strnlen(str, N); }
    size_t max_size() const noexcept { return N-1; }

    bool operator==(const FixedString& o) const noexcept
    { return strncmp(str, o.str, N) == 0; }
    bool operator<(const FixedString& o) const noexcept
    { return strncmp(str, o.str, N) < 0; }

    bool operator==(const char* o) const noexcept
    { return strcmp(str, o) == 0; }
    bool operator<(const char* o) const noexcept
    { return strcmp(str, o) < 0; }

    template <size_t M>
    bool operator==(const FixedString<M>& o) const noexcept
    { return strcmp(str, o.str) == 0; }
    template <size_t M>
    bool operator<=(const FixedString<M>& o) const noexcept
    { return strcmp(str, o.str) <= 0; }
private:
    char str[N];
};

template <size_t N>
inline std::ostream& operator<<(std::ostream& os, const FixedString<N>& str)
{ return os << str.data(); }

/*
template <size_t N>
inline std::istream& operator>>(std::istream& is, FixedString<N>& str)
{ is.width(N-1); return is >> &str[0]; }
*/

}
#endif

ext: hpp

max_stars_repo_name: clienthax/neptools

max_issues_repo_name: clienthax/neptools

max_forks_repo_name: clienthax/neptools


text:
#include <boost/test/unit_test.hpp>
#include <cstdlib>

using boost::unit_test::test_suite;

void Vektor3Test1() { }

test_suite* Vektor3_test_suite()
{
  test_suite *test = BOOST_TEST_SUITE("Vektor3 test suite");
  test->add(BOOST_TEST_CASE(&Vektor3Test1));

  return test;
}

test_suite* init_unit_test_suite(int, char *[])
{
  std::system("true");
  // leads to "Test setup error: child has exited; pid: 1001; uid: 30540; exit value: 0"

  test_suite *test = BOOST_TEST_SUITE("Master test suite");
  test->add(Vektor3_test_suite());
  return test;
}


text:
#include <boost/config.hpp>

//
//  bind_fwd2_test.cpp - forwarding test for 2 arguments
//
//  Copyright (c) 2015 Peter Dimov
//
//  Distributed under the Boost Software License, Version 1.0.
//  See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt
//

#include <boost/bind/bind.hpp>
#include <boost/core/lightweight_test.hpp>

using namespace boost::placeholders;

//

int fv1( int const & a )
{
    return a;
}

void fv2_1( int & a, int const & b )
{
    a = b;
}

void fv2_2( int const & a, int & b )
{
    b = a;
}

int fv2_3( int const & a, int const & b )
{
    return a+b;
}

void test()
{
    {
        int const a = 1;
        int r = boost::bind( fv1, _1 )( a );
        BOOST_TEST( r == 1 );
    }

    {
        int r = boost::bind( fv1, _1 )( 1 );
        BOOST_TEST( r == 1 );
    }

    {
        int a = 1;
        int const b = 2;

        boost::bind( fv2_1, _1, _2 )( a, b );

        BOOST_TEST( a == 2 );
    }

    {
        int a = 1;

        boost::bind( fv2_1, _1, _2 )( a, 2 );

        BOOST_TEST( a == 2 );
    }

    {
        int const a = 1;
        int b = 2;

        boost::bind( fv2_2, _1, _2 )( a, b );

        BOOST_TEST( b == 1 );
    }

    {
        int b = 2;

        boost::bind( fv2_2, _1, _2 )( 1, b );

        BOOST_TEST( b == 1 );
    }

    {
        int const a = 1;
        int const b = 2;

        int r = boost::bind( fv2_3, _1, _2 )( a, b );

        BOOST_TEST( r == 3 );
    }

    {
        int const a = 1;

        int r = boost::bind( fv2_3, _1, _2 )( a, 2 );

        BOOST_TEST( r == 3 );
    }

    {
        int const b = 2;

        int r = boost::bind( fv2_3, _1, _2 )( 1, b );

        BOOST_TEST( r == 3 );
    }

    {
        int r = boost::bind( fv2_3, _1, _2 )( 1, 2 );

        BOOST_TEST( r == 3 );
    }
}

int main()
{
    test();
    return boost::report_errors();
}

max_issues_repo_name: qingkouwei/mediaones


text:
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef avro_SymbolMap_hh__
#define avro_SymbolMap_hh__

#include <map>
#include <boost/noncopyable.hpp>

#include "Node.hh"
#include "Schema.hh"

namespace avro {

/// Avro schemas can include types that were previously defined with names in
/// the same avro schema.  In order to identify new types, they are stored in a
/// map so that the actual type may be identified by name.  This class
/// implements the symbolic name to node mapping.
///

class SymbolMap : private boost::noncopyable
{

  public:

    SymbolMap()
    {}

    bool registerSymbol(const NodePtr &node) {

        if(node->type() == AVRO_SYMBOLIC) {
            throw Exception("Node must not be a symbolic name");
        }
        const std::string name = node->name();
        if(name.empty()) {
            throw Exception("Node must have a name to be registered");
        }
        bool added = false;
        MapImpl::iterator lb = map_.lower_bound(name);

        if(lb == map_.end() || map_.key_comp()(name, lb->first)) {
            map_.insert(lb, std::make_pair(name, node));
            added = true;
        }
        return added;
    }

    bool hasSymbol(const std::string &name) const {
        return map_.find(name) != map_.end();
    }

    NodePtr locateSymbol(const std::string &name) const {
        MapImpl::const_iterator iter = map_.find(name);
        return (iter == map_.end()) ? NodePtr() : iter->second;
    }

  private:

    typedef std::map<std::string, NodePtr> MapImpl;

    MapImpl map_;
};

} // namespace avro

#endif

ext: hh

max_stars_repo_name: SpotifyUsedOpenSourceCode/avro

max_issues_repo_name: SpotifyUsedOpenSourceCode/avro

max_forks_repo_name: SpotifyUsedOpenSourceCode/avro


text:
#include "HttpServer.h"

#include "RequestHandlerFactory.h"
#include "Configuration.h"

#include <Poco/Util/Option.h>
#include <Poco/Util/HelpFormatter.h>
#include <Poco/Net/ServerSocket.h>
#include <Poco/Net/HTTPServer.h>
#include <Poco/Net/HTTPServerParams.h>

#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>

#include <iostream>
#include <cstdlib>

using Poco::Util::Option;
using Poco::Util::HelpFormatter;
using Poco::Net::HTTPServerParams;
using Poco::Net::ServerSocket;
using Poco::Net::HTTPServer;

const std::string  DefaultConfFile = "server.xml";
const std::string DefaultWebConfPath = "vhosts.d";
const char* TESTING_ENVIRONMENT_NAME = "HTTP_SERVER_DEBUG_TESTING";

void HttpServer::Blocker::wait(HttpServer* server)
{
	assert(server);
	server->waitForTerminationRequest();
}

HttpServer::HttpServer(std::shared_ptr<Blocker> blocker) :
 m_blocker(blocker),
 m_helpRequested(false),
 m_testArgs(false)
{
}

HttpServer::~HttpServer()
{
}

void HttpServer::initialize(Application &self)
{
    loadConfiguration(); // load default configuration files, if present
    ServerApplication::initialize(self);
}

void HttpServer::uninitialize()
{
    ServerApplication::uninitialize();
}

void HttpServer::defineOptions(Poco::Util::OptionSet &options)
{
    ServerApplication::defineOptions(options);

	options.addOption(
		Option("configuration", "c", "Path to configuration directory")
		.required(false)
		.repeatable(false)
		.argument("dir")
	);

	options.addOption(
		Option("port", "p", "Port")
		.required(false)
		.repeatable(false)
		.argument("port")
	);
	options.addOption(
		Option("test", "t", "Test")
		.required(false)
		.repeatable(false)
	);

	options.addOption(
		Option("help", "h", "Display help information on command line arguments")
		.required(false)
		.repeatable(false)
		.callback(Poco::Util::OptionCallback<HttpServer>(this, &HttpServer::handleHelp))
	);
}

void HttpServer::handleOption(const std::string &name, const std::string &value)
{
    ServerApplication::handleOption(name, value);

	//Poco::Util::Application &app = Poco::Util::Application::instance();

	if (name == "help") {
		m_helpRequested = true;
	}  else if (name == "configuration") {
		m_confDir = value;
	} else if (name == "port") {
		m_configuration.m_port = std::stoi(value);
	} else if (name == "test") {
		m_testArgs = true;
	}
}

void HttpServer::displayHelp()
{
    HelpFormatter helpFormatter(options());
    helpFormatter.setCommand(commandName());
    helpFormatter.setUsage("OPTIONS");
    helpFormatter.setHeader("A web server that show content of directories.");
    helpFormatter.format(std::cout);
}

void HttpServer::handleHelp(const std::string &, const std::string &)
{
    m_helpRequested = true;
    displayHelp();
    stopOptionsProcessing();
}

int HttpServer::loadServerConfiguration()
{
	if (!m_confDir.empty()) {
		int port = m_configuration.m_port;
		if (!m_configuration.load(m_confDir + boost::filesystem::path::preferred_separator + DefaultConfFile)) {
			return Application::EXIT_CONFIG;
		}

		if (port != 0) {
			m_configuration.m_port = port;
		}
	}

	if (!m_configuration.validate()) {
		return Application::EXIT_CONFIG;
	}

	return Application::EXIT_OK;
}

int HttpServer::loadWebDirectoryConfiguration(const std::vector<std::string> & args)
{
	Poco::Util::Application &app = Poco::Util::Application::instance();

	if (!m_confDir.empty()) {
		const std::string dir = m_confDir + boost::filesystem::path::preferred_separator + DefaultWebConfPath;

		if (!loadWebDirectories(dir, m_webDirectories)) {
			return Application::EXIT_CONFIG;
		}
	}

	std::for_each (args.begin(), args.end(), [this, &app](const std::string& path) {
		WebDirectory dir;

		dir.m_path = path;
		dir.m_followSymLink = true;
		dir.m_showContent = true;

		if (dir.validate()) {
			m_webDirectories.push_back(dir);
		} else {
			app.logger().warning("Directory \"" + path + "\" from command line ignored, as it doesn't validated");
		}
	});

	return Application::EXIT_OK;
}

int HttpServer::port() const
{
	return m_configuration.m_port;
}

int HttpServer::main(const std::vector<std::string> & args)
{
	int status;

    if (m_helpRequested)
        return Application::EXIT_USAGE;

	status = loadServerConfiguration();
	if (status != Application::EXIT_OK)
		return status;

	status = loadWebDirectoryConfiguration(args);
	if (status != Application::EXIT_OK)
		return status;

    HTTPServerParams *params = new HTTPServerParams;
    //params->setServerName(ServerName);
    //params->setSoftwareVersion(ServerVersion);

	m_configuration.convert(params);

    // set-up a server socket
    ServerSocket svs(m_configuration.m_port);

    // set-up a HTTPServer instance
    HTTPServer srv(new RequestHandlerFactory(m_webDirectories), svs, params);

	if (m_testArgs) {
		return Application::EXIT_OK;
	}

	// start the HTTPServer
    srv.start();

	assert(m_blocker.get());
    // wait for CTRL-C or kill
    m_blocker->wait(this);
    // Stop the HTTPServer
    srv.stop();

    return Application::EXIT_OK;
}

max_stars_repo_name: feniksa/tinyhttpd

max_issues_repo_name: feniksa/tinyhttpd

max_forks_repo_name: feniksa/tinyhttpd


text:
#include <cassert>
#include <cstdlib> // abs
#include <cstdint> // uint32_t
#include <ios>
#include <iomanip>
#include <unordered_map>
#include <utility>

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/log/trivial.hpp>
#include <boost/serialization/map.hpp>
#include <boost/serialization/set.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/unordered_map.hpp>

#include <graphtyper/graph/absolute_position.hpp>
#include <graphtyper/graph/graph.hpp>
#include <graphtyper/graph/reference_depth.hpp>
#include <graphtyper/graph/var_record.hpp>
#include <graphtyper/typer/variant.hpp>
#include <graphtyper/typer/variant_map.hpp>
#include <graphtyper/typer/variant_support.hpp>
#include <graphtyper/typer/vcf.hpp>
#include <graphtyper/utilities/io.hpp>
#include <graphtyper/utilities/options.hpp>
#include <graphtyper/utilities/type_conversions.hpp>

namespace
{

std::pair<double, long>
get_abs_dev_strand_bias(std::vector<gyper::VariantSupport> const & supports)
{
  long total_depth = 0;
  long total_reversed = 0;

  for (auto const & sup : supports)
  {
    total_depth += sup.lq_support + sup.hq_support;
    total_reversed += sup.sequence_reversed;
  }

  assert(total_depth > 0);
  double abs_dev_sb = std::abs(static_cast<double>(total_reversed) / static_cast<double>(total_depth) - 0.5);
  return std::pair<double, long>(abs_dev_sb, total_depth);
}

std::pair<double, long>
get_abs_dev_read_bias(std::vector<gyper::VariantSupport> const & supports)
{
  long total_depth = 0;
  long total_first_read = 0;

  for (auto const & sup : supports)
  {
    total_depth += sup.lq_support + sup.hq_support;
    total_first_read += sup.first_in_pairs;
  }

  double abs_dev_rb = std::abs(static_cast<double>(total_first_read) / static_cast<double>(total_depth) - 0.5);
  return std::pair<double, long>(abs_dev_rb, total_depth);
}

} // anon namespace

namespace gyper
{

void
VariantMap::add_variants(std::vector<VariantCandidate> && vars, long const sample_index)
{
  assert(varmaps.size() > 0);
  assert(sample_index < static_cast<long>(varmaps.size()));
  auto & varmap = varmaps[sample_index];

  for (auto & var : vars)
  {
    assert(var.seqs.size() >= 2);
    assert(var.is_normalized());
    assert(var.seqs[0].size() > 0);
    assert(var.seqs[1].size() > 0);

    // Extract all required information
    uint32_t const ORIGINAL_POS = var.original_pos;

    auto it = varmap.find(var);

    if (it == varmap.end())
    {
      it = varmap.insert(std::make_pair<VariantCandidate, VariantSupport>(
                           VariantCandidate(var),
                           VariantSupport())).first;

      // Expand to learn the true size
      it->second.is_indel = var.seqs[0].size() != var.seqs[1].size();
      long const old_size = std::max(var.seqs[0].size(), var.seqs[1].size()) - 1;
      var.expanded_normalized();
      assert(var.seqs[0].size() > 0);
      assert(var.seqs[1].size() > 0);
      it->second.var_size = std::max(var.seqs[0].size(), var.seqs[1].size()) - 1;
      it->second.growth = std::max(0l, static_cast<long>(it->second.var_size) - old_size);
    }

    ++it->second.depth; // Increment depth
    it->second.lq_support += ((var.flags & IS_LOW_BASE_QUAL) != 0);
    it->second.hq_support += ((var.flags & IS_LOW_BASE_QUAL) == 0);
    it->second.proper_pairs += ((var.flags & IS_PROPER_PAIR) != 0);

    if ((var.flags & IS_MAPQ_BAD) == 0)
      it->second.is_any_mapq_good = true;

    it->second.first_in_pairs += ((var.flags & IS_FIRST_IN_PAIR) != 0);
    it->second.sequence_reversed += ((var.flags & IS_SEQ_REVERSED) != 0);
    it->second.clipped += ((var.flags & IS_CLIPPED) != 0);
    it->second.unique_positions.insert(ORIGINAL_POS);
  }
}

void
VariantMap::create_varmap_for_all(ReferenceDepth const & reference_depth)
{
  assert(varmaps.size() == reference_depth.depths.size());
  long const NUM_SAMPLES = static_cast<long>(varmaps.size());

  for (long i = 0; i < NUM_SAMPLES; ++i)
  {
    auto & varmap = varmaps[i];
    long new_min_support = minimum_variant_support;

    if (varmap.size() > 50)
    {
      while (new_min_support < 15)
      {
        long transitions = 0;
        long transversions = 0;
        long num_above_threshold = 0;

        for (auto it = varmap.cbegin(); it != varmap.cend(); ++it)
        {
          if (it->second.is_support_above_cutoff(new_min_support))
          {
            ++num_above_threshold;
            int ret = it->first.is_transition_or_transversion();

            if (ret == 1)
              ++transitions;
            else if (ret == 2)
              ++transversions;
          }
        }

        if ((num_above_threshold > 50 && transversions > transitions) ||
            (num_above_threshold > 75 &&
             (static_cast<double>(transitions) / static_cast<double>(transversions)) < 1.2) ||
            (num_above_threshold > 150 &&
             (static_cast<double>(transitions) / static_cast<double>(transversions)) < 1.3))
        {
          ++new_min_support;
        }
        else
        {
          break;
        }
      } // while(true)

      #ifndef NDEBUG
      if (new_min_support > minimum_variant_support)
        BOOST_LOG_TRIVIAL(debug) << "[graphtyper::variant_map] Min support increased to " << new_min_support;
      #endif // NDEBUG
    } // if (varmap.size() > 50)

    double const min_ratio = minimum_variant_support_ratio;

    for (auto map_it = varmap.begin(); map_it != varmap.end(); ++map_it)
    {
      VariantCandidate const & var = map_it->first;
      VariantSupport & new_var_support = map_it->second;

      // Check if support is above cutoff and some reasonable hard filters
      if (new_var_support.is_support_above_cutoff(new_min_support))
      {
        new_var_support.set_depth(reference_depth.get_read_depth(map_it->first, i));

        // Check if ratio is above cutoff
        if (new_var_support.is_ratio_above_cutoff(min_ratio))
        {
          // In this case, we can add the variant
          #ifndef NDEBUG
          new_var_support.pn_index = static_cast<uint32_t>(i);
          #endif // NDEBUG

          pool_varmap[var].push_back(new_var_support);
        }
      }
    }
  } // for (long i = 0; i < NUM_SAMPLES; ++i)

#ifndef NDEBUG
  BOOST_LOG_TRIVIAL(debug) << "Pool varmap size " << pool_varmap.size();

  for (auto it = pool_varmap.begin(); it != pool_varmap.end(); ++it)
  {
    BOOST_LOG_TRIVIAL(debug) << it->first.print();
  }
#endif // NDEBUG
}

void
VariantMap::filter_varmap_for_all()
{
  BOOST_LOG_TRIVIAL(debug) << "[graphtyper::variant_map] Number of variants above minimum cutoff is "
                           << pool_varmap.size();

  // No need to filter no variants, in fact it will segfault
  if (pool_varmap.size() == 0)
    return;

  // Filter on strand bias
  //*
  for (auto it = pool_varmap.begin(); it != pool_varmap.end();)  // no increment
  {
    // Skip read bias on indels
    bool is_any_hq = false;

    for (auto const & sup : it->second)
    {
      if (sup.hq_support >= 5 && sup.proper_pairs >= 5)
      {
        is_any_hq = true;
        break;
      }
    }

    assert(it->second.size() > 0);
    bool const is_indel = it->second[0].is_indel;

    // Filter on strand bias
    if (!is_any_hq)
    {
      std::pair<double, long> abs_dev_sb_depth = get_abs_dev_strand_bias(it->second);
      double abs_dev_sb = abs_dev_sb_depth.first;
      long depth = abs_dev_sb_depth.second;

      if (is_indel && abs_dev_sb > 0.07)
        abs_dev_sb -= 0.07;

      if ((abs_dev_sb > 0.49999) ||
          (abs_dev_sb > 0.45 && depth > 30) ||
          (abs_dev_sb > 0.40 && depth > 80) ||
          (abs_dev_sb > 0.37 && depth > 200) ||
          (abs_dev_sb > 0.34 && depth > 500))
      {
#ifndef NDEBUG
        BOOST_LOG_TRIVIAL(debug) << "Strand bias removed " << it->first.print() << " " << abs_dev_sb << " " << depth;
#endif // NDEBUG
        it = pool_varmap.erase(it);
        continue;
      }

      // Filter on read bias
      if (!is_indel)
      {
        std::pair<double, long> abs_dev_rb_depth = get_abs_dev_read_bias(it->second);
        double abs_dev_rb = abs_dev_rb_depth.first;
        long depth = abs_dev_rb_depth.second;

        if ((abs_dev_rb > 0.49999 && depth > 10) ||
            (abs_dev_rb > 0.45 && depth > 40) ||
            (abs_dev_rb > 0.40 && depth > 100) ||
            (abs_dev_rb > 0.35 && depth > 500))
        {
#ifndef NDEBUG
          BOOST_LOG_TRIVIAL(debug) << "Read bias removed " << it->first.print() << " " << abs_dev_rb << " " << depth;
#endif // NDEBUG
          it = pool_varmap.erase(it);
          continue;
        }
      }
    }

    ++it;
  }
  //*/

  /// Limit to the number of variants in a 100 bp window
  if (static_cast<long>(pool_varmap.size()) > Options::const_instance()->soft_cap_of_variants_in_100_bp_window)
  {
    long const max_variants_in_100bp_window = Options::const_instance()->soft_cap_of_variants_in_100_bp_window;
    assert(max_variants_in_100bp_window > 0);

    BOOST_LOG_TRIVIAL(debug) << "[graphtyper::variant_map] Soft cap of variants in 100 bp window is "
                             << Options::const_instance()->soft_cap_of_variants_in_100_bp_window;

    // Gather how many variants fall into each bucket (which covers 100 bp)
    std::vector<long> max_scores_in_bucket;
    auto window_it = pool_varmap.begin();
    long current_bucket = window_it->first.abs_pos / 100l;

    auto filter_window =
      [&max_scores_in_bucket, max_variants_in_100bp_window, this](PoolVarMap::iterator window_it)
      {
        BOOST_LOG_TRIVIAL(debug) << "[graphtyper::variant_map] Too many variants! " << max_scores_in_bucket.size();

        // Find the minimum max score for passing a variant in this region
        std::vector<long> sorted_max_scores_in_bucket(max_scores_in_bucket);
        std::sort(sorted_max_scores_in_bucket.begin(), sorted_max_scores_in_bucket.end());
        long const index_with_min_score = sorted_max_scores_in_bucket.size() -
                                          max_variants_in_100bp_window;
        assert(index_with_min_score <= static_cast<long>(sorted_max_scores_in_bucket.size()));
        long const min_score_pass = std::min(50l, sorted_max_scores_in_bucket[index_with_min_score]);

        for (long s{0}; s < static_cast<long>(max_scores_in_bucket.size()); ++s)
        {
          if (max_scores_in_bucket[s] >= min_score_pass)
          {
            ++window_it;
          }
          else
          {
#ifndef NDEBUG
            BOOST_LOG_TRIVIAL(debug) << "Due to too high graph complexity I erased the variant: "
                                     << window_it->first.print()
                                     << ". It had support in " << window_it->second.size()
                                     << " samples and score of " << max_scores_in_bucket[s];
#endif // NDEBUG
            window_it = pool_varmap.erase(window_it);
          }
        }

        return window_it; // return the new iterator
      };

    for (auto map_it = pool_varmap.begin(); map_it != pool_varmap.end(); ++map_it)
    {
      long const bucket = map_it->first.abs_pos / 100l;
      assert(bucket >= current_bucket); // We should never go backwards in order

      if (bucket != current_bucket) // Let's continue looping through the variants until we reach the next bucket
      {
        assert(bucket > current_bucket);

        // We have reached a new bucket, check if there are too many variants in the bucket
        if (static_cast<long>(max_scores_in_bucket.size()) > max_variants_in_100bp_window)
        {
          assert(std::distance(window_it, map_it) == static_cast<long>(max_scores_in_bucket.size()));
          window_it = filter_window(window_it);
          map_it = window_it;
        }
        else
        {
          window_it = map_it;
        }

        // Update current bucket
        current_bucket = bucket;
        max_scores_in_bucket.clear();
      }

      // Add the max variant score
      long max_score{0};

      // Get the max score for this variant
      for (VariantSupport const & supp : map_it->second)
      {
        long const score = supp.get_score();

        if (score > max_score)
          max_score = score;
      }

      max_scores_in_bucket.push_back(max_score);
    }

    // check final bucket
    if (window_it != pool_varmap.end() &&
        static_cast<long>(max_scores_in_bucket.size()) > max_variants_in_100bp_window)
    {
      filter_window(window_it);
    }
  }

  /** Break down variants to check if they are duplicates */
  //std::vector<VariantCandidate> broken_vars_to_add;
  //std::vector<std::vector<VariantSupport> > supports_to_add;

  for (auto map_it = pool_varmap.begin(); map_it != pool_varmap.end();)  // no increment
  {
    VariantCandidate var(map_it->first);

    {
      long constexpr EXTRA_BASES_TO_ADD{5};

      for (long i{0}; i < EXTRA_BASES_TO_ADD; ++i)
        if (!var.add_base_in_front(false)) // false is add_N
          break;

      for (long i{0}; i < EXTRA_BASES_TO_ADD; ++i)
        if (!var.add_base_in_back(false)) // false is add_N
          break;
    }

    Variant var_cp;
    var_cp.abs_pos = var.abs_pos;
    var_cp.seqs = var.seqs;

    // Force that no variant is overlapping
    long const reach{-1};
    bool const is_no_variant_overlapping{true}; // Force it to be true so we wont use skyr
    bool const is_all_biallelic{false};
    Variant new_var(var_cp);
    new_var.infos["SBF1"] = ".";

    std::vector<Variant> new_broken_down_vars =
      break_down_variant(std::move(new_var), reach, is_no_variant_overlapping, is_all_biallelic);

    assert(new_broken_down_vars.size() != 0);

    if (new_broken_down_vars.size() == 1)
    {
      ++map_it; // The variant could not be broken down, move on
      continue;
    }

    for (auto & broken_var : new_broken_down_vars)
      broken_var.normalize();

    // Change Variant -> VariantCandidate
    std::vector<VariantCandidate> new_broken_down_var_candidates(new_broken_down_vars.size());

    for (long i = 0; i < static_cast<long>(new_broken_down_vars.size()); ++i)
    {
      new_broken_down_var_candidates[i].abs_pos = new_broken_down_vars[i].abs_pos;
      new_broken_down_var_candidates[i].seqs = std::move(new_broken_down_vars[i].seqs);
      new_broken_down_var_candidates[i].flags = var.flags;
    }

    // Remove broken variants which are already in the varmap
    new_broken_down_var_candidates.erase(
      std::remove_if(new_broken_down_var_candidates.begin(),
                     new_broken_down_var_candidates.end(),
                     [&](VariantCandidate const & broken_var){
        return pool_varmap.find(broken_var) != pool_varmap.end();
      }), new_broken_down_var_candidates.end());

    // Remove broken variants that are already in the graph
    new_broken_down_var_candidates.erase(
      std::remove_if(new_broken_down_var_candidates.begin(),
                     new_broken_down_var_candidates.end(),
                     [&](VariantCandidate const & broken_var){
        return graph.is_variant_in_graph(Variant(broken_var));
      }), new_broken_down_var_candidates.end());

    if (new_broken_down_var_candidates.size() == 0)
    {
#ifndef NDEBUG
      BOOST_LOG_TRIVIAL(debug) << __HERE__ << " Erased " << map_it->first.print();
#endif // NDEBUG
      map_it = pool_varmap.erase(map_it); // Erase the old variant
    }
    else
    {
      ++map_it; // Don't erase the old variant, just move on to the next one
    }
  }
  //*/
}

void
VariantMap::clear()
{
  samples.clear();
  pool_varmap.clear();
  varmaps.clear();
}

#ifndef NDEBUG
void
VariantMap::write_stats(std::string const & prefix)
{
  auto const & pn = samples[0];

  // Determine filename
  std::string const discovery_fn =
    Options::const_instance()->stats + "/" + prefix + "_" + pn + "_discovery.tsv.gz";

  // Open the stream
  std::stringstream discovery_ss;

  // Write header to the stream
  discovery_ss << "CHROM\tPOS\tREF\tALT\tSAMPLE\tGT\tAD\tDP\tHQ\tLQ\tPP\tMQ0\tCR\tVS\tG\tUP\tREV\t1\tSUP\tSUPR\tPL\n";

  //for (auto const & sample_name : samples)
  //  discovery_ss << "\t" << sample_name;

  //discovery_ss << '\n';

  for (auto map_it = pool_varmap.begin(); map_it != pool_varmap.end(); ++map_it)
  {
    Variant var(map_it->first);
    assert(var.seqs.size() == 2);
    auto contig_pos = absolute_pos.get_contig_position(var.abs_pos, gyper::graph.contigs);
    discovery_ss << contig_pos.first << "\t" << contig_pos.second << "\t";

    // REF
    std::copy(var.seqs[0].begin(), var.seqs[0].end(), std::ostream_iterator<char>(discovery_ss));
    discovery_ss << '\t';

    // ALT
    std::copy(var.seqs[1].begin(), var.seqs[1].end(), std::ostream_iterator<char>(discovery_ss));
    discovery_ss << '\t';

    // Add naive genotype calls
    std::sort(map_it->second.begin(),
              map_it->second.end(),
              [](VariantSupport const & a, VariantSupport const & b) -> bool
      {
        return a.pn_index < b.pn_index;
      }
              );

    assert(Options::const_instance()->stats.size() > 0);
    std::size_t m = 0; // Index in map->second

    // Loop over all samples
    for (std::size_t p = 0; p < varmaps.size(); ++p)
    {
      // SAMPLE
      discovery_ss << samples[p] << '\t';

      std::vector<uint16_t> coverage(2);
      std::vector<uint16_t> phred(3);

      if (m >= map_it->second.size() || p < map_it->second[m].pn_index)
      {
        // Add empty call
        discovery_ss << ".\t";
      }
      else
      {
        auto const & var_support = map_it->second[m];
        assert(p == var_support.pn_index);

        coverage[1] = var_support.lq_support + var_support.hq_support;
        coverage[0] = var_support.depth - coverage[1];
        uint32_t phred0 = coverage[1] * 25u;
        uint32_t phred1 = var_support.depth * 3u;
        uint32_t phred2 = coverage[0] * 25u;
        uint32_t const normalizer = std::min(phred0, std::min(phred1, phred2));
        phred0 -= normalizer;
        phred1 -= normalizer;
        phred2 -= normalizer;
        assert(phred0 == 0 || phred1 == 0 || phred2 == 0);
        phred[0] = phred0 < 255 ? phred0 : 255;
        phred[1] = phred1 < 255 ? phred1 : 255;
        phred[2] = phred2 < 255 ? phred2 : 255;

        long const gt = std::distance(phred.begin(),
                                      std::find(phred.begin(), phred.end(), 0u)
                                      );

        if (gt == 0)
          discovery_ss << "0/0\t";
        else if (gt == 1)
          discovery_ss << "0/1\t";
        else
          discovery_ss << "1/1\t";

        discovery_ss.setf(std::ios::fixed);
        discovery_ss.precision(2);
        discovery_ss << coverage[0] << ',' << coverage[1] << '\t'
                     << var_support.depth << '\t'
                     << var_support.hq_support << '\t'
                     << var_support.lq_support << '\t'
                     << var_support.proper_pairs << '\t'
//                     << var_support.mapq0 << '\t'
//                     << var_support.clipped << '\t'
                     << var_support.var_size << '\t'
                     << var_support.growth << '\t'
                     << var_support.unique_positions.size() << '\t'
                     << var_support.sequence_reversed << '\t'
                     << var_support.first_in_pairs << '\t'
                     << var_support.get_corrected_support() << '\t'
                     << var_support.get_ratio() << '\t'
                     << phred[0] << ',' << phred[1] << ',' << phred[2]
                     << '\t';

        ++m;
      }
    }

    discovery_ss << '\n';
    //new_variant_vcf.variants.push_back(std::move(var));
  }

  write_gzipped_to_file(discovery_ss, discovery_fn, false /*append*/);
}

#endif // NDEBUG

void
VariantMap::get_vcf(Vcf & new_variant_vcf, std::string const & output_name)
{
  new_variant_vcf.open(WRITE_BGZF_MODE, output_name);

#ifndef NDEBUG
  BOOST_LOG_TRIVIAL(debug) << "[graphtyper::variant_map] Writing " << pool_varmap.size() << " variants.";
#endif // NDEBUG

  for (auto map_it = pool_varmap.begin(); map_it != pool_varmap.end(); ++map_it)
  {
    new_variant_vcf.variants.push_back(Variant(map_it->first));
  }
}

void
save_variant_map(std::string const & path, VariantMap const & map)
{
  std::ofstream ofs(path.c_str(), std::ios::binary);

  if (!ofs.is_open())
  {
    BOOST_LOG_TRIVIAL(error) << "[graphtyper::variant_map] Could not save variant_map at '"
                             << path
                             << "'";

    std::exit(1);
  }

  boost::archive::binary_oarchive oa(ofs);
  oa << map;
}

VariantMap
load_variant_map(std::string const & path)
{
  VariantMap map;
  std::ifstream ifs(path.c_str(), std::ios::binary);

  if (!ifs.is_open())
  {
    BOOST_LOG_TRIVIAL(error) << "[graphtyper::constructor] Could not load variant_map at '"
                             << path
                             << "'";

    std::exit(1);
  }

  boost::archive::binary_iarchive ia(ifs);
  ia >> map;
  return map;
}

void
VariantMap::set_samples(std::vector<std::string> const & new_samples)
{
  samples = new_samples;
  varmaps.resize(samples.size());
}

void
VariantMap::load_many_variant_maps(std::string const & path)
{
  std::vector<std::string> paths;
  std::ifstream ifs(path.c_str());

  for (std::string line; std::getline(ifs, line);)
    paths.push_back(line);

  this->load_many_variant_maps(paths);
}

void
VariantMap::load_many_variant_maps(std::vector<std::string> const & paths)
{
  this->pool_varmap.clear();  // Should be empty initially

  for (long p = 0; p < static_cast<long>(paths.size()); ++p)
  {
    auto const & file_path = paths[p];
    VariantMap new_map = load_variant_map(file_path);

    if (p == 0)
    {
      minimum_variant_support = new_map.minimum_variant_support;
      minimum_variant_support_ratio = new_map.minimum_variant_support_ratio;
    }

#ifndef NDEBUG
    // Sample index offset
    long sample_index_offset = samples.size();
#endif // NDEBUG

    // Copy samples
    std::copy(new_map.samples.begin(), new_map.samples.end(), std::back_inserter(samples));

    for (auto it = new_map.pool_varmap.begin(); it != new_map.pool_varmap.end(); ++it)
    {
#ifndef NDEBUG
      // Change all pn_indexes with offset
      for (auto variant_support : it->second)
        variant_support.pn_index += sample_index_offset;
#endif // NDEBUG

      auto find_it = pool_varmap.find(it->first);

      if (find_it == pool_varmap.end())
      {
        // Not found
        pool_varmap[it->first] = it->second;
      }
      else
      {
        // Found
        std::copy(it->second.begin(), it->second.end(), std::back_inserter(find_it->second));
      }
    }
  }
}

} // namespace gyper


text:
#include <boost/test/unit_test.hpp>

#include <algorithm>
#include <deque>
#include <string>
#include <string_view>

#include <wordring/whatwg/infra/infra.hpp>

#ifndef __cpp_lib_char8_t
using char8_t = char;
namespace std { using u8string = string; }
#endif

BOOST_AUTO_TEST_SUITE(whatwg_infra_test)

// ----------------------------------------------------------------------------

// 4.3. Bytes -----------------------------------------------------------------

BOOST_AUTO_TEST_CASE(is_ascii_byte__1)
{
	using namespace wordring::whatwg;

	for (unsigned char i = 0; i <= 127; i++) BOOST_CHECK(is_ascii_byte(i) == true);
	for (unsigned int i = 128; i <= 255; i++) BOOST_CHECK(is_ascii_byte(static_cast<unsigned char>(i)) == false);
}

// 4.4. Byte sequences --------------------------------------------------------

BOOST_AUTO_TEST_CASE(to_byte_lowercase__1)
{
	using namespace wordring::whatwg;

	char const * lower = "abcdefghijklmnopqrstuvwxyz0123456789";
	char const * upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
	char const * mix = "abcdEFGhijklmnopqrstuvwxyz0123456789";

	std::string_view lower_a{ lower };
	std::string_view upper_a{ upper };
	std::string_view mix_a{ mix };

	std::deque<char> lower_b{ lower_a.begin(), lower_a.end() };
	std::deque<char> upper_b{ upper_a.begin(), upper_a.end() };
	std::deque<char> mix_b{ mix_a.begin(), mix_a.end() };

	std::deque<char> result{};

	to_byte_lowercase(mix_b.begin() + 4, mix_b.begin() + 7, mix_b.begin() + 4);
	BOOST_CHECK(mix_b == lower_b);
}

BOOST_AUTO_TEST_CASE(to_byte_uppercase__1)
{
	using namespace wordring::whatwg;

	char const * lower = "abcdefghijklmnopqrstuvwxyz0123456789";
	char const * upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
	char const * mix = "ABCDefgHIJKLMNOPQRSTUVWXYZ0123456789";

	std::string_view lower_a{ lower };
	std::string_view upper_a{ upper };
	std::string_view mix_a{ mix };

	std::deque<char> lower_b{ lower_a.begin(), lower_a.end() };
	std::deque<char> upper_b{ upper_a.begin(), upper_a.end() };
	std::deque<char> mix_b{ mix_a.begin(), mix_a.end() };

	std::deque<char> result{};

	to_byte_uppercase(mix_b.begin() + 4, mix_b.begin() + 7, mix_b.begin() + 4);
	BOOST_CHECK(mix_b == upper_b);
}

BOOST_AUTO_TEST_CASE(is_byte_case_insensitive_match__1)
{
	using namespace wordring::whatwg;

	char const * lower = "abcdefghijklmnopqrstuvwxyz0123456789";
	char const * upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
	char const * fail = "ABCD";

	std::string_view lower_a{ lower };
	std::string_view upper_a{ upper };
	std::string_view fail_a{ fail };

	std::deque<char> lower_b{ lower_a.begin(), lower_a.end() };
	std::deque<char> upper_b{ upper_a.begin(), upper_a.end() };
	std::deque<char> fail_b{ fail_a.begin(), fail_a.end() };

	BOOST_CHECK(is_byte_case_insensitive_match(lower_a.begin(), lower_a.end(), upper_b.begin(), upper_b.end()) == true);
}

BOOST_AUTO_TEST_CASE(isomorphic_decode__1)
{
	using namespace wordring::whatwg;

	char const * lower = "abcdefghijklmnopqrstuvwxyz0123456789";
	std::string_view lower_a{ lower };
	std::deque<char> lower_b{ lower_a.begin(), lower_a.end() };

	std::deque<char32_t> result{};

	isomorphic_decode(lower_b.begin(), lower_b.end(), std::back_inserter(result));
	BOOST_CHECK(std::equal(lower_b.begin(), lower_b.end(), result.begin(), result.end()));
}

// 4.5. Code points -----------------------------------------------------------

BOOST_AUTO_TEST_CASE(is_surrogate__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_surrogate(U'\u0000') == false);
	BOOST_CHECK(is_surrogate(U'\uD7FF') == false);

	BOOST_CHECK(is_surrogate(static_cast<char32_t>(0xD800)) == true);
	BOOST_CHECK(is_surrogate(static_cast<char32_t>(0xDFFF)) == true);

	BOOST_CHECK(is_surrogate(U'\uE000') == false);
	BOOST_CHECK(is_surrogate(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_scalar_value__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_scalar_value(U'\u0000') == true);
	BOOST_CHECK(is_scalar_value(U'\uD7FF') == true);

	BOOST_CHECK(is_scalar_value(static_cast<char32_t>(0xD800)) == false);
	BOOST_CHECK(is_scalar_value(static_cast<char32_t>(0xDFFF)) == false);

	BOOST_CHECK(is_scalar_value(U'\uE000') == true);
	BOOST_CHECK(is_scalar_value(static_cast<char32_t>(0xFFFFFFFF)) == true);
}

BOOST_AUTO_TEST_CASE(is_noncharacter__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_noncharacter(U'\uFDD0') == true);
	BOOST_CHECK(is_noncharacter(U'\uFDEF') == true);
}

BOOST_AUTO_TEST_CASE(is_ascii_code_point__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_code_point(U'\u0000') == true);
	BOOST_CHECK(is_ascii_code_point(U'\u007F') == true);

	BOOST_CHECK(is_ascii_code_point(U'\u0080') == false);
	BOOST_CHECK(is_ascii_code_point(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_tab_or_newline__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_tab_or_newline(U'\u0009') == true);
	BOOST_CHECK(is_ascii_tab_or_newline(U'\u000A') == true);
	BOOST_CHECK(is_ascii_tab_or_newline(U'\u000D') == true);

	BOOST_CHECK(is_ascii_tab_or_newline(U'a') == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_white_space__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_white_space(U'\u0009') == true);
	BOOST_CHECK(is_ascii_white_space(U'\u000A') == true);
	BOOST_CHECK(is_ascii_white_space(U'\u000C') == true);
	BOOST_CHECK(is_ascii_white_space(U'\u000D') == true);
	BOOST_CHECK(is_ascii_white_space(U'\u0020') == true);

	BOOST_CHECK(is_ascii_white_space(U'a') == false);
}

BOOST_AUTO_TEST_CASE(is_c0_control__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_c0_control(U'\u0000') == true);
	BOOST_CHECK(is_c0_control(U'\u001F') == true);

	BOOST_CHECK(is_c0_control(U'\u0020') == false);
	BOOST_CHECK(is_c0_control(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_digit__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_digit(U'\u0000') == false);
	BOOST_CHECK(is_ascii_digit(U'\u0029') == false);

	BOOST_CHECK(is_ascii_digit(U'0') == true);
	BOOST_CHECK(is_ascii_digit(U'9') == true);

	BOOST_CHECK(is_ascii_digit(U'\u003A') == false);
	BOOST_CHECK(is_ascii_digit(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_upper_hex_digit__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_upper_hex_digit(U'\u0000') == false);

	BOOST_CHECK(is_ascii_upper_hex_digit(U'\u0029') == false);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'0') == true);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'9') == true);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'\u003A') == false);

	BOOST_CHECK(is_ascii_upper_hex_digit(U'\u0040') == false);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'A') == true);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'F') == true);
	BOOST_CHECK(is_ascii_upper_hex_digit(U'\u0047') == false);

	BOOST_CHECK(is_ascii_upper_hex_digit(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_lower_hex_digit__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_lower_hex_digit(U'\u0000') == false);

	BOOST_CHECK(is_ascii_lower_hex_digit(U'\u0029') == false);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'0') == true);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'9') == true);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'\u003A') == false);

	BOOST_CHECK(is_ascii_lower_hex_digit(U'\u0060') == false);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'a') == true);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'f') == true);
	BOOST_CHECK(is_ascii_lower_hex_digit(U'\u0067') == false);

	BOOST_CHECK(is_ascii_lower_hex_digit(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_hex_digit__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_hex_digit(U'\u0000') == false);

	BOOST_CHECK(is_ascii_hex_digit(U'\u0029') == false);
	BOOST_CHECK(is_ascii_hex_digit(U'0') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'9') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'\u003A') == false);

	BOOST_CHECK(is_ascii_hex_digit(U'\u0040') == false);
	BOOST_CHECK(is_ascii_hex_digit(U'A') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'F') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'\u0047') == false);

	BOOST_CHECK(is_ascii_hex_digit(U'\u0060') == false);
	BOOST_CHECK(is_ascii_hex_digit(U'a') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'f') == true);
	BOOST_CHECK(is_ascii_hex_digit(U'\u0067') == false);

	BOOST_CHECK(is_ascii_hex_digit(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_upper_alpha__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_upper_alpha(U'\u0000') == false);
	BOOST_CHECK(is_ascii_upper_alpha(U'\u0040') == false);

	BOOST_CHECK(is_ascii_upper_alpha(U'A') == true);
	BOOST_CHECK(is_ascii_upper_alpha(U'Z') == true);

	BOOST_CHECK(is_ascii_upper_alpha(U'\u005B') == false);
	BOOST_CHECK(is_ascii_upper_alpha(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_lower_alpha__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_lower_alpha(U'\u0000') == false);
	BOOST_CHECK(is_ascii_lower_alpha(U'\u0060') == false);

	BOOST_CHECK(is_ascii_lower_alpha(U'a') == true);
	BOOST_CHECK(is_ascii_lower_alpha(U'z') == true);

	BOOST_CHECK(is_ascii_lower_alpha(U'\u007B') == false);
	BOOST_CHECK(is_ascii_lower_alpha(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_alpha__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_alpha(U'\u0000') == false);

	BOOST_CHECK(is_ascii_alpha(U'\u0040') == false);
	BOOST_CHECK(is_ascii_alpha(U'A') == true);
	BOOST_CHECK(is_ascii_alpha(U'Z') == true);
	BOOST_CHECK(is_ascii_alpha(U'\u005B') == false);

	BOOST_CHECK(is_ascii_alpha(U'\u0060') == false);
	BOOST_CHECK(is_ascii_alpha(U'a') == true);
	BOOST_CHECK(is_ascii_alpha(U'z') == true);
	BOOST_CHECK(is_ascii_alpha(U'\u007B') == false);

	BOOST_CHECK(is_ascii_alpha(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

BOOST_AUTO_TEST_CASE(is_ascii_alphanumeric__1)
{
	using namespace wordring::whatwg;

	BOOST_CHECK(is_ascii_alphanumeric(U'\u0000') == false);

	BOOST_CHECK(is_ascii_alphanumeric(U'\u0029') == false);
	BOOST_CHECK(is_ascii_alphanumeric(U'0') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'9') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'\u003A') == false);

	BOOST_CHECK(is_ascii_alphanumeric(U'\u0040') == false);
	BOOST_CHECK(is_ascii_alphanumeric(U'A') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'Z') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'\u005B') == false);

	BOOST_CHECK(is_ascii_alphanumeric(U'\u0060') == false);
	BOOST_CHECK(is_ascii_alphanumeric(U'a') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'z') == true);
	BOOST_CHECK(is_ascii_alphanumeric(U'\u007B') == false);

	BOOST_CHECK(is_ascii_alphanumeric(static_cast<char32_t>(0xFFFFFFFF)) == false);
}

// 4.6. Strings ---------------------------------------------------------------

BOOST_AUTO_TEST_CASE(isomorphic_encode__1)
{
	using namespace wordring::whatwg;

	std::u32string a1{ U"abcdefg" };
	std::string b1{ "abcdefg" };
	std::vector<char> o1;

	isomorphic_encode(a1.begin(), a1.end(), std::back_inserter(o1));
	BOOST_CHECK(std::equal(b1.begin(), b1.end(), o1.begin(), o1.end()) == true);

	o1.clear();
	isomorphic_encode(b1.begin(), b1.end(), std::back_inserter(o1));
	BOOST_CHECK(std::equal(b1.begin(), b1.end(), o1.begin(), o1.end()) == true);
}

BOOST_AUTO_TEST_CASE(is_ascii_string__1)
{
	using namespace wordring::whatwg;

	// utf-8
	std::u8string const s8_true{ u8"abcdefg" };
	std::u8string const s8_false{ u8"abcdefgあいうえお" };
	BOOST_CHECK(is_ascii_string(s8_true.begin(), s8_true.end()) == true);
	BOOST_CHECK(is_ascii_string(s8_false.begin(), s8_false.end()) == false);

	// utf-16
	std::u16string const s16_true{ u"abcdefg" };
	std::u16string const s16_false{ u"abcdefgあいうえお" };
	BOOST_CHECK(is_ascii_string(s16_true.begin(), s16_true.end()) == true);
	BOOST_CHECK(is_ascii_string(s16_false.begin(), s16_false.end()) == false);

	// utf-32
	std::u32string const s32_true{ U"abcdefg" };
	std::u32string const s32_false{ U"abcdefgあいうえお" };
	BOOST_CHECK(is_ascii_string(s32_true.begin(), s32_true.end()) == true);
	BOOST_CHECK(is_ascii_string(s32_false.begin(), s32_false.end()) == false);
}

BOOST_AUTO_TEST_CASE(to_ascii_lowercase__1)
{
	using namespace wordring::whatwg;

	// utf-8
	std::u8string const s8_original{ u8"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string const s8_correct{ u8"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string s8_result{};
	to_ascii_lowercase(s8_original.begin(), s8_original.end(), std::back_inserter(s8_result));
	BOOST_CHECK(s8_correct == s8_result);

	// utf-16
	std::u16string const s16_original{ u"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u16string const s16_correct{ u"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u16string s16_result{};
	to_ascii_lowercase(s16_original.begin(), s16_original.end(), std::back_inserter(s16_result));
	BOOST_CHECK(s16_correct == s16_result);

	// utf-32
	std::u32string const s32_original{ U"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u32string const s32_correct{ U"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u32string s32_result{};
	to_ascii_lowercase(s32_original.begin(), s32_original.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_correct == s32_result);
}

BOOST_AUTO_TEST_CASE(to_ascii_uppercase__1)
{
	using namespace wordring::whatwg;

	// utf-8
	std::u8string const s8_original{ u8"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string const s8_correct{ u8"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string s8_result{};
	to_ascii_uppercase(s8_original.begin(), s8_original.end(), std::back_inserter(s8_result));
	BOOST_CHECK(s8_correct == s8_result);

	// utf-16
	std::u16string const s16_original{ u"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u16string const s16_correct{ u"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u16string s16_result{};
	to_ascii_uppercase(s16_original.begin(), s16_original.end(), std::back_inserter(s16_result));
	BOOST_CHECK(s16_correct == s16_result);

	// utf-32
	std::u32string const s32_original{ U"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u32string const s32_correct{ U"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u32string s32_result{};
	to_ascii_uppercase(s32_original.begin(), s32_original.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_correct == s32_result);
}

BOOST_AUTO_TEST_CASE(is_ascii_case_insensitive_match__1)
{
	using namespace wordring::whatwg;

	// utf-8
	std::u8string const s8_original{ u8"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string const s8_correct{ u8"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u8string const s8_incorrect_1{ u8"abcdeあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠abcde" };
	std::u8string const s8_incorrect_2{ u8"12345あいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	BOOST_CHECK(is_ascii_case_insensitive_match(s8_original.begin(), s8_original.end(), s8_correct.begin(), s8_correct.end()) == true);
	BOOST_CHECK(is_ascii_case_insensitive_match(s8_original.begin(), s8_original.end(), s8_incorrect_1.begin(), s8_incorrect_1.end()) == false);
	BOOST_CHECK(is_ascii_case_insensitive_match(s8_original.begin(), s8_original.end(), s8_incorrect_2.begin(), s8_incorrect_2.end()) == false);
}

BOOST_AUTO_TEST_CASE(ascii_encode__1)
{
	using namespace wordring::whatwg;

	std::u16string const s16_original{ u"ABCDE" };
	std::u16string const s16_correct{ u"ABCDE" };
	std::u8string s8_result{};
	ascii_encode(s16_original.begin(), s16_original.end(), std::back_inserter(s8_result));
	BOOST_CHECK(std::equal(s16_correct.begin(), s16_correct.end(), s8_result.begin(), s8_result.end()));
}

BOOST_AUTO_TEST_CASE(ascii_decode__1)
{
	using namespace wordring::whatwg;

	std::string const s_original{ "ABCDE" };
	std::string const s_correct{ "ABCDE" };
	std::u16string s16_result{};
	ascii_decode(s_original.begin(), s_original.end(), std::back_inserter(s16_result));
	BOOST_CHECK(std::equal(s_correct.begin(), s_correct.end(), s16_result.begin(), s16_result.end()));
}

BOOST_AUTO_TEST_CASE(strip_newlines__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original{ U"ABCDE\nあいうえ\r\nお𠀋𡈽𡌛\r𡑮𡢽Àⱥ𐊠" };
	std::u32string const s32_correct{ U"ABCDEあいうえお𠀋𡈽𡌛𡑮𡢽Àⱥ𐊠" };
	std::u32string s32_result{};
	strip_newlines(s32_original.begin(), s32_original.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct);
}

BOOST_AUTO_TEST_CASE(normalize_newlines__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original{ U"A\rA\nA\r\n\n\n\r\r\n" };
	std::u32string const s32_correct{ U"A\nA\nA\n\n\n\n\n" };
	std::u32string s32_result{};
	normalize_newlines(s32_original.begin(), s32_original.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct);
}

BOOST_AUTO_TEST_CASE(strip_leading_and_trailing_ascii_whitespace__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"  ABC 123  " };
	std::u32string const s32_correct_1{ U"ABC 123" };
	std::u32string s32_result{};
	strip_leading_and_trailing_ascii_whitespace(s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct_1);

	std::u32string const s32_original_2{ U"ABC 123" };
	std::u32string const s32_correct_2{ U"ABC 123" };
	s32_result.clear();
	strip_leading_and_trailing_ascii_whitespace(s32_original_2.begin(), s32_original_2.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct_2);
}

BOOST_AUTO_TEST_CASE(strip_and_collapse_ascii_whitespace__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"  ABC   123  " };
	std::u32string const s32_correct_1{ U"ABC 123" };
	std::u32string s32_result{};
	strip_and_collapse_ascii_whitespace(s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct_1);

	std::u32string const s32_original_2{ U"ABC 123" };
	std::u32string const s32_correct_2{ U"ABC 123" };
	s32_result.clear();
	strip_and_collapse_ascii_whitespace(s32_original_2.begin(), s32_original_2.end(), std::back_inserter(s32_result));
	BOOST_CHECK(s32_result == s32_correct_2);
}

BOOST_AUTO_TEST_CASE(collect_sequence_of_code_points__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"  ABC   123  " };
	std::u32string const s32_correct_1{ U"  " };
	std::u32string s32_result_1{};
	auto it_result_1 = collect_sequence_of_code_points(
		s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result_1), is_ascii_white_space);
	BOOST_CHECK(*it_result_1 == U'A');
	BOOST_CHECK(s32_result_1 == s32_correct_1);

	std::u32string const s32_original_2{ U"ABC   123  " };
	std::u32string const s32_correct_2{ U"" };
	std::u32string s32_result_2{};
	auto it_result_2 = collect_sequence_of_code_points(
		s32_original_2.begin(), s32_original_2.end(), std::back_inserter(s32_result_2), is_ascii_white_space);
	BOOST_CHECK(*it_result_2 == U'A');
	BOOST_CHECK(s32_result_2 == s32_correct_2);

	std::u32string const s32_original_3{ U"   " };
	std::u32string const s32_correct_3{ U"   " };
	std::u32string s32_result_3{};
	auto it_result_3 = collect_sequence_of_code_points(
		s32_original_3.begin(), s32_original_3.end(), std::back_inserter(s32_result_3), is_ascii_white_space);
	BOOST_CHECK(it_result_3 == s32_original_3.end());
	BOOST_CHECK(s32_result_3 == s32_correct_3);
}

BOOST_AUTO_TEST_CASE(skip_ascii_whitespace__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"  ABC" };
	std::u32string const s32_correct_1{ U"ABC" };
	auto it_result_1 = skip_ascii_whitespace(s32_original_1.begin(), s32_original_1.end());
	std::u32string s32_result_1{ it_result_1, s32_original_1.end() };
	BOOST_CHECK(*it_result_1 == U'A');
	BOOST_CHECK(s32_result_1 == s32_correct_1);
}

BOOST_AUTO_TEST_CASE(strictly_split_on_a_particular_delimiter__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"ABC,DEF,GHI" };
	std::vector<std::u32string> const s32_correct_1{ U"ABC", U"DEF", U"GHI" };
	std::vector<std::u32string> s32_result_1{};
	strictly_split_on_particular_delimiter(
		s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result_1), U',');
	BOOST_CHECK(s32_result_1 == s32_correct_1);

	std::u32string const s32_original_2{};
	std::vector<std::u32string> const s32_correct_2{ std::u32string{} };
	std::vector<std::u32string> s32_result_2{};
	strictly_split_on_particular_delimiter(
		s32_original_2.begin(), s32_original_2.end(), std::back_inserter(s32_result_2), U',');
	BOOST_CHECK(s32_result_2 == s32_correct_2);

	std::u32string const s32_original_3{ U"," };
	std::vector<std::u32string> const s32_correct_3{ std::u32string{}, std::u32string{} };
	std::vector<std::u32string> s32_result_3{};
	strictly_split_on_particular_delimiter(
		s32_original_3.begin(), s32_original_3.end(), std::back_inserter(s32_result_3), U',');
	BOOST_CHECK(s32_result_3 == s32_correct_3);
}

BOOST_AUTO_TEST_CASE(split_on_ascii_whitespace__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U" ABC DEF GHI " };
	std::vector<std::u32string> const s32_correct_1{ U"ABC", U"DEF", U"GHI" };
	std::vector<std::u32string> s32_result_1{};
	split_on_ascii_whitespace(
		s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result_1));
	BOOST_CHECK(s32_result_1 == s32_correct_1);
}

BOOST_AUTO_TEST_CASE(split_on_commas__1)
{
	using namespace wordring::whatwg;

	std::u32string const s32_original_1{ U"ABC,DEF,GHI" };
	std::vector<std::u32string> const s32_correct_1{ U"ABC", U"DEF", U"GHI" };
	std::vector<std::u32string> s32_result_1{};
	split_on_commas(
		s32_original_1.begin(), s32_original_1.end(), std::back_inserter(s32_result_1));
	BOOST_CHECK(s32_result_1 == s32_correct_1);

	std::u32string const s32_original_2{};
	std::vector<std::u32string> const s32_correct_2{};
	std::vector<std::u32string> s32_result_2{};
	split_on_commas(
		s32_original_2.begin(), s32_original_2.end(), std::back_inserter(s32_result_2));
	BOOST_CHECK(s32_result_2 == s32_correct_2);

	std::u32string const s32_original_3{ U"," };
	std::vector<std::u32string> const s32_correct_3{ std::u32string{} };
	std::vector<std::u32string> s32_result_3{};
	split_on_commas(
		s32_original_3.begin(), s32_original_3.end(), std::back_inserter(s32_result_3));
	BOOST_CHECK(s32_result_3 == s32_correct_3);
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: wordring/wordring

max_issues_repo_name: wordring/libwordring

max_forks_repo_name: wordring/libwordring


text:
#include <k4a/k4a.hpp>

#include <iostream>
#include <fstream>
#include <iomanip>
#include <string>
#include <vector>
#include <array>

#include <Eigen/Dense>

#include <pcl/point_cloud.h>
#include <pcl/point_types.h>
#include <pcl/visualization/pcl_visualizer.h>
#include <pcl/common/transforms.h>
#include <pcl/io/ply_io.h>
#include <pcl/filters/passthrough.h>
#include <pcl/filters/bilateral.h>
#include <pcl/filters/fast_bilateral.h>

#include <glm/glm.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtc/type_ptr.hpp>

#include <Open3D/Geometry/PointCloud.h>
#include <Open3d/Registration/ColoredICP.h>
#include <Open3D/Open3D.h>

#include "k4a_grabber.h"

using namespace std;
using namespace boost;
using namespace pcl;
using namespace Eigen;
using namespace glm;

typedef pcl::PointXYZRGB PointType;

template<typename DataType>
Eigen::Matrix<DataType, Eigen::Dynamic, Eigen::Dynamic> load_csv(const std::string & path)
{
	std::ifstream indata;
	indata.open(path);
	std::string line;
	std::vector<DataType> values;
	unsigned int rows = 0;
	while (std::getline(indata, line))
	{
		std::stringstream lineStream(line);
		std::string cell;
		while (std::getline(lineStream, cell, ','))
		{
			values.push_back(std::stod(cell));
		}
		++rows;
	}
	return Eigen::Map<const Eigen::Matrix<typename Eigen::Matrix<DataType, Eigen::Dynamic, Eigen::Dynamic>::Scalar, Eigen::Matrix<DataType, Eigen::Dynamic, Eigen::Dynamic>::RowsAtCompileTime, Eigen::Matrix<DataType, Eigen::Dynamic, Eigen::Dynamic>::ColsAtCompileTime, Eigen::RowMajor>>(values.data(), rows, values.size() / rows);
}

void convertToOpenCDPointCloud(pcl::PointCloud<PointType>::ConstPtr cloud, open3d::geometry::PointCloud &pointCloud)
{
	double x, y, z, r, g, b;
	pointCloud.Clear();
	for (const auto &point : cloud->points)
	{
		x = point.x;
		y = point.y;
		z = point.z;
		r = point.r / 256.0;
		g = point.g / 256.0;
		b = point.b / 256.0;

		pointCloud.points_.push_back(Eigen::Vector3d(x, y, z));
		pointCloud.colors_.push_back(Eigen::Vector3d(r, g, b));
	}
}

Eigen::Matrix4d colorCloudRegistrationOptimization(pcl::PointCloud<PointType>::ConstPtr pcl_source, pcl::PointCloud<PointType>::ConstPtr pcl_target, const Eigen::Matrix4d &init_transformation)
{
	open3d::geometry::PointCloud open3d_source_original, open3d_target_original;
	open3d::geometry::PointCloud open3d_source_current, open3d_target_current;

	convertToOpenCDPointCloud(pcl_source, open3d_source_original);
	convertToOpenCDPointCloud(pcl_target, open3d_target_original);

	open3d::registration::RegistrationResult result;
	Eigen::Matrix4d current_transformation = init_transformation;

	open3d_source_current = open3d_source_original;
	open3d_target_current = open3d_target_original;

	open3d::geometry::PointCloud open3d_source_down;
	open3d::geometry::PointCloud open3d_target_down;

	double radius = 0.01;

	open3d_source_down = *open3d_source_current.VoxelDownSample(radius);
	open3d_target_down = *open3d_target_current.VoxelDownSample(radius);

	open3d_source_down.EstimateNormals(open3d::geometry::KDTreeSearchParamHybrid(radius * 2, 30));
	open3d_target_down.EstimateNormals(open3d::geometry::KDTreeSearchParamHybrid(radius * 2, 30));

	result = open3d::registration::RegistrationColoredICP(open3d_source_down, open3d_target_down,
		radius, current_transformation,
		open3d::registration::ICPConvergenceCriteria(1e-16, 1e-16, 500));

	return result.transformation_;
}

int main(int argc, char **argv)
{
	string frame_sub_master_file = "frame_sub_master.csv";
	cout << "Reading frame_sub_master.csv" << endl;
	Transform<double, 3, Affine> transformation_sub_master;
	transformation_sub_master.matrix() = load_csv<double>(frame_sub_master_file);

	const uint32_t deviceCount = k4a::device::get_installed_count();
	if (deviceCount == 0)
	{
		cout << "no azure kinect devices detected!" << endl;
	}

	// PCL Visualizer
	boost::shared_ptr<pcl::visualization::PCLVisualizer> viewer(new pcl::visualization::PCLVisualizer("Point Cloud Viewer"));

	// Point Cloud
	pcl::PointCloud<PointType>::Ptr cloud_master;
	pcl::PointCloud<PointType>::Ptr cloud_sub;

	// Retrieved Point Cloud Callback Function
	boost::mutex mutex_master;
	boost::function<void(const pcl::PointCloud<PointType>::ConstPtr&)> function_master =
		[&cloud_master, &mutex_master](const pcl::PointCloud<PointType>::ConstPtr& ptr)
	{
		boost::mutex::scoped_lock lock(mutex_master);

		/* Point Cloud Processing */

		cloud_master = ptr->makeShared();

	};

	boost::mutex mutex_sub;
	boost::function<void(const pcl::PointCloud<PointType>::ConstPtr&)> function_sub =
		[&cloud_sub, &mutex_sub](const pcl::PointCloud<PointType>::ConstPtr& ptr)
	{
		boost::mutex::scoped_lock lock(mutex_sub);

		/* Point Cloud Processing */

		cloud_sub = ptr->makeShared();

	};

	// KinectAzureDKGrabber
	boost::shared_ptr<pcl::Grabber> grabber_master =
		boost::make_shared<pcl::KinectAzureDKGrabber>(0, K4A_DEPTH_MODE_NFOV_UNBINNED, K4A_IMAGE_FORMAT_COLOR_BGRA32, K4A_COLOR_RESOLUTION_720P);
	boost::shared_ptr<pcl::Grabber> grabber_sub =
		boost::make_shared<pcl::KinectAzureDKGrabber>(1, K4A_DEPTH_MODE_NFOV_UNBINNED, K4A_IMAGE_FORMAT_COLOR_BGRA32, K4A_COLOR_RESOLUTION_720P);

	boost::shared_ptr<pcl::KinectAzureDKGrabber> grabber_ = boost::dynamic_pointer_cast<pcl::KinectAzureDKGrabber>(grabber_master);

	// Register Callback Function
	boost::signals2::connection connection_sub = grabber_sub->registerCallback(function_sub);
	boost::signals2::connection connection_master = grabber_master->registerCallback(function_master);

	// Start Grabber
	grabber_sub->start();
	grabber_master->start();

	k4a::calibration calibration = grabber_->getCalibration();
	k4a_calibration_intrinsic_parameters_t *intrinsics = &calibration.color_camera_calibration.intrinsics.parameters;
	Eigen::Matrix3f intrinsics_eigen;
	intrinsics_eigen <<
		intrinsics->param.fx, 0.0f, intrinsics->param.cx,
		0.0f, intrinsics->param.fy, intrinsics->param.cy,
		0.0f, 0.0f, 1.0f;
	Eigen::Matrix4f extrinsics_eigen = Eigen::Matrix4f::Identity();
	viewer->setCameraParameters(intrinsics_eigen, extrinsics_eigen);

	pcl::PointCloud<PointType>::Ptr cloud(new PointCloud<PointType>);

	float aspect = 0.475f;

	transformation_sub_master.matrix()(0, 3) *= aspect;
	transformation_sub_master.matrix()(1, 3) *= aspect;
	transformation_sub_master.matrix()(2, 3) *= aspect;

	bool refineResult = false;

	Eigen::Matrix4d init_transformation;
	init_transformation = transformation_sub_master.matrix();

	Eigen::Matrix4d refinedTransformation;

	while (!viewer->wasStopped())
	{
		// Update Viewer
		viewer->spinOnce();
		boost::mutex::scoped_try_lock lock_master(mutex_master);
		boost::mutex::scoped_try_lock lock_sub(mutex_sub);

		if (lock_master.owns_lock() && cloud_master &&
			lock_sub.owns_lock() && cloud_sub)
		{
			if (!refineResult)
			{
				std::cout << "Start Calibration Optimization." << std::endl;
				refinedTransformation = colorCloudRegistrationOptimization(cloud_master, cloud_sub, init_transformation);
				refineResult = true;
				std::cout << "Calibration optimization process over." << std::endl;
			}

			transformPointCloud(*cloud_master, *cloud_master, refinedTransformation, true);

			*cloud = *cloud_sub + *cloud_master;

			// Update Point Cloud
			if (!viewer->updatePointCloud(cloud, "cloud"))
			{
				viewer->addPointCloud(cloud, "cloud");
			}
		}
	}
	// Stop Grabber
	grabber_sub->stop();
	grabber_master->stop();

	// Disconnect Callback Function
	if (connection_master.connected() || connection_sub.connected())
	{
		connection_sub.disconnect();
		connection_master.disconnect();
	}

	return 0;
}