text:
// Copyright (c) Facebook, Inc. and its affiliates.
//
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
#include "compiler_gym/envs/mlir/service/MlirSession.h"

#include <cpuinfo.h>
#include <fmt/format.h>
#include <glog/logging.h>

#include <boost/process.hpp>
#include <chrono>
#include <future>
#include <iomanip>
#include <optional>
#include <string>

#include "boost/algorithm/string.hpp"
#include "boost/filesystem.hpp"
#include "compiler_gym/envs/mlir/service/ActionSpace.h"
#include "compiler_gym/envs/mlir/service/Benchmark.h"
#include "compiler_gym/envs/mlir/service/BenchmarkFactory.h"
#include "compiler_gym/envs/mlir/service/MlirUtils.h"
#include "compiler_gym/envs/mlir/service/Observation.h"
#include "compiler_gym/envs/mlir/service/ObservationSpaces.h"
#include "compiler_gym/util/EnumUtil.h"
#include "compiler_gym/util/GrpcStatusMacros.h"
#include "compiler_gym/util/RunfilesPath.h"
#include "llvm/Support/TargetSelect.h"
#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
#include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
#include "mlir/Conversion/Passes.h"
#include "mlir/Conversion/SCFToStandard/SCFToStandard.h"
#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Linalg/Transforms/CodegenStrategy.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/StandardOps/IR/Ops.h"
#include "mlir/Dialect/StandardOps/Transforms/Passes.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/ExecutionEngine/OptUtils.h"
#include "mlir/IR/AsmState.h"
#include "mlir/IR/Dialect.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/InitAllDialects.h"
#include "mlir/InitAllPasses.h"
#include "mlir/Parser.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Support/FileUtilities.h"
#include "mlir/Support/MlirOptMain.h"
#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Export.h"
#include "mlir/Target/LLVMIR/ModuleTranslation.h"
#include "mlir/Transforms/Passes.h"
#include "nlohmann/json.hpp"

namespace fs = boost::filesystem;
namespace bp = boost::process;

namespace compiler_gym::mlir_service {

using grpc::Status;
using grpc::StatusCode;
using nlohmann::json;

using BenchmarkProto = compiler_gym::Benchmark;
using ActionSpaceProto = compiler_gym::ActionSpace;

std::string MlirSession::getCompilerVersion() const {
  std::stringstream ss;
  // MLIR doesn't maintain a version as it lives in the llvm-project.
  ss << "LLVM " << LLVM_VERSION_STRING << " patch=" << LLVM_VERSION_PATCH << " "
     << llvm::Triple::normalize(LLVM_DEFAULT_TARGET_TRIPLE);
  return ss.str();
}

std::vector<ActionSpace> MlirSession::getActionSpaces() const { return getMlirActionSpaceList(); }

std::vector<ObservationSpace> MlirSession::getObservationSpaces() const {
  return getMlirObservationSpaceList();
}

MlirSession::MlirSession(const boost::filesystem::path& workingDirectory)
    : CompilationSession(workingDirectory),
      observationSpaceNames_(util::createPascalCaseToEnumLookupTable<MlirObservationSpace>()),
      spaceContainsEventChecker_(makeDefaultSpaceContainsEventChecker()) {
  cpuinfo_initialize();
}

Status MlirSession::init(const ActionSpace& actionSpace, const BenchmarkProto& benchmark) {
  BenchmarkFactory& benchmarkFactory = BenchmarkFactory::getSingleton(workingDirectory());

  // Get the benchmark or return an error.
  std::unique_ptr<Benchmark> mlirBenchmark;
  RETURN_IF_ERROR(benchmarkFactory.getBenchmark(benchmark, &mlirBenchmark));

  // Verify the benchmark now to catch errors early.
  RETURN_IF_ERROR(mlirBenchmark->verify_module());

  MlirActionSpace actionSpaceEnum;
  RETURN_IF_ERROR(util::pascalCaseToEnum(actionSpace.name(), &actionSpaceEnum));

  return init(actionSpaceEnum, std::move(mlirBenchmark));
}

Status MlirSession::init(CompilationSession* other) {
  // TODO: Static cast?
  auto mlirOther = static_cast<MlirSession*>(other);
  return init(mlirOther->actionSpace(), mlirOther->benchmark().clone(workingDirectory()));
}

Status MlirSession::init(const MlirActionSpace& actionSpace, std::unique_ptr<Benchmark> benchmark) {
  benchmark_ = std::move(benchmark);
  actionSpace_ = actionSpace;
  actionSpaces_ = this->getActionSpaces();
  actions_count_ = 0;

  return Status::OK;
}

Status MlirSession::applyAction(const Event& action, bool& endOfEpisode,
                                std::optional<ActionSpace>& newActionSpace,
                                bool& actionHadNoEffect) {
  ++actions_count_;
  endOfEpisode = true;
  DCHECK(benchmark_) << "Calling applyAction() before init()";

  spaceContainsEventChecker_.checkContains(actionSpaces_[0].space(), action);
  return mlir::performLinalgCodegen(action, benchmark().module());
}

Status MlirSession::endOfStep(bool actionHadNoEffect, bool& endOfEpisode,
                              std::optional<ActionSpace>& newActionSpace) {
  if (actionHadNoEffect) {
    return Status::OK;
  } else {
    return benchmark().verify_module();
  }
}

Status MlirSession::computeObservation(const ObservationSpace& observationSpace,
                                       Event& observation) {
  DCHECK(benchmark_) << "Calling computeObservation() before init()";

  const auto& it = observationSpaceNames_.find(observationSpace.name());
  if (it == observationSpaceNames_.end()) {
    return Status(
        StatusCode::INVALID_ARGUMENT,
        fmt::format("Could not interpret observation space name: {}", observationSpace.name()));
  }
  const MlirObservationSpace observationSpaceEnum = it->second;

  if (observationSpaceEnum == MlirObservationSpace::RUNTIME && actions_count_ == 0) {
    // Do nothing before first action.
    *observation.mutable_double_tensor()->mutable_value()->Add() = 0;
    *observation.mutable_double_tensor()->mutable_shape()->Add() = 1;
    return Status::OK;
  } else {
    return setObservation(observationSpaceEnum, workingDirectory(), benchmark(), observation);
  }
}

Status MlirSession::handleSessionParameter(const std::string& key, const std::string& value,
                                           std::optional<std::string>& reply) {
  if (key == "mlir.set_runtimes_per_observation_count") {
    const int ivalue = std::stoi(value);
    if (ivalue < 1) {
      return Status(
          StatusCode::INVALID_ARGUMENT,
          fmt::format("runtimes_per_observation_count must be >= 1. Received: {}", ivalue));
    }
    benchmark().setRuntimesPerObservationCount(ivalue);
    reply = value;
  } else if (key == "mlir.get_runtimes_per_observation_count") {
    reply = fmt::format("{}", benchmark().getRuntimesPerObservationCount());
  } else if (key == "mlir.set_warmup_runs_count_per_runtime_observation") {
    const int ivalue = std::stoi(value);
    if (ivalue < 0) {
      return Status(
          StatusCode::INVALID_ARGUMENT,
          fmt::format("warmup_runs_count_per_runtime_observation must be >= 0. Received: {}",
                      ivalue));
    }
    benchmark().setWarmupRunsPerRuntimeObservationCount(ivalue);
    reply = value;
  } else if (key == "mlir.get_warmup_runs_count_per_runtime_observation") {
    reply = fmt::format("{}", benchmark().getWarmupRunsPerRuntimeObservationCount());
  } else if (key == "mlir.set_buildtimes_per_observation_count") {
    const int ivalue = std::stoi(value);
    if (ivalue < 1) {
      return Status(
          StatusCode::INVALID_ARGUMENT,
          fmt::format("buildtimes_per_observation_count must be >= 1. Received: {}", ivalue));
    }
    benchmark().setBuildtimesPerObservationCount(ivalue);
    reply = value;
  } else if (key == "mlir.get_buildtimes_per_observation_count") {
    reply = fmt::format("{}", benchmark().getBuildtimesPerObservationCount());
  }
  return Status::OK;
}

}  // namespace compiler_gym::mlir_service

ext: cc

max_stars_repo_name: thecoblack/CompilerGym

max_issues_repo_name: thecoblack/CompilerGym

max_forks_repo_name: thecoblack/CompilerGym


text:
// Copyright (c) 2019 Graphcore Ltd. All rights reserved.
#define BOOST_TEST_MODULE NonLinearityGradSweepTest
// Test for the Non-Linearity Gradient Operations
// Used to verify the accuracy of Non-Linearity backward processing functions
// over any part of the Half-Precision range for both inputs.

#include <poplar/CSRFunctions.hpp>
#include <poplar/Engine.hpp>
#include <poplibs_support/TestDevice.hpp>
#include <popops/Zero.hpp>

#include "poputil/VertexTemplates.hpp"

#include "../lib/popops/ExprOpUtil.hpp"
#include "popops/ElementWise.hpp"
#include <poplibs_test/NonLinearity.hpp>
#include <poplibs_test/Util.hpp>
#include <popnn/NonLinearity.hpp>
#include <popnn/NonLinearityDefUtil.hpp>
#include <popnn/codelets.hpp>
#include <popops/codelets.hpp>
#include <poputil/TileMapping.hpp>

#include <boost/program_options.hpp>

#include <exception>

using namespace poplar;
using namespace poplar::program;
using namespace poputil;
using namespace poplibs_test::util;
using namespace popops;
using namespace popnn;
using namespace poplibs_support;

const poplar::OptionFlags options{{"debug.executionProfile", "compute_sets"}};

#define TOL 0.1 // tolerance of 0.1%
#define FLOAT_ATOL 1e-20
#define HALF_ATOL 1e-7

// The number of activation sweeps executed per tile
#define MAX_ACT_SWEEP_PER_TILE 512

// The largest number that can be represented in half-precision
#define HALF_0_TO_6 (0x45ff)

//*****************************************************************************
// Execute the Non-Linearity Gradient function over every possible range of
// valid Half-Precision value, and optionally stream the function output to
// stdout.
//
// The Non-Linearity Gradient function is a function of two tensors:
//    1. Tensor of activation inputs
//    2. Tensor of Gradient Outputs
//
// One sweep of the activation inputs is performed for every Gradient output
// value. Alternatively it is also possible to decimate the range of Gradient
// outputs used. In order to speed up execution, multiple activation sweeps
// can be packed into a single Tensor, with the following caveats/restrictions:
//   - The number of gradient output points in a single tensor must be a
//     power of 2.
//   - Every execution can only process the same multiple of activation sweeps.
//     Therefore if there are not sufficient activation sweeps left over for
//     the final execution, the final execution will be abandoned.
//
// Exclude the following values:
//   o     NaN: Exponent=0b11111 && Fraction=0
//   o  +/-Inf: Exponent=0b11111 && Fraction!=0
//
// The set of valid half-precision codes with the exponent in the range
// [0b00000 - 0b11110]. The full range can be defined by sweeping across the
// following ranges:
//   o codes from 0 to 0b0111101111111111 (0x7bff) and
//   o codes from -0 to 0b1111101111111111 (-0x7bff).
//
// The constant HALF_MAX_CODE is defined as 0x7bff. The full range of inputs is
// stored in the tensor in the following order:
//
//    Index into Tensor       Half-Precision Value
//    -----------------       --------------------
//          0                  -HALF_MAX_CODE
//          1                  -(HALF_MAX_CODE-1)
//          .                        .
//          .                        .
//          .                        .
//    HALF_MAX_CODE                 -0
//    HALF_MAX_CODE+1               +0
//          .                        .
//          .                        .
//          .                        .
//   (HALF_MAX_CODE+1)*2-1      HALF_MAX_CODE
//
//
// Format of the CSV output:
//   - The elements in each row are in increasing order with unit decimation.
//   - Each row corresponds to one particular Gradient Output code.
//   - The rows are arranged in decreasing order of Gradient Outputs, beginning
//     at the upper limit and proceeding to as close to the lower limit as
//     possible.
//
//*****************************************************************************
bool doNonLinearityGradSweep(const DeviceType &deviceType,
                             const unsigned ntiles, NonLinearityType nlType,
                             const unsigned activation_max_range,
                             const unsigned gradout_lower_code,
                             const unsigned gradout_upper_code,
                             const unsigned stride_vertex,
                             const unsigned stride_delta, bool doPrintTensors,
                             bool ignoreData) {

  unsigned sweep_activation = (activation_max_range + 1) * 2;
  unsigned num_elements = sweep_activation;

  auto invertHalfCodeSign = [](const unsigned code) { return code ^ 0x8000; };

  auto halfPrecisionToIndex = [invertHalfCodeSign](const unsigned code) {
    int index = ((code & (1 << 15)) ? (-invertHalfCodeSign(code) - 1) : code);
    return index;
  };

  auto indexToHalfPrecision = [invertHalfCodeSign](const int index) {
    unsigned code = ((index >= 0) ? index : invertHalfCodeSign((-index) - 1));
    return code;
  };

  int sweep_upper = halfPrecisionToIndex(gradout_upper_code);
  int sweep_lower = halfPrecisionToIndex(gradout_lower_code);
  if (sweep_lower >= sweep_upper) {
    throw std::invalid_argument("range-hi must be greater than range-lo!");
  }

  const std::size_t xSize = num_elements * stride_vertex;

  auto device = createTestDevice(deviceType, 1, ntiles);
  auto &target = device.getTarget();
  Graph graph(target);
  popnn::addCodelets(graph);

  // layer parameters
  auto actH = graph.addVariable(HALF, {xSize}, "actH");
  auto deltaH = graph.addVariable(HALF, {xSize}, "deltaH");

  // arbitrary mappings
  mapTensorLinearly(graph, actH);
  mapTensorLinearly(graph, deltaH);

  graph.createHostWrite("inH", actH);
  graph.createHostWrite("inDeltaH", deltaH);
  graph.createHostRead("outDeltaH", deltaH);

  //
  // Sweep_Full the full range of valid half values
  //
  // Exclude the following values:
  //   o     NaN: Exponent=0b11111 && Fraction=0
  //   o  +/-Inf: Exponent=0b11111 && Fraction!=0
  //
  // The set of valid half codes is the set of all values with the
  // exponent in the range [0b00000 - 0b11110]
  //
  std::vector<uint16_t> rawHActInH(xSize);
  std::vector<uint16_t> rawHDeltaInH(xSize);
  std::vector<uint16_t> rawHDeltaOutH(xSize);

  for (unsigned block = 0; block < stride_vertex; block++) {
    for (unsigned col = 0; (col < sweep_activation / 2) && (col < num_elements);
         col++) {
      rawHActInH[(block * sweep_activation) + col] =
          (((sweep_activation / 2) - 1 - col) | (1 << 15));
    }

    for (unsigned col = 0; (col < sweep_activation / 2) &&
                           ((col + (sweep_activation / 2)) < num_elements);
         col++) {
      rawHActInH[(block * sweep_activation) + (sweep_activation / 2) + col] =
          col;
    }
  }

  // reference results calculated in harness
  const auto batchSize = 1;
  boost::multi_array<double, 2> hRefActIn(boost::extents[batchSize][xSize]),
      hRefDeltaOut(boost::extents[batchSize][xSize]),
      hDeltaOutH(boost::extents[batchSize][xSize]);

  // Save the inputs
  poplar::copyDeviceHalfToDouble(target, rawHActInH.data(), &hRefActIn[0][0],
                                 xSize);

  int stride = stride_vertex * stride_delta;
  unsigned row_code = 0;
  unsigned total_rows = [](const unsigned a, const unsigned b) {
    return (a / b) * b;
  }((sweep_upper - sweep_lower), stride);
  for (int tile_row = sweep_upper; tile_row >= (sweep_lower + stride);
       tile_row -= stride) {

    std::cerr << std::hex << "GradOut:" << row_code << std::dec << "; Progress "
              << (sweep_upper - tile_row) << "/" << total_rows << "\n";

    // Construct activations within limited ranged, one stride_vertex of column
    // sweeps at a time
    for (unsigned block = 0; block < stride_vertex; block++) {
      for (unsigned col = 0; col < num_elements; col++) {
        int row = tile_row - (block * stride_delta);
        row_code = indexToHalfPrecision(row);
        rawHDeltaInH[(block * sweep_activation) + col] = row_code;
      }
    }

    // Obtain backward gradient calculation for reference
    poplar::copyDeviceHalfToDouble(target, rawHDeltaInH.data(),
                                   &hRefDeltaOut[0][0], xSize);
    poplibs_test::bwdNonLinearity(nlType, hRefActIn, hRefDeltaOut);

    // build and run the target code
    auto bwdProg = Sequence();

    //
    // Exceptions:
    //
    //    - Invalid Instruction    Enabled
    //    - Divide by 0            Enabled
    //    - Overflow               Disabled  // Overflow Expected
    //    - Stochastic Rounding    Disabled  // For deterministic results
    //    - NANOO                  Disabled  // Overflow Expected
    //
    setFloatingPointBehaviour(graph, bwdProg, {true, true, false, false, false},
                              "Set");

    auto deltaHH =
        nonLinearityInputGradient(graph, nlType, actH, deltaH, bwdProg);
    bwdProg.add(Copy(deltaHH, deltaH));
    Engine bwdEng(graph, bwdProg);
    device.bind([&](const Device &d) {
      bwdEng.load(d);
      bwdEng.writeTensor("inH", rawHActInH.data(),
                         rawHActInH.data() + rawHActInH.size());
      bwdEng.writeTensor("inDeltaH", rawHDeltaInH.data(),
                         rawHDeltaInH.data() + rawHDeltaInH.size());
      bwdEng.run();
      bwdEng.readTensor("outDeltaH", rawHDeltaOutH.data(),
                        rawHDeltaOutH.data() + rawHDeltaOutH.size());
    });

    poplar::copyDeviceHalfToDouble(target, rawHDeltaOutH.data(),
                                   &hDeltaOutH[0][0], xSize);

    /* Print output as a matrix */
    if (doPrintTensors) {
      for (auto block = 0U; block < xSize; block += sweep_activation) {
        unsigned i;
        for (i = 0U; i < num_elements - 1; ++i) {
          std::cout << hDeltaOutH[0][block + i] << ",";
        }
        std::cout << hDeltaOutH[0][block + i] << "\n";
      }
    }

    /* Check result */
    if (!ignoreData &&
        !checkIsClose("outDeltaH", hDeltaOutH, hRefDeltaOut, TOL, HALF_ATOL)) {
      return false;
    }
  }

  return true;
}

//******************************************************************************
int main(int argc, char **argv) {
  namespace po = boost::program_options;

  DeviceType deviceType;
  Type dataType;

  bool doPrintTensors = false;
  bool ignoreData = false;
  unsigned tiles = 1;
  unsigned stride = 1;
  std::string act_max_code_string;
  unsigned act_max_code = HALF_0_TO_6;
  NonLinearityType nlType;
  unsigned gradout_upper_code = 0x3C00;
  unsigned gradout_lower_code = 0x3BFF;
  std::string range_lo;
  std::string range_hi;

  po::options_description desc("Options");

  // clang-format off
  desc.add_options()
    ("help", "Print help")
    ("print",
     po::value<bool>(&doPrintTensors)->default_value(doPrintTensors),
     "Print the tensors")
    ("ignoreCheck",
     po::value<bool>(&ignoreData)->default_value(ignoreData),
     "Ignore mismatches from the expected results")
    ("device-type",
     po::value<DeviceType>(&deviceType)->required(),
     "Device Type")
    ("tiles",
     po::value<unsigned>(&tiles)->default_value(tiles),
     "Number of tiles to use")
    ("nl-type",
     po::value<NonLinearityType>(&nlType)->required(),
     "Non-linearity type")
    ("act-max-code",
     po::value<std::string>(&act_max_code_string)
         ->value_name(act_max_code_string),
     "The limit of the sweep of activations, as the Half-precision limit")
    ("stride",
     po::value<unsigned>(&stride)->default_value(stride),
     "Sweep Interval")
    ("range-lo",
     po::value<std::string>(&range_lo)->value_name("range-lo"),
     "Lower range of the sweep")
    ("range-hi",
     po::value<std::string>(&range_hi)->value_name("range-hi"),
     "Upper range of the sweep");
  // clang-format on

  po::variables_map vm;
  try {
    po::store(po::parse_command_line(argc, argv, desc), vm);
    if (vm.count("help")) {
      std::cout << desc << "\n\n";
      return 1;
    }
    po::notify(vm);
  } catch (std::exception &e) {
    std::cerr << "error: " << e.what() << "\n";
    return 1;
  }

  if (stride & (stride - 1)) {
    throw std::invalid_argument("stride must be a power of 2!");
  }

  if (stride > MAX_ACT_SWEEP_PER_TILE) {
    throw std::invalid_argument("stride exceeds maximum!");
  }

  if (vm.count("act-max-code")) {
    act_max_code = std::stoul(act_max_code_string, nullptr, 0);
  }

  if (vm.count("range-lo")) {
    gradout_lower_code = std::stoul(range_lo, nullptr, 0);
  }

  if (vm.count("range-hi")) {
    gradout_upper_code = std::stoul(range_hi, nullptr, 0);
  }

  unsigned stride_vertex = (gradout_upper_code - gradout_lower_code) / stride;
  if (stride_vertex > MAX_ACT_SWEEP_PER_TILE) {
    stride_vertex = MAX_ACT_SWEEP_PER_TILE;
  }

  std::cerr << "Input arguments:"
            << "\n";
  std::cerr << "  Activation range-lo = 0x" << std::hex
            << [](const unsigned code) { return code ^ 0x8000; }(act_max_code)
            << std::dec << "\n";

  std::cerr << "  Activation range-hi = 0x" << std::hex << act_max_code
            << std::dec << "\n";
  std::cerr << "  GradOut range-lo = 0x" << std::hex << gradout_lower_code
            << std::dec << "\n";
  std::cerr << "  GradOut range-hi = 0x" << std::hex << gradout_upper_code
            << std::dec << "\n";
  std::cerr << "  GradOut decimation factor = " << stride << "\n";
  std::cerr << "  Activation Sweeps per Vertex = " << stride_vertex << "\n\n";

  return !doNonLinearityGradSweep(
      deviceType, tiles, nlType, act_max_code, gradout_lower_code,
      gradout_upper_code, stride_vertex, stride, doPrintTensors, ignoreData);
}

ext: cpp

max_stars_repo_name: graphcore/poplibs


text:
#ifndef SOLVER_PARAMS_HPP
#define SOLVER_PARAMS_HPP

#include <Eigen/Dense>

// struct par_solver
// {
//   ///// Will not change between iterations
//   double x_min = -std::numeric_limits<double>::max();
//   double x_max = std::numeric_limits<double>::max();

//   double y_min = -std::numeric_limits<double>::max();
//   double y_max = std::numeric_limits<double>::max();

//   double z_min = -std::numeric_limits<double>::max();
//   double z_max = std::numeric_limits<double>::max();
//   Eigen::Vector3d v_max;
//   Eigen::Vector3d a_max;
//   double ydot_max;
//   double dc;
//   double dist_to_use_straight_guess;
//   int a_star_samp_x;
//   int a_star_samp_y;
//   int a_star_samp_z;
//   double a_star_fraction_voxel_size;
//   int num_seg;
//   int deg_pos;
//   int deg_yaw;
//   int num_max_of_obst;

//   std::string basis;
//   double a_star_bias;
//   bool allow_infeasible_guess;
//   double Ra;

//   double alpha_shrink;

//   // weights
//   double c_pos_smooth;
//   double c_yaw_smooth;
//   double c_fov;
//   double c_final_pos;
// };

#endif

ext: hpp

max_stars_repo_name: t-thanh/panther

max_issues_repo_name: t-thanh/panther

max_forks_repo_name: t-thanh/panther


text:
#include <boost/uuid/uuid.hpp>            // uuid class
#include <boost/uuid/uuid_generators.hpp> // generators
#include <boost/uuid/uuid_io.hpp>         // streaming operators etc.
#include <string>
#include <boost/lexical_cast.hpp>
#include "uuid.h"

using std::string;

/*class Uuid{

public:
string generate_uuid(void);
Uuid(){
	}
};*/

string Uuid::generate_uuid(){
    	boost::uuids::uuid uuid = boost::uuids::random_generator()();
	//string s=uuid;
//	return s;
const std::string tmp = boost::lexical_cast<std::string>(uuid);
	return tmp;

	//std::cout << uuid << std::endl;
}
/*int main(){
	Uuid* s=new Uuid();
	std::cout<<s->generate_uuid();

}*/


text:
/*
 * MIT License
 *
 * Copyright (c) 2020 Robert Grupp
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "xregOpenCLSys.h"

#include <boost/compute/system.hpp>

#include "xregStringUtils.h"

std::map<std::string,boost::compute::device> xreg::BuildDevIDStrsToDevMap()
{
  OpenCLNameDevMap id_str_to_devs;

  const auto all_devs = boost::compute::system::devices();

  for (const auto& cur_dev : all_devs)
  {
    std::string cur_dev_id = StringRemoveAll(cur_dev.name());  // removes any whitespace

    const auto exts = cur_dev.extensions();

    // check for NVIDIA extensions
    const auto nv_ext = std::find(exts.begin(), exts.end(), "cl_nv_device_attribute_query");
    if (nv_ext != exts.end())
    {
      int pci_bus_id_nv = 0;
      size_t pci_bus_id_nv_ret_size = 0;
      if (CL_SUCCESS == clGetDeviceInfo(cur_dev.id(),
                                        static_cast<cl_device_info>(0x4008),
                                        sizeof(pci_bus_id_nv), &pci_bus_id_nv,
                                        &pci_bus_id_nv_ret_size))
      {
        cur_dev_id += "-";
        cur_dev_id += ToString(pci_bus_id_nv);
      }
    }
    // TODO: AMD
    // TODO: Intel

    id_str_to_devs.emplace(cur_dev_id, cur_dev);
  }

  return id_str_to_devs;
}

std::vector<std::string> xreg::DevIDStrs(const bool use_cpu)
{
  const auto id_dev_map = BuildDevIDStrsToDevMap();

  std::vector<std::string> id_strs;
  id_strs.reserve(id_dev_map.size());

  for (const auto& id_dev_kv : id_dev_map)
  {
    if (use_cpu || (id_dev_kv.second.type() == boost::compute::device::gpu))
    {
      id_strs.push_back(id_dev_kv.first);
    }
  }

  return id_strs;
}


text:
#include "cli_args.hpp"

#include "connection.hpp"
#include "routing.hpp"
#include "utils.hpp"

#include <iostream>
#include <map>
#include <sstream>
#include <string>

#include <boost/program_options.hpp>

// Option strings.
#define K_S "K"
#define MLC_S "mlc"
#define NET_S "net"
#define ST_S "st"
#define POPULATION_S "population"
#define GD_S "gd"
#define BF_S "bf"
#define EE_S "ee"

using namespace std;
namespace po = boost::program_options;

cli_args
process_cli_args(int argc, const char *argv[])
{
  cli_args result;

  try
    {
      // General options.
      po::options_description gen("General options");
      gen.add_options()
        ("help,h", "produce help message");

      // Network options.
      po::options_description net("Network options");
      net.add_options()
        (NET_S, po::value<string>()->required(),
         "the network file name")

        ("units", po::value<int>()->required(),
         "the number of units")

        (GD_S, "run the generic Dijkstra search")
        (BF_S, "corroborate with the brute force search")
        (EE_S, "run the edge exclusion search");

      // Traffic options.
      po::options_description tra("Traffic options");
      tra.add_options()
        ("ol", po::value<double>()->required(),
         "the offered load")

        ("mht", po::value<double>()->required(),
         "the mean holding time")

        ("mnu", po::value<double>()->required(),
         "the mean number of units");

      // Simulation options.
      po::options_description sim("Simulation options");
      sim.add_options()
        ("seed", po::value<int>()->default_value(1),
         "the seed of the random number generator")

        (POPULATION_S, po::value<string>()->required(),
         "the population name");

      po::options_description all("Allowed options");
      all.add(gen).add(net).add(tra).add(sim);

      po::variables_map vm;
      po::store(po::command_line_parser(argc, argv).options(all).run(), vm);

      if (vm.count("help"))
        {
          cout << all << "\n";
          exit(0);
        }

      // If there is something wrong with parameters, we will get
      // notified automatically and the program will exit.
      po::notify(vm);

      // The search options.
      if (vm.count(GD_S))
        result.gd = true;

      // The search options.
      if (vm.count(BF_S))
        result.bf = true;

      // The search options.
      if (vm.count(EE_S))
        result.ee = true;

      // Let's check the combinations of the search algorithms.
      if (!result.gd && !result.ee)
        {
          cout << "You have the following search options:\n";
          cout << "* --gd to run the generic Dijkstra search only,\n";
          cout << "* --ee to run the edge exclusion search only,\n";
          cout << "* --gd --ee to run both searches.\n\n";
          cout <<
            "Along with --gd, you can also add --bf to corraborate\n"
            "the exactness of the generic Dijkstra search with the\n"
            "brute force search.  Use for small networks only.\n\n";
          cout <<
            "If you use both --gd and --ee, the connection will be\n"
            "established with the path found by the generic\n"
            "Dijkstra search.\n";
          cout.flush();
          exit(0);
        }

      // The network options.
      result.net = vm[NET_S].as<string>();

      result.units = vm["units"].as<int>();

      // The traffic options.
      result.ol = vm["ol"].as<double>();
      result.mht = vm["mht"].as<double>();
      result.mnu = vm["mnu"].as<double>();

      // The simulation options.
      result.seed = vm["seed"].as<int>();
      result.population = vm[POPULATION_S].as<string>();
    }
  catch(const std::exception& e)
    {
      cerr << e.what() << std::endl;
      exit(1);
    }

  return result;
}

max_stars_repo_name: iszczesniak/ddpp

max_issues_repo_name: iszczesniak/ddpp

max_forks_repo_name: iszczesniak/ddpp


text:
//
// Copyright (c) YugaByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
// in compliance with the License.  You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied.  See the License for the specific language governing permissions and limitations
// under the License.
//
//

#include "yb/util/opid.h"

#include <iostream>

#include <boost/functional/hash.hpp>

#include <glog/logging.h>

namespace yb {

constexpr int64_t OpId::kUnknownTerm;

void OpId::MakeAtLeast(const OpId& rhs) {
  if (rhs.empty()) {
    return;
  }
  if (empty()) {
    *this = rhs;
    return;
  }
  if (rhs.term > term || (rhs.term == term && rhs.index > index)) {
    *this = rhs;
  }
}

void OpId::MakeAtMost(const OpId& rhs) {
  if (rhs.empty()) {
    return;
  }
  if (empty()) {
    *this = rhs;
    return;
  }
  if (rhs.term < term || (rhs.term == term && rhs.index < index)) {
    *this = rhs;
  }
}

std::ostream& operator<<(std::ostream& out, const OpId& op_id) {
  return out << "{ term: " << op_id.term << " index: " << op_id.index << " }";
}

size_t hash_value(const OpId& op_id) noexcept {
  size_t result = 0;

  boost::hash_combine(result, op_id.term);
  boost::hash_combine(result, op_id.index);

  return result;
}

} // namespace yb

max_stars_repo_name: fabiocmazzo/yugabyte-db

max_issues_repo_name: fabiocmazzo/yugabyte-db

max_forks_repo_name: fabiocmazzo/yugabyte-db


text:
/* Output File Format:
 * 1st int: Number of classes
 * 2nd int: output feature vector length N
 * Follow P (unknown) patches:
 *      int object_id,
 *      float yaw, pitch, roll
 *      float x, y, z (in object coordinates)
 *      float[N] output vector
 */

#include <iostream>

#include <train_patch_generator.h>
#include <glog/logging.h>
#include <lmdb.h>
#include <boost/algorithm/string.hpp>

#include <cv.h>
#include <highgui.h>
#include <fstream>

void train_patch_generator::generate_train_patches(){

    //check caffe files
    CHECK_GT(caffe_model_definition_filename_.size(), 0) << "No caffe definition model defined.";
    CHECK_GT(caffe_model_weights_filename_.size(), 0) << "No caffe weights model defined.";

    //caffe::Caffe::set_phase(caffe::Caffe::TEST);  // used with previous version of Caffe
    caffe::Net<float> caffe_net(caffe_model_definition_filename_, caffe::TEST);
    caffe_net.CopyTrainedLayersFrom(caffe_model_weights_filename_);

    //create lmdb database
    MDB_env* mdb_env;
    MDB_dbi mdb_dbi;
    MDB_txn* mdb_txn;
    MDB_cursor* mdb_cursor;
    MDB_val mdb_key, mdb_value;
    CHECK_GT(input_lmdb_.size(), 0) << "No lmdb input specified.";
    CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
    CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS);  // 1TB
    CHECK_EQ(mdb_env_open(mdb_env,
             input_lmdb_.c_str(),
             MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
    CHECK_EQ(mdb_txn_begin(mdb_env, NULL, MDB_RDONLY, &mdb_txn), MDB_SUCCESS)
        << "mdb_txn_begin failed";
    CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
        << "mdb_open failed";
    CHECK_EQ(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor), MDB_SUCCESS)
        << "mdb_cursor_open failed";
    CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST),
        MDB_SUCCESS) << "mdb_cursor_get failed";

    //annotation file
    std::ifstream fannot((input_lmdb_ + "/patch_annotation_lmdb.txt").c_str());
    CHECK(fannot) << "Cannot open annotation file: " << input_lmdb_ + "/patch_annotation_lmdb.txt";

    CHECK(output_file_.size() > 0) << "No output file specified";

    //output file
    std::ofstream fout(output_file_.c_str(), std::ios::out | std::ios::binary);
    CHECK(fout) << "Could not open output file " << output_file_ << " for writing.";

    //Get number of classes
    int num_classes;
    fannot >> num_classes;
    //write number of classes
    fout.write((char*)&num_classes, sizeof(int));

    bool output_vector_length_written = false;
    int num_patches = 0;
    while(true){

        bool data_end = false;
        std::vector<Annotation> annot_vec;
        std::vector<float> float_data;
        for(int b=0; b<batch_size_; ++b){

            //get patch from lmdb
            caffe::Datum datum;
            datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size);
            std::string key((char*)mdb_key.mv_data);
            key.resize(13); //format: xxxx_yyyyyyyy -> x: obj number, y: patch number

            //get annotation
            Annotation annot;
            CHECK(fannot >> annot.annot_key >> annot.yaw >> annot.pitch >> annot.roll >> annot.obj_x >> annot.obj_y >> annot.obj_z)
                    << "Couldn't read annotation. Maybe not enough entries?";

            CHECK_EQ(key, annot.annot_key) << "annotation and database key mismatch";

            annot_vec.push_back(annot);

            //normalize & copy patch to Net
            //std::vector<float> float_data((unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
            int start = float_data.size();
            float_data.insert(float_data.end(), (unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
            //apply same scale to the input as in training
            for(int i=start; i<float_data.size(); ++i)
                float_data[i] /= 255.0f;

            if(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_NEXT) != MDB_SUCCESS){
                data_end = true;
                break;
            }

        }

        if(data_end)
            break;

        if(use_gpu_)
            caffe::caffe_copy(caffe_net.input_blobs()[0]->count(), &(float_data[0]), caffe_net.input_blobs()[0]->mutable_gpu_data());
        else
            caffe::caffe_copy(caffe_net.input_blobs()[0]->count(), &(float_data[0]), caffe_net.input_blobs()[0]->mutable_cpu_data());

        //Get feature from Net - forward image patch
        //output_blob[0]: contains the result
        //output_blob[1]: contains the input
        const std::vector<caffe::Blob<float>* > output_blob = caffe_net.ForwardPrefilled();

        for(int b=0; b<batch_size_; ++b){

            int feature_vector_length = output_blob[0]->count() / batch_size_;

            //write vector length only once
            if(!output_vector_length_written){
                int output_vector_length = feature_vector_length;
                fout.write((char*)&output_vector_length, sizeof(int));
                output_vector_length_written = true;
            }

            //write annotation & feature vector
            std::vector<std::string> object_id_str;
            boost::split(object_id_str, annot_vec[b].annot_key, boost::is_any_of("_"));
            int obj_id = boost::lexical_cast<int>(object_id_str[0].c_str(), object_id_str[0].size());

            fout.write((char*)&obj_id,  sizeof(int));
            fout.write((char*)&annot_vec[b].yaw,     sizeof(float));
            fout.write((char*)&annot_vec[b].pitch,   sizeof(float));
            fout.write((char*)&annot_vec[b].roll,    sizeof(float));
            fout.write((char*)&annot_vec[b].obj_x,   sizeof(float));
            fout.write((char*)&annot_vec[b].obj_y,   sizeof(float));
            fout.write((char*)&annot_vec[b].obj_z,   sizeof(float));

            for(int i=0; i<feature_vector_length; ++i){
                float f = output_blob[0]->data_at(b, i, 0, 0);
                fout.write((char*)&f, sizeof(float));
            }

//    //      visualize patches and output, works only with full network, i.e. full reconstruction
//            cv::Mat rgb1(patch_size_, patch_size_, CV_8UC3);
//            cv::Mat rgb2(patch_size_, patch_size_, CV_8UC3);
//            for(int c=0; c<3; ++c){
//                for(int h=0; h<patch_size_; ++h){
//                    for(int w=0; w<patch_size_; ++w){
//                        int idx = c*patch_size_*patch_size_ + h*patch_size_ + w;
//                        rgb1.at<cv::Vec3b>(h, w)[c] = static_cast<unsigned char>(output_blob[0]->data_at(b, idx, 0, 0) * 255.0f);
//                        rgb2.at<cv::Vec3b>(h, w)[c] = static_cast<unsigned char>(output_blob[1]->data_at(b, idx, 0, 0) * 255.0f);
//                    }
//                }
//            }

//            cv::resize(rgb1, rgb1, cv::Size(100, 100));
//            cv::resize(rgb2, rgb2, cv::Size(100, 100));
//            int k=-1;
//            while(k==-1){
//                cv::imshow("rgb1", rgb1);
//                cv::imshow("rgb2", rgb2);
//                k = cv::waitKey(30);
//            }

        }

        num_patches += batch_size_;
        std::cout << "Training samples written: " << num_patches << "\r";

    }
    std::cout << std::endl << "Finished!" << std::endl;

    //close lmdb database
    mdb_cursor_close(mdb_cursor);
    mdb_close(mdb_env, mdb_dbi);
    mdb_txn_abort(mdb_txn);
    mdb_env_close(mdb_env);

    //close files
    fannot.close();
    fout.close();

}

void train_patch_generator::generate_train_patches_pixeltests(){

    //create lmdb database
    MDB_env* mdb_env;
    MDB_dbi mdb_dbi;
    MDB_txn* mdb_txn;
    MDB_cursor* mdb_cursor;
    MDB_val mdb_key, mdb_value;
    CHECK_GT(input_lmdb_.size(), 0) << "No lmdb input specified.";
    CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
    CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS);  // 1TB
    CHECK_EQ(mdb_env_open(mdb_env,
             input_lmdb_.c_str(),
             MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
    CHECK_EQ(mdb_txn_begin(mdb_env, NULL, MDB_RDONLY, &mdb_txn), MDB_SUCCESS)
        << "mdb_txn_begin failed";
    CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
        << "mdb_open failed";
    CHECK_EQ(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor), MDB_SUCCESS)
        << "mdb_cursor_open failed";
    CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST),
        MDB_SUCCESS) << "mdb_cursor_get failed";

    //annotation file
    std::ifstream fannot((input_lmdb_ + "/patch_annotation_lmdb.txt").c_str());
    CHECK(fannot) << "Cannot open annotation file: " << input_lmdb_ + "/patch_annotation_lmdb.txt";

    CHECK(output_file_.size() > 0) << "No output file specified";

    //output file
    std::ofstream fout(output_file_.c_str(), std::ios::out | std::ios::binary);
    CHECK(fout) << "Could not open output file " << output_file_ << " for writing.";

    //Get number of classes
    int num_classes;
    fannot >> num_classes;
    //write number of classes
    fout.write((char*)&num_classes, sizeof(int));

    bool output_vector_length_written = false;
    int num_patches = 0;

    int feature_vector_length = 0;

    while(true){

        bool data_end = false;
        std::vector<Annotation> annot_vec;
        std::vector<float> float_data;
        for(int b=0; b<batch_size_; ++b){

            //get patch from lmdb
            caffe::Datum datum;
            datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size);
            std::string key((char*)mdb_key.mv_data);
            key.resize(13); //format: xxxx_yyyyyyyy -> x: obj number, y: patch number

            //get annotation
            Annotation annot;
            CHECK(fannot >> annot.annot_key >> annot.yaw >> annot.pitch >> annot.roll >> annot.obj_x >> annot.obj_y >> annot.obj_z)
                    << "Couldn't read annotation. Maybe not enough entries?";

            CHECK_EQ(key, annot.annot_key) << "annotation and database key mismatch";

            annot_vec.push_back(annot);

            //normalize & copy patch to Net
            //std::vector<float> float_data((unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
            int start = float_data.size();
            float_data.insert(float_data.end(), (unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
            //apply same scale to the input as in training
            for(int i=start; i<float_data.size(); ++i)
                float_data[i] /= 255.0f;

            if(feature_vector_length == 0)
                feature_vector_length = datum.data().size();

            if(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_NEXT) != MDB_SUCCESS){
                data_end = true;
                break;
            }

        }

        if(data_end)
            break;

        for(int b=0; b<batch_size_; ++b){

            //write vector length only once
            if(!output_vector_length_written){
                int output_vector_length = feature_vector_length;
                fout.write((char*)&output_vector_length, sizeof(int));
                output_vector_length_written = true;
            }

            //write annotation & feature vector
            std::vector<std::string> object_id_str;
            boost::split(object_id_str, annot_vec[b].annot_key, boost::is_any_of("_"));
            int obj_id = boost::lexical_cast<int>(object_id_str[0].c_str(), object_id_str[0].size());

            fout.write((char*)&obj_id,  sizeof(int));
            fout.write((char*)&annot_vec[b].yaw,     sizeof(float));
            fout.write((char*)&annot_vec[b].pitch,   sizeof(float));
            fout.write((char*)&annot_vec[b].roll,    sizeof(float));
            fout.write((char*)&annot_vec[b].obj_x,   sizeof(float));
            fout.write((char*)&annot_vec[b].obj_y,   sizeof(float));
            fout.write((char*)&annot_vec[b].obj_z,   sizeof(float));

            fout.write((char*)&float_data[b*feature_vector_length], feature_vector_length * sizeof(float));

        }

        num_patches += batch_size_;
        std::cout << "Training samples written: " << num_patches << "\r";

    }
    std::cout << std::endl << "Finished!" << std::endl;

    //close lmdb database
    mdb_cursor_close(mdb_cursor);
    mdb_close(mdb_env, mdb_dbi);
    mdb_txn_abort(mdb_txn);
    mdb_env_close(mdb_env);

    //close files
    fannot.close();
    fout.close();

}

void train_patch_generator::generate_train_patches_kmeans_centers(){

    //create lmdb database
    MDB_env* mdb_env;
    MDB_dbi mdb_dbi;
    MDB_txn* mdb_txn;
    MDB_cursor* mdb_cursor;
    MDB_val mdb_key, mdb_value;
    CHECK_GT(input_lmdb_.size(), 0) << "No lmdb input specified.";
    CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
    CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS);  // 1TB
    CHECK_EQ(mdb_env_open(mdb_env,
             input_lmdb_.c_str(),
             MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
    CHECK_EQ(mdb_txn_begin(mdb_env, NULL, MDB_RDONLY, &mdb_txn), MDB_SUCCESS)
        << "mdb_txn_begin failed";
    CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
        << "mdb_open failed";
    CHECK_EQ(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor), MDB_SUCCESS)
        << "mdb_cursor_open failed";
    CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST),
        MDB_SUCCESS) << "mdb_cursor_get failed";

    //annotation file
    std::ifstream fannot((input_lmdb_ + "/patch_annotation_lmdb.txt").c_str());
    CHECK(fannot) << "Cannot open annotation file: " << input_lmdb_ + "/patch_annotation_lmdb.txt";

    CHECK(output_file_.size() > 0) << "No output file specified";

    //Get number of classes
    int num_classes;
    fannot >> num_classes;

    int num_patches = 0;

    int feature_vector_length = 0;

    std::vector<float> float_data;
    std::vector<Annotation> annot_vec;

    while(true){

        bool data_end = false;

        //get patch from lmdb
        caffe::Datum datum;
        datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size);
        std::string key((char*)mdb_key.mv_data);
        key.resize(13); //format: xxxx_yyyyyyyy -> x: obj number, y: patch number

        //get annotation
        Annotation annot;
        CHECK(fannot >> annot.annot_key >> annot.yaw >> annot.pitch >> annot.roll >> annot.obj_x >> annot.obj_y >> annot.obj_z)
                << "Couldn't read annotation. Maybe not enough entries?";

        CHECK_EQ(key, annot.annot_key) << "annotation and database key mismatch";

        annot_vec.push_back(annot);

        //normalize & copy patch to Net
        //std::vector<float> float_data((unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
        int start = float_data.size();
        float_data.insert(float_data.end(), (unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
        //apply same scale to the input as in training
        for(int i=start; i<float_data.size(); ++i)
            float_data[i] /= 255.0f;

        if(feature_vector_length == 0)
            feature_vector_length = datum.data().size();

        if(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_NEXT) != MDB_SUCCESS){
            data_end = true;
            break;
        }

        if(data_end)
            break;

    }

    int nvectors = float_data.size() / feature_vector_length;
    cv::Mat data(nvectors, feature_vector_length, CV_32FC1);
    std::cout << "nvectors: " << nvectors << std::endl;

    //nvectors = 800;
    int nfeatures = 400;

    for(int row = 0; row < nvectors; ++row)
        for(int col = 0; col < feature_vector_length; ++col)
            data.at<float>(row, col) = float_data[row*feature_vector_length + col];

    cv::Mat bestLabels;
    cv::TermCriteria criteria(cv::TermCriteria::COUNT, 100, 0.001);
    cv::Mat centers;
    cv::kmeans(data, nfeatures, bestLabels, criteria, 1, cv::KMEANS_PP_CENTERS, centers);

    cv::FileStorage fout_centers(output_file_ + ".centers.xml", cv::FileStorage::WRITE);
    fout_centers << "centers" << centers;
    fout_centers.release();

    std::cout << std::endl << "Finished!" << std::endl;

    //close lmdb database
    mdb_cursor_close(mdb_cursor);
    mdb_close(mdb_env, mdb_dbi);
    mdb_txn_abort(mdb_txn);
    mdb_env_close(mdb_env);

    //close files
    fannot.close();

}

void train_patch_generator::generate_train_patches_kmeans_vectors(){

    //create lmdb database
    MDB_env* mdb_env;
    MDB_dbi mdb_dbi;
    MDB_txn* mdb_txn;
    MDB_cursor* mdb_cursor;
    MDB_val mdb_key, mdb_value;
    CHECK_GT(input_lmdb_.size(), 0) << "No lmdb input specified.";
    CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed";
    CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS);  // 1TB
    CHECK_EQ(mdb_env_open(mdb_env,
             input_lmdb_.c_str(),
             MDB_RDONLY|MDB_NOTLS, 0664), MDB_SUCCESS) << "mdb_env_open failed";
    CHECK_EQ(mdb_txn_begin(mdb_env, NULL, MDB_RDONLY, &mdb_txn), MDB_SUCCESS)
        << "mdb_txn_begin failed";
    CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS)
        << "mdb_open failed";
    CHECK_EQ(mdb_cursor_open(mdb_txn, mdb_dbi, &mdb_cursor), MDB_SUCCESS)
        << "mdb_cursor_open failed";
    CHECK_EQ(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_FIRST),
        MDB_SUCCESS) << "mdb_cursor_get failed";

    //annotation file
    std::ifstream fannot((input_lmdb_ + "/patch_annotation_lmdb.txt").c_str());
    CHECK(fannot) << "Cannot open annotation file: " << input_lmdb_ + "/patch_annotation_lmdb.txt";

    CHECK(output_file_.size() > 0) << "No output file specified";

    //output file
    std::ofstream fout(output_file_.c_str(), std::ios::out | std::ios::binary);
    CHECK(fout) << "Could not open output file " << output_file_ << " for writing.";

    //Get number of classes
    int num_classes;
    fannot >> num_classes;
    //write number of classes
    fout.write((char*)&num_classes, sizeof(int));

    int num_patches = 0;

    int feature_vector_length = 0;

    std::vector<float> float_data;
    std::vector<Annotation> annot_vec;

    while(true){

        bool data_end = false;

        //get patch from lmdb
        caffe::Datum datum;
        datum.ParseFromArray(mdb_value.mv_data, mdb_value.mv_size);
        std::string key((char*)mdb_key.mv_data);
        key.resize(13); //format: xxxx_yyyyyyyy -> x: obj number, y: patch number

        //get annotation
        Annotation annot;
        CHECK(fannot >> annot.annot_key >> annot.yaw >> annot.pitch >> annot.roll >> annot.obj_x >> annot.obj_y >> annot.obj_z)
                << "Couldn't read annotation. Maybe not enough entries?";

        CHECK_EQ(key, annot.annot_key) << "annotation and database key mismatch";

        annot_vec.push_back(annot);

        //normalize & copy patch to Net
        //std::vector<float> float_data((unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
        int start = float_data.size();
        float_data.insert(float_data.end(), (unsigned char*)&datum.data()[0], (unsigned char*)&datum.data()[0] + datum.data().size());
        //apply same scale to the input as in training
        for(int i=start; i<float_data.size(); ++i)
            float_data[i] /= 255.0f;

        if(feature_vector_length == 0)
            feature_vector_length = datum.data().size();

        if(mdb_cursor_get(mdb_cursor, &mdb_key, &mdb_value, MDB_NEXT) != MDB_SUCCESS){
            data_end = true;
            break;
        }

        if(data_end)
            break;

    }

    int nvectors = float_data.size() / feature_vector_length;
    cv::Mat data(nvectors, feature_vector_length, CV_32FC1);
    std::cout << "nvectors: " << nvectors << std::endl;

    //nvectors = 800;
    int nfeatures = 400;

    for(int row = 0; row < nvectors; ++row)
        for(int col = 0; col < feature_vector_length; ++col)
            data.at<float>(row, col) = float_data[row*feature_vector_length + col];

    cv::Mat centers;
    cv::FileStorage fout_centers(output_file_ + ".centers.xml", cv::FileStorage::READ);
    fout_centers["centers"] >> centers;
    fout_centers.release();

    fout.write((char*)&nfeatures, sizeof(int));

    for(int b=0; b<nvectors; ++b){
        //write annotation & feature vector
        std::vector<std::string> object_id_str;
        boost::split(object_id_str, annot_vec[b].annot_key, boost::is_any_of("_"));
        int obj_id = boost::lexical_cast<int>(object_id_str[0].c_str(), object_id_str[0].size());

        fout.write((char*)&obj_id,  sizeof(int));
        fout.write((char*)&annot_vec[b].yaw,     sizeof(float));
        fout.write((char*)&annot_vec[b].pitch,   sizeof(float));
        fout.write((char*)&annot_vec[b].roll,    sizeof(float));
        fout.write((char*)&annot_vec[b].obj_x,   sizeof(float));
        fout.write((char*)&annot_vec[b].obj_y,   sizeof(float));
        fout.write((char*)&annot_vec[b].obj_z,   sizeof(float));

        std::vector<float> z(nfeatures);
        float meanz = 0;
        for(int k=0; k<nfeatures; ++k){
            z[k] = 0;
            for(int i=0; i<feature_vector_length; ++i)
                z[k] += pow(data.at<float>(b, i) - centers.at<float>(k, i), 2);
            z[k] = sqrt(z[k]);
            meanz += z[k] / nfeatures;
        }

        std::vector<float> fvec(nfeatures, 0);
        for(int k=0; k<nfeatures; ++k)
            fvec[k] = std::max((float)0, float(meanz - z[k]));

        fout.write((char*)&(fvec[0]), nfeatures * sizeof(float));

        num_patches++;
        std::cout << "Training samples written: " << num_patches << "\r";
    }

    std::cout << std::endl << "Finished!" << std::endl;

    //close lmdb database
    mdb_cursor_close(mdb_cursor);
    mdb_close(mdb_env, mdb_dbi);
    mdb_txn_abort(mdb_txn);
    mdb_env_close(mdb_env);

    //close files
    fannot.close();
    fout.close();

}