text:
/*
 * Copyright 2017 MapD Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "StringDictionary.h"
#include "StringDictionaryClient.h"
#include "../Shared/sqltypes.h"
#include "../Utils/StringLike.h"
#include "../Utils/Regexp.h"
#include "Shared/thread_count.h"

#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <glog/logging.h>
#include <sys/fcntl.h>

#include <thread>

namespace {
const int PAGE_SIZE = getpagesize();

size_t file_size(const int fd) {
  struct stat buf;
  fstat(fd, &buf);
  return buf.st_size;
}

int checked_open(const char* path, const bool recover) {
  auto fd = open(path, O_RDWR | O_CREAT | (recover ? O_APPEND : O_TRUNC), 0644);
  CHECK_GE(fd, 0);
  return fd;
}

void* checked_mmap(const int fd, const size_t sz) {
  auto ptr = mmap(nullptr, sz, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
  CHECK(ptr != reinterpret_cast<void*>(-1));
#ifdef __linux__
#ifdef MADV_HUGEPAGE
  madvise(ptr, sz, MADV_RANDOM | MADV_WILLNEED | MADV_HUGEPAGE);
#else
  madvise(ptr, sz, MADV_RANDOM | MADV_WILLNEED);
#endif
#endif
  return ptr;
}

void checked_munmap(void* addr, size_t length) {
  CHECK_EQ(0, munmap(addr, length));
}

const uint32_t round_up_p2(const size_t num) {
  uint32_t in = num;
  in--;
  in |= in >> 1;
  in |= in >> 2;
  in |= in >> 4;
  in |= in >> 8;
  in |= in >> 16;
  in++;
  // TODO MAT deal with case where filesize has been increased but reality is
  // we are constrained to 2^30.
  // In that situation this calculation will wrap to zero
  if (in == 0) {
    in = 1 << 31;
  }
  return in;
}

size_t rk_hash(const std::string& str) {
  size_t str_hash = 1;
  for (size_t i = 0; i < str.size(); ++i) {
    str_hash = str_hash * 997 + str[i];
  }
  return str_hash;
}
}  // namespace

const int32_t StringDictionary::INVALID_STR_ID{-1};

StringDictionary::StringDictionary(const std::string& folder, const bool recover, size_t initial_capacity) noexcept
    : str_count_(0),
      str_ids_(initial_capacity, INVALID_STR_ID),
      payload_fd_(-1),
      offset_fd_(-1),
      offset_map_(nullptr),
      payload_map_(nullptr),
      offset_file_size_(0),
      payload_file_size_(0),
      payload_file_off_(0) {
  if (folder.empty()) {
    return;
  }
  // initial capacity must be a power of two for efficient bucket computation
  CHECK_EQ(size_t(0), (initial_capacity & (initial_capacity - 1)));
  boost::filesystem::path storage_path(folder);
  offsets_path_ = (storage_path / boost::filesystem::path("DictOffsets")).string();
  const auto payload_path = (storage_path / boost::filesystem::path("DictPayload")).string();
  payload_fd_ = checked_open(payload_path.c_str(), recover);
  offset_fd_ = checked_open(offsets_path_.c_str(), recover);
  payload_file_size_ = file_size(payload_fd_);
  if (payload_file_size_ == 0) {
    addPayloadCapacity();
  }
  offset_file_size_ = file_size(offset_fd_);
  if (offset_file_size_ == 0) {
    addOffsetCapacity();
  }
  payload_map_ = reinterpret_cast<char*>(checked_mmap(payload_fd_, payload_file_size_));
  offset_map_ = reinterpret_cast<StringIdxEntry*>(checked_mmap(offset_fd_, offset_file_size_));
  if (recover) {
    const size_t bytes = file_size(offset_fd_);
    const unsigned str_count = bytes / sizeof(StringIdxEntry);
    // at this point we know the size of the StringDict we need to load
    // so lets reallocate the vector to the correct size
    const uint32_t max_entries = round_up_p2(str_count * 2 + 1);
    std::vector<int32_t> new_str_ids(max_entries, INVALID_STR_ID);
    str_ids_.swap(new_str_ids);
    unsigned string_id = 0;
    mapd_lock_guard<mapd_shared_mutex> write_lock(rw_mutex_);
    for (string_id = 0; string_id < str_count; ++string_id) {
      const auto recovered = getStringFromStorage(string_id);
      if (std::get<2>(recovered)) {
        // hit the canary, recovery finished
        break;
      }
      getOrAddImpl(std::string(std::get<0>(recovered), std::get<1>(recovered)), true);
    }
    if (bytes % sizeof(StringIdxEntry) != 0) {
      LOG(WARNING) << "Offsets " << offsets_path_ << " file is truncated";
    }
  }
}

StringDictionary::StringDictionary(const LeafHostInfo& host, const int dict_id)
    : client_(new StringDictionaryClient(host, dict_id, true)),
      client_no_timeout_(new StringDictionaryClient(host, dict_id, false)) {}

StringDictionary::~StringDictionary() noexcept {
  if (payload_map_) {
    CHECK(offset_map_);
    checked_munmap(payload_map_, payload_file_size_);
    checked_munmap(offset_map_, offset_file_size_);
    CHECK_GE(payload_fd_, 0);
    close(payload_fd_);
    CHECK_GE(offset_fd_, 0);
    close(offset_fd_);
  }
}

int32_t StringDictionary::getOrAdd(const std::string& str) noexcept {
  if (client_) {
    std::vector<int32_t> string_ids;
    client_->get_or_add_bulk(string_ids, {str});
    CHECK_EQ(size_t(1), string_ids.size());
    return string_ids.front();
  }
  mapd_lock_guard<mapd_shared_mutex> write_lock(rw_mutex_);
  return getOrAddImpl(str, false);
}

namespace {

template <class T>
void log_encoding_error(const std::string& str) {
  LOG(ERROR) << "Could not encode string: " << str << ", the encoded value doesn't fit in " << sizeof(T) * 8
             << " bits. Will store NULL instead.";
}

}  // namespace

template <class T>
void StringDictionary::getOrAddBulk(const std::vector<std::string>& string_vec, T* encoded_vec) {
  if (client_no_timeout_) {
    getOrAddBulkRemote(string_vec, encoded_vec);
    return;
  }
  mapd_lock_guard<mapd_shared_mutex> write_lock(rw_mutex_);
  size_t out_idx{0};
  for (const auto& str : string_vec) {
    const auto string_id = getOrAddImpl(str, false);
    const bool invalid = string_id > max_valid_int_value<T>();
    if (invalid || string_id == inline_int_null_value<int32_t>()) {
      if (invalid) {
        log_encoding_error<T>(str);
      }
      encoded_vec[out_idx++] = inline_int_null_value<T>();
      continue;
    }
    encoded_vec[out_idx++] = string_id;
  }
}

template void StringDictionary::getOrAddBulk(const std::vector<std::string>& string_vec, uint8_t* encoded_vec);
template void StringDictionary::getOrAddBulk(const std::vector<std::string>& string_vec, uint16_t* encoded_vec);
template void StringDictionary::getOrAddBulk(const std::vector<std::string>& string_vec, int32_t* encoded_vec);

template <class T>
void StringDictionary::getOrAddBulkRemote(const std::vector<std::string>& string_vec, T* encoded_vec) {
  CHECK(client_no_timeout_);
  std::vector<int32_t> string_ids;
  client_no_timeout_->get_or_add_bulk(string_ids, string_vec);
  size_t out_idx{0};
  for (size_t i = 0; i < string_ids.size(); ++i) {
    const auto string_id = string_ids[i];
    const bool invalid = string_id > max_valid_int_value<T>();
    if (invalid || string_id == inline_int_null_value<int32_t>()) {
      if (invalid) {
        log_encoding_error<T>(string_vec[i]);
      }
      encoded_vec[out_idx++] = inline_int_null_value<T>();
      continue;
    }
    encoded_vec[out_idx++] = string_id;
  }
}

template void StringDictionary::getOrAddBulkRemote(const std::vector<std::string>& string_vec, uint8_t* encoded_vec);
template void StringDictionary::getOrAddBulkRemote(const std::vector<std::string>& string_vec, uint16_t* encoded_vec);
template void StringDictionary::getOrAddBulkRemote(const std::vector<std::string>& string_vec, int32_t* encoded_vec);

int32_t StringDictionary::getIdOfString(const std::string& str) const {
  mapd_shared_lock<mapd_shared_mutex> read_lock(rw_mutex_);
  if (client_) {
    return client_->get(str);
  }
  return getUnlocked(str);
}

int32_t StringDictionary::getUnlocked(const std::string& str) const noexcept {
  auto str_id = str_ids_[computeBucket(str, str_ids_, false)];
  return str_id;
}

std::string StringDictionary::getString(int32_t string_id) const {
  mapd_shared_lock<mapd_shared_mutex> read_lock(rw_mutex_);
  if (client_) {
    std::string ret;
    client_->get_string(ret, string_id);
    return ret;
  }
  return getStringUnlocked(string_id);
}

std::string StringDictionary::getStringUnlocked(int32_t string_id) const noexcept {
  CHECK_LT(string_id, static_cast<int32_t>(str_count_));
  return getStringChecked(string_id);
}

std::pair<char*, size_t> StringDictionary::getStringBytes(int32_t string_id) const noexcept {
  mapd_shared_lock<mapd_shared_mutex> read_lock(rw_mutex_);
  CHECK(!client_);
  CHECK_LE(0, string_id);
  CHECK_LT(string_id, static_cast<int32_t>(str_count_));
  return getStringBytesChecked(string_id);
}

size_t StringDictionary::storageEntryCount() const {
  mapd_shared_lock<mapd_shared_mutex> read_lock(rw_mutex_);
  if (client_) {
    return client_->storage_entry_count();
  }
  return str_count_;
}

namespace {

bool is_like(const std::string& str,
             const std::string& pattern,
             const bool icase,
             const bool is_simple,
             const char escape) {
  return icase ? (is_simple ? string_ilike_simple(str.c_str(), str.size(), pattern.c_str(), pattern.size())
                            : string_ilike(str.c_str(), str.size(), pattern.c_str(), pattern.size(), escape))
               : (is_simple ? string_like_simple(str.c_str(), str.size(), pattern.c_str(), pattern.size())
                            : string_like(str.c_str(), str.size(), pattern.c_str(), pattern.size(), escape));
}

}  // namespace

std::vector<std::string> StringDictionary::getLike(const std::string& pattern,
                                                   const bool icase,
                                                   const bool is_simple,
                                                   const char escape,
                                                   const size_t generation) const {
  mapd_lock_guard<mapd_shared_mutex> write_lock(rw_mutex_);
  if (client_) {
    return client_->get_like(pattern, icase, is_simple, escape, generation);
  }
  const auto cache_key = std::make_tuple(pattern, icase, is_simple, escape);
  const auto it = like_cache_.find(cache_key);
  if (it != like_cache_.end()) {
    return it->second;
  }
  std::vector<std::string> result;
  std::vector<std::thread> workers;
  int worker_count = cpu_threads();
  CHECK_GT(worker_count, 0);
  std::vector<std::vector<std::string>> worker_results(worker_count);
  CHECK_LE(generation, str_count_);
  for (int worker_idx = 0; worker_idx < worker_count; ++worker_idx) {
    workers.emplace_back(
        [&worker_results, &pattern, generation, icase, is_simple, escape, worker_idx, worker_count, this]() {
          for (size_t string_id = worker_idx; string_id < generation; string_id += worker_count) {
            const auto str = getStringUnlocked(string_id);
            if (is_like(str, pattern, icase, is_simple, escape)) {
              worker_results[worker_idx].push_back(str);
            }
          }
        });
  }
  for (auto& worker : workers) {
    worker.join();
  }
  for (const auto& worker_result : worker_results) {
    result.insert(result.end(), worker_result.begin(), worker_result.end());
  }
  // place result into cache for reuse if similar query
  const auto it_ok = like_cache_.insert(std::make_pair(cache_key, result));
  CHECK(it_ok.second);
  return result;
}

namespace {

bool is_regexp_like(const std::string& str, const std::string& pattern, const char escape) {
  return regexp_like(str.c_str(), str.size(), pattern.c_str(), pattern.size(), escape);
}

}  // namespace

std::vector<std::string> StringDictionary::getRegexpLike(const std::string& pattern,
                                                         const char escape,
                                                         const size_t generation) const {
  mapd_lock_guard<mapd_shared_mutex> write_lock(rw_mutex_);
  if (client_) {
    return client_->get_regexp_like(pattern, escape, generation);
  }
  const auto cache_key = std::make_pair(pattern, escape);
  const auto it = regex_cache_.find(cache_key);
  if (it != regex_cache_.end()) {
    return it->second;
  }
  std::vector<std::string> result;
  std::vector<std::thread> workers;
  int worker_count = cpu_threads();
  CHECK_GT(worker_count, 0);
  std::vector<std::vector<std::string>> worker_results(worker_count);
  CHECK_LE(generation, str_count_);
  for (int worker_idx = 0; worker_idx < worker_count; ++worker_idx) {
    workers.emplace_back([&worker_results, &pattern, generation, escape, worker_idx, worker_count, this]() {
      for (size_t string_id = worker_idx; string_id < generation; string_id += worker_count) {
        const auto str = getStringUnlocked(string_id);
        if (is_regexp_like(str, pattern, escape)) {
          worker_results[worker_idx].push_back(str);
        }
      }
    });
  }
  for (auto& worker : workers) {
    worker.join();
  }
  for (const auto& worker_result : worker_results) {
    result.insert(result.end(), worker_result.begin(), worker_result.end());
  }
  const auto it_ok = regex_cache_.insert(std::make_pair(cache_key, result));
  CHECK(it_ok.second);
  return result;
}

bool StringDictionary::fillRateIsHigh() const noexcept {
  return str_ids_.size() <= str_count_ * 2;
}

void StringDictionary::increaseCapacity() noexcept {
  const size_t MAX_STRCOUNT = 1 << 30;
  if (str_count_ >= MAX_STRCOUNT) {
    LOG(FATAL) << "Maximum number (" << str_count_
               << ") of Dictionary encoded Strings reached for this column, offset path for column is  "
               << offsets_path_;
  }
  std::vector<int32_t> new_str_ids(str_ids_.size() * 2, INVALID_STR_ID);
  for (size_t i = 0; i < str_count_; ++i) {
    const auto str = getStringChecked(i);
    int32_t bucket = computeBucket(str, new_str_ids, true);
    new_str_ids[bucket] = i;
  }
  str_ids_.swap(new_str_ids);
}

int32_t StringDictionary::getOrAddImpl(const std::string& str, bool recover) noexcept {
  // @TODO(wei) treat empty string as NULL for now
  if (str.size() == 0)
    return inline_int_null_value<int32_t>();
  CHECK(str.size() <= MAX_STRLEN);
  int32_t bucket = computeBucket(str, str_ids_, recover);
  if (str_ids_[bucket] == INVALID_STR_ID) {
    if (fillRateIsHigh()) {
      // resize when more than 50% is full
      increaseCapacity();
      bucket = computeBucket(str, str_ids_, recover);
    }
    if (recover) {
      payload_file_off_ += str.size();
    } else {
      appendToStorage(str);
    }
    str_ids_[bucket] = static_cast<int32_t>(str_count_);
    ++str_count_;
    invalidateInvertedIndex();
  }
  return str_ids_[bucket];
}

std::string StringDictionary::getStringChecked(const int string_id) const noexcept {
  const auto str_canary = getStringFromStorage(string_id);
  CHECK(!std::get<2>(str_canary));
  return std::string(std::get<0>(str_canary), std::get<1>(str_canary));
}

std::pair<char*, size_t> StringDictionary::getStringBytesChecked(const int string_id) const noexcept {
  const auto str_canary = getStringFromStorage(string_id);
  CHECK(!std::get<2>(str_canary));
  return std::make_pair(std::get<0>(str_canary), std::get<1>(str_canary));
}

int32_t StringDictionary::computeBucket(const std::string& str,
                                        const std::vector<int32_t>& data,
                                        const bool unique) const noexcept {
  auto bucket = rk_hash(str) & (data.size() - 1);
  while (true) {
    if (data[bucket] == INVALID_STR_ID) {  // In this case it means the slot is available for use
      break;
    }
    // if records are unique I don't need to do this test as I know it will not be the same
    if (!unique) {
      const auto old_str = getStringChecked(data[bucket]);
      if (str.size() == old_str.size() && !memcmp(str.c_str(), old_str.c_str(), str.size())) {
        // found the string
        break;
      }
    }
    // wrap around
    if (++bucket == data.size()) {
      bucket = 0;
    }
  }
  return bucket;
}

void StringDictionary::appendToStorage(const std::string& str) noexcept {
  CHECK_GE(payload_fd_, 0);
  CHECK_GE(offset_fd_, 0);
  // write the payload
  if (payload_file_off_ + str.size() > payload_file_size_) {
    checked_munmap(payload_map_, payload_file_size_);
    addPayloadCapacity();
    CHECK(payload_file_off_ + str.size() <= payload_file_size_);
    payload_map_ = reinterpret_cast<char*>(checked_mmap(payload_fd_, payload_file_size_));
  }
  memcpy(payload_map_ + payload_file_off_, str.c_str(), str.size());
  // write the offset and length
  size_t offset_file_off = str_count_ * sizeof(StringIdxEntry);
  StringIdxEntry str_meta{static_cast<uint64_t>(payload_file_off_), str.size()};
  payload_file_off_ += str.size();
  if (offset_file_off + sizeof(str_meta) >= offset_file_size_) {
    checked_munmap(offset_map_, offset_file_size_);
    addOffsetCapacity();
    CHECK(offset_file_off + sizeof(str_meta) <= offset_file_size_);
    offset_map_ = reinterpret_cast<StringIdxEntry*>(checked_mmap(offset_fd_, offset_file_size_));
  }
  memcpy(offset_map_ + str_count_, &str_meta, sizeof(str_meta));
}

std::tuple<char*, size_t, bool> StringDictionary::getStringFromStorage(const int string_id) const noexcept {
  CHECK_GE(payload_fd_, 0);
  CHECK_GE(offset_fd_, 0);
  CHECK_GE(string_id, 0);
  const StringIdxEntry* str_meta = offset_map_ + string_id;
  if (str_meta->size == 0xffff) {
    // hit the canary
    return std::make_tuple(nullptr, 0, true);
  }
  return std::make_tuple(payload_map_ + str_meta->off, str_meta->size, false);
}

void StringDictionary::addPayloadCapacity() noexcept {
  payload_file_size_ += addStorageCapacity(payload_fd_);
}

void StringDictionary::addOffsetCapacity() noexcept {
  offset_file_size_ += addStorageCapacity(offset_fd_);
}

size_t StringDictionary::addStorageCapacity(int fd) noexcept {
  static const ssize_t CANARY_BUFF_SIZE = 1024 * PAGE_SIZE;
  if (!CANARY_BUFFER) {
    CANARY_BUFFER = static_cast<char*>(malloc(CANARY_BUFF_SIZE));
    CHECK(CANARY_BUFFER);
    memset(CANARY_BUFFER, 0xff, CANARY_BUFF_SIZE);
  }
  CHECK_NE(lseek(fd, 0, SEEK_END), -1);
  CHECK(write(fd, CANARY_BUFFER, CANARY_BUFF_SIZE) == CANARY_BUFF_SIZE);
  return CANARY_BUFF_SIZE;
}

void StringDictionary::invalidateInvertedIndex() noexcept {
  if (!like_cache_.empty()) {
    decltype(like_cache_)().swap(like_cache_);
  }
  if (!regex_cache_.empty()) {
    decltype(regex_cache_)().swap(regex_cache_);
  }
}

char* StringDictionary::CANARY_BUFFER{nullptr};

bool StringDictionary::checkpoint() noexcept {
  if (client_) {
    try {
      return client_->checkpoint();
    } catch (...) {
      return false;
    }
  }
  bool ret = true;
  ret = ret && (msync((void*)offset_map_, offset_file_size_, MS_SYNC) == 0);
  ret = ret && (msync((void*)payload_map_, payload_file_size_, MS_SYNC) == 0);
  ret = ret && (fsync(offset_fd_) == 0);
  ret = ret && (fsync(payload_fd_) == 0);
  return ret;
}

void translate_string_ids(std::vector<int32_t>& dest_ids,
                          const LeafHostInfo& dict_server_host,
                          const int32_t dest_dict_id,
                          const std::vector<int32_t>& source_ids,
                          const int32_t source_dict_id,
                          const int32_t dest_generation) {
  StringDictionaryClient string_client(dict_server_host, -1, true);
  string_client.translate_string_ids(dest_ids, dest_dict_id, source_ids, source_dict_id, dest_generation);
}

max_stars_repo_name: bowlofstew/mapd-core

max_issues_repo_name: ngaut/mapd-core

max_forks_repo_name: bowlofstew/mapd-core


text:
// Copyright (c) YugaByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
// in compliance with the License.  You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied.  See the License for the specific language governing permissions and limitations
// under the License.
//

#include "yb/gen_yrpc/model.h"

#include <boost/algorithm/string/predicate.hpp>

#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>

#include "yb/gutil/macros.h"
#include "yb/gutil/strings/join.h"
#include "yb/gutil/strings/split.h"

#include "yb/rpc/service.pb.h"

using google::protobuf::internal::WireFormatLite;

namespace yb {
namespace gen_yrpc {

namespace {

const std::string kProtoExtension = ".proto";

}

WireFormatLite::FieldType FieldType(const google::protobuf::FieldDescriptor* field) {
  return static_cast<WireFormatLite::FieldType>(field->type());
}

WireFormatLite::WireType WireType(const google::protobuf::FieldDescriptor* field) {
  return WireFormatLite::WireTypeForFieldType(FieldType(field));
}

size_t FixedSize(const google::protobuf::FieldDescriptor* field) {
  switch (FieldType(field)) {
    case WireFormatLite::TYPE_DOUBLE: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_FIXED64: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_SFIXED64:
      return 8;
    case WireFormatLite::TYPE_FLOAT: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_FIXED32: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_SFIXED32:
      return 4;
    case WireFormatLite::TYPE_BOOL:
      return 1;
    case WireFormatLite::TYPE_INT64: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_UINT64: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_INT32: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_STRING: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_BYTES: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_UINT32: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_ENUM: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_SINT32: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_SINT64: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_MESSAGE: FALLTHROUGH_INTENDED;
    case WireFormatLite::TYPE_GROUP: FALLTHROUGH_INTENDED;
    default:
      return 0;
  }
}

std::string MakeLightweightName(const std::string& input) {
  auto idx = input.find_last_of('.');
  if (idx != std::string::npos) {
    ++idx;
  } else {
    idx = 0;
  }
  return input.substr(0, idx) + "LW" + input.substr(idx);
}

bool IsLightweightMethod(const google::protobuf::MethodDescriptor* method, rpc::RpcSides side) {
  const auto& options = method->options().GetExtension(rpc::lightweight_method);
  if (side == rpc::RpcSides::BOTH) {
    return options.sides() != rpc::RpcSides::NONE;
  }
  return options.sides() == side || options.sides() == rpc::RpcSides::BOTH;
}

bool IsTrivialMethod(const google::protobuf::MethodDescriptor* method) {
  return method->options().GetExtension(rpc::trivial);
}

bool HasLightweightMethod(const google::protobuf::ServiceDescriptor* service, rpc::RpcSides side) {
  for (int i = 0; i != service->method_count(); ++i) {
    if (IsLightweightMethod(service->method(i), side)) {
      return true;
    }
  }
  return false;
}

bool HasLightweightMethod(const google::protobuf::FileDescriptor* file, rpc::RpcSides side) {
  for (int i = 0; i != file->service_count(); ++i) {
    if (HasLightweightMethod(file->service(i), side)) {
      return true;
    }
  }

  return false;
}

std::string ReplaceNamespaceDelimiters(const std::string& arg_full_name) {
  return JoinStrings(strings::Split(arg_full_name, "."), "::");
}

// Strips the package from method arguments if they are in the same package as
// the service, otherwise leaves them so that we can have fully qualified
// namespaces for method arguments.
std::string RelativeClassPath(const std::string& clazz, const std::string& service) {
  GStringPiece service_package(service);
  auto last_dot = service_package.rfind('.');
  if (last_dot != GStringPiece::npos) {
    service_package = service_package.substr(0, last_dot + 1);

    GStringPiece clazz_path(clazz);
    if (clazz_path.starts_with(service_package)) {
      return ReplaceNamespaceDelimiters(clazz_path.substr(service_package.length()).ToString());
    }
  }

  return "::" + ReplaceNamespaceDelimiters(clazz);
}

std::string UnnestedName(
    const google::protobuf::Descriptor* message, Lightweight lightweight, bool full_path) {
  auto name = lightweight ? MakeLightweightName(message->name()) : message->name();
  if (message->options().map_entry()) {
    name += "_DoNotUse";
  }
  if (message->containing_type()) {
    return UnnestedName(message->containing_type(), lightweight, full_path) + "_" + name;
  }
  if (full_path) {
    const auto& full_name = message->full_name();
    auto idx = full_name.find_last_of('.');
    if (idx != std::string::npos) {
      return full_name.substr(0, idx + 1) + name;
    }
  }
  return name;
}

std::string MapFieldType(const google::protobuf::FieldDescriptor* field, Lightweight lightweight) {
  switch (WireFormatLite::FieldTypeToCppType(FieldType(field))) {
    case WireFormatLite::CppType::CPPTYPE_INT32:
      return "int32_t";
    case WireFormatLite::CppType::CPPTYPE_INT64:
      return "int64_t";
    case WireFormatLite::CppType::CPPTYPE_UINT32:
      return "uint32_t";
    case WireFormatLite::CppType::CPPTYPE_UINT64:
      return "uint64_t";
    case WireFormatLite::CppType::CPPTYPE_DOUBLE:
      return "double";
    case WireFormatLite::CppType::CPPTYPE_FLOAT:
      return "float";
    case WireFormatLite::CppType::CPPTYPE_BOOL:
      return "bool";
    case WireFormatLite::CppType::CPPTYPE_ENUM:
      return RelativeClassPath(
          field->enum_type()->containing_type()
              ? UnnestedName(field->enum_type()->containing_type(), lightweight, true) + "." +
                    field->enum_type()->name()
              : field->enum_type()->full_name(),
          field->containing_type()->full_name());
    case WireFormatLite::CppType::CPPTYPE_STRING:
      return lightweight ? "::yb::Slice" : "std::string";
    case WireFormatLite::CppType::CPPTYPE_MESSAGE:
      if (IsPbAny(field->message_type()) && lightweight) {
        return "::yb::rpc::LWAny";
      }
      return RelativeClassPath(UnnestedName(field->message_type(), lightweight, true),
                               field->containing_type()->full_name());
    default:
      break;
  }
  return "UNSUPPORTED TYPE";
}

bool IsMessage(const google::protobuf::FieldDescriptor* field) {
  return field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE;
}

bool IsSimple(const google::protobuf::FieldDescriptor* field) {
  return !field->is_repeated() && !IsMessage(field);
}

bool NeedArena(const google::protobuf::Descriptor* message) {
  const auto& options = message->options().GetExtension(rpc::lightweight_message);
  if (options.force_arena()) {
    return true;
  }
  for (int i = 0; i != message->field_count(); ++i) {
    const auto* field = message->field(i);
    if (IsMessage(field) || StoredAsSlice(field)) {
      return true;
    }
  }
  return false;
}

bool IsPointerField(const google::protobuf::FieldDescriptor* field) {
  const auto& lightweight_field_options = field->options().GetExtension(rpc::lightweight_field);
  return lightweight_field_options.pointer();
}

bool StoredAsSlice(const google::protobuf::FieldDescriptor* field) {
  auto type = field->type();
  return type == google::protobuf::FieldDescriptor::TYPE_BYTES ||
         type == google::protobuf::FieldDescriptor::TYPE_STRING;
}

bool IsPbAny(const google::protobuf::Descriptor* message) {
  return message->full_name() == "google.protobuf.Any";
}

bool IsLwAny(const google::protobuf::Descriptor* message) {
  return message->full_name() == "yb.rpc.Any";
}

std::string RemoveProtoExtension(const std::string& fname) {
  if (boost::ends_with(fname, kProtoExtension)) {
    return fname.substr(0, fname.length() - kProtoExtension.length());
  } else {
    return fname;
  }
}

std::vector<std::string> ListDependencies(const google::protobuf::FileDescriptor* file) {
  std::vector<std::string> result;
  for (int i = 0; i != file->dependency_count(); ++i) {
    std::string dependency_path = RemoveProtoExtension(file->dependency(i)->name());
    if (dependency_path == "google/protobuf/any") {
      result.push_back("yb/rpc/any");
      continue;
    }
    if (dependency_path == "yb/rpc/rpc_header" ||
        dependency_path == "yb/rpc/lightweight_message" ||
        dependency_path == "yb/rpc/service" ||
        boost::starts_with(dependency_path, "google/protobuf/")) {
      continue;
    }
    result.push_back(std::move(dependency_path));
  }
  return result;
}

} // namespace gen_yrpc
} // namespace yb

ext: cc

max_stars_repo_name: mikecaat/yugabyte-db

max_issues_repo_name: mikecaat/yugabyte-db

max_forks_repo_name: aatrey-yb/yugabyte-db


text:
/*
Copyright (c) Members of the EGEE Collaboration. 2004.
See http://www.eu-egee.org/partners for details on the
copyright holders.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// File: ldap-utils.cpp
// Author: Salvatore Monforte
// Copyright (c) 2002 EU DataGrid.

// $Id: ldap-utils-g2.cpp,v 1.1.2.22 2012/06/22 11:51:32 mcecchi Exp $
#include <sys/time.h>
#include <ldap.h>
#include <lber.h>

#include <iostream>
#include <algorithm>
#include <string>
#include <vector>
#include <set>
#include <map>
#include <functional>

#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/shared_array.hpp>
#include <boost/tokenizer.hpp>
#include <boost/progress.hpp>
#include <boost/any.hpp>
#include <boost/regex.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/erase.hpp>
#include <boost/algorithm/string/regex.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/assign/list_of.hpp>
#include <boost/assign/std/vector.hpp>
#include <boost/assign/list_inserter.hpp>
#include "glite/wms/ism/purchaser/common.h"
#include "glite/wms/common/logger/logger_utils.h"
#include "glite/wmsutils/classads/classad_utils.h"

#include "ldap-dn-utils-g2.h"
#include "schema_utils.h"

namespace cu  = glite::wmsutils::classads;
namespace ba  = boost::algorithm;
namespace bas = boost::assign;
namespace ism = glite::wms::ism;

namespace glite {
namespace wms {
namespace ism {
namespace purchaser {

namespace {

struct ManagerInfo
{
  ad_ptr ad;
};
typedef std::map<std::string, ManagerInfo> ManagerInfoMap;

struct ServiceInfo;
typedef std::map<std::string, ServiceInfo> ServiceInfoMap;

struct ShareInfo;
typedef std::map<std::string, ShareInfo> ShareInfoMap;

struct AccessPolicyInfo {
  ad_ptr ad;
};
typedef std::map<std::string, AccessPolicyInfo> AccessPolicyInfoMap;

class EndpointInfo
{
public:
  ad_ptr ad;
  std::vector<classad::ExprTree*> policy_rules;
  bool has_servicelnk_iterator;
  bool has_policylnk_iterator;
  bool has_shareslnk_iterators;
  ServiceInfoMap::iterator service_lnk;
  AccessPolicyInfoMap::iterator policy_lnk;
  std::vector<ShareInfoMap::iterator> shares_lnk;
  EndpointInfo()
    : has_servicelnk_iterator(false), has_policylnk_iterator(false), has_shareslnk_iterators(false) { }
};
typedef std::map<std::string, EndpointInfo> EndpointInfoMap;

struct BenchMarkInfo
{
  ad_ptr ad;
};
typedef std::map<std::string, BenchMarkInfo> BenchMarkInfoMap;

struct ExecEnvInfo
{
  ad_ptr ad;
  std::set<std::string> applications;
  std::vector<BenchMarkInfoMap::iterator> bmarks_lnk;
};

struct DataStoreInfo
{
  ad_ptr ad;
};

typedef std::map<std::string, ExecEnvInfo> ExecEnvInfoMap;
typedef std::map<std::string, boost::any> ResourceInfoMap;

struct ServiceInfo
{
public:
  ServiceInfo()
    : has_managerlnk(false), has_shareslnk_iterators(false) { }
  ad_ptr ad;
  ManagerInfoMap::iterator manager_lnk;
  bool has_managerlnk;
  bool has_shareslnk_iterators;

  // The following is required since for StorageServices
  // the relevant Shares are most likely bound to the only
  // existing EndPoint for a given Service and do not
  // publish the ForeignKey describing such a relation.
  std::vector<ShareInfoMap::iterator> shares_lnk;

  // The following applies only to StorageServices
  std::vector<ad_ptr> access_protocols;
};

struct ShareInfo
{
  ad_ptr ad;
  std::vector<classad::ExprTree*> policy_rules;
  std::vector<ResourceInfoMap::iterator> resources_lnk;
};

//typedef std::map<std::string, ad_ptr> SubClusterInfoMap;
/*
struct ClusterInfo {
  std::string site_id;
  std::vector<CEInfoMap::iterator> ces_links;
  std::vector<SubClusterInfoMap::iterator> subclusters_links;
};
typedef std::map<std::string, ClusterInfo> ClusterInfoMap;

struct VoViewInfo
{
  VoViewInfo(std::string const& i, ad_ptr a)
    : id(i), ad(a)
  {
  }
  std::string id;
  ad_ptr ad;
};

typedef std::map<
  std::string, // ce id
  std::vector<VoViewInfo> // list of ce views per vo
> VoViewInfoMap;

struct SEInfo
{
  ad_ptr ad;
  std::vector<classad::ExprTree*> storage_areas;
  std::vector<classad::ExprTree*> control_protocols;
  std::vector<classad::ExprTree*> access_protocols;
  std::vector<ad_ptr> ces_binds;
};

typedef std::map<std::string, SEInfo> SEInfoMap;

*/

struct BDII_info
{
  ShareInfoMap shares;
  ServiceInfoMap services;
  ManagerInfoMap managers;
  EndpointInfoMap endpoints;
  AccessPolicyInfoMap policies;
  ResourceInfoMap resources;
  BenchMarkInfoMap benchmarks;
};

struct is_a_literal_node_starting_with
{
  is_a_literal_node_starting_with(std::string const& p) : prefix(p) {}
  bool operator()(classad::ExprTree* e) const {
    classad::Value v;
    std::string s;
    return (
      cu::is_literal(e) &&
      e->Evaluate(v) && v.IsStringValue(s) && ba::starts_with(s,prefix)
    );
  }
  std::string prefix;
};

struct is_a_literal_node_equals_to
{

  is_a_literal_node_equals_to(std::string const& v) : value(v) {}
  bool operator()(classad::ExprTree* e) const {
    classad::Value v;
    std::string s;
    return (
      cu::is_literal(e) &&
      e->Evaluate(v) && v.IsStringValue(s) && ba::equals(s,value)
    );
  }
  std::string value;
};

inline bool is_objectclass(std::string const& item, const classad::ClassAd& ad)
{
  bool result = false;
  classad::ExprList* e = 0;
  ad.EvaluateAttrList("objectClass",e);
  if(e) result = std::find_if(e->begin(),e->end(), is_a_literal_node_equals_to(item)) != e->end();
  return result;
}

inline bool is_glue2_computing_share(const classad::ClassAd& ad)
{
  return is_objectclass("GLUE2ComputingShare", ad);
}

inline bool is_glue2_storage_share(const classad::ClassAd& ad)
{
  return is_objectclass("GLUE2StorageShare", ad);
}

inline bool is_glue2_execenv_resource(const classad::ClassAd& ad)
{
  return is_objectclass("GLUE2ExecutionEnvironment", ad);
}

inline bool is_glue2_datastore_resource(const classad::ClassAd& ad)
{
  return is_objectclass("GLUE2DataStore", ad);
}

void process_glue2_storage_access_protocol_info(
 std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const protocol_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const service_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );

  ServiceInfoMap::iterator it;
  bool insert;
  boost::tie(it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  cleanup_glue_info(ad,
    bas::list_of("StorageServiceForeignKey")
  );
  it->second.access_protocols.push_back(
    ad
  );
}

void process_glue2_service_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const service_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  ServiceInfoMap::iterator it;
  bool insert;
  boost::tie(it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("AdminDomainForeignKey")
      ("OtherInfo")
  );
  if (insert) { // process_glue2_service_capacity_info might have inserted its ad before
    it->second.ad.reset(new classad::ClassAd());
    it->second.ad->Insert(
      "Service",
      ad->Copy()
    );
  }
  else {
    dynamic_cast<classad::ClassAd*>(
      it->second.ad->Lookup("Service")
    )->Update(*ad);
  }
}

void process_glue2_manager_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const manager_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const service_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );
  ManagerInfoMap::iterator manager_it;
  bool insert;
  boost::tie(manager_it, insert) = bdii_info.managers.insert(
    std::make_pair(manager_id, ManagerInfo())
  );
  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("ComputingServiceForeignKey")
      ("ServiceForeignKey")
      ("OtherInfo")
  );
  manager_it->second.ad.reset(new classad::ClassAd());
  manager_it->second.ad->Insert(
    "Manager",
    ad->Copy()
  );

  ServiceInfoMap::iterator service_it;
  boost::tie(service_it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  service_it->second.manager_lnk = manager_it;
  service_it->second.has_managerlnk = true;
}

template<typename T>
void extract_glue2_info_value_in(
  classad::ClassAd const& ad,
  std::string const& from,
  std::string const& what,
  classad::ClassAd* in)
{
  classad::ExprList* el = dynamic_cast<classad::ExprList*>(
    ad.Lookup(from)
  );
  if (el) {
    std::vector<classad::ExprTree*>::const_iterator it(
      std::find_if(
        el->begin(), el->end(),
        is_a_literal_node_starting_with(what+"=")
      )
    );
    if (it != el->end()) {
      classad::Value v;
      std::string s;
      (*it)->Evaluate(v) && v.IsStringValue(s);
      in->InsertAttr(what,boost::lexical_cast<T>(s.substr(s.find("=")+1)));
    }
  }
}

void process_glue2_resource_fk(
  ShareInfoMap::iterator& share_it,
  ad_ptr ad,
  BDII_info& bdii_info)
{
  classad::ExprList* el = dynamic_cast<classad::ExprList*>(
    ad->Lookup("ResourceForeignKey")
  );
  if (el && el->size()) {
    classad::Value v;
    std::string s;
    (*el->begin())->Evaluate(v) && v.IsStringValue(s);

    bool insert;
    boost::any resource;
    ResourceInfoMap::iterator resource_it;

    if (is_glue2_computing_share(*ad)) resource = ExecEnvInfo();
    else if (is_glue2_storage_share(*ad)) resource = DataStoreInfo();
    else {
      Error("Cannot discriminate between Computing or Storage share: " << *ad);
      return ;
    }
    boost::tie(resource_it, insert) = bdii_info.resources.insert(
      std::make_pair(s, resource)
    );
    share_it->second.resources_lnk.push_back(resource_it);
  }
}

EndpointInfoMap::iterator
process_glue2_endpoint_fk_entry(
  BDII_info& bdii_info,
  classad::ExprTree* e)
{
  classad::Value v;
  std::string s;
  EndpointInfoMap::iterator endpoint_it(
    bdii_info.endpoints.end()
  );
  if (e->Evaluate(v) && v.IsStringValue(s)) {
    bool insert;
    boost::tie(endpoint_it, insert) = bdii_info.endpoints.insert(
      std::make_pair(s, EndpointInfo())
    );
  }
  return endpoint_it;
}

bool process_glue2_endpoint_fk(
  ShareInfoMap::iterator& share_it,
  ad_ptr ad,
  BDII_info& bdii_info)
{
  std::string const fk(
    is_glue2_computing_share(*ad) ?
      "ComputingEndpointForeignKey" : "StorageEndpointForeignKey"
  );
  bool result = false;
  classad::ExprList* el = dynamic_cast<classad::ExprList*>(
    ad->Lookup(fk)
  );

  if (el && el->size()) {
    for( // link each endpoint to the share it binds to
      classad::ExprList::iterator el_it = el->begin() ;
      el_it != el->end(); ++el_it
    ) {
      EndpointInfoMap::iterator endpoint_it(
        process_glue2_endpoint_fk_entry(bdii_info, *el_it)
      );
      if (endpoint_it != bdii_info.endpoints.end()) {
         endpoint_it->second.shares_lnk.push_back(share_it);
         result = true;
         endpoint_it->second.has_shareslnk_iterators = true;
      }
    }
  }
  return result;
}
void bind_share_to_service(
  ShareInfoMap::iterator& share_it,
  std::string const& service_id,
  BDII_info& bdii_info
)
{
  bool insert;
  ServiceInfoMap::iterator service_it;
  boost::tie(service_it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  service_it->second.shares_lnk.push_back(share_it);
  service_it->second.has_shareslnk_iterators = true;
}

void process_glue2_share_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const share_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const service_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );
  ShareInfoMap::iterator share_it;
  bool insert;
  boost::tie(share_it, insert) = bdii_info.shares.insert(
    std::make_pair(share_id, ShareInfo())
  );

  process_glue2_resource_fk(share_it, ad, bdii_info);

  if( !process_glue2_endpoint_fk(share_it, ad, bdii_info) ) {

    // For this Share the FK describing the relation between a Share
    // and the Endpoint exsposing it is missing.
    bind_share_to_service(share_it, service_id, bdii_info);
  }

  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("ComputingEndpointForeignKey")
      ("ComputingServiceForeignKey")
      ("EndpointForeignKey")
      ("ResourceForeignKey")
      ("ServiceForeignKey")
      ("ExecutionEnvironmentForeignKey")
  );
  if (insert) { // process_glue2_share_capacity_info might have inserted its ad before
    share_it->second.ad.reset(new classad::ClassAd());
    share_it->second.ad->Insert(
      "Share",
      ad->Copy()
    );
  } else {
    classad::ClassAd* shareAd = 0;
    share_it->second.ad->EvaluateAttrClassAd("Share",shareAd);
    shareAd->Update(*ad);
  }
}

void process_glue2_endpoint_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const endpoint_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const service_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );
  EndpointInfoMap::iterator endpoint_it;
  bool insert;
  boost::tie(endpoint_it, insert) = bdii_info.endpoints.insert(
    std::make_pair(endpoint_id, EndpointInfo())
  );
  classad::ClassAd* oi = new classad::ClassAd;
  extract_glue2_info_value_in<std::string>(*ad, "OtherInfo", "HostDN", oi);
  extract_glue2_info_value_in<std::string>(*ad, "OtherInfo", "MiddlewareName", oi);
  extract_glue2_info_value_in<std::string>(*ad, "OtherInfo", "MiddlewareVersion", oi);
  ad->Insert("OtherInfo", oi);

  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("ComputingServiceForeignKey")
      ("ServiceForeignKey")
  );

  endpoint_it->second.ad.reset(new classad::ClassAd());
  endpoint_it->second.ad->Insert(
    "Endpoint",
    ad->Copy()
  );
  ServiceInfoMap::iterator service_it;
  boost::tie(service_it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  endpoint_it->second.service_lnk = service_it;
  endpoint_it->second.has_servicelnk_iterator = true;
}

void process_glue2_execenv_info(
std::string const resource_id,
std::string const service_id,
ad_ptr ad,
  BDII_info& bdii_info
) {

  bool insert;
  ResourceInfoMap::iterator resource_it;
  boost::tie(resource_it, insert) = bdii_info.resources.insert(
    std::make_pair(resource_id, ExecEnvInfo())
  );

  classad::ClassAd* oi = new classad::ClassAd;
  extract_glue2_info_value_in<short>(*ad, "OtherInfo", "SmpSize", oi);
  extract_glue2_info_value_in<short>(*ad, "OtherInfo", "Cores", oi);
  ad->Insert("OtherInfo", oi);

  cleanup_glue_info(ad, bas::list_of("CreationTime")
    ("ComputingManagerForeignKey")
    ("ManagerForeignKey")
  );

  ad_ptr resource_ad(new classad::ClassAd());
  resource_ad->Insert(
    "ExecutionEnvironment", ad->Copy()
  );
  ExecEnvInfo& eei = boost::any_cast<ExecEnvInfo&>(resource_it->second);
  eei.ad = resource_ad;
}

void process_glue2_datastore_info(
  std::string const resource_id,
  std::string const service_id,
  ad_ptr ad,
  BDII_info& bdii_info
) {
  bool insert;
  ResourceInfoMap::iterator resource_it;
  boost::tie(resource_it, insert) = bdii_info.resources.insert(
    std::make_pair(resource_id, DataStoreInfo())
  );

  cleanup_glue_info(ad, bas::list_of("CreationTime")
    ("ComputingManagerForeignKey")
    ("ManagerForeignKey")
  );

  ad_ptr resource_ad(new classad::ClassAd());
  resource_ad->Insert(
    "DataStore", ad->Copy()
  );
  DataStoreInfo& dsi = boost::any_cast<DataStoreInfo&>(resource_it->second);
  dsi.ad = resource_ad;
}

void process_glue2_resource_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const resource_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const service_id(
   ldap_dn_tokens[2].substr(ldap_dn_tokens[2].find("=")+1)
  );
  if (is_glue2_execenv_resource(*ad)) {
    process_glue2_execenv_info(resource_id, service_id, ad, bdii_info);
  }
  else if (is_glue2_datastore_resource(*ad)) {
    process_glue2_datastore_info(resource_id, service_id, ad, bdii_info);
  }
}

void process_glue2_application_env_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const appenv_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const resource_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );

  std::string application =
    cu::evaluate_attribute(*ad,"AppName");

  bool insert;
  ResourceInfoMap::iterator resource_it;
  boost::tie(resource_it, insert) = bdii_info.resources.insert(
    std::make_pair(resource_id, ExecEnvInfo())
  );

  boost::any_cast<ExecEnvInfo&>(
    resource_it->second
  ).applications.insert(application);
}

void process_glue2_benchmark_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const bmark_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );

  std::string const resource_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );

  bool insert;
  BenchMarkInfoMap::iterator bmark_it;
  boost::tie(bmark_it, insert) = bdii_info.benchmarks.insert(
    std::make_pair(bmark_id, BenchMarkInfo())
  );

  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("ComputingManagerForeignKey")
      ("ExecutionEnvironmentForeignKey")
      ("OtherInfo")
  );
  // The schema defines the BenchmarkValue to be a 32-bit float
  // However LDAP is rather deficient in data types and the only
  // numeric type is integer, so it has to be stored as a string.
  try {
    std::string v = cu::evaluate_attribute(*ad, "Value");
    ad->InsertAttr("Value", boost::lexical_cast<float>(v));
  } catch(boost::bad_lexical_cast &) {}

  bmark_it->second.ad = ad;

  ResourceInfoMap::iterator resource_it;
  boost::tie(resource_it, insert) = bdii_info.resources.insert(
    std::make_pair(resource_id, ExecEnvInfo())
  );
  boost::any_cast<ExecEnvInfo&>(
    resource_it->second
  ).bmarks_lnk.push_back(bmark_it);
}

void process_glue2_mapping_policy_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const share_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );
  // Policy rules are inserted directly into the relevant ShareInfo
  bool insert;
  ShareInfoMap::iterator share_it;
  boost::tie(share_it, insert) = bdii_info.shares.insert(
    std::make_pair(share_id, ShareInfo())
  );
  if (!share_it->second.ad) {
    share_it->second.ad.reset(new classad::ClassAd);
    share_it->second.ad->Insert(
      "Share",
      new classad::ClassAd()
    );
  }
  share_it->second.ad->DeepInsert(
    share_it->second.ad->Lookup("Share"),
    "Policy",
    ad->Lookup("Rule")->Copy()
  );
  /*
  classad::ExprList* el = dynamic_cast<classad::ExprList*>(
    ad->Lookup("Rule")
  );
  std::transform(
    el->begin(), el->end(),
    std::inserter(
      share_it->second.policy_rules,
      share_it->second.policy_rules.end()
    ),
    boost::bind(&classad::ExprTree::Copy, _1)
  );
  */
}

void process_glue2_access_policy_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const policy_id(
   ldap_dn_tokens[0].substr(ldap_dn_tokens[0].find("=")+1)
  );
  std::string const endpoint_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );
  bool insert;
  AccessPolicyInfoMap::iterator policy_it;
  boost::tie(policy_it, insert) = bdii_info.policies.insert(
    std::make_pair(policy_id, AccessPolicyInfo())
  );

  cleanup_glue_info(ad,
    bas::list_of("CreationTime")
      ("UserDomainForeignKey")
      ("EndpointForeignKey")
      ("OtherInfo")
  );

  policy_it->second.ad = ad;

  EndpointInfoMap::iterator endpoint_it;
  boost::tie(endpoint_it, insert) = bdii_info.endpoints.insert(
    std::make_pair(endpoint_id, EndpointInfo())
  );
  endpoint_it->second.policy_lnk = policy_it;
  endpoint_it->second.has_policylnk_iterator = true;
}

void process_glue2_service_capacity_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const service_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );

  ServiceInfoMap::iterator it;
  bool insert;
  boost::tie(it, insert) = bdii_info.services.insert(
    std::make_pair(service_id, ServiceInfo())
  );
  if (ad) {
    cleanup_glue_info(ad,
      bas::list_of("StorageServiceForeignKey")
    );
  }
  // process_glue2_service_info might have inserted its ad before
  if (insert) {
    it->second.ad.reset(new classad::ClassAd());
    it->second.ad->Insert("Service", new classad::ClassAd());
  }
  if (it->second.ad && ad) {
    it->second.ad->DeepInsert(
      it->second.ad->Lookup("Service"),
      "Capacity",
      ad->Copy()
    );
  }
}

void process_glue2_share_capacity_info(
  std::vector<std::string> const& ldap_dn_tokens,
  ad_ptr ad,
  BDII_info& bdii_info
)
{
  std::string const share_id(
   ldap_dn_tokens[1].substr(ldap_dn_tokens[1].find("=")+1)
  );

  ShareInfoMap::iterator it;
  bool insert;
  boost::tie(it, insert) = bdii_info.shares.insert(
    std::make_pair(share_id, ShareInfo())
  );
  cleanup_glue_info(ad,
    bas::list_of("StorageShareForeignKey")
  );
  // process_glue2_service_info might have inserted its ad before
  if (insert) {
    it->second.ad.reset(new classad::ClassAd());
    it->second.ad->Insert("Share", new classad::ClassAd());
  }
  it->second.ad->DeepInsert(
    it->second.ad->Lookup("Share"),
    "Capacity",
    ad->Copy()
  );
}

inline classad::ExprTree* make_literal(std::string const& s)
{
  classad::Value v;
  v.SetStringValue(s);
  return classad::Literal::MakeLiteral(v);
}

inline classad::ExprTree* parse_expression(std::string const& s)
{
  classad::ClassAdParser parser;
  return parser.ParseExpression(s);
}
inline classad::ExprTree* copy_benchmark_info(BenchMarkInfoMap::const_iterator bi)
{
  return dynamic_cast<classad::ExprTree*>(bi->second.ad->Copy());
}

typedef std::list<std::string> glue2_stripping_prefix;
typedef boost::function<
  bool(std::vector<std::string> const&)
> glue2_dn_test_predicate;

typedef boost::function<void(
  std::vector<std::string> const&,
  ad_ptr,
  BDII_info&
)> glue2_info_processing_fn;

typedef boost::tuple<
  glue2_dn_test_predicate,
  glue2_stripping_prefix,
  glue2_info_processing_fn
> glue2_info_processor_tuple;

const glue2_stripping_prefix
  service_pfx  = bas::list_of("GLUE2Entity")("GLUE2Service")("GLUE2StorageServiceCapacity"),
  manager_pfx  = bas::list_of("GLUE2Entity")("GLUE2Manager")("GLUE2ComputingManager")("GLUE2StorageManager"),
  share_pfx    = bas::list_of("GLUE2Entity")("GLUE2Share")("GLUE2ComputingShare")("GLUE2StorageShareCapacity")("GLUE2StorageShare"),
  endpoint_pfx = bas::list_of("GLUE2Entity")("GLUE2Endpoint")("GLUE2ComputingEndpoint"),
  resource_pfx = bas::list_of("GLUE2Entity")("GLUE2Resource")("GLUE2ExecutionEnvironment")("GLUE2DataStore"),
  mpolicy_pfx  = bas::list_of("GLUE2Entity")("GLUE2Policy")("GLUE2MappingPolicy"),
  apolicy_pfx  = bas::list_of("GLUE2Entity")("GLUE2Policy")("GLUE2AccessPolicy"),
  appenv_pfx   = bas::list_of("GLUE2Entity")("GLUE2ApplicationEnvironment"),
  bmark_pfx    = bas::list_of("GLUE2Entity")("GLUE2Benchmark"),
  proto_pfx    = bas::list_of("GLUE2Entity")("GLUE2StorageAccessProtocol");

std::list<glue2_info_processor_tuple>
glue2_info_processors = bas::tuple_list_of
  (is_glue2_service_dn, service_pfx, process_glue2_service_info)
  (is_glue2_manager_dn, manager_pfx, process_glue2_manager_info)
  (is_glue2_share_dn, share_pfx, process_glue2_share_info)
  (is_glue2_endpoint_dn, endpoint_pfx, process_glue2_endpoint_info)
  (is_glue2_resource_dn, resource_pfx, process_glue2_resource_info)
  (is_glue2_mapping_policy_dn, mpolicy_pfx, process_glue2_mapping_policy_info)
  (is_glue2_access_policy_dn, apolicy_pfx, process_glue2_access_policy_info)
  (is_glue2_application_env_dn, appenv_pfx, process_glue2_application_env_info)
  (is_glue2_benchmark_dn, bmark_pfx, process_glue2_benchmark_info)
  (is_glue2_service_capacity_dn, service_pfx, process_glue2_service_capacity_info)
  (is_glue2_share_capacity_dn, share_pfx, process_glue2_share_capacity_info)
  (is_glue2_storage_access_protocol_dn, proto_pfx, process_glue2_storage_access_protocol_info)
;
  // The Helper expects to find some attribute
  // required to support LCGCE and GLUE13

  typedef std::pair< std::string, std::string > str_pair;
  struct parse_expression_and_insert
  {
    parse_expression_and_insert(ad_ptr _ad) : ad(_ad) {}
    inline void operator()(str_pair const& p) {
      ad->Insert(p.first, parse_expression(p.second));
    }
    ad_ptr ad;
  };
  std::vector<str_pair> glue13_attr_refs = bas::list_of<str_pair>
    ("GlueCEInfoHostName", "GLUE2.Computing.Share.OtherInfo.InfoProviderHost")
    ("GlueHostApplicationSoftwareRunTimeEnvironment","GLUE2.ApplicationEnvironment.AppName")
    ("GlueCEAccessControlBaseRule","GLUE2.Computing.Endpoint.Policy")
    ("QueueName","GLUE2.Computing.Share.MappingQueue")
    ("LRMSType","GLUE2.Computing.Manager.ProductName")
    ("GlueCEUniqueID","CEid")
    ("GlobusResourceContactString","CEid")
    ;

} // anonymous namespace

void
fetch_bdii_se_info_g2(
  std::string const& host,
  size_t port,
  std::string const& dn,
  time_t timeout,
  std::string const& ldap_se_filter_ext,
  gluece_info_container_type& se_info_container)
{
  LDAP* ld = 0;
  int result = ldap_initialize(
    &ld,
    std::string("ldap://" + host + ":" + boost::lexical_cast<std::string>(port)).c_str()
  );
  boost::shared_ptr<void> ld_guard(
    static_cast<void*>(0),
    boost::bind(ldap_unbind_ext, ld, (LDAPControl**)(0), (LDAPControl**)(0))
  );
  if (result != LDAP_SUCCESS ) {

    throw LDAPException(
      std::string("ldap_simple_bind_s error: ").append(
        ldap_err2string(result)
      )
    );
  }

  struct timeval to;
  to.tv_sec = timeout;
  to.tv_usec = 0L;

  ldap_set_option(ld, LDAP_OPT_NETWORK_TIMEOUT, &to);

  LDAPMessage *ldresult = 0;
  if ( (result = ldap_search_ext_s(
    ld,
    dn.c_str(),
    LDAP_SCOPE_SUBTREE,
    ldap_se_filter_ext.c_str(),
    0,
    false,
    0,
    0,
    &to,
    LDAP_NO_LIMIT,
    &ldresult) != LDAP_SUCCESS)) {

    throw LDAPException(
      std::string("ldap_search error: ").append(
        ldap_err2string(result)
      )
    );
  }
  struct BDII_info bdii_info;

  boost::shared_ptr<void> ldresult_guard(
    ldresult, ldap_msgfree
  );

  time_t const t0 = std::time(0);
  size_t n_entries = 0;
  for (
    LDAPMessage* lde = ldap_first_entry(ld, ldresult);
    lde != 0; lde = ldap_next_entry(ld, lde)
  ) {

    boost::shared_ptr<char> dn_str(
      ldap_get_dn( ld, lde ),
      ber_memfree
    );
    ++n_entries;
    std::vector<std::string> ldap_dn_tokens;
    tokenize_ldap_dn(dn_str.get(), ldap_dn_tokens);
    for (
      std::list<glue2_info_processor_tuple>::const_iterator p_it = glue2_info_processors.begin() ;
      p_it != glue2_info_processors.end() ; ++p_it ) {

      glue2_dn_test_predicate check = p_it->get<0>();

      if (check(ldap_dn_tokens)) {

        ad_ptr ad(
          create_classad_from_ldap_entry(
            ld, lde, p_it->get<1>(), true
        ));
        glue2_info_processing_fn process = p_it->get<2>();
        process(ldap_dn_tokens, ad, bdii_info);
        break ;
      }
    }
  }
  Debug("#" << n_entries << " LDAP entries received in " << std::time(0) - t0 << " seconds");

  EndpointInfoMap::const_iterator ep_it(bdii_info.endpoints.begin());
  EndpointInfoMap::const_iterator const ep_e(bdii_info.endpoints.end());

  time_t const t1 = std::time(0);
  size_t n_shares = 0;

for( ; ep_it != ep_e; ++ep_it) {

    ad_ptr storageAd(new classad::ClassAd);
    if (ep_it->second.ad) {
      storageAd->Update(*ep_it->second.ad);
      if (ep_it->second.has_servicelnk_iterator) {
        if (ep_it->second.service_lnk->second.ad) {
          storageAd->Update(*ep_it->second.service_lnk->second.ad);
        }
        if (ep_it->second.service_lnk->second.has_managerlnk &&
          ep_it->second.service_lnk->second.manager_lnk->second.ad) {
          storageAd->Update(*ep_it->second.service_lnk->second.manager_lnk->second.ad);
        }
      }
    }
    if (ep_it->second.has_policylnk_iterator &&
      ep_it->second.policy_lnk->second.ad &&
      storageAd->Lookup("Endpoint")) {
      storageAd->DeepInsert(
        storageAd->Lookup("Endpoint"),
        "Policy",
        ep_it->second.policy_lnk->second.ad->Lookup("Rule")->Copy()
      );
    }
    if (ep_it->second.has_policylnk_iterator &&
      ep_it->second.policy_lnk->second.ad &&
      storageAd->Lookup("Endpoint")) {
      storageAd->DeepInsert(
        storageAd->Lookup("Endpoint"),
        "Policy",
        ep_it->second.policy_lnk->second.ad->Lookup("Rule")->Copy()
      );
    }
    if (ep_it->second.has_servicelnk_iterator) {

      if (ep_it->second.service_lnk->second.ad)
        storageAd->Update(
          *ep_it->second.service_lnk->second.ad
        );

      std::vector<classad::ExprTree*> protocols;
      std::transform(
        ep_it->second.service_lnk->second.access_protocols.begin(),
        ep_it->second.service_lnk->second.access_protocols.end(),
        std::inserter(protocols, protocols.end()),
        boost::bind(&classad::ExprTree::Copy, _1)
      );
      storageAd->Insert(
        "Protocols",
        classad::ExprList::MakeExprList(protocols)
      );
    }

    // If there is no Share bound to the Endpoint then we have to choose
    // Shares directly from the Service

    std::vector<ShareInfoMap::iterator> shares;
    if (ep_it->second.shares_lnk.empty()) {
      if (ep_it->second.has_servicelnk_iterator && ep_it->second.service_lnk->second.has_shareslnk_iterators) {
        shares = ep_it->second.service_lnk->second.shares_lnk;
      } else {
        continue;
      }
    } else {
      if (ep_it->second.has_shareslnk_iterators) {
        shares = ep_it->second.shares_lnk;
      } else {
        continue;
      }
    }

    std::vector<ShareInfoMap::iterator>::const_iterator sh_it(shares.begin());
    std::vector<ShareInfoMap::iterator>::const_iterator const sh_e(shares.end());
    for( ; sh_it != sh_e; ++sh_it) {

      classad::ClassAd* storageAd_copy(
        dynamic_cast<classad::ClassAd*>(storageAd->Copy())
      );
      classad::ExprTree* shareAd_copy(
        (*sh_it)->second.ad->Lookup("Share")->Copy()
      );
      if (
        (*sh_it)->second.ad &&
        (*sh_it)->second.ad->Lookup("Rule")
      ) {
        dynamic_cast<classad::ClassAd*>(shareAd_copy)->Insert(
          "Policy",
          (*sh_it)->second.ad->Lookup("Rule")->Copy()
        );
      }
      storageAd_copy->Insert("Share", shareAd_copy);

      if( !(*sh_it)->second.resources_lnk.empty() ) {
        DataStoreInfo const& dsi(
          boost::any_cast<DataStoreInfo const&>(
            (*sh_it)->second.resources_lnk.front()->second
          )
        );

        storageAd_copy->Update(*dsi.ad);
      }

      classad::ClassAd* g2Ad = new classad::ClassAd;

      g2Ad->Insert("Storage", storageAd_copy);
      ad_ptr result( new classad::ClassAd );
      result->Insert("GLUE2", g2Ad);

      std::string id;
      try {
        id = std::string(
          cu::evaluate_expression(
            *result,
            "GLUE2.Storage.Share.ID"
          )
        );
      } catch (std::exception) {
        Info("purchased entry missing GLUE2.Storage.Share.ID");
        continue;
      }
      se_info_container.insert(std::make_pair(id, result));
      n_shares++;
    }
  }

  Debug("#" << n_shares << " GLUE2StorageShare's ClassAd(s) generated in " << std::time(0) - t1 << " seconds");
}

void
fetch_bdii_ce_info_g2(
  std::string const& host,
  size_t port,
  std::string const& dn,
  time_t timeout,
  std::string const& ldap_ce_filter_ext,
  gluece_info_container_type& ce_info_container)
{
  LDAP* ld = 0;
  int result = ldap_initialize(
    &ld,
    std::string("ldap://" + host + ":" + boost::lexical_cast<std::string>(port)).c_str()
  );
  boost::shared_ptr<void> ld_guard(
    static_cast<void*>(0),
    boost::bind(ldap_unbind_ext, ld, (LDAPControl**)(0), (LDAPControl**)(0))
  );
  if (result != LDAP_SUCCESS ) {

    throw LDAPException(
      std::string("ldap_simple_bind_s error: ").append(
        ldap_err2string(result)
      )
    );
  }

  struct timeval to;
  to.tv_sec = timeout;
  to.tv_usec = 0L;

  ldap_set_option(ld, LDAP_OPT_NETWORK_TIMEOUT, &to);

  LDAPMessage *ldresult = 0;
  if ( (result = ldap_search_ext_s(
    ld,
    dn.c_str(),
    LDAP_SCOPE_SUBTREE,
    ldap_ce_filter_ext.c_str(),
    0,
    false,
    0,
    0,
    &to,
    LDAP_NO_LIMIT,
    &ldresult) != LDAP_SUCCESS)) {

    throw LDAPException(
      std::string("ldap_search error: ").append(
        ldap_err2string(result)
      )
    );
  }

  boost::shared_ptr<void> ldresult_guard(
    ldresult, ldap_msgfree
  );

  BDII_info bdii_info;

  time_t const t0 = std::time(0);
  size_t n_entries = 0;
  for (
    LDAPMessage* lde = ldap_first_entry(ld, ldresult);
    lde != 0; lde = ldap_next_entry(ld, lde)
  ) {

    boost::shared_ptr<char> dn_str(
      ldap_get_dn( ld, lde ),
      ber_memfree
    );
    ++n_entries;
    std::vector<std::string> ldap_dn_tokens;
    tokenize_ldap_dn(dn_str.get(), ldap_dn_tokens);
    for (
      std::list<glue2_info_processor_tuple>::const_iterator p_it = glue2_info_processors.begin() ;
      p_it != glue2_info_processors.end() ; ++p_it ) {

      glue2_dn_test_predicate check = p_it->get<0>();

      if (check(ldap_dn_tokens)) {

        ad_ptr ad(
          create_classad_from_ldap_entry(
            ld, lde, p_it->get<1>(), true
        ));
        glue2_info_processing_fn process = p_it->get<2>();
        process(ldap_dn_tokens, ad, bdii_info);
        break ;
      }
    }
  }
  Debug("#" << n_entries << " LDAP entries received in " << std::time(0) - t0 << " seconds");

  EndpointInfoMap::const_iterator ep_it(bdii_info.endpoints.begin());
  EndpointInfoMap::const_iterator const ep_e(bdii_info.endpoints.end());

  time_t const t1 = std::time(0);
  size_t n_shares = 0;
  for( ; ep_it != ep_e; ++ep_it) {

    if ( ep_it->second.shares_lnk.empty() ) { // Not an Endpoint bound to Shares
       continue;
    }
    ad_ptr computingAd(new classad::ClassAd);

    if (ep_it->second.ad) {
      computingAd->Update(*ep_it->second.ad);
    }

    if (!ep_it->second.has_servicelnk_iterator) {
       continue;
    }

    computingAd->Update(*ep_it->second.service_lnk->second.ad);
    computingAd->Update(*ep_it->second.service_lnk->second.manager_lnk->second.ad);

    computingAd->DeepInsert(
      computingAd->Lookup("Endpoint"),
      "Policy",
       ep_it->second.policy_lnk->second.ad->Lookup("Rule")->Copy()
    );

   std::vector<ShareInfoMap::iterator>::const_iterator sh_it(
      ep_it->second.shares_lnk.begin()
    );
    std::vector<ShareInfoMap::iterator>::const_iterator const sh_e(
      ep_it->second.shares_lnk.end()
    );

    for( ; sh_it != sh_e; ++sh_it) {

      classad::ClassAd* computingAd_copy(
        dynamic_cast<classad::ClassAd*>(computingAd->Copy())
      );
      computingAd_copy->Update(*(*sh_it)->second.ad);
      ExecEnvInfo const& eei(
        boost::any_cast<ExecEnvInfo>((*sh_it)->second.resources_lnk.front()->second)
      );

      std::vector<classad::ExprTree*> apps_el;
      std::transform(
        eei.applications.begin(),
        eei.applications.end(),
        std::back_inserter(apps_el),
        make_literal
      );
      classad::ClassAd* appenvAd = new classad::ClassAd();
      appenvAd->Insert("AppName", classad::ExprList::MakeExprList(apps_el));

      std::vector<classad::ExprTree*> bmark_ets;
      std::transform(
        eei.bmarks_lnk.begin(),
        eei.bmarks_lnk.end(),
        std::back_inserter(bmark_ets),
        copy_benchmark_info
       );

      classad::ClassAd* g2Ad = new classad::ClassAd;

      g2Ad->Insert("Benchmark", classad::ExprList::MakeExprList(bmark_ets));
      g2Ad->Insert("ApplicationEnvironment", appenvAd);
      if (eei.ad) {
        g2Ad->Update(*eei.ad);
      }
      g2Ad->Insert("Computing", computingAd_copy);
      if ((*sh_it)->second.ad &&
        (*sh_it)->second.ad->Lookup("Rule")) {
        g2Ad->DeepInsert(
          computingAd_copy->Lookup("Share"),
          "Policy",
          classad::ExprList::MakeExprList((*sh_it)->second.policy_rules)
        );
      }
      std::string interface_name = cu::evaluate_expression(
        *ep_it->second.ad, "EndPoint.InterfaceName"
      );
     classad::ClassAd* shareAd = dynamic_cast<classad::ClassAd*>(
        computingAd_copy->Lookup("Share")
      );

      classad::ExprList* el;
      shareAd->EvaluateAttrList("OtherInfo", el);
      // Find the CREAMCEId bound to the relevant EndPoint via InterfaceName
      std::vector<classad::ExprTree*>::const_iterator oi_it(
        std::find_if(el->begin(), el->end(),
          is_a_literal_node_starting_with(
            "CREAMCEId=" + ba::erase_first_copy(ep_it->first, "_"+interface_name)
          )
        )
      );
      // Handling OtherInfo conversion to ClassAd for the Share
      classad::ClassAd* oi = new classad::ClassAd;

      if (oi_it != el->end()) {
        classad::Value v;
        std::string s;
        (*oi_it)->Evaluate(v) && v.IsStringValue(s);
        oi->InsertAttr("CREAMCEId", s.substr(s.find("=")+1));

      }
      extract_glue2_info_value_in<std::string>(
        *shareAd  , "OtherInfo", "InfoProviderName", oi
      );
      extract_glue2_info_value_in<std::string>(
        *shareAd, "OtherInfo", "InfoProviderVersion", oi
      );
      extract_glue2_info_value_in<std::string>(
        *shareAd, "OtherInfo", "InfoProviderHost", oi
      );
      g2Ad->DeepInsert(
        computingAd_copy->Lookup("Share"),
        "OtherInfo", oi
      );

      ad_ptr result( new classad::ClassAd );
      result->Insert("GLUE2", g2Ad);

      std::string const id = cu::evaluate_expression(*result,
        "IsUndefined(GLUE2.Computing.Share.OtherInfo.CREAMCEId) ?"
        "    GLUE2.Computing.Share.ID : GLUE2.Computing.Share.OtherInfo.CREAMCEId"
      );
      std::string const policy = cu::evaluate_expression(*result,
        "GLUE2.Computing.Share.Policy[0]"
      );

      std::string const glue13Id(
        id + "/" + ba::erase_regex_copy(policy, boost::regex(".+:"))
      );
      std::for_each(
        glue13_attr_refs.begin(), glue13_attr_refs.end(),
        parse_expression_and_insert(result)
      );
      result->InsertAttr("CEid", id);
      ce_info_container.insert(std::make_pair(glue13Id, result));
      n_shares++;
    }
  }
  Debug("#" << n_shares << " GLUE2ComputingShare's ClassAd(s) generated in " << std::time(0) - t1 << " seconds");
}

void fetch_bdii_info_g2(
  std::string const& hostname,
  size_t port,
  std::string const& dn,
  time_t timeout,
  std::string const& ldap_ce_filter_ext,
  std::string const& ldap_se_filter_ext,
  gluece_info_container_type& gluece_info_container,
  gluece_info_container_type& gluese_info_container)
{
  fetch_bdii_ce_info_g2(hostname, port, dn, timeout, ldap_ce_filter_ext, gluece_info_container);
  fetch_bdii_se_info_g2(hostname, port, dn, timeout, ldap_se_filter_ext, gluese_info_container);
}

}}}}

ext: cpp

max_stars_repo_name: italiangrid/wms

max_issues_repo_name: italiangrid/wms

max_forks_repo_name: italiangrid/wms


text:
//==================================================================================================
/*!
  @file

  @copyright 2016 NumScale SAS
  @copyright 2016 J.T. Lapreste

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#ifndef BOOST_SIMD_ARCH_COMMON_SIMD_FUNCTION_IS_EQUAL_WITH_EQUAL_NANS_HPP_INCLUDED
#define BOOST_SIMD_ARCH_COMMON_SIMD_FUNCTION_IS_EQUAL_WITH_EQUAL_NANS_HPP_INCLUDED
#include <boost/simd/detail/overload.hpp>

#include <boost/simd/meta/hierarchy/simd.hpp>
#include <boost/simd/meta/as_logical.hpp>
#include <boost/simd/function/simd/is_equal.hpp>
#include <boost/simd/function/simd/is_nan.hpp>
#include <boost/simd/function/simd/logical_and.hpp>
#include <boost/simd/function/simd/logical_or.hpp>

namespace boost { namespace simd { namespace ext
{
   namespace bd = boost::dispatch;
   namespace bs = boost::simd;
   BOOST_DISPATCH_OVERLOAD(is_equal_with_equal_nans_
                          , (typename A0, typename X)
                          , bd::cpu_
                          , bs::pack_<bd::floating_<A0>, X>
                          , bs::pack_<bd::floating_<A0>, X>
                          )
   {
      BOOST_FORCEINLINE bs::as_logical_t<A0>  operator()( const A0& a0, const A0& a1) const BOOST_NOEXCEPT
      {
        return logical_or(is_equal(a0,a1), logical_and(bs::is_nan(a0), bs::is_nan(a1)));
      }
   };

   BOOST_DISPATCH_OVERLOAD(is_equal_with_equal_nans_
                          , (typename A0, typename X)
                          , bd::cpu_
                          , bs::pack_<bd::fundamental_<A0>, X>
                          , bs::pack_<bd::fundamental_<A0>, X>
                          )
   {
      BOOST_FORCEINLINE bs::as_logical_t<A0>   operator()( const A0& a0, const A0& a1) const BOOST_NOEXCEPT
      {
        return is_equal(a0,a1);
      }
   };

} } }

#endif

ext: hpp


text:
/*
 * Copyright (c) 2011, Peter Thorson. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the WebSocket++ Project nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL PETER THORSON BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include "chat.hpp"

#include "../../src/websocketpp.hpp"
#include <boost/asio.hpp>

#include <iostream>

using namespace websocketchat;
using websocketpp::server;

int main(int argc, char* argv[]) {
    short port = 9003;

    if (argc == 2) {
        // TODO: input validation?
        port = atoi(argv[1]);
    }

    try {
        // create an instance of our handler
        server::handler::ptr handler(new chat_server_handler());

        // create a server that listens on port `port` and uses our handler
        server endpoint(handler);

        endpoint.alog().set_level(websocketpp::log::alevel::CONNECT);
        endpoint.alog().set_level(websocketpp::log::alevel::DISCONNECT);

        endpoint.elog().set_level(websocketpp::log::elevel::RERROR);
        endpoint.elog().set_level(websocketpp::log::elevel::FATAL);

        // setup server settings
        // Chat server should only be receiving small text messages, reduce max
        // message size limit slightly to save memory, improve performance, and
        // guard against DoS attacks.
        //server->set_max_message_size(0xFFFF); // 64KiB

        std::cout << "Starting chat server on port " << port << std::endl;

        endpoint.listen(port);
    } catch (std::exception& e) {
        std::cerr << "Exception: " << e.what() << std::endl;
    }

    return 0;
}

max_stars_repo_name: SAN-CHAIN/sand

max_issues_repo_name: SAN-CHAIN/sand

max_forks_repo_name: SAN-CHAIN/sand