text:
/*
 * Copyright 2017 MapD Technologies, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <algorithm>
#include <boost/variant.hpp>
#include <boost/variant/get.hpp>
#include <limits>
#include <mutex>
#include <string>
#include <vector>

#include "Catalog/Catalog.h"
#include "DataMgr/DataMgr.h"
#include "DataMgr/FixedLengthArrayNoneEncoder.h"
#include "Fragmenter/InsertOrderFragmenter.h"
#include "QueryEngine/TargetValue.h"
#include "Shared/ConfigResolve.h"
#include "Shared/DateConverters.h"
#include "Shared/Logger.h"
#include "Shared/TypedDataAccessors.h"
#include "Shared/thread_count.h"
#include "TargetValueConvertersFactories.h"

namespace Fragmenter_Namespace {

inline void wait_cleanup_threads(std::vector<std::future<void>>& threads) {
  for (auto& t : threads) {
    t.get();
  }
  threads.clear();
}

FragmentInfo& InsertOrderFragmenter::getFragmentInfoFromId(const int fragment_id) {
  auto fragment_it = std::find_if(
      fragmentInfoVec_.begin(), fragmentInfoVec_.end(), [=](const auto& f) -> bool {
        return f.fragmentId == fragment_id;
      });
  CHECK(fragment_it != fragmentInfoVec_.end());
  return *fragment_it;
}

inline bool is_integral(const SQLTypeInfo& t) {
  return t.is_integer() || t.is_boolean() || t.is_time() || t.is_timeinterval();
}

bool FragmentInfo::unconditionalVacuum_{false};

void InsertOrderFragmenter::updateColumn(const Catalog_Namespace::Catalog* catalog,
                                         const std::string& tab_name,
                                         const std::string& col_name,
                                         const int fragment_id,
                                         const std::vector<uint64_t>& frag_offsets,
                                         const std::vector<ScalarTargetValue>& rhs_values,
                                         const SQLTypeInfo& rhs_type,
                                         const Data_Namespace::MemoryLevel memory_level,
                                         UpdelRoll& updel_roll) {
  const auto td = catalog->getMetadataForTable(tab_name);
  CHECK(td);
  const auto cd = catalog->getMetadataForColumn(td->tableId, col_name);
  CHECK(cd);
  td->fragmenter->updateColumn(catalog,
                               td,
                               cd,
                               fragment_id,
                               frag_offsets,
                               rhs_values,
                               rhs_type,
                               memory_level,
                               updel_roll);
}

void InsertOrderFragmenter::updateColumn(const Catalog_Namespace::Catalog* catalog,
                                         const TableDescriptor* td,
                                         const ColumnDescriptor* cd,
                                         const int fragment_id,
                                         const std::vector<uint64_t>& frag_offsets,
                                         const ScalarTargetValue& rhs_value,
                                         const SQLTypeInfo& rhs_type,
                                         const Data_Namespace::MemoryLevel memory_level,
                                         UpdelRoll& updel_roll) {
  updateColumn(catalog,
               td,
               cd,
               fragment_id,
               frag_offsets,
               std::vector<ScalarTargetValue>(1, rhs_value),
               rhs_type,
               memory_level,
               updel_roll);
}

static int get_chunks(const Catalog_Namespace::Catalog* catalog,
                      const TableDescriptor* td,
                      const FragmentInfo& fragment,
                      const Data_Namespace::MemoryLevel memory_level,
                      std::vector<std::shared_ptr<Chunk_NS::Chunk>>& chunks) {
  for (int cid = 1, nc = 0; nc < td->nColumns; ++cid) {
    if (const auto cd = catalog->getMetadataForColumn(td->tableId, cid)) {
      ++nc;
      if (!cd->isVirtualCol) {
        auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cid);
        CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
        ChunkKey chunk_key{
            catalog->getCurrentDB().dbId, td->tableId, cid, fragment.fragmentId};
        auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                               &catalog->getDataMgr(),
                                               chunk_key,
                                               memory_level,
                                               0,
                                               chunk_meta_it->second.numBytes,
                                               chunk_meta_it->second.numElements);
        chunks.push_back(chunk);
      }
    }
  }
  return chunks.size();
}

struct ChunkToInsertDataConverter {
 public:
  virtual ~ChunkToInsertDataConverter() {}

  virtual void convertToColumnarFormat(size_t row, size_t indexInFragment) = 0;

  virtual void addDataBlocksToInsertData(
      Fragmenter_Namespace::InsertData& insertData) = 0;
};

template <typename BUFFER_DATA_TYPE, typename INSERT_DATA_TYPE>
struct ScalarChunkConverter : public ChunkToInsertDataConverter {
  using ColumnDataPtr =
      std::unique_ptr<INSERT_DATA_TYPE, CheckedMallocDeleter<INSERT_DATA_TYPE>>;

  const Chunk_NS::Chunk* chunk_;
  ColumnDataPtr column_data_;
  const ColumnDescriptor* column_descriptor_;
  const BUFFER_DATA_TYPE* data_buffer_addr_;

  ScalarChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk)
      : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) {
    column_data_ = ColumnDataPtr(reinterpret_cast<INSERT_DATA_TYPE*>(
        checked_malloc(num_rows * sizeof(INSERT_DATA_TYPE))));
    data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->get_buffer()->getMemoryPtr();
  }

  ~ScalarChunkConverter() override {}

  void convertToColumnarFormat(size_t row, size_t indexInFragment) override {
    auto buffer_value = data_buffer_addr_[indexInFragment];
    auto insert_value = static_cast<INSERT_DATA_TYPE>(buffer_value);
    column_data_.get()[row] = insert_value;
  }

  void addDataBlocksToInsertData(Fragmenter_Namespace::InsertData& insertData) override {
    DataBlockPtr dataBlock;
    dataBlock.numbersPtr = reinterpret_cast<int8_t*>(column_data_.get());
    insertData.data.push_back(dataBlock);
    insertData.columnIds.push_back(column_descriptor_->columnId);
  }
};

struct FixedLenArrayChunkConverter : public ChunkToInsertDataConverter {
  const Chunk_NS::Chunk* chunk_;
  const ColumnDescriptor* column_descriptor_;

  std::unique_ptr<std::vector<ArrayDatum>> column_data_;
  int8_t* data_buffer_addr_;
  size_t fixed_array_length_;

  FixedLenArrayChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk)
      : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) {
    column_data_ = std::make_unique<std::vector<ArrayDatum>>(num_rows);
    data_buffer_addr_ = chunk->get_buffer()->getMemoryPtr();
    fixed_array_length_ = chunk->get_column_desc()->columnType.get_size();
  }

  ~FixedLenArrayChunkConverter() override {}

  void convertToColumnarFormat(size_t row, size_t indexInFragment) override {
    auto src_value_ptr = data_buffer_addr_ + (indexInFragment * fixed_array_length_);
    (*column_data_)[row] =
        ArrayDatum(fixed_array_length_, (int8_t*)src_value_ptr, DoNothingDeleter());
  }

  void addDataBlocksToInsertData(Fragmenter_Namespace::InsertData& insertData) override {
    DataBlockPtr dataBlock;
    dataBlock.arraysPtr = column_data_.get();
    insertData.data.push_back(dataBlock);
    insertData.columnIds.push_back(column_descriptor_->columnId);
  }
};

struct ArrayChunkConverter : public FixedLenArrayChunkConverter {
  StringOffsetT* index_buffer_addr_;

  ArrayChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk)
      : FixedLenArrayChunkConverter(num_rows, chunk) {
    index_buffer_addr_ =
        (StringOffsetT*)(chunk->get_index_buf() ? chunk->get_index_buf()->getMemoryPtr()
                                                : nullptr);
  }

  ~ArrayChunkConverter() override {}

  void convertToColumnarFormat(size_t row, size_t indexInFragment) override {
    size_t src_value_size =
        index_buffer_addr_[indexInFragment + 1] - index_buffer_addr_[indexInFragment];
    auto src_value_ptr = data_buffer_addr_ + index_buffer_addr_[indexInFragment];
    (*column_data_)[row] =
        ArrayDatum(src_value_size, (int8_t*)src_value_ptr, DoNothingDeleter());
  }
};

struct StringChunkConverter : public ChunkToInsertDataConverter {
  const Chunk_NS::Chunk* chunk_;
  const ColumnDescriptor* column_descriptor_;

  std::unique_ptr<std::vector<std::string>> column_data_;
  const int8_t* data_buffer_addr_;
  const StringOffsetT* index_buffer_addr_;

  StringChunkConverter(size_t num_rows, const Chunk_NS::Chunk* chunk)
      : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) {
    column_data_ = std::make_unique<std::vector<std::string>>(num_rows);
    data_buffer_addr_ = chunk->get_buffer()->getMemoryPtr();
    index_buffer_addr_ =
        (StringOffsetT*)(chunk->get_index_buf() ? chunk->get_index_buf()->getMemoryPtr()
                                                : nullptr);
  }

  ~StringChunkConverter() override {}

  void convertToColumnarFormat(size_t row, size_t indexInFragment) override {
    size_t src_value_size =
        index_buffer_addr_[indexInFragment + 1] - index_buffer_addr_[indexInFragment];
    auto src_value_ptr = data_buffer_addr_ + index_buffer_addr_[indexInFragment];
    (*column_data_)[row] = std::string((const char*)src_value_ptr, src_value_size);
  }

  void addDataBlocksToInsertData(Fragmenter_Namespace::InsertData& insertData) override {
    DataBlockPtr dataBlock;
    dataBlock.stringsPtr = column_data_.get();
    insertData.data.push_back(dataBlock);
    insertData.columnIds.push_back(column_descriptor_->columnId);
  }
};

template <typename BUFFER_DATA_TYPE>
struct DateChunkConverter : public ChunkToInsertDataConverter {
  using ColumnDataPtr = std::unique_ptr<int64_t, CheckedMallocDeleter<int64_t>>;

  const Chunk_NS::Chunk* chunk_;
  ColumnDataPtr column_data_;
  const ColumnDescriptor* column_descriptor_;
  const BUFFER_DATA_TYPE* data_buffer_addr_;

  DateChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk)
      : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) {
    column_data_ = ColumnDataPtr(
        reinterpret_cast<int64_t*>(checked_malloc(num_rows * sizeof(int64_t))));
    data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->get_buffer()->getMemoryPtr();
  }

  ~DateChunkConverter() override {}

  void convertToColumnarFormat(size_t row, size_t indexInFragment) override {
    auto buffer_value = data_buffer_addr_[indexInFragment];
    auto insert_value = static_cast<int64_t>(buffer_value);
    column_data_.get()[row] = DateConverters::get_epoch_seconds_from_days(insert_value);
  }

  void addDataBlocksToInsertData(Fragmenter_Namespace::InsertData& insertData) override {
    DataBlockPtr dataBlock;
    dataBlock.numbersPtr = reinterpret_cast<int8_t*>(column_data_.get());
    insertData.data.push_back(dataBlock);
    insertData.columnIds.push_back(column_descriptor_->columnId);
  }
};

void InsertOrderFragmenter::updateColumns(
    const Catalog_Namespace::Catalog* catalog,
    const TableDescriptor* td,
    const int fragmentId,
    const std::vector<TargetMetaInfo> sourceMetaInfo,
    const std::vector<const ColumnDescriptor*> columnDescriptors,
    const RowDataProvider& sourceDataProvider,
    const size_t indexOffFragmentOffsetColumn,
    const Data_Namespace::MemoryLevel memoryLevel,
    UpdelRoll& updelRoll) {
  if (!is_feature_enabled<VarlenUpdates>()) {
    throw std::runtime_error("varlen UPDATE path not enabled.");
  }

  updelRoll.is_varlen_update = true;
  updelRoll.catalog = catalog;
  updelRoll.logicalTableId = catalog->getLogicalTableId(td->tableId);
  updelRoll.memoryLevel = memoryLevel;

  size_t num_entries = sourceDataProvider.getEntryCount();
  size_t num_rows = sourceDataProvider.getRowCount();

  if (0 == num_rows) {
    // bail out early
    return;
  }

  TargetValueConverterFactory factory;

  auto& fragment = getFragmentInfoFromId(fragmentId);
  std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks;
  get_chunks(catalog, td, fragment, memoryLevel, chunks);
  std::vector<std::unique_ptr<TargetValueConverter>> sourceDataConverters(
      columnDescriptors.size());
  std::vector<std::unique_ptr<ChunkToInsertDataConverter>> chunkConverters;

  std::shared_ptr<Chunk_NS::Chunk> deletedChunk;
  for (size_t indexOfChunk = 0; indexOfChunk < chunks.size(); indexOfChunk++) {
    auto chunk = chunks[indexOfChunk];
    const auto chunk_cd = chunk->get_column_desc();

    if (chunk_cd->isDeletedCol) {
      deletedChunk = chunk;
      continue;
    }

    auto targetColumnIt = std::find_if(columnDescriptors.begin(),
                                       columnDescriptors.end(),
                                       [=](const ColumnDescriptor* cd) -> bool {
                                         return cd->columnId == chunk_cd->columnId;
                                       });

    if (targetColumnIt != columnDescriptors.end()) {
      auto indexOfTargetColumn = std::distance(columnDescriptors.begin(), targetColumnIt);

      auto sourceDataMetaInfo = sourceMetaInfo[indexOfTargetColumn];
      auto targetDescriptor = columnDescriptors[indexOfTargetColumn];

      ConverterCreateParameter param{num_rows,
                                     *catalog,
                                     sourceDataMetaInfo,
                                     targetDescriptor,
                                     targetDescriptor->columnType,
                                     !targetDescriptor->columnType.get_notnull(),
                                     sourceDataProvider.getLiteralDictionary()};
      auto converter = factory.create(param);
      sourceDataConverters[indexOfTargetColumn] = std::move(converter);

      if (targetDescriptor->columnType.is_geometry()) {
        // geometry columns are composites
        // need to skip chunks, depending on geo type
        switch (targetDescriptor->columnType.get_type()) {
          case kMULTIPOLYGON:
            indexOfChunk += 5;
            break;
          case kPOLYGON:
            indexOfChunk += 4;
            break;
          case kLINESTRING:
            indexOfChunk += 2;
            break;
          case kPOINT:
            indexOfChunk += 1;
            break;
          default:
            CHECK(false);  // not supported
        }
      }
    } else {
      if (chunk_cd->columnType.is_varlen() || chunk_cd->columnType.is_fixlen_array()) {
        std::unique_ptr<ChunkToInsertDataConverter> converter;

        if (chunk_cd->columnType.is_fixlen_array()) {
          converter =
              std::make_unique<FixedLenArrayChunkConverter>(num_rows, chunk.get());
        } else if (chunk_cd->columnType.is_string()) {
          converter = std::make_unique<StringChunkConverter>(num_rows, chunk.get());
        } else if (chunk_cd->columnType.is_geometry()) {
          // the logical geo column is a string column
          converter = std::make_unique<StringChunkConverter>(num_rows, chunk.get());
        } else {
          converter = std::make_unique<ArrayChunkConverter>(num_rows, chunk.get());
        }

        chunkConverters.push_back(std::move(converter));

      } else if (chunk_cd->columnType.is_date_in_days()) {
        /* Q: Why do we need this?
           A: In variable length updates path we move the chunk content of column
           without decoding. Since it again passes through DateDaysEncoder
           the expected value should be in seconds, but here it will be in days.
           Therefore, using DateChunkConverter chunk values are being scaled to
           seconds which then ultimately encoded in days in DateDaysEncoder.
        */
        std::unique_ptr<ChunkToInsertDataConverter> converter;
        const size_t physical_size = chunk_cd->columnType.get_size();
        if (physical_size == 2) {
          converter =
              std::make_unique<DateChunkConverter<int16_t>>(num_rows, chunk.get());
        } else if (physical_size == 4) {
          converter =
              std::make_unique<DateChunkConverter<int32_t>>(num_rows, chunk.get());
        } else {
          CHECK(false);
        }
        chunkConverters.push_back(std::move(converter));
      } else {
        std::unique_ptr<ChunkToInsertDataConverter> converter;
        SQLTypeInfo logical_type = get_logical_type_info(chunk_cd->columnType);
        int logical_size = logical_type.get_size();
        int physical_size = chunk_cd->columnType.get_size();

        if (logical_type.is_string()) {
          // for dicts -> logical = physical
          logical_size = physical_size;
        }

        if (8 == physical_size) {
          converter = std::make_unique<ScalarChunkConverter<int64_t, int64_t>>(
              num_rows, chunk.get());
        } else if (4 == physical_size) {
          if (8 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int32_t, int64_t>>(
                num_rows, chunk.get());
          } else {
            converter = std::make_unique<ScalarChunkConverter<int32_t, int32_t>>(
                num_rows, chunk.get());
          }
        } else if (2 == chunk_cd->columnType.get_size()) {
          if (8 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int16_t, int64_t>>(
                num_rows, chunk.get());
          } else if (4 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int16_t, int32_t>>(
                num_rows, chunk.get());
          } else {
            converter = std::make_unique<ScalarChunkConverter<int16_t, int16_t>>(
                num_rows, chunk.get());
          }
        } else if (1 == chunk_cd->columnType.get_size()) {
          if (8 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int8_t, int64_t>>(
                num_rows, chunk.get());
          } else if (4 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int8_t, int32_t>>(
                num_rows, chunk.get());
          } else if (2 == logical_size) {
            converter = std::make_unique<ScalarChunkConverter<int8_t, int16_t>>(
                num_rows, chunk.get());
          } else {
            converter = std::make_unique<ScalarChunkConverter<int8_t, int8_t>>(
                num_rows, chunk.get());
          }
        } else {
          CHECK(false);  // unknown
        }

        chunkConverters.push_back(std::move(converter));
      }
    }
  }

  static boost_variant_accessor<ScalarTargetValue> SCALAR_TARGET_VALUE_ACCESSOR;
  static boost_variant_accessor<int64_t> OFFSET_VALUE__ACCESSOR;

  updelRoll.dirtyChunks[deletedChunk.get()] = deletedChunk;
  ChunkKey chunkey{updelRoll.catalog->getCurrentDB().dbId,
                   deletedChunk->get_column_desc()->tableId,
                   deletedChunk->get_column_desc()->columnId,
                   fragment.fragmentId};
  updelRoll.dirtyChunkeys.insert(chunkey);
  bool* deletedChunkBuffer =
      reinterpret_cast<bool*>(deletedChunk->get_buffer()->getMemoryPtr());

  std::atomic<size_t> row_idx{0};

  auto row_converter = [&sourceDataProvider,
                        &sourceDataConverters,
                        &indexOffFragmentOffsetColumn,
                        &chunkConverters,
                        &deletedChunkBuffer,
                        &row_idx](size_t indexOfEntry) -> void {
    // convert the source data
    const auto row = sourceDataProvider.getEntryAt(indexOfEntry);
    if (row.empty()) {
      return;
    }

    size_t indexOfRow = row_idx.fetch_add(1);

    for (size_t col = 0; col < sourceDataConverters.size(); col++) {
      if (sourceDataConverters[col]) {
        const auto& mapd_variant = row[col];
        sourceDataConverters[col]->convertToColumnarFormat(indexOfRow, &mapd_variant);
      }
    }

    auto scalar = checked_get(
        indexOfRow, &row[indexOffFragmentOffsetColumn], SCALAR_TARGET_VALUE_ACCESSOR);
    auto indexInChunkBuffer = *checked_get(indexOfRow, scalar, OFFSET_VALUE__ACCESSOR);

    // convert the remaining chunks
    for (size_t idx = 0; idx < chunkConverters.size(); idx++) {
      chunkConverters[idx]->convertToColumnarFormat(indexOfRow, indexInChunkBuffer);
    }

    // now mark the row as deleted
    deletedChunkBuffer[indexInChunkBuffer] = true;
  };

  bool can_go_parallel = num_rows > 20000;

  if (can_go_parallel) {
    const size_t num_worker_threads = cpu_threads();
    std::vector<std::future<void>> worker_threads;
    for (size_t i = 0,
                start_entry = 0,
                stride = (num_entries + num_worker_threads - 1) / num_worker_threads;
         i < num_worker_threads && start_entry < num_entries;
         ++i, start_entry += stride) {
      const auto end_entry = std::min(start_entry + stride, num_rows);
      worker_threads.push_back(std::async(
          std::launch::async,
          [&row_converter](const size_t start, const size_t end) {
            for (size_t indexOfRow = start; indexOfRow < end; ++indexOfRow) {
              row_converter(indexOfRow);
            }
          },
          start_entry,
          end_entry));
    }

    for (auto& child : worker_threads) {
      child.wait();
    }

  } else {
    for (size_t entryIdx = 0; entryIdx < num_entries; entryIdx++) {
      row_converter(entryIdx);
    }
  }

  Fragmenter_Namespace::InsertData insert_data;
  insert_data.databaseId = catalog->getCurrentDB().dbId;
  insert_data.tableId = td->tableId;

  for (size_t i = 0; i < chunkConverters.size(); i++) {
    chunkConverters[i]->addDataBlocksToInsertData(insert_data);
    continue;
  }

  for (size_t i = 0; i < sourceDataConverters.size(); i++) {
    if (sourceDataConverters[i]) {
      sourceDataConverters[i]->addDataBlocksToInsertData(insert_data);
    }
    continue;
  }

  insert_data.numRows = num_rows;
  insertDataNoCheckpoint(insert_data);

  // update metdata
  if (!deletedChunk->get_buffer()->has_encoder) {
    deletedChunk->init_encoder();
  }
  deletedChunk->get_buffer()->encoder->updateStats(static_cast<int64_t>(true), false);

  auto& shadowDeletedChunkMeta =
      fragment.shadowChunkMetadataMap[deletedChunk->get_column_desc()->columnId];
  if (shadowDeletedChunkMeta.numElements >
      deletedChunk->get_buffer()->encoder->getNumElems()) {
    // the append will have populated shadow meta data, otherwise use existing num
    // elements
    deletedChunk->get_buffer()->encoder->setNumElems(shadowDeletedChunkMeta.numElements);
  }
  deletedChunk->get_buffer()->setUpdated();
}

void InsertOrderFragmenter::updateColumn(const Catalog_Namespace::Catalog* catalog,
                                         const TableDescriptor* td,
                                         const ColumnDescriptor* cd,
                                         const int fragment_id,
                                         const std::vector<uint64_t>& frag_offsets,
                                         const std::vector<ScalarTargetValue>& rhs_values,
                                         const SQLTypeInfo& rhs_type,
                                         const Data_Namespace::MemoryLevel memory_level,
                                         UpdelRoll& updel_roll) {
  updel_roll.catalog = catalog;
  updel_roll.logicalTableId = catalog->getLogicalTableId(td->tableId);
  updel_roll.memoryLevel = memory_level;

  const size_t ncore = cpu_threads();
  const auto nrow = frag_offsets.size();
  const auto n_rhs_values = rhs_values.size();
  if (0 == nrow) {
    return;
  }
  CHECK(nrow == n_rhs_values || 1 == n_rhs_values);

  auto& fragment = getFragmentInfoFromId(fragment_id);
  auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(cd->columnId);
  CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
  ChunkKey chunk_key{
      catalog->getCurrentDB().dbId, td->tableId, cd->columnId, fragment.fragmentId};
  auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                         &catalog->getDataMgr(),
                                         chunk_key,
                                         Data_Namespace::CPU_LEVEL,
                                         0,
                                         chunk_meta_it->second.numBytes,
                                         chunk_meta_it->second.numElements);

  std::vector<int8_t> has_null_per_thread(ncore, 0);
  std::vector<double> max_double_per_thread(ncore, std::numeric_limits<double>::lowest());
  std::vector<double> min_double_per_thread(ncore, std::numeric_limits<double>::max());
  std::vector<int64_t> max_int64t_per_thread(ncore, std::numeric_limits<int64_t>::min());
  std::vector<int64_t> min_int64t_per_thread(ncore, std::numeric_limits<int64_t>::max());

  // parallel update elements
  std::vector<std::future<void>> threads;

  const auto segsz = (nrow + ncore - 1) / ncore;
  auto dbuf = chunk->get_buffer();
  auto dbuf_addr = dbuf->getMemoryPtr();
  dbuf->setUpdated();
  {
    std::lock_guard<std::mutex> lck(updel_roll.mutex);
    if (updel_roll.dirtyChunks.count(chunk.get()) == 0) {
      updel_roll.dirtyChunks.emplace(chunk.get(), chunk);
    }

    ChunkKey chunkey{updel_roll.catalog->getCurrentDB().dbId,
                     cd->tableId,
                     cd->columnId,
                     fragment.fragmentId};
    updel_roll.dirtyChunkeys.insert(chunkey);
  }
  for (size_t rbegin = 0, c = 0; rbegin < nrow; ++c, rbegin += segsz) {
    threads.emplace_back(std::async(
        std::launch::async,
        [=,
         &has_null_per_thread,
         &min_int64t_per_thread,
         &max_int64t_per_thread,
         &min_double_per_thread,
         &max_double_per_thread,
         &frag_offsets,
         &rhs_values] {
          SQLTypeInfo lhs_type = cd->columnType;

          // !! not sure if this is a undocumented convention or a bug, but for a sharded
          // table the dictionary id of a encoded string column is not specified by
          // comp_param in physical table but somehow in logical table :) comp_param in
          // physical table is always 0, so need to adapt accordingly...
          auto cdl = (shard_ < 0)
                         ? cd
                         : catalog->getMetadataForColumn(
                               catalog->getLogicalTableId(td->tableId), cd->columnId);
          CHECK(cdl);
          DecimalOverflowValidator decimalOverflowValidator(lhs_type);
          NullAwareValidator<DecimalOverflowValidator> nullAwareDecimalOverflowValidator(
              lhs_type, &decimalOverflowValidator);
          DateDaysOverflowValidator dateDaysOverflowValidator(lhs_type);
          NullAwareValidator<DateDaysOverflowValidator> nullAwareDateOverflowValidator(
              lhs_type, &dateDaysOverflowValidator);

          StringDictionary* stringDict{nullptr};
          if (lhs_type.is_string()) {
            CHECK(kENCODING_DICT == lhs_type.get_compression());
            auto dictDesc = const_cast<DictDescriptor*>(
                catalog->getMetadataForDict(cdl->columnType.get_comp_param()));
            CHECK(dictDesc);
            stringDict = dictDesc->stringDict.get();
            CHECK(stringDict);
          }

          for (size_t r = rbegin; r < std::min(rbegin + segsz, nrow); r++) {
            const auto roffs = frag_offsets[r];
            auto data_ptr = dbuf_addr + roffs * get_element_size(lhs_type);
            auto sv = &rhs_values[1 == n_rhs_values ? 0 : r];
            ScalarTargetValue sv2;

            // Subtle here is on the two cases of string-to-string assignments, when
            // upstream passes RHS string as a string index instead of a preferred "real
            // string".
            //   case #1. For "SET str_col = str_literal", it is hard to resolve temp str
            //   index
            //            in this layer, so if upstream passes a str idx here, an
            //            exception is thrown.
            //   case #2. For "SET str_col1 = str_col2", RHS str idx is converted to LHS
            //   str idx.
            if (rhs_type.is_string()) {
              if (const auto vp = boost::get<int64_t>(sv)) {
                auto dictDesc = const_cast<DictDescriptor*>(
                    catalog->getMetadataForDict(rhs_type.get_comp_param()));
                if (nullptr == dictDesc) {
                  throw std::runtime_error(
                      "UPDATE does not support cast from string literal to string "
                      "column.");
                }
                auto stringDict = dictDesc->stringDict.get();
                CHECK(stringDict);
                sv2 = NullableString(stringDict->getString(*vp));
                sv = &sv2;
              }
            }

            if (const auto vp = boost::get<int64_t>(sv)) {
              auto v = *vp;
              if (lhs_type.is_string()) {
                throw std::runtime_error("UPDATE does not support cast to string.");
              }
              put_scalar<int64_t>(data_ptr, lhs_type, v, cd->columnName, &rhs_type);
              if (lhs_type.is_decimal()) {
                nullAwareDecimalOverflowValidator.validate<int64_t>(v);
                int64_t decimal_val;
                get_scalar<int64_t>(data_ptr, lhs_type, decimal_val);
                tabulate_metadata(lhs_type,
                                  min_int64t_per_thread[c],
                                  max_int64t_per_thread[c],
                                  has_null_per_thread[c],
                                  (v == inline_int_null_value<int64_t>() &&
                                   lhs_type.get_notnull() == false)
                                      ? v
                                      : decimal_val);
                auto const positive_v_and_negative_d = (v >= 0) && (decimal_val < 0);
                auto const negative_v_and_positive_d = (v < 0) && (decimal_val >= 0);
                if (positive_v_and_negative_d || negative_v_and_positive_d) {
                  throw std::runtime_error(
                      "Data conversion overflow on " + std::to_string(v) +
                      " from DECIMAL(" + std::to_string(rhs_type.get_dimension()) + ", " +
                      std::to_string(rhs_type.get_scale()) + ") to (" +
                      std::to_string(lhs_type.get_dimension()) + ", " +
                      std::to_string(lhs_type.get_scale()) + ")");
                }
              } else if (is_integral(lhs_type)) {
                if (lhs_type.is_date_in_days()) {
                  // Store meta values in seconds
                  if (lhs_type.get_size() == 2) {
                    nullAwareDateOverflowValidator.validate<int16_t>(v);
                  } else {
                    nullAwareDateOverflowValidator.validate<int32_t>(v);
                  }
                  int64_t days;
                  get_scalar<int64_t>(data_ptr, lhs_type, days);
                  const auto seconds = DateConverters::get_epoch_seconds_from_days(days);
                  tabulate_metadata(lhs_type,
                                    min_int64t_per_thread[c],
                                    max_int64t_per_thread[c],
                                    has_null_per_thread[c],
                                    (v == inline_int_null_value<int64_t>() &&
                                     lhs_type.get_notnull() == false)
                                        ? NullSentinelSupplier()(lhs_type, v)
                                        : seconds);
                } else {
                  int64_t target_value;
                  if (rhs_type.is_decimal()) {
                    target_value = round(decimal_to_double(rhs_type, v));
                  } else {
                    target_value = v;
                  }
                  tabulate_metadata(lhs_type,
                                    min_int64t_per_thread[c],
                                    max_int64t_per_thread[c],
                                    has_null_per_thread[c],
                                    target_value);
                }
              } else {
                tabulate_metadata(
                    lhs_type,
                    min_double_per_thread[c],
                    max_double_per_thread[c],
                    has_null_per_thread[c],
                    rhs_type.is_decimal() ? decimal_to_double(rhs_type, v) : v);
              }
            } else if (const auto vp = boost::get<double>(sv)) {
              auto v = *vp;
              if (lhs_type.is_string()) {
                throw std::runtime_error("UPDATE does not support cast to string.");
              }
              put_scalar<double>(data_ptr, lhs_type, v, cd->columnName);
              if (lhs_type.is_integer()) {
                tabulate_metadata(lhs_type,
                                  min_int64t_per_thread[c],
                                  max_int64t_per_thread[c],
                                  has_null_per_thread[c],
                                  int64_t(v));
              } else if (lhs_type.is_fp()) {
                tabulate_metadata(lhs_type,
                                  min_double_per_thread[c],
                                  max_double_per_thread[c],
                                  has_null_per_thread[c],
                                  double(v));
              } else {
                UNREACHABLE() << "Unexpected combination of a non-floating or integer "
                                 "LHS with a floating RHS.";
              }
            } else if (const auto vp = boost::get<float>(sv)) {
              auto v = *vp;
              if (lhs_type.is_string()) {
                throw std::runtime_error("UPDATE does not support cast to string.");
              }
              put_scalar<float>(data_ptr, lhs_type, v, cd->columnName);
              if (lhs_type.is_integer()) {
                tabulate_metadata(lhs_type,
                                  min_int64t_per_thread[c],
                                  max_int64t_per_thread[c],
                                  has_null_per_thread[c],
                                  int64_t(v));
              } else {
                tabulate_metadata(lhs_type,
                                  min_double_per_thread[c],
                                  max_double_per_thread[c],
                                  has_null_per_thread[c],
                                  double(v));
              }
            } else if (const auto vp = boost::get<NullableString>(sv)) {
              const auto s = boost::get<std::string>(vp);
              const auto sval = s ? *s : std::string("");
              if (lhs_type.is_string()) {
                decltype(stringDict->getOrAdd(sval)) sidx;
                {
                  std::unique_lock<std::mutex> lock(temp_mutex_);
                  sidx = stringDict->getOrAdd(sval);
                }
                put_scalar<int32_t>(data_ptr, lhs_type, sidx, cd->columnName);
                tabulate_metadata(lhs_type,
                                  min_int64t_per_thread[c],
                                  max_int64t_per_thread[c],
                                  has_null_per_thread[c],
                                  int64_t(sidx));
              } else if (sval.size() > 0) {
                auto dval = std::atof(sval.data());
                if (lhs_type.is_boolean()) {
                  dval = sval == "t" || sval == "true" || sval == "T" || sval == "True";
                } else if (lhs_type.is_time()) {
                  throw std::runtime_error(
                      "Date/Time/Timestamp update not supported through translated "
                      "string path.");
                }
                if (lhs_type.is_fp() || lhs_type.is_decimal()) {
                  put_scalar<double>(data_ptr, lhs_type, dval, cd->columnName);
                  tabulate_metadata(lhs_type,
                                    min_double_per_thread[c],
                                    max_double_per_thread[c],
                                    has_null_per_thread[c],
                                    double(dval));
                } else {
                  put_scalar<int64_t>(data_ptr, lhs_type, dval, cd->columnName);
                  tabulate_metadata(lhs_type,
                                    min_int64t_per_thread[c],
                                    max_int64t_per_thread[c],
                                    has_null_per_thread[c],
                                    int64_t(dval));
                }
              } else {
                put_null(data_ptr, lhs_type, cd->columnName);
                has_null_per_thread[c] = true;
              }
            } else {
              CHECK(false);
            }
          }
        }));
    if (threads.size() >= (size_t)cpu_threads()) {
      wait_cleanup_threads(threads);
    }
  }
  wait_cleanup_threads(threads);

  // for unit test
  if (Fragmenter_Namespace::FragmentInfo::unconditionalVacuum_) {
    if (cd->isDeletedCol) {
      const auto deleted_offsets = getVacuumOffsets(chunk);
      if (deleted_offsets.size() > 0) {
        compactRows(catalog, td, fragment_id, deleted_offsets, memory_level, updel_roll);
        return;
      }
    }
  }
  bool has_null_per_chunk{false};
  double max_double_per_chunk{std::numeric_limits<double>::lowest()};
  double min_double_per_chunk{std::numeric_limits<double>::max()};
  int64_t max_int64t_per_chunk{std::numeric_limits<int64_t>::min()};
  int64_t min_int64t_per_chunk{std::numeric_limits<int64_t>::max()};
  for (size_t c = 0; c < ncore; ++c) {
    has_null_per_chunk = has_null_per_chunk || has_null_per_thread[c];
    max_double_per_chunk =
        std::max<double>(max_double_per_chunk, max_double_per_thread[c]);
    min_double_per_chunk =
        std::min<double>(min_double_per_chunk, min_double_per_thread[c]);
    max_int64t_per_chunk =
        std::max<int64_t>(max_int64t_per_chunk, max_int64t_per_thread[c]);
    min_int64t_per_chunk =
        std::min<int64_t>(min_int64t_per_chunk, min_int64t_per_thread[c]);
  }
  updateColumnMetadata(cd,
                       fragment,
                       chunk,
                       has_null_per_chunk,
                       max_double_per_chunk,
                       min_double_per_chunk,
                       max_int64t_per_chunk,
                       min_int64t_per_chunk,
                       cd->columnType,
                       updel_roll);
}

void InsertOrderFragmenter::updateColumnMetadata(const ColumnDescriptor* cd,
                                                 FragmentInfo& fragment,
                                                 std::shared_ptr<Chunk_NS::Chunk> chunk,
                                                 const bool has_null_per_chunk,
                                                 const double max_double_per_chunk,
                                                 const double min_double_per_chunk,
                                                 const int64_t max_int64t_per_chunk,
                                                 const int64_t min_int64t_per_chunk,
                                                 const SQLTypeInfo& rhs_type,
                                                 UpdelRoll& updel_roll) {
  auto td = updel_roll.catalog->getMetadataForTable(cd->tableId);
  auto key = std::make_pair(td, &fragment);
  std::lock_guard<std::mutex> lck(updel_roll.mutex);
  if (0 == updel_roll.chunkMetadata.count(key)) {
    updel_roll.chunkMetadata[key] = fragment.getChunkMetadataMapPhysical();
  }
  if (0 == updel_roll.numTuples.count(key)) {
    updel_roll.numTuples[key] = fragment.shadowNumTuples;
  }
  auto& chunkMetadata = updel_roll.chunkMetadata[key];

  auto buffer = chunk->get_buffer();
  const auto& lhs_type = cd->columnType;

  auto update_stats = [& encoder = buffer->encoder](auto min, auto max, auto has_null) {
    static_assert(std::is_same<decltype(min), decltype(max)>::value,
                  "Type mismatch on min/max");
    if (has_null) {
      encoder->updateStats(decltype(min)(), true);
    }
    if (max < min) {
      return;
    }
    encoder->updateStats(min, false);
    encoder->updateStats(max, false);
  };

  if (is_integral(lhs_type) || (lhs_type.is_decimal() && rhs_type.is_decimal())) {
    update_stats(min_int64t_per_chunk, max_int64t_per_chunk, has_null_per_chunk);
  } else if (lhs_type.is_fp()) {
    update_stats(min_double_per_chunk, max_double_per_chunk, has_null_per_chunk);
  } else if (lhs_type.is_decimal()) {
    update_stats((int64_t)(min_double_per_chunk * pow(10, lhs_type.get_scale())),
                 (int64_t)(max_double_per_chunk * pow(10, lhs_type.get_scale())),
                 has_null_per_chunk);
  } else if (!lhs_type.is_array() && !lhs_type.is_geometry() &&
             !(lhs_type.is_string() && kENCODING_DICT != lhs_type.get_compression())) {
    update_stats(min_int64t_per_chunk, max_int64t_per_chunk, has_null_per_chunk);
  }
  buffer->encoder->getMetadata(chunkMetadata[cd->columnId]);

  // removed as @alex suggests. keep it commented in case of any chance to revisit
  // it once after vacuum code is introduced. fragment.invalidateChunkMetadataMap();
}

void InsertOrderFragmenter::updateMetadata(const Catalog_Namespace::Catalog* catalog,
                                           const MetaDataKey& key,
                                           UpdelRoll& updel_roll) {
  mapd_unique_lock<mapd_shared_mutex> writeLock(fragmentInfoMutex_);
  if (updel_roll.chunkMetadata.count(key)) {
    auto& fragmentInfo = *key.second;
    const auto& chunkMetadata = updel_roll.chunkMetadata[key];
    fragmentInfo.shadowChunkMetadataMap = chunkMetadata;
    fragmentInfo.setChunkMetadataMap(chunkMetadata);
    fragmentInfo.shadowNumTuples = updel_roll.numTuples[key];
    fragmentInfo.setPhysicalNumTuples(fragmentInfo.shadowNumTuples);
    // TODO(ppan): When fragment-level compaction is enable, the following code
    // should suffice. When not (ie. existing code), we'll revert to update
    // InsertOrderFragmenter::varLenColInfo_
    /*
    for (const auto cit : chunkMetadata) {
      const auto& cd = *catalog->getMetadataForColumn(td->tableId, cit.first);
      if (cd.columnType.get_size() < 0)
        fragmentInfo.varLenColInfox[cd.columnId] = cit.second.numBytes;
    }
    */
  }
}

auto InsertOrderFragmenter::getChunksForAllColumns(
    const TableDescriptor* td,
    const FragmentInfo& fragment,
    const Data_Namespace::MemoryLevel memory_level) {
  std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks;
  // coming from updateColumn (on '$delete$' column) we dont have chunks for all columns
  for (int col_id = 1, ncol = 0; ncol < td->nColumns; ++col_id) {
    if (const auto cd = catalog_->getMetadataForColumn(td->tableId, col_id)) {
      ++ncol;
      if (!cd->isVirtualCol) {
        auto chunk_meta_it = fragment.getChunkMetadataMapPhysical().find(col_id);
        CHECK(chunk_meta_it != fragment.getChunkMetadataMapPhysical().end());
        ChunkKey chunk_key{
            catalog_->getCurrentDB().dbId, td->tableId, col_id, fragment.fragmentId};
        auto chunk = Chunk_NS::Chunk::getChunk(cd,
                                               &catalog_->getDataMgr(),
                                               chunk_key,
                                               memory_level,
                                               0,
                                               chunk_meta_it->second.numBytes,
                                               chunk_meta_it->second.numElements);
        chunks.push_back(chunk);
      }
    }
  }
  return chunks;
}

// get a sorted vector of offsets of rows to vacuum
const std::vector<uint64_t> InsertOrderFragmenter::getVacuumOffsets(
    const std::shared_ptr<Chunk_NS::Chunk>& chunk) {
  const auto data_buffer = chunk->get_buffer();
  const auto data_addr = data_buffer->getMemoryPtr();
  const size_t nrows_in_chunk = data_buffer->size();
  const size_t ncore = cpu_threads();
  const size_t segsz = (nrows_in_chunk + ncore - 1) / ncore;
  std::vector<std::vector<uint64_t>> deleted_offsets;
  deleted_offsets.resize(ncore);
  std::vector<std::future<void>> threads;
  for (size_t rbegin = 0; rbegin < nrows_in_chunk; rbegin += segsz) {
    threads.emplace_back(std::async(std::launch::async, [=, &deleted_offsets] {
      const auto rend = std::min<size_t>(rbegin + segsz, nrows_in_chunk);
      const auto ithread = rbegin / segsz;
      CHECK(ithread < deleted_offsets.size());
      deleted_offsets[ithread].reserve(segsz);
      for (size_t r = rbegin; r < rend; ++r) {
        if (data_addr[r]) {
          deleted_offsets[ithread].push_back(r);
        }
      }
    }));
  }
  wait_cleanup_threads(threads);
  std::vector<uint64_t> all_deleted_offsets;
  for (size_t i = 0; i < ncore; ++i) {
    all_deleted_offsets.insert(
        all_deleted_offsets.end(), deleted_offsets[i].begin(), deleted_offsets[i].end());
  }
  return all_deleted_offsets;
}

template <typename T>
static void set_chunk_stats(const SQLTypeInfo& col_type,
                            int8_t* data_addr,
                            int8_t& has_null,
                            T& min,
                            T& max) {
  T v;
  const auto can_be_null = !col_type.get_notnull();
  const auto is_null = get_scalar<T>(data_addr, col_type, v);
  if (is_null) {
    has_null = has_null || (can_be_null && is_null);
  } else {
    set_minmax(min, max, v);
  }
}

static void set_chunk_metadata(const Catalog_Namespace::Catalog* catalog,
                               FragmentInfo& fragment,
                               const std::shared_ptr<Chunk_NS::Chunk>& chunk,
                               const size_t nrows_to_keep,
                               UpdelRoll& updel_roll) {
  auto cd = chunk->get_column_desc();
  auto td = catalog->getMetadataForTable(cd->tableId);
  auto data_buffer = chunk->get_buffer();
  std::lock_guard<std::mutex> lck(updel_roll.mutex);
  const auto key = std::make_pair(td, &fragment);
  if (0 == updel_roll.chunkMetadata.count(key)) {
    updel_roll.chunkMetadata[key] = fragment.getChunkMetadataMapPhysical();
  }
  auto& chunkMetadata = updel_roll.chunkMetadata[key];
  chunkMetadata[cd->columnId].numElements = nrows_to_keep;
  chunkMetadata[cd->columnId].numBytes = data_buffer->size();
  if (updel_roll.dirtyChunks.count(chunk.get()) == 0) {
    updel_roll.dirtyChunks.emplace(chunk.get(), chunk);
  }
}

auto InsertOrderFragmenter::vacuum_fixlen_rows(
    const FragmentInfo& fragment,
    const std::shared_ptr<Chunk_NS::Chunk>& chunk,
    const std::vector<uint64_t>& frag_offsets) {
  const auto cd = chunk->get_column_desc();
  const auto& col_type = cd->columnType;
  auto data_buffer = chunk->get_buffer();
  auto data_addr = data_buffer->getMemoryPtr();
  auto element_size =
      col_type.is_fixlen_array() ? col_type.get_size() : get_element_size(col_type);
  int64_t irow_of_blk_to_keep = 0;  // head of next row block to keep
  int64_t irow_of_blk_to_fill = 0;  // row offset to fit the kept block
  size_t nbytes_fix_data_to_keep = 0;
  auto nrows_to_vacuum = frag_offsets.size();
  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
  for (size_t irow = 0; irow <= nrows_to_vacuum; irow++) {
    auto is_last_one = irow == nrows_to_vacuum;
    auto irow_to_vacuum = is_last_one ? nrows_in_fragment : frag_offsets[irow];
    auto maddr_to_vacuum = data_addr;
    int64_t nrows_to_keep = irow_to_vacuum - irow_of_blk_to_keep;
    if (nrows_to_keep > 0) {
      auto nbytes_to_keep = nrows_to_keep * element_size;
      if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
        // move curr fixlen row block toward front
        memmove(maddr_to_vacuum + irow_of_blk_to_fill * element_size,
                maddr_to_vacuum + irow_of_blk_to_keep * element_size,
                nbytes_to_keep);
      }
      irow_of_blk_to_fill += nrows_to_keep;
      nbytes_fix_data_to_keep += nbytes_to_keep;
    }
    irow_of_blk_to_keep = irow_to_vacuum + 1;
  }
  return nbytes_fix_data_to_keep;
}

auto InsertOrderFragmenter::vacuum_varlen_rows(
    const FragmentInfo& fragment,
    const std::shared_ptr<Chunk_NS::Chunk>& chunk,
    const std::vector<uint64_t>& frag_offsets) {
  auto data_buffer = chunk->get_buffer();
  auto index_buffer = chunk->get_index_buf();
  auto data_addr = data_buffer->getMemoryPtr();
  auto indices_addr = index_buffer ? index_buffer->getMemoryPtr() : nullptr;
  auto index_array = (StringOffsetT*)indices_addr;
  int64_t irow_of_blk_to_keep = 0;  // head of next row block to keep
  int64_t irow_of_blk_to_fill = 0;  // row offset to fit the kept block
  size_t nbytes_fix_data_to_keep = 0;
  size_t nbytes_var_data_to_keep = 0;
  auto nrows_to_vacuum = frag_offsets.size();
  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
  for (size_t irow = 0; irow <= nrows_to_vacuum; irow++) {
    auto is_last_one = irow == nrows_to_vacuum;
    auto irow_to_vacuum = is_last_one ? nrows_in_fragment : frag_offsets[irow];
    auto maddr_to_vacuum = data_addr;
    int64_t nrows_to_keep = irow_to_vacuum - irow_of_blk_to_keep;
    if (nrows_to_keep > 0) {
      auto ibyte_var_data_to_keep = nbytes_var_data_to_keep;
      auto nbytes_to_keep =
          (is_last_one ? data_buffer->size() : index_array[irow_to_vacuum]) -
          index_array[irow_of_blk_to_keep];
      if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
        // move curr varlen row block toward front
        memmove(data_addr + ibyte_var_data_to_keep,
                data_addr + index_array[irow_of_blk_to_keep],
                nbytes_to_keep);

        const auto index_base = index_array[irow_of_blk_to_keep];
        for (int64_t i = 0; i < nrows_to_keep; ++i) {
          auto& index = index_array[irow_of_blk_to_keep + i];
          index = ibyte_var_data_to_keep + (index - index_base);
        }
      }
      nbytes_var_data_to_keep += nbytes_to_keep;
      maddr_to_vacuum = indices_addr;

      constexpr static auto index_element_size = sizeof(StringOffsetT);
      nbytes_to_keep = nrows_to_keep * index_element_size;
      if (irow_of_blk_to_fill != irow_of_blk_to_keep) {
        // move curr fixlen row block toward front
        memmove(maddr_to_vacuum + irow_of_blk_to_fill * index_element_size,
                maddr_to_vacuum + irow_of_blk_to_keep * index_element_size,
                nbytes_to_keep);
      }
      irow_of_blk_to_fill += nrows_to_keep;
      nbytes_fix_data_to_keep += nbytes_to_keep;
    }
    irow_of_blk_to_keep = irow_to_vacuum + 1;
  }
  return nbytes_var_data_to_keep;
}

void InsertOrderFragmenter::compactRows(const Catalog_Namespace::Catalog* catalog,
                                        const TableDescriptor* td,
                                        const int fragment_id,
                                        const std::vector<uint64_t>& frag_offsets,
                                        const Data_Namespace::MemoryLevel memory_level,
                                        UpdelRoll& updel_roll) {
  auto& fragment = getFragmentInfoFromId(fragment_id);
  auto chunks = getChunksForAllColumns(td, fragment, memory_level);
  const auto ncol = chunks.size();

  std::vector<int8_t> has_null_per_thread(ncol, 0);
  std::vector<double> max_double_per_thread(ncol, std::numeric_limits<double>::lowest());
  std::vector<double> min_double_per_thread(ncol, std::numeric_limits<double>::max());
  std::vector<int64_t> max_int64t_per_thread(ncol, std::numeric_limits<uint64_t>::min());
  std::vector<int64_t> min_int64t_per_thread(ncol, std::numeric_limits<uint64_t>::max());

  // parallel delete columns
  std::vector<std::future<void>> threads;
  auto nrows_to_vacuum = frag_offsets.size();
  auto nrows_in_fragment = fragment.getPhysicalNumTuples();
  auto nrows_to_keep = nrows_in_fragment - nrows_to_vacuum;

  for (size_t ci = 0; ci < chunks.size(); ++ci) {
    auto chunk = chunks[ci];
    const auto cd = chunk->get_column_desc();
    const auto& col_type = cd->columnType;
    auto data_buffer = chunk->get_buffer();
    auto index_buffer = chunk->get_index_buf();
    auto data_addr = data_buffer->getMemoryPtr();
    auto indices_addr = index_buffer ? index_buffer->getMemoryPtr() : nullptr;
    auto index_array = (StringOffsetT*)indices_addr;
    bool is_varlen = col_type.is_varlen_indeed();

    auto fixlen_vacuum = [=,
                          &has_null_per_thread,
                          &max_double_per_thread,
                          &min_double_per_thread,
                          &min_int64t_per_thread,
                          &max_int64t_per_thread,
                          &updel_roll,
                          &frag_offsets,
                          &fragment] {
      size_t nbytes_fix_data_to_keep;
      nbytes_fix_data_to_keep = vacuum_fixlen_rows(fragment, chunk, frag_offsets);

      data_buffer->encoder->setNumElems(nrows_to_keep);
      data_buffer->setSize(nbytes_fix_data_to_keep);
      data_buffer->setUpdated();

      set_chunk_metadata(catalog, fragment, chunk, nrows_to_keep, updel_roll);

      auto daddr = data_addr;
      auto element_size =
          col_type.is_fixlen_array() ? col_type.get_size() : get_element_size(col_type);
      for (size_t irow = 0; irow < nrows_to_keep; ++irow, daddr += element_size) {
        if (col_type.is_fixlen_array()) {
          auto encoder =
              dynamic_cast<FixedLengthArrayNoneEncoder*>(data_buffer->encoder.get());
          CHECK(encoder);
          encoder->updateMetadata((int8_t*)daddr);
        } else if (col_type.is_fp()) {
          set_chunk_stats(col_type,
                          data_addr,
                          has_null_per_thread[ci],
                          min_double_per_thread[ci],
                          max_double_per_thread[ci]);
        } else {
          set_chunk_stats(col_type,
                          data_addr,
                          has_null_per_thread[ci],
                          min_int64t_per_thread[ci],
                          max_int64t_per_thread[ci]);
        }
      }
    };

    auto varlen_vacuum = [=, &updel_roll, &frag_offsets, &fragment] {
      size_t nbytes_var_data_to_keep;
      nbytes_var_data_to_keep = vacuum_varlen_rows(fragment, chunk, frag_offsets);

      data_buffer->encoder->setNumElems(nrows_to_keep);
      data_buffer->setSize(nbytes_var_data_to_keep);
      data_buffer->setUpdated();

      index_array[nrows_to_keep] = data_buffer->size();
      index_buffer->setSize(sizeof(*index_array) *
                            (nrows_to_keep ? 1 + nrows_to_keep : 0));
      index_buffer->setUpdated();

      set_chunk_metadata(catalog, fragment, chunk, nrows_to_keep, updel_roll);
    };

    if (is_varlen) {
      threads.emplace_back(std::async(std::launch::async, varlen_vacuum));
    } else {
      threads.emplace_back(std::async(std::launch::async, fixlen_vacuum));
    }
    if (threads.size() >= (size_t)cpu_threads()) {
      wait_cleanup_threads(threads);
    }
  }

  wait_cleanup_threads(threads);

  auto key = std::make_pair(td, &fragment);
  updel_roll.numTuples[key] = nrows_to_keep;
  for (size_t ci = 0; ci < chunks.size(); ++ci) {
    auto chunk = chunks[ci];
    auto cd = chunk->get_column_desc();
    if (!cd->columnType.is_fixlen_array()) {
      updateColumnMetadata(cd,
                           fragment,
                           chunk,
                           has_null_per_thread[ci],
                           max_double_per_thread[ci],
                           min_double_per_thread[ci],
                           max_int64t_per_thread[ci],
                           min_int64t_per_thread[ci],
                           cd->columnType,
                           updel_roll);
    }
  }
}

}  // namespace Fragmenter_Namespace

void UpdelRoll::commitUpdate() {
  if (nullptr == catalog) {
    return;
  }
  const auto td = catalog->getMetadataForTable(logicalTableId);
  CHECK(td);
  // checkpoint all shards regardless, or epoch becomes out of sync
  if (td->persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
    catalog->checkpoint(logicalTableId);
  }
  // for each dirty fragment
  for (auto& cm : chunkMetadata) {
    cm.first.first->fragmenter->updateMetadata(catalog, cm.first, *this);
  }
  dirtyChunks.clear();
  // flush gpu dirty chunks if update was not on gpu
  if (memoryLevel != Data_Namespace::MemoryLevel::GPU_LEVEL) {
    for (const auto& chunkey : dirtyChunkeys) {
      catalog->getDataMgr().deleteChunksWithPrefix(
          chunkey, Data_Namespace::MemoryLevel::GPU_LEVEL);
    }
  }
}

void UpdelRoll::cancelUpdate() {
  if (nullptr == catalog) {
    return;
  }

  if (is_varlen_update) {
    int databaseId = catalog->getCurrentDB().dbId;
    int32_t tableEpoch = catalog->getTableEpoch(databaseId, logicalTableId);

    dirtyChunks.clear();
    const_cast<Catalog_Namespace::Catalog*>(catalog)->setTableEpoch(
        databaseId, logicalTableId, tableEpoch);
  } else {
    const auto td = catalog->getMetadataForTable(logicalTableId);
    CHECK(td);
    if (td->persistenceLevel != memoryLevel) {
      for (auto dit : dirtyChunks) {
        catalog->getDataMgr().free(dit.first->get_buffer());
        dit.first->set_buffer(nullptr);
      }
    }
  }
}

max_stars_repo_name: eliask/omniscidb

max_issues_repo_name: eliask/omniscidb

max_forks_repo_name: eliask/omniscidb


text:
#include "server.hpp"

#include "rapidjson/istreamwrapper.h"
#include "rapidjson/schema.h"
#include "spdlog/sinks/basic_file_sink.h"
#include "spdlog/sinks/stdout_color_sinks.h"
#include "spdlog/spdlog.h"
#include <boost/dll.hpp>

#include <chrono>
#include <fstream>

namespace beast = boost::beast;   // from <boost/beast.hpp>
namespace http = beast::http;     // from <boost/beast/http.hpp>
namespace net = boost::asio;      // from <boost/asio.hpp>
using tcp = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>

using namespace mahjong;

Server server;

Server::Server() : pool_(3) {}

/**
 * @brief JSON データを解析する。
 *
 * @param[in] doc ドキュメント
 * @return (成功したかどうか, リクエストデータ)
 */
std::tuple<bool, RequestData> Server::parse_json(const rapidjson::Document &doc)
{
    RequestData req = parse_request(doc);

    std::string dora_indicators = "";
    for (const auto &tile : req.dora_indicators)
        dora_indicators += Tile::Name.at(tile) + (&tile != &req.dora_indicators.back() ? " " : "");
    spdlog::get("logger")->info("IP: {}, 場風牌: {}, 自風牌: {}, 巡目: {}, 手牌の種類: {}, 手牌: "
                                "{}, フラグ: {}, ドラ表示牌: {}",
                                req.ip, Tile::Name.at(req.bakaze), Tile::Name.at(req.zikaze),
                                req.turn, req.syanten_type, req.hand.to_string(), req.flag,
                                dora_indicators);

    auto counts = ExpectedValueCalculator::count_left_tiles(req.hand, req.dora_indicators);
    for (auto x : counts) {
        if (x < 0)
            return {false, req};
    }

    return {true, req};
}

std::string Server::process_request(const std::string &json)
{
    // std::cout << json << std::endl;
    // std::cout << "===========================" << std::endl;

    rapidjson::Document doc(rapidjson::kObjectType);

    // リクエストデータを読み込む。
    rapidjson::Document req_doc;
    req_doc.Parse(json.c_str());
    if (req_doc.HasParseError()) {
        doc.AddMember("success", false, doc.GetAllocator());
        doc.AddMember("err_msg", "Failed to parse request data (invalid json format).",
                      doc.GetAllocator());
        return to_json_str(doc);
    }

    std::string schema_path =
        (boost::dll::program_location().parent_path() / "request_schema.json").string();
    std::ifstream ifs(schema_path);
    rapidjson::IStreamWrapper isw(ifs);
    rapidjson::Document sd;
    sd.ParseStream(isw);
    rapidjson::SchemaDocument schema(sd);
    rapidjson::SchemaValidator validator(schema);

    if (!req_doc.Accept(validator)) {
        // rapidjson::StringBuffer sb;
        // validator.GetInvalidSchemaPointer().StringifyUriFragment(sb);
        // printf("Invalid schema: %s\n", sb.GetString());
        // printf("Invalid keyword: %s\n", validator.GetInvalidSchemaKeyword());
        doc.AddMember("success", false, doc.GetAllocator());
        doc.AddMember("request", req_doc.GetObject(), doc.GetAllocator());
        doc.AddMember("err_msg", "Failed to parse request data (invalid value found).",
                      doc.GetAllocator());
        return to_json_str(doc);
    }

    // JSON を解析する。
    auto [success, req] = parse_json(req_doc);
    if (!success) {
        doc.AddMember("success", false, doc.GetAllocator());
        doc.AddMember("request", req_doc.GetObject(), doc.GetAllocator());
        doc.AddMember("err_msg", "Failed to parse request data (invalid hand found).",
                      doc.GetAllocator());
        return to_json_str(doc);
    }

    // 向聴数を計算する。
    auto [syanten_type, syanten] = SyantenCalculator::calc(req.hand, req.syanten_type);
    if (syanten == -1) {
        doc.AddMember("success", false, doc.GetAllocator());
        doc.AddMember("request", req_doc.GetObject(), doc.GetAllocator());
        doc.AddMember("err_msg", "和了形です。", doc.GetAllocator());
        return to_json_str(doc);
    }

    // 計算する。
    rapidjson::Value res_doc = create_response(req, doc);

    // 出力用 JSON を作成する。
    doc.AddMember("success", true, doc.GetAllocator());
    doc.AddMember("request", req_doc.GetObject(), doc.GetAllocator());
    doc.AddMember("response", res_doc, doc.GetAllocator());

    return to_json_str(doc);
}

// This function produces an HTTP response for the given
// request. The type of the response object depends on the
// contents of the request, so the interface requires the
// caller to pass a generic lambda for receiving the response.
template <class Body, class Allocator, class Send>
void handle_request(beast::string_view doc_root,
                    http::request<Body, http::basic_fields<Allocator>> &&req, Send &&send)
{
    // Returns a bad request response
    auto const bad_request = [&req](beast::string_view why) {
        http::response<http::string_body> res{http::status::bad_request, req.version()};
        res.set(http::field::server, BOOST_BEAST_VERSION_STRING);
        res.set(http::field::content_type, "text/html");
        res.keep_alive(req.keep_alive());
        res.body() = std::string(why);
        res.prepare_payload();
        return res;
    };

    // Returns a not found response
    auto const not_found = [&req](beast::string_view target) {
        http::response<http::string_body> res{http::status::not_found, req.version()};
        res.set(http::field::server, BOOST_BEAST_VERSION_STRING);
        res.set(http::field::content_type, "text/html");
        res.keep_alive(req.keep_alive());
        res.body() = "The resource '" + std::string(target) + "' was not found.";
        res.prepare_payload();
        return res;
    };

    // Returns a server error response
    auto const server_error = [&req](beast::string_view what) {
        http::response<http::string_body> res{http::status::internal_server_error, req.version()};
        res.set(http::field::server, BOOST_BEAST_VERSION_STRING);
        res.set(http::field::content_type, "text/html");
        res.keep_alive(req.keep_alive());
        res.body() = "An error occurred: '" + std::string(what) + "'";
        res.prepare_payload();
        return res;
    };

    // Make sure we can handle the method
    if (req.method() != http::verb::post)
        return send(bad_request("Unknown HTTP-method"));

    // Request path must be absolute and not contain "..".
    if (req.target().empty() || req.target()[0] != '/' ||
        req.target().find("..") != beast::string_view::npos)
        return send(bad_request("Illegal request-target"));

    auto future = server.pool_.enqueue([&] { return server.process_request(req.body()); });
    http::string_body::value_type body;
    body = future.get();

    // Respond to GET request
    http::response<http::string_body> res{std::piecewise_construct,
                                          std::make_tuple(std::move(body)),
                                          std::make_tuple(http::status::ok, req.version())};
    res.set(http::field::server, BOOST_BEAST_VERSION_STRING);
    res.set(http::field::content_type, "application/json");
    res.set(http::field::access_control_allow_origin, "*");
    res.prepare_payload();
    res.keep_alive(req.keep_alive());
    return send(std::move(res));
}

// Report a failure
void fail(beast::error_code ec, char const *what)
{
    std::cerr << what << ": " << ec.message() << "\n";
}

// This is the C++11 equivalent of a generic lambda.
// The function object is used to send an HTTP message.
template <class Stream> struct send_lambda
{
    Stream &stream_;
    bool &close_;
    beast::error_code &ec_;

    explicit send_lambda(Stream &stream, bool &close, beast::error_code &ec)
        : stream_(stream), close_(close), ec_(ec)
    {
    }

    template <bool isRequest, class Body, class Fields>
    void operator()(http::message<isRequest, Body, Fields> &&msg) const
    {
        // Determine if we should close the connection after
        close_ = msg.need_eof();

        // We need the serializer here because the serializer requires
        // a non-const file_body, and the message oriented version of
        // http::write only works with const messages.
        http::serializer<isRequest, Body, Fields> sr{msg};
        http::write(stream_, sr, ec_);
    }
};

// Handles an HTTP server connection
void do_session(tcp::socket &socket, std::shared_ptr<std::string const> const &doc_root)
{
    bool close = false;
    beast::error_code ec;

    // This buffer is required to persist across reads
    beast::flat_buffer buffer;

    // This lambda is used to send messages
    send_lambda<tcp::socket> lambda{socket, close, ec};

    for (;;) {
        // Read a request
        http::request<http::string_body> req;
        http::read(socket, buffer, req, ec);
        if (ec == http::error::end_of_stream)
            break;
        if (ec)
            return fail(ec, "read");

        // Send the response
        handle_request(*doc_root, std::move(req), lambda);
        if (ec)
            return fail(ec, "write");
        if (close) {
            // This means we should close the connection, usually because
            // the response indicated the "Connection: close" semantic.
            break;
        }
    }

    // Send a TCP shutdown
    socket.shutdown(tcp::socket::shutdown_send, ec);

    // At this point the connection is closed gracefully
}

int Server::run()
{
    spdlog::get("logger")->info("Launch server.");

    try {
        auto const address = net::ip::make_address("0.0.0.0");
        auto const port = static_cast<unsigned short>(std::atoi("8888"));
        auto const doc_root = std::make_shared<std::string>(".");

        // The io_context is required for all I/O
        net::io_context ioc{1};

        // The acceptor receives incoming connections
        tcp::acceptor acceptor{ioc, {address, port}};
        for (;;) {
            // This will receive the new connection
            tcp::socket socket{ioc};

            // Block until we get a connection
            acceptor.accept(socket);

            // Launch the session, transferring ownership of the socket
            std::thread{std::bind(&do_session, std::move(socket), doc_root)}.detach();
        }
    }
    catch (const std::exception &e) {
        spdlog::get("logger")->error("Error: {}", e.what());
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}

int main()
{
    auto console_sink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
    auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>("log.txt", false);
    std::vector<spdlog::sink_ptr> sinks = {console_sink, file_sink};
    auto logger = std::make_shared<spdlog::logger>("logger", sinks.begin(), sinks.end());
    spdlog::register_logger(logger);
    spdlog::flush_every(std::chrono::seconds(3));

    return server.run();
}

ext: cpp

max_stars_repo_name: Muratam/mahjong-cpp

max_issues_repo_name: Muratam/mahjong-cpp

max_forks_repo_name: Muratam/mahjong-cpp


text:
#ifndef SHIFT_PROTO_NAMESCOPE_HPP
#define SHIFT_PROTO_NAMESCOPE_HPP

#include <cstddef>
#include <cstdint>
#include <list>
#include <stdexcept>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <shift/core/boost_disable_warnings.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/variant/get.hpp>
#include <boost/variant/recursive_wrapper.hpp>
#include <boost/variant/variant.hpp>
#include <shift/core/boost_restore_warnings.hpp>
#include "shift/proto/types.hpp"
#include "shift/proto/node.hpp"

namespace shift::proto
{
struct namescope : public node
{
  namescope() = default;
  namescope(const namescope&) = default;
  namescope(namescope&&) = default;
  ~namescope() override;
  namescope& operator=(const namescope&) = default;
  namescope& operator=(namescope&&) = default;

  /// @see node::find_type.
  proto::type* find_type(const namescope_path& namescope_path,
                         std::size_t current_depth = 0) override;

  std::vector<proto::type> static_children;
  std::vector<std::shared_ptr<proto::type>> dynamic_children;

  std::vector<namescope*> namescopes;
  std::vector<alias*> aliases;
  std::vector<enumeration*> enumerations;
  std::vector<message*> messages;
  std::vector<interface*> interfaces;
  std::vector<service*> services;

  /// Parses a source document into this namescope.
  void parse(std::string_view content);

  /// Assigns the 'source_filename' attribute of all nodes which don't have it
  /// set, yet. This is particularly useful to track where a node is defined.
  void symbol_source(const std::string& source_path);

  /// Collapse all name scopes sharing the same name into a single one.
  void merge();

  /// Converts static_children to dynamic_children, because we want to store
  /// pointers to type and still need to be able to resize the type vectors
  /// (which might invalidate all pointers on reallocation).
  void convert_type_vectors();

  /// Unpacks all nested name scopes into a more handy structure.
  void unpack();

  /// Sorts all vectors by the name of their elements. You should call this
  /// after namescope::unpack to get stable results.
  void sort();

  /// Semantically check the collection of all nodes.
  bool analyze();

  /// @see node::generate_uids.
  void generate_uids() override;
};
}

#endif

ext: hpp

max_stars_repo_name: cspanier/shift

max_issues_repo_name: cspanier/shift

max_forks_repo_name: cspanier/shift


text:
#pragma once
#ifndef CANNON_RAY_CAMERA_H
#define CANNON_RAY_CAMERA_H

/*!
 * \file cannon/ray/camera.hpp
 * \brief File containing Camera class definition.
 */

#include <Eigen/Dense>

#include <cannon/math/random_double.hpp>
#include <cannon/utils/class_forward.hpp>

using namespace Eigen;

using namespace cannon::math;

namespace cannon {
  namespace ray {

    CANNON_CLASS_FORWARD(Ray);

    /*!
     * \brief Struct representing the sampled random values necessary for
     * generating a camera ray.
     */
    struct CameraSample {
      Vector2d p_film;
      Vector2d p_lens;
      double time;
    };

    /*!
     * \brief Class representing a camera generating rays into a scene.
     */
    class Camera {
      public:

        /*!
         * Constructor taking look point, lookat point, camera up vector, field
         * of view (in radians) and aspect ratio.
         */
        Camera(const Vector3d& look_from, const Vector3d& look_at, const
            Vector3d& vup, double vfov, double aspect_ratio, double aperture,
            double focus_dist, double time_0 = 0.0, double time_1 = 1.0) {
          auto h = std::tan(vfov / 2);
          auto viewport_height = 2.0 * h;
          auto viewport_width = aspect_ratio * viewport_height;

          w_ = (look_from - look_at).normalized();
          u_ = vup.cross(w_).normalized();
          v_ = w_.cross(u_);

          origin_ = look_from;
          horizontal_ = focus_dist * viewport_width * u_;
          vertical_ = focus_dist * viewport_height * v_;
          lower_left_corner_ = origin_ - horizontal_/2 - vertical_/2 - focus_dist * w_;

          lens_radius_ = aperture / 2;

          time_0_ = time_0;
          time_1_ = time_1;
        }

        /*!
         * Get a ray corresponding to a particular direction into the scene.
         *
         * \param s Horizontal component of ray direction along view.
         * \param t Vertical component of ray direction along view.
         *
         * \returns The sampled ray.
         */
        Ray get_ray(double s, double t) const;

        /*!
         * Get a ray for a particular pixel using the input camera sample.
         *
         * \param s Horizontal component of ray direction along view.
         * \param t Vertical component of ray direction along view.
         * \param sample CameraSample to use for random samples
         *
         * \returns The sampled ray.
         */
        Ray get_ray(double s, double t, const CameraSample& sample) const;

      private:
        Vector3d origin_; //!< Camera origin
        Vector3d lower_left_corner_; //!< Lower-left corner of camera view plane
        Vector3d horizontal_; //!< Horizontal extent of camera view plane
        Vector3d vertical_; //!< Vertical extent of camera view plane
        Vector3d u_, v_, w_; //!< Basis vectors for camera
        double lens_radius_; //!< Simulated thin lens radius
        double time_0_, time_1_; //!< Shutter open and close time
    };

  } // namespace ray
} // namespace cannon

#endif /* ifndef CANNON_RAY_CAMERA_H */

max_stars_repo_name: cannontwo/cannon

max_issues_repo_name: cannontwo/cannon

max_forks_repo_name: cannontwo/cannon


text:
#pragma once

#include <stdexcept>
#include <string>

#include <boost/lexical_cast.hpp>
#include <utility>

class KeyError: public std::exception {
public:
  KeyError(std::string key, std::string object = "")
    : key(std::move(key)), object(std::move(object)) { }

  const char *what() const noexcept override { return key.c_str(); }

  std::string key;
  std::string object;
};

class IndexError: public std::exception {
public:
  IndexError(int index, std::string object = "")
    : index(index), object(std::move(object)) {
    indexstr = boost::lexical_cast<std::string>(index);
  }

  const char *what() const noexcept override { return indexstr.c_str(); }

  int index;
  std::string indexstr;
  std::string object;
};

ext: hh

max_stars_repo_name: gnafit/gna

max_issues_repo_name: gnafit/gna

max_forks_repo_name: gnafit/gna


text:
/*
 *    Copyright (c) 2008-2021 Nuovation System Designs, LLC
 *    All rights reserved.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

/**
 *    @file
 *      This file defines a Nuovations Log Utilities concrete writer
 *      implementation for an open, appendable System V Release 4
 *      (SVR4)-, 4.3BSD-, POSIX.1-2001-, or POSIX.1-2008-conformant
 *      file descriptor.
 */

#ifndef LOGUTILITIES_LOGWRITERDESCRIPTOR_HPP
#define LOGUTILITIES_LOGWRITERDESCRIPTOR_HPP

#include <stdint.h>

#include <boost/shared_ptr.hpp>

#include "LogWriterStdio.hpp"

namespace Nuovations
{

    namespace Log
    {

        namespace Writer
        {

            /**
             *  @brief
             *    Log writer object for an open, appendable System V
             *    Release 4 (SVR4)-, 4.3BSD-, POSIX.1-2001-, or
             *    POSIX.1-2008-conformant file descriptor.
             *
             *  @ingroup writer
             *
             */
            class Descriptor :
                public Stdio
            {
            public:
                /**
                 *  @brief
                 *    Descriptor management flags.
                 *
                 *    Descriptor management flags which determine how
                 *    the writer interacts with the descriptor. These
                 *    flags may be bitwise-or'd with one another to
                 *    create compound management behaviors.
                 */
#if __cplusplus >= 201103L
                enum class Flags : uint8_t {
#else
                enum Flags {
#endif // __cplusplus >= 201103L
                    kNone    = 0,      //!< Specify no special descriptor management behavior.
                    kNoClose = 1 << 0, //!< Do not attempt to close the descriptor associated with the writer when the writer is destroyed.
                    kNoFlush = 1 << 1  //!< Do not attempt to flush the buffers associated with the writer when the writer is destroyed.
                };

            public:
                Descriptor(int inDescriptor);
                Descriptor(int inDescriptor, Flags inFlags);
                Descriptor(const Descriptor & inWriter);
                virtual ~Descriptor(void);

            protected:
                Descriptor(void);

                void SetDescriptor(int inDescriptor);
                void SetDescriptor(int inDescriptor, Flags inFlags);

            private:
                struct Implementation;

                /**
                 *  A shared, reference-counted pointer to the writer
                 *  implementation.
                 */
                boost::shared_ptr<Implementation> mImplementation;
            };

            /**
             *  @brief
             *    Bitwise or operator overload to support combining
             *    descriptor management flags.
             *
             *  @param[in]  inFirst   The operator left hand descriptor
             *                        management flags to combine.
             *  @param[in]  inSecond  The operator right hand descriptor
             *                        management flags to combine.
             *
             *  @returns
             *    The combined descriptor management flags.
             *
             */
            inline Descriptor::Flags operator |(const Descriptor::Flags &inFirst, const Descriptor::Flags &inSecond)
            {
                using underlying_type = typename std::underlying_type<Descriptor::Flags>::type;

                const Descriptor::Flags lFlags =
                    static_cast<Descriptor::Flags>(static_cast<underlying_type>(inFirst) |
                                                   static_cast<underlying_type>(inSecond));

                return (lFlags);
            }

        }; // namespace Writer

    }; // namespace Log

}; // namespace Nuovations

#endif /* LOGUTILITIES_LOGWRITERDESCRIPTOR_HPP */

max_stars_repo_name: Nuovations/LogUtilities

max_issues_repo_name: Nuovations/LogUtilities

max_forks_repo_name: Nuovations/LogUtilities


text:
/*    Copyright (c) 2010-2018, Delft University of Technology
 *    All rigths reserved
 *
 *    This file is part of the Tudat. Redistribution and use in source and
 *    binary forms, with or without modification, are permitted exclusively
 *    under the terms of the Modified BSD license. You should have received
 *    a copy of the license with this file. If not, please or visit:
 *    http://tudat.tudelft.nl/LICENSE.
 *
 */

#define BOOST_TEST_MAIN

#include <string>

#include <boost/make_shared.hpp>
#include <boost/test/unit_test.hpp>

#include "Tudat/Mathematics/BasicMathematics/linearAlgebra.h"
#include "Tudat/Astrodynamics/BasicAstrodynamics/physicalConstants.h"
#include "Tudat/Astrodynamics/BasicAstrodynamics/orbitalElementConversions.h"

#include "Tudat/External/SpiceInterface/spiceInterface.h"
#include "Tudat/Mathematics/NumericalIntegrators/rungeKuttaCoefficients.h"
#include "Tudat/Astrodynamics/BasicAstrodynamics/accelerationModel.h"
#include "Tudat/Astrodynamics/BasicAstrodynamics/keplerPropagator.h"
#include "Tudat/SimulationSetup/tudatSimulationHeader.h"
#include "Tudat/SimulationSetup/PropagationSetup/dynamicsSimulator.h"

namespace tudat
{

namespace unit_tests
{

//! Using declarations.
using namespace interpolators;
using namespace numerical_integrators;
using namespace spice_interface;
using namespace ephemerides;
using namespace simulation_setup;
using namespace basic_astrodynamics;
using namespace orbital_element_conversions;
using namespace propagators;

BOOST_AUTO_TEST_SUITE( test_multi_arc_dynamics )

BOOST_AUTO_TEST_CASE( testKeplerMultiArcDynamics )
{
    //Load spice kernels.
    spice_interface::loadStandardSpiceKernels( );

    for( unsigned testCase = 0; testCase < 3; testCase++ )
    {
        std::vector< std::string > bodyNames;
        bodyNames.push_back( "Earth" );
        bodyNames.push_back( "Moon" );

        // Specify initial time
        double initialEphemerisTime = 1.0E7;
        double finalEphemerisTime = 2.0E7;
        double maximumTimeStep = 3600.0;
        double buffer = 5.0 * maximumTimeStep;

        // Create bodies needed in simulation
        std::map< std::string, std::shared_ptr< BodySettings > > bodySettings =
                getDefaultBodySettings( bodyNames, initialEphemerisTime - buffer, finalEphemerisTime + buffer );
        std::dynamic_pointer_cast< InterpolatedSpiceEphemerisSettings >( bodySettings[ "Moon" ]->ephemerisSettings )->
                resetFrameOrigin( "Earth" );
        bodySettings[ "Moon" ]->ephemerisSettings->resetMakeMultiArcEphemeris( true );
        bodySettings[ "Earth" ]->ephemerisSettings = std::make_shared< ConstantEphemerisSettings >(
                    Eigen::Vector6d::Zero( ) );

        NamedBodyMap bodyMap = createBodies( bodySettings );

        setGlobalFrameBodyEphemerides( bodyMap, "SSB", "ECLIPJ2000" );

        // Set accelerations between bodies that are to be taken into account.
        SelectedAccelerationMap accelerationMap;
        std::map< std::string, std::vector< std::shared_ptr< AccelerationSettings > > > accelerationsOfMoon;
        accelerationsOfMoon[ "Earth" ].push_back( std::make_shared< AccelerationSettings >( central_gravity ) );
        accelerationMap[ "Moon" ] = accelerationsOfMoon;

        std::vector< std::string > bodiesToIntegrate, centralBodies;
        bodiesToIntegrate.push_back( "Moon" );
        centralBodies.push_back( "SSB" );

        std::vector< double > integrationArcStarts, integrationArcEnds;

        double integrationStartTime = initialEphemerisTime + 1.0E4;
        double integrationEndTime = finalEphemerisTime - 1.0E4;
        double arcDuration = 1.0E6;
        double arcOverlap = 1.0E4;

        double currentStartTime = integrationStartTime;
        double currentEndTime = integrationStartTime + arcDuration;

        do
        {
            integrationArcStarts.push_back( currentStartTime );
            integrationArcEnds.push_back( currentEndTime );

            currentStartTime = currentEndTime - arcOverlap;
            currentEndTime = currentStartTime + arcDuration;
        }
        while( currentEndTime < integrationEndTime );

        unsigned int numberOfIntegrationArcs = integrationArcStarts.size( );

        std::vector< Eigen::VectorXd > systemInitialStates;
        std::vector< Eigen::Vector6d > initialKeplerElements;

        systemInitialStates.resize( numberOfIntegrationArcs );
        initialKeplerElements.resize( numberOfIntegrationArcs );

        double earthGravitationalParameter =  bodyMap.at( "Earth" )->getGravityFieldModel( )->getGravitationalParameter( );
        for(  unsigned int j = 0; j < numberOfIntegrationArcs; j++ )
        {
            systemInitialStates[ j ]  = spice_interface::getBodyCartesianStateAtEpoch(
                        bodiesToIntegrate[ 0 ], "Earth", "ECLIPJ2000", "NONE", integrationArcStarts[ j ] );
            initialKeplerElements[ j ] = (
                        orbital_element_conversions::convertCartesianToKeplerianElements(
                            Eigen::Vector6d( systemInitialStates[ j ] ),
                            earthGravitationalParameter ) );
        }

        AccelerationMap accelerationModelMap = createAccelerationModelsMap(
                    bodyMap, accelerationMap, bodiesToIntegrate, centralBodies );

        std::vector< std::shared_ptr< SingleArcPropagatorSettings< double > > > arcPropagationSettingsList;
        for( unsigned int i = 0; i < numberOfIntegrationArcs; i++ )
        {
            arcPropagationSettingsList.push_back(
                        std::make_shared< TranslationalStatePropagatorSettings< double > >
                        ( centralBodies, accelerationModelMap, bodiesToIntegrate,
                          systemInitialStates.at( i ), integrationArcEnds.at( i ) ) );
        }

        // For case 0: test multi-arc estimation with same integration settings for each arc
        if( testCase == 0 )
        {
            std::shared_ptr< IntegratorSettings< > > integratorSettings =
                    std::make_shared< IntegratorSettings< > >
                    ( rungeKutta4, initialEphemerisTime, 120.0 );
            MultiArcDynamicsSimulator< > dynamicsSimulator(
                        bodyMap, integratorSettings, std::make_shared< MultiArcPropagatorSettings< double > >(
                            arcPropagationSettingsList ), integrationArcStarts );
        }
        // For case 1: test multi-arc estimation with different integration settings object for each arc
        else if( testCase == 1 )
        {
            std::vector< std::shared_ptr< IntegratorSettings< > > > integratorSettingsList;
            for( unsigned int i = 0; i < numberOfIntegrationArcs; i++ )
            {
                integratorSettingsList.push_back( std::make_shared< IntegratorSettings< > >
                                                  ( rungeKutta4, integrationArcStarts.at( i ), 120.0 ) );
            }
            MultiArcDynamicsSimulator< > dynamicsSimulator(
                        bodyMap, integratorSettingsList, std::make_shared< MultiArcPropagatorSettings< double > >(
                            arcPropagationSettingsList ) );
        }
        // For case 0: test multi-arc estimation with same integration settings for each arc, and arc initial state interpolated
        // from previous state
        else  if( testCase == 2 )
        {
            std::shared_ptr< IntegratorSettings< > > integratorSettings =
                    std::make_shared< IntegratorSettings< > >
                    ( rungeKutta4, initialEphemerisTime, 120.0 );
            MultiArcDynamicsSimulator< > dynamicsSimulator(
                        bodyMap, integratorSettings, std::make_shared< MultiArcPropagatorSettings< double > >(
                            arcPropagationSettingsList, true ), integrationArcStarts );
        }

        std::shared_ptr< Ephemeris > moonEphemeris = bodyMap.at( "Moon" )->getEphemeris( );

        double testStartTime, testEndTime;
        double testTimeStep = 10000.0;
        double timeBuffer = 1000.0;

        Eigen::Vector6d stateDifference;

        for( unsigned int i = 0; i < numberOfIntegrationArcs ; i++ )
        {
            if( i == 0 )
            {
                testStartTime = integrationArcStarts.at( i ) + timeBuffer;
            }
            else
            {
                testStartTime = integrationArcEnds.at( i - 1 ) + timeBuffer;
            }

            if( i == numberOfIntegrationArcs - 1 )
            {
                testEndTime = integrationArcEnds.at( i ) - timeBuffer;
            }
            else
            {
                testEndTime = integrationArcStarts.at( i + 1 ) - timeBuffer;
            }

            // Check if output corresponds to expected analytical solution
            if( testCase < 2 || i == 0 )
            {
                double currentTestTime = testStartTime;
                while( currentTestTime < testEndTime )
                {
                    stateDifference = ( moonEphemeris->getCartesianState( currentTestTime ) ) -
                            ( orbital_element_conversions::convertKeplerianToCartesianElements(
                                  propagateKeplerOrbit( initialKeplerElements.at( i ), currentTestTime - integrationArcStarts.at( i ),
                                                        earthGravitationalParameter ), earthGravitationalParameter ) );
                    for( int i = 0; i < 3; i++ )
                    {
                        BOOST_CHECK_SMALL( stateDifference( i ), 1.0E-4 );
                        BOOST_CHECK_SMALL( stateDifference( i + 3 ), 1.0E-10 );

                    }
                    currentTestTime += testTimeStep;
                }
            }
            // Check if arc information is properly passed to next arc
            else
            {
                double currentTestTime = testStartTime;
                while( currentTestTime < testEndTime )
                {
                    stateDifference = ( moonEphemeris->getCartesianState( currentTestTime ) ) -
                            ( orbital_element_conversions::convertKeplerianToCartesianElements(
                                  propagateKeplerOrbit( initialKeplerElements.at( 0 ), currentTestTime - integrationArcStarts.at( 0 ),
                                                        earthGravitationalParameter ), earthGravitationalParameter ) );
                    for( int i = 0; i < 3; i++ )
                    {
                        BOOST_CHECK_SMALL( stateDifference( i ), 1.0E-3);
                        BOOST_CHECK_SMALL( stateDifference( i + 3 ), 1.0E-9 );

                    }
                    currentTestTime += testTimeStep;
                }
            }
        }
    }
}

BOOST_AUTO_TEST_SUITE_END( )

}

}

max_stars_repo_name: J-Westin/tudat

max_issues_repo_name: J-Westin/tudat

max_forks_repo_name: J-Westin/tudat


text:
//==================================================================================================
/*!
  @file

  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#include <boost/simd/function/insert.hpp>
#include <boost/simd/detail/unroll.hpp>
#include <boost/simd/logical.hpp>
#include <boost/simd/pack.hpp>
#include <simd_test.hpp>
#include <array>

namespace bs = boost::simd;
namespace bd = boost::dispatch;

template<typename A, typename P, typename... N>
void f( nsm::list<N...> const&, A& a, P& p)
{
  BOOST_SIMD_LOCAL_UNROLL( a[N::value] = N::value%2 ? true : false );
  BOOST_SIMD_LOCAL_UNROLL( (bs::insert(p, N::value, N::value%2 ? true : false )) );
}

template <typename T, std::size_t N, typename Env>
void test_st(Env& runtime)
{
  bs::pack<T, N>  p;
  std::array<bool,N> ref;

  f( nsm::range<std::size_t, 0, N>(),ref,p);

  for(std::size_t i = 0;i < N; ++i) STF_EQUAL( p[i], ref[i] );
}

STF_CASE_TPL("Check static insert on pack" , STF_NUMERIC_TYPES)
{
  static const std::size_t N = boost::simd::pack<T>::static_size;

  test_st<bs::logical<T>, N>(runtime);
  test_st<bs::logical<T>, N/2>(runtime);
  test_st<bs::logical<T>, N*2>(runtime);
}


text:
#include <blitz/timer.h>
#include <iostream>
#include <fstream>

BZ_USING_NAMESPACE(blitz)
BZ_USING_NAMESPACE(std)

extern "C" {
    void echo_f77tuned(int& N, int& niters, float& check, int& blockSize);
}

int main()
{
    int N = 1024;
    int niters = 48;
    float check;
    double Mflops = niters * 9;
    Timer timer;

    ofstream ofs("echotune.log");

    cout << "This program decides on the best block size for a typical 2D "
         << endl << "stencil operation.  Pick the block size which has the "
         << endl << "maximum Mflops/s." << endl << endl;

    cout << "Block size\tMflops/s" << endl;

    int blockSize;

    for (blockSize=1; blockSize < 32; ++blockSize)
    {
        timer.start();
        echo_f77tuned(N, niters, check, blockSize);
        timer.stop();
        cout << blockSize << "\t" << (Mflops/timer.elapsedSeconds()) << endl;
        ofs << blockSize << "\t" << (Mflops/timer.elapsedSeconds()) << endl;
    }
    for (; blockSize < 1024; blockSize += 32)
    {
        timer.start();
        echo_f77tuned(N, niters, check, blockSize);
        timer.stop();
        cout << blockSize << "\t" << (Mflops/timer.elapsedSeconds()) << endl;
        ofs << blockSize << "\t" << (Mflops/timer.elapsedSeconds()) << endl;
    }

    return 0;
}

max_stars_repo_name: fraguela/depspawn

max_issues_repo_name: MSV-Project/IBAMR

max_forks_repo_name: MSV-Project/IBAMR


text:
#include <string>             // string
#include <list>               // list
#include <algorithm>          // for_each()
#include <vector>             // vector
#include <iostream>           // cout
#include <boost/foreach.hpp>  // BOOST_FOREACH()
using namespace std;

struct MY_FUNC : public unary_function<int,void> {
	public:
		void operator() (int elem)
		{
			cout << elem;
		}
};

int main()
{
	// string
	string hello( "Hello, world!" );

	// list
	list<int> lst;
	lst.push_back(1);
	lst.push_back(2);
	lst.push_back(3);

	// vector
	vector<int> vec;
	vec.push_back(1);
	vec.push_back(2);
	vec.push_back(3);

	BOOST_FOREACH( char ch, hello ) {
		cout << ch;
	}

	for_each(lst.begin(), lst.end(), MY_FUNC());

	BOOST_FOREACH( int i, lst ) {
		cout << i;
	}

	return 0;
}

ext: cc

max_stars_repo_name: quchunguang/test


text:
/*
 * Output.cpp
 *
 *  Created on: 2015/07/22
 *      Author: kryozahiro
 */

#include "Output.h"

#include <cassert>
#include <boost/filesystem.hpp>
#include <boost/log/sinks/sync_frontend.hpp>
#include <boost/log/sinks/text_ostream_backend.hpp>
#include <boost/log/expressions/predicates.hpp>
#include <boost/log/utility/setup/console.hpp>
#include <boost/log/utility/setup/file.hpp>
#include "cpputil/GenericIo.h"
using namespace std;
using namespace cpputil;
namespace pt = boost::property_tree;
namespace fs = boost::filesystem;
namespace lg = boost::log;

Output::Output(const boost::property_tree::ptree& outputTree) {
	for (const pt::ptree::value_type& kvp : outputTree) {
		if (kvp.first == "<xmlattr>") {
			continue;
		}
		assert(kvp.first == "Logger");

		const pt::ptree& loggerTree = kvp.second;
		Logger logger;
		logger.range = loggerTree.get<std::pair<int, int>>("Range");
		logger.interval = loggerTree.get<int>("Interval");
		logger.filename = loggerTree.get<string>("Filename");
		assert(logger.interval > 0);

		string target = loggerTree.get<string>("Target");
		if (target == "Summary") {
			summaryLogger = logger;
		} else if (target == "Relation") {
			relationLogger = logger;
		} else if (target == "Evaluation") {
			evaluationLogger = logger;
		} else if (target == "Program") {
			programLogger = logger;
		} else if (target == "Solution") {
			solutionLogger = logger;
		} else if (target == "Validator") {
			validatorLogger = logger;
		}
	}
}

void Output::setSink(int stage, const std::string& stageName) {
	auto core = lg::core::get();

	if (stageName == "SolverStage") {
		core->remove_all_sinks();

		fs::path summaryPath(summaryLogger.filename);
		string fullname = summaryPath.parent_path().generic_string() + string("/s") + to_string(stage) + summaryPath.filename().generic_string();
		lg::add_file_log(lg::keywords::file_name = fullname, lg::keywords::filter = lg::expressions::is_in_range<int>("Summary", 0, INT_MAX));
		lg::add_console_log(std::cerr, lg::keywords::filter = lg::expressions::is_in_range<int>("Summary", 0, INT_MAX));

		fs::path relationPath(relationLogger.filename);
		relationFile = relationPath.parent_path().generic_string() + string("/s") + to_string(stage) + relationPath.filename().generic_string();

		setMutableSink(stage, "Evaluation", evaluationLogger);
		setMutableSink(stage, "Program", programLogger);
		setMutableSink(stage, "Solution", solutionLogger);

	} else if (stageName == "ValidatorStage") {
		core->remove_all_sinks();
		setMutableSink(stage, "Validator", validatorLogger);
	}
}

std::string Output::getRelationFile() const {
	return relationFile;
}

std::pair<int, int> Output::getEvaluationLoggerRange() const {
	return evaluationLogger.range;
}

void Output::setMutableSink(int stage, const std::string& target, Logger& logger) {
	fs::path path(logger.filename);
	for (int i = logger.range.first; i < logger.range.second; i += logger.interval) {
		string fullname = path.parent_path().generic_string() + string("/s") + to_string(stage) + path.stem().generic_string() + to_string(i) + path.extension().generic_string();
		lg::add_file_log(lg::keywords::file_name = fullname, lg::keywords::filter = lg::expressions::is_in_range<int>(target, i, i + 1));
	}
}

max_stars_repo_name: kryozahiro/gamesolver

max_issues_repo_name: kryozahiro/gamesolver

max_forks_repo_name: kryozahiro/gamesolver