text:
#include "stdafx.h"

#include "../../common.shared/url_parser/url_parser.h"

#include "../tools/file_sharing.h"
#include "../tools/system.h"

#include "archive_index.h"
#include "contact_archive.h"
#include "history_message.h"
#include "messages_data.h"
#include "storage.h"

#include "image_cache.h"

#include <boost/range/adaptor/map.hpp>

namespace
{
    const std::wstring tmp_to_add_extension = L".a.tmp";
    const std::wstring tmp_to_delete_extension = L".d.tmp";

    const int32_t max_block_size        = 1000;

    const int32_t fetch_size            = 30;

    enum tlv_fields : uint32_t
    {
        tlv_img_pack                = 1,

        tlv_msg_id                  = 2,
        tlv_msg_url                 = 3,
        tlv_msg_is_filesharing      = 4
    };
}

core::archive::image_data::image_data()
    : msgid_(0)
    , is_filesharing_(false)
{
}

core::archive::image_data::image_data(int64_t _msgid)
    : msgid_(_msgid)
    , is_filesharing_(false)
{
}

core::archive::image_data::image_data(int64_t _msgid, std::string _url, bool _is_filesharing)
    : msgid_(_msgid)
    , url_(std::move(_url))
    , is_filesharing_(_is_filesharing)
{
}

int64_t core::archive::image_data::get_msgid() const
{
    return msgid_;
}

void core::archive::image_data::set_msgid(int64_t _value)
{
    msgid_ = _value;
}

std::string core::archive::image_data::get_url() const
{
    return url_;
}

void core::archive::image_data::set_url(const std::string& _value)
{
    url_ = _value;
}

bool core::archive::image_data::get_is_filesharing() const
{
    return is_filesharing_;
}

void core::archive::image_data::set_is_filesharing(bool _value)
{
    is_filesharing_ = _value;
}

void core::archive::image_data::serialize(core::tools::binary_stream& _data) const
{
    core::tools::tlvpack msg_pack;

    msg_pack.push_child(core::tools::tlv(tlv_fields::tlv_msg_id, (int64_t) msgid_));
    msg_pack.push_child(core::tools::tlv(tlv_fields::tlv_msg_url, (std::string) url_));
    msg_pack.push_child(core::tools::tlv(tlv_fields::tlv_msg_is_filesharing, (bool) is_filesharing_));

    msg_pack.serialize(_data);
}

bool core::archive::image_data::unserialize(core::tools::binary_stream& _data)
{
    core::tools::tlvpack msg_pack;

    if (!msg_pack.unserialize(_data))
        return false;

    for (auto tlv_field = msg_pack.get_first(); tlv_field; tlv_field = msg_pack.get_next())
    {
        switch (static_cast<tlv_fields>(tlv_field->get_type()))
        {
        case tlv_fields::tlv_msg_id:
            msgid_ = tlv_field->get_value<int64_t>();
            break;
        case tlv_fields::tlv_msg_url:
            url_ = tlv_field->get_value<std::string>();
            break;
        case tlv_fields::tlv_msg_is_filesharing:
            is_filesharing_ = tlv_field->get_value<bool>();
            break;
        default:
            assert(!"invalid field");
        }
    }

    return true;
}

core::archive::image_cache::image_cache(const std::wstring& _file_name)
    : storage_(std::make_unique<storage>(_file_name))
    , tmp_to_add_storage_(std::make_unique<storage>(_file_name + tmp_to_add_extension))
    , tmp_to_delete_storage_(std::make_unique<storage>(_file_name + tmp_to_delete_extension))
    , building_in_progress_(false)
    , tree_is_consistent_(false)
{
}

core::archive::image_cache::~image_cache()
{
}

bool core::archive::image_cache::load_from_local(const contact_archive& _archive)
{
    const bool success = read_file(*storage_, image_by_msgid_);
    if (success)
    {
        tree_is_consistent_ = update_file_from_tmp_files();
        return tree_is_consistent_;
    }

    // file doesn't exists or corruped so rebuild anyway
    return build(_archive);
}

void core::archive::image_cache::get_images(int64_t _from, int64_t _count, image_list& _images) const
{
    _images.clear();

    images_map_t::const_iterator it;

    {
        std::unique_lock<std::mutex> lock(mutex_);

        if (_from == -1)
        {
            while (building_in_progress_ && image_by_msgid_.empty())
            {
                data_ready_.wait(lock);
            }

            it = image_by_msgid_.end();
        }
        else
        {
            it = image_by_msgid_.lower_bound(_from);
            while (building_in_progress_ && it == image_by_msgid_.begin())
            {
                data_ready_.wait(lock);
                it = image_by_msgid_.lower_bound(_from);
            }

            if (it == image_by_msgid_.end())
            {
                assert(!"invalid index number");
                return;
            }
        }
    }

    for (auto begin = image_by_msgid_.begin(); it != begin; )
    {
        --it;

        if (_count != -1 && _images.size() >= _count)
            return;

        _images.emplace_front(it->second);
    }
}

bool core::archive::image_cache::update(const history_block& _block)
{
    if (!tree_is_consistent_)
    {
        image_vector_t to_add;
        image_vector_t to_delete;

        for (const auto& it : _block)
        {
            const history_message& message = *it;
            if (message.is_patch())
                to_delete.emplace_back(image_data(message.get_msgid()));
            else
                extract_images(message, to_add);
        }

        const bool add_result = append_to_file(*tmp_to_add_storage_, to_add);
        const bool delete_result = append_to_file(*tmp_to_delete_storage_, to_delete);

        return add_result && delete_result;
    }

    const auto end = image_by_msgid_.end();

    bool need_to_save_all = false;

    image_vector_t to_add;

    for (const auto& it : _block)
    {
        const history_message& message = *it;
        const auto msgid = message.get_msgid();

        if (message.is_patch())
        {
            auto to_delete = image_by_msgid_.find(msgid);
            if (to_delete != end)
            {
                need_to_save_all = true;
                image_by_msgid_.erase(to_delete);
            }
        }
        else
        {
            extract_images(message, to_add);
        }
    }

    if (need_to_save_all)
    {
        return save_all();
    }
    else if (!to_add.empty())
    {
        archive::storage_mode mode;
        mode.flags_.write_ = true;
        mode.flags_.append_ = true;
        if (!storage_->open(mode))
            return false;

        core::tools::auto_scope lb([this]{ storage_->close(); });

        add_images_to_tree(to_add);

        return serialize_block(*storage_, to_add);
    }

    return true;
}

bool core::archive::image_cache::synchronize(const archive_index& _index)
{
    erase_deleted_from_tree(_index);
    return save_all();
}

bool core::archive::image_cache::build(const contact_archive& _archive)
{
    cancel_build_.clear();

    tree_is_consistent_ = false;
    building_in_progress_ = true;

    image_by_msgid_.clear();

    history_block messages;

    int64_t from = -1;
    while (true)
    {
        if (cancel_build_.test_and_set())
        {
            return false;
        }

        cancel_build_.clear();

        _archive.get_messages(from, fetch_size, -1, messages, contact_archive::get_message_policy::skip_patches_and_deleted);
        if (messages.empty())
        {
            building_in_progress_ = false;
            tree_is_consistent_ = true;

            update_tree_from_tmp_files();

            data_ready_.notify_all();

            if (save_all())
            {
                delete_tmp_files();
                return true;
            }

            return false;
        }

        from = (*messages.begin())->get_msgid();

        const auto images = extract_images(messages);
        if (!images.empty())
        {
            {
                std::lock_guard<std::mutex> lock(mutex_);
                add_images_to_tree(images);
            }
            data_ready_.notify_all();
        }
    }
}

void core::archive::image_cache::cancel_build()
{
    cancel_build_.test_and_set();
}

template<typename T>
static core::tools::tlvpack make_block(const T& _images)
{
    core::tools::tlvpack block;

    for (const auto& img : _images)
    {
        core::tools::binary_stream img_data;
        img.serialize(img_data);
        block.push_child(core::tools::tlv(tlv_fields::tlv_img_pack, img_data));
    }
    return block;
}

static bool serialize_block_impl(core::archive::storage& _storage, const core::tools::tlvpack& block)
{
    core::tools::binary_stream stream;
    block.serialize(stream);

    int64_t offset = 0;
    return _storage.write_data_block(stream, offset);
}

bool core::archive::image_cache::serialize_block(storage& _storage, const image_vector_t& _images) const
{
    assert(!_images.empty());
    return serialize_block_impl(_storage, make_block(_images));
}

bool core::archive::image_cache::serialize_block(storage& _storage, const images_map_t& _images) const
{
    assert(!_images.empty());
    return serialize_block_impl(_storage, make_block(boost::adaptors::values(_images)));
}

bool core::archive::image_cache::unserialize_block(core::tools::binary_stream& _stream, images_map_t& _images) const
{
    while (_stream.available())
    {
        core::tools::tlvpack block;
        if (!block.unserialize(_stream))
            return false;

        for (auto item = block.get_first(); item; item = block.get_next())
        {
            image_data img;
            auto stream = item->get_value<core::tools::binary_stream>();
            if (!img.unserialize(stream))
                return false;

            const auto id = img.get_msgid();
            _images.emplace(id, std::move(img));
        }
    }

    return true;
}

bool core::archive::image_cache::save_all() const
{
    archive::storage_mode mode;
    mode.flags_.write_ = true;
    mode.flags_.truncate_ = true;

    if (!storage_->open(mode))
        return false;

    core::tools::auto_scope lb([this]{ storage_->close(); });

    image_vector_t block;
    for (const auto& image : image_by_msgid_)
    {
        block.emplace_back(image.second);
        if (block.size() >= max_block_size)
        {
            if (!serialize_block(*storage_, block))
                return false;
            block.clear();
        }
    }

    if (!block.empty())
    {
        return serialize_block(*storage_, block);
    }

    return true;
}

void core::archive::image_cache::extract_images(const history_message& _message, image_vector_t& _images) const
{
    const auto& quote_list = _message.get_quotes();
    if (quote_list.empty())
    {
        extract_images(_message.get_msgid(), _message.get_text(), _images);
    }
    else
    {
        for (const auto& quote : quote_list)
            extract_images(_message.get_msgid(), quote.get_text(), _images);
    }
}

void core::archive::image_cache::extract_images(int64_t _msgid, const std::string& _text, image_vector_t& _images) const
{
    const auto urls = common::tools::url_parser::parse_urls(_text);

    for (const auto& url_info : urls)
    {
        if (url_info.is_filesharing())
        {
            auto content_type = core::file_sharing_content_type::undefined;

            if (!tools::get_content_type_from_uri(url_info.url_, content_type))
            {
                continue;
            }

            if ((content_type == file_sharing_content_type::image) ||
                (content_type == file_sharing_content_type::gif) ||
                (content_type == file_sharing_content_type::video))
            {
                _images.emplace_back(_msgid, url_info.url_, url_info.is_filesharing());
            }
        }

        if (url_info.is_image())
        {
            _images.emplace_back(_msgid, url_info.url_, url_info.is_filesharing());
        }
    }
}

core::archive::image_cache::image_vector_t core::archive::image_cache::extract_images(const history_block& _block) const
{
    image_vector_t images;

    for (const auto& it : _block)
    {
        const history_message& message = *it;
        if (!message.is_patch())
            extract_images(message, images);
    }

    return images;
}

void core::archive::image_cache::add_images_to_tree(const image_vector_t& _images)
{
    for (const auto& img : _images)
        image_by_msgid_.emplace(img.get_msgid(), img);
}

void core::archive::image_cache::add_images_to_tree(const images_map_t& _images)
{
    image_by_msgid_.insert(_images.begin(), _images.end());
}

void core::archive::image_cache::erase_deleted_from_tree(const archive_index& _index)
{
    std::vector<int64_t> msg_ids;

    headers_list headers;
    int64_t from = -1;
    while (true)
    {
        _index.serialize_from(from, fetch_size, -1, headers);
        if (headers.empty())
            break;

        from = headers.begin()->get_id();

        for (const auto& msg : headers)
            msg_ids.push_back(msg.get_id());

        headers.clear();
    }

    std::vector<int64_t> to_delete;

    int64_t prev = 0;
    for (const auto& id : image_by_msgid_)
    {
        if (id.first == prev)
            continue;

        prev = id.first;

        if (!std::binary_search(msg_ids.begin(), msg_ids.end(), id.first)) // messages always sorted by design
            to_delete.push_back(id.first);
    }

    for (auto id : to_delete)
        image_by_msgid_.erase(id);
}

bool core::archive::image_cache::read_file(storage& _storage, images_map_t& _images) const
{
    archive::storage_mode mode;
    mode.flags_.read_ = true;

    if (!_storage.open(mode))
        return false;

    core::tools::auto_scope lb([&_storage]{ _storage.close(); });

    core::tools::binary_stream stream;
    while (_storage.read_data_block(-1, stream))
    {
        if (!unserialize_block(stream, _images))
            return false;

        stream.reset();
    }

    return _storage.get_last_error() == archive::error::end_of_file;
}

static bool open_storage_for_append(core::archive::storage& _storage)
{
    core::archive::storage_mode mode;
    mode.flags_.write_ = true;
    mode.flags_.append_ = true;

    return _storage.open(mode);
}

bool core::archive::image_cache::append_to_file(storage& _storage, const image_vector_t& _images) const
{
    if (_images.empty())
        return true;

    if (!open_storage_for_append(_storage))
        return false;

    core::tools::auto_scope lb([&_storage] { _storage.close(); });

    return serialize_block(_storage, _images);
}

bool core::archive::image_cache::append_to_file(storage& _storage, const images_map_t& _images) const
{
    if (_images.empty())
        return true;

    if (!open_storage_for_append(_storage))
        return false;

    core::tools::auto_scope lb([&_storage] { _storage.close(); });

    return serialize_block(_storage, _images);
}

bool core::archive::image_cache::update_tree_from_tmp_files()
{
    images_map_t to_add;
    if (!read_file(*tmp_to_add_storage_, to_add))
        return false;

    add_images_to_tree(to_add);

    bool need_to_save_all = false;
    return process_deleted(need_to_save_all);
}

bool core::archive::image_cache::process_deleted(bool& _need_to_save_all)
{
    images_map_t to_delete;
    if (!read_file(*tmp_to_delete_storage_, to_delete))
        return false;

    const auto end = image_by_msgid_.end();
    for (const auto& it : to_delete)
    {
        auto to_delete_iter = image_by_msgid_.find(it.second.get_msgid());
        if (to_delete_iter != end)
        {
            _need_to_save_all = true;
            image_by_msgid_.erase(to_delete_iter);
        }
    }

    return true;
}

bool core::archive::image_cache::update_file_from_tmp_files()
{
    const bool to_add_exists = tools::system::is_exist(tmp_to_add_storage_->get_file_name());
    if (to_add_exists)
    {
        images_map_t to_add;
        if (!read_file(*tmp_to_add_storage_, to_add))
            return false;

        add_images_to_tree(to_add);

        if (!append_to_file(*storage_, to_add))
            return false;
    }

    const bool to_delete_exists = tools::system::is_exist(tmp_to_delete_storage_->get_file_name());
    if (to_delete_exists)
    {
        bool need_to_save_all = false;
        if (!process_deleted(need_to_save_all))
            return false;

        if (need_to_save_all && !save_all())
            return false;
    }

    delete_tmp_files();

    return true;
}

void core::archive::image_cache::delete_tmp_files()
{
    tools::system::delete_file(tmp_to_add_storage_->get_file_name());
    tools::system::delete_file(tmp_to_delete_storage_->get_file_name());
}


text:
#include <fs/fs.h>
#include "dataset.h"

#include <gtest/gtest.h>
#include <boost/filesystem.hpp>

#include <atomic>
#include <chrono>
#include <thread>

namespace {

class ReadWorker
{
public:

    ReadWorker(const DataSet& dataSet)
        : _dataSet{dataSet}
    {}

    void start() {

        _count = 0;
        _isRun = true;
        _thread = std::thread{&ReadWorker::worker, this};
    }

    void stop() {

        _isRun = false;
        _thread.join();
        std::cout << "ReadWorker, iterations count: " << _count << std::endl;
    }

private:

    void worker() {

        while (_isRun) {

            const oda::fs::Path& path = _dataSet.getRandomPath();
            const std::string content = oda::fs::readFile(path);

            _count++;
        }
    }

    const DataSet& _dataSet;

    std::size_t _count;
    std::atomic_bool _isRun;
    std::thread _thread;
};

} // anonimous namespace

TEST(rwFile, readOnly_1) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 1};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();
}

TEST(rwFile, readOnly_3) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 3};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();
}

TEST(rwFile, readOnly_10k) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 10000};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();
}

#ifndef ODA_FS_FAKE_LOCK

namespace {

class WriteWorker
{
public:

    WriteWorker(DataSet& dataSet)
        : _dataSet{dataSet},
          _length{_dataSet.getData(_dataSet.getRandomPath()).length()}
    {}

    void start() {

        _count = 0;
        _isRun = true;
        _thread = std::thread{&WriteWorker::worker, this};
    }

    void stop() {

        _isRun = false;
        _thread.join();
        std::cout << "WriteWorker, iterations count: " << _count << std::endl;
    }

private:

    void worker() {

        std::string data;
        data.resize(_length);

        while (_isRun) {

            generateData(&data[0], data.length());

            const oda::fs::Path& path = _dataSet.getRandomPath();
            _dataSet.update(path, data);

            oda::fs::writeFile(path, data, std::ios_base::trunc);

            _count++;
        }
    }

    DataSet& _dataSet;
    const std::size_t _length;

    std::size_t _count;
    std::atomic_bool _isRun;
    std::thread _thread;
};

} // anonimous namespace

TEST(rwFile, rw_1) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 1};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    WriteWorker writeWorker{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    writeWorker.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();

    writeWorker.stop();
}

TEST(rwFile, rw_3) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 3};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    WriteWorker writeWorker{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    writeWorker.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();

    writeWorker.stop();
}

TEST(rwFile, rw_10k) {

    const oda::fs::Path prefixPath = oda::fs::currentDirectory() / "test_odaFS_dir";
    boost::filesystem::create_directories(prefixPath);
    DataSet dataSet{prefixPath, 10000};
    DataSetFiles files{dataSet};

    ReadWorker readWorker_1{dataSet};
    ReadWorker readWorker_2{dataSet};
    ReadWorker readWorker_3{dataSet};
    ReadWorker readWorker_4{dataSet};

    WriteWorker writeWorker{dataSet};

    readWorker_1.start();
    readWorker_2.start();
    readWorker_3.start();
    readWorker_4.start();

    writeWorker.start();

    std::this_thread::sleep_for(std::chrono::seconds{1});

    readWorker_1.stop();
    readWorker_2.stop();
    readWorker_3.stop();
    readWorker_4.stop();

    writeWorker.stop();
}

#endif

ext: cpp


text:
#ifndef PYBNESIAN_FACTORS_DISCRETE_DISCRETE_INDICES_HPP
#define PYBNESIAN_FACTORS_DISCRETE_DISCRETE_INDICES_HPP

#include <arrow/compute/api.h>
#include <Eigen/Dense>
#include <dataset/dataset.hpp>
#include <factors/assignment.hpp>
#include <util/hash_utils.hpp>

using dataset::DataFrame;
using Eigen::VectorXi;

namespace factors::discrete {

void check_is_string_dictionary(const std::shared_ptr<arrow::DictionaryArray>& dict, const std::string& variable);

template <typename ArrowType>
void sum_to_discrete_indices_null(VectorXi& accum_indices,
                                  Array_ptr& indices,
                                  int stride,
                                  Buffer_ptr& combined_bitmap) {
    using ArrayType = typename arrow::TypeTraits<ArrowType>::ArrayType;
    auto dwn_indices = std::static_pointer_cast<ArrayType>(indices);

    auto raw_combined_bitmap = combined_bitmap->data();
    for (auto i = 0, j = 0; i < indices->length(); ++i) {
        if (util::bit_util::GetBit(raw_combined_bitmap, i)) {
            accum_indices(j++) += dwn_indices->Value(i) * stride;
        }
    }
}

void sum_to_discrete_indices_null(VectorXi& accum_indices, Array_ptr& indices, int stride, Buffer_ptr& combined_bitmap);

template <typename ArrowType>
void sum_to_discrete_indices(VectorXi& accum_indices, Array_ptr& indices, int stride) {
    using ArrayType = typename arrow::TypeTraits<ArrowType>::ArrayType;
    using EigenMap = Map<const Matrix<typename ArrowType::c_type, Dynamic, 1>>;
    auto dwn_indices = std::static_pointer_cast<ArrayType>(indices);
    auto* raw_values = dwn_indices->raw_values();
    const EigenMap map_eigen(raw_values, indices->length());
    accum_indices += map_eigen.template cast<int>() * stride;
}

void sum_to_discrete_indices(VectorXi& accum_indices, Array_ptr& indices, int stride);

template <bool contains_null>
VectorXi discrete_indices(const DataFrame& df,
                          const std::string& variable,
                          const std::vector<std::string>& evidence,
                          const VectorXi& strides) {
    if constexpr (contains_null) {
        auto combined_bitmap = df.combined_bitmap(variable, evidence);

        auto valid_rows = util::bit_util::non_null_count(combined_bitmap, df->num_rows());

        VectorXi indices = VectorXi::Zero(valid_rows);

        auto dict_variable = std::static_pointer_cast<arrow::DictionaryArray>(df.col(variable));
        auto variable_indices = dict_variable->indices();

        sum_to_discrete_indices_null(indices, variable_indices, strides(0), combined_bitmap);

        int i = 1;
        for (auto it = evidence.begin(), end = evidence.end(); it != end; ++it, ++i) {
            auto dict_evidence = std::static_pointer_cast<arrow::DictionaryArray>(df.col(*it));
            auto evidence_indices = dict_evidence->indices();
            sum_to_discrete_indices_null(indices, evidence_indices, strides(i), combined_bitmap);
        }

        return indices;
    } else {
        VectorXi indices = VectorXi::Zero(df->num_rows());

        auto dict_variable = std::static_pointer_cast<arrow::DictionaryArray>(df.col(variable));
        auto variable_indices = dict_variable->indices();

        sum_to_discrete_indices(indices, variable_indices, strides(0));

        int i = 1;
        for (auto it = evidence.begin(), end = evidence.end(); it != end; ++it, ++i) {
            auto dict_evidence = std::static_pointer_cast<arrow::DictionaryArray>(df.col(*it));
            auto evidence_indices = dict_evidence->indices();
            sum_to_discrete_indices(indices, evidence_indices, strides(i));
        }

        return indices;
    }
}

VectorXi discrete_indices(const DataFrame& df,
                          const std::string& variable,
                          const std::vector<std::string>& evidence,
                          const VectorXi& strides);

template <bool contains_null>
VectorXi discrete_indices(const DataFrame& df, const std::vector<std::string>& variables, const VectorXi& strides) {
    if constexpr (contains_null) {
        auto combined_bitmap = df.combined_bitmap(variables);

        auto valid_rows = util::bit_util::non_null_count(combined_bitmap, df->num_rows());

        VectorXi indices = VectorXi::Zero(valid_rows);

        int i = 0;
        for (auto it = variables.begin(), end = variables.end(); it != end; ++it, ++i) {
            auto dict_variable = std::static_pointer_cast<arrow::DictionaryArray>(df.col(*it));
            auto variable_indices = dict_variable->indices();
            sum_to_discrete_indices_null(indices, variable_indices, strides(i), combined_bitmap);
        }

        return indices;
    } else {
        VectorXi indices = VectorXi::Zero(df->num_rows());

        int i = 0;
        for (auto it = variables.begin(), end = variables.end(); it != end; ++it, ++i) {
            auto dict_variable = std::static_pointer_cast<arrow::DictionaryArray>(df.col(*it));
            auto variable_indices = dict_variable->indices();
            sum_to_discrete_indices(indices, variable_indices, strides(i));
        }

        return indices;
    }
}

VectorXi discrete_indices(const DataFrame& df, const std::vector<std::string>& variables, const VectorXi& strides);

std::pair<VectorXi, VectorXi> create_cardinality_strides(const DataFrame& df,
                                                         const std::string& variable,
                                                         const std::vector<std::string>& evidence);
std::pair<VectorXi, VectorXi> create_cardinality_strides(const DataFrame& df,
                                                         const std::vector<std::string>& variables);

VectorXi joint_counts(const DataFrame& df,
                      const std::string& variable,
                      const std::vector<std::string>& evidence,
                      const VectorXi& cardinality,
                      const VectorXi& strides);

VectorXi marginal_counts(const VectorXi& joint_counts, int index, const VectorXi& cardinality, const VectorXi& strides);

std::vector<Array_ptr> discrete_slice_indices(const DataFrame& df,
                                              const std::vector<std::string>& discrete_vars,
                                              const VectorXi& indices,
                                              int num_factors);

void check_domain_variable(const DataFrame& df,
                           const std::string& variable,
                           const std::vector<std::string>& variable_values);

}  // namespace factors::discrete

#endif  // PYBNESIAN_FACTORS_DISCRETE_DISCRETE_INDICES_HPP

ext: hpp

max_stars_repo_name: vishalbelsare/PyBNesian

max_issues_repo_name: vishalbelsare/PyBNesian

max_forks_repo_name: vishalbelsare/PyBNesian


text:
/*
 * EncryptUtils.cpp
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "flow/EncryptUtils.h"
#include "flow/Trace.h"

#include <boost/format.hpp>

std::string getEncryptDbgTraceKey(std::string_view prefix,
                                  EncryptCipherDomainId domainId,
                                  Optional<EncryptCipherBaseKeyId> baseCipherId) {
	// Construct the TraceEvent field key ensuring its uniqueness and compliance to TraceEvent field validator and log
	// parsing tools
	if (baseCipherId.present()) {
		boost::format fmter("%s.%lld.%llu");
		return boost::str(boost::format(fmter % prefix % domainId % baseCipherId.get()));
	} else {
		boost::format fmter("%s.%lld");
		return boost::str(boost::format(fmter % prefix % domainId));
	}
}

max_stars_repo_name: sfc-gh-bvr/foundationdb

max_issues_repo_name: sfc-gh-bvr/foundationdb

max_forks_repo_name: sfc-gh-bvr/foundationdb


text:
/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com)     |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/
#include "hphp/runtime/base/file-util.h"

#include <vector>
#include <fstream>

#include <boost/filesystem.hpp>

#include <sys/types.h>
#include <dirent.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <libgen.h>

#include <folly/String.h>

#include "hphp/runtime/base/runtime-error.h"
#include "hphp/util/lock.h"
#include "hphp/util/logger.h"
#include "hphp/util/exception.h"
#include "hphp/util/network.h"
#include "hphp/util/compatibility.h"

namespace HPHP {
///////////////////////////////////////////////////////////////////////////////

using std::string;
using std::vector;
namespace fs = boost::filesystem;

bool FileUtil::mkdir(const std::string &path, int mode /* = 0777 */) {
  if (path.empty()) {
    return false;
  }
  size_t pos = path.rfind('/');
  if (pos != string::npos) {
    // quick test whole path exists
    if (access(path.substr(0, pos).c_str(), F_OK) >= 0) {
      return false;
    }
    for (pos = path.find('/'); pos != string::npos;
         pos = path.find('/', pos + 1)) {
      string subpath = path.substr(0, pos);
      if (subpath.empty()) continue;
      if (access(subpath.c_str(), F_OK) < 0 &&
          ::mkdir(subpath.c_str(), mode) < 0) {
        Logger::Error("unable to mkdir %s", subpath.c_str());
        return false;
      }
    }
  }
  return true;
}

static bool same(const char *file1, const char *file2) {
  FILE *f1 = fopen(file1, "r");
  if (f1 == nullptr) {
    Logger::Error("unable to read %s", file1);
    return false;
  }
  FILE *f2 = fopen(file2, "r");
  if (f2 == nullptr) {
    fclose(f1);
    Logger::Error("unable to read %s", file2);
    return false;
  }

  bool ret = false;
  char buf1[8192];
  char buf2[sizeof(buf1)];
  int n1;
  while ((n1 = fread(buf1, 1, sizeof(buf1), f1))) {
    int toread = n1;
    int pos = 0;
    while (toread) {
      int n2 = fread(buf2 + pos, 1, toread, f2);
      if (n2 <= 0) {
        goto exit_false;
      }
      toread -= n2;
      pos += n2;
    }
    if (memcmp(buf1, buf2, n1) != 0) {
      goto exit_false;
    }
  }
  if (fread(buf2, 1, 1, f2) == 0) {
    ret = true;
  }
 exit_false:
  fclose(f2);
  fclose(f1);
  return ret;
}

void FileUtil::syncdir(const std::string &dest_, const std::string &src_,
                       bool keepSrc /* = false */) {
  if (src_.empty() || dest_.empty()) return;

  string src = src_;
  if (src[src.size() - 1] != '/') src += '/';
  string dest = dest_;
  if (dest[dest.size() - 1] != '/') dest += '/';

  DIR *ddest = opendir(dest.c_str());
  if (ddest == nullptr) {
    Logger::Error("syncdir: unable to open dest %s", dest.c_str());
    return;
  }

  DIR *dsrc = opendir(src.c_str());
  if (dsrc == nullptr) {
    closedir(ddest);
    Logger::Error("syncdir: unable to open src %s", src.c_str());
    return;
  }

  dirent *e;

  std::set<string> todelete;
  while ((e = readdir(ddest))) {
    if (strcmp(e->d_name, ".") == 0 ||
        strcmp(e->d_name, "..") == 0) {
      continue;
    }
    string fsrc = src + e->d_name;
    string fdest = dest + e->d_name;

    // delete files/directories that are only in dest
    if (access(fsrc.c_str(), F_OK) < 0) {
      size_t pos = fdest.rfind('.');
      if (pos != string::npos) {
        string ext = fdest.substr(pos + 1);
        // do not delete intermediate files
        if (ext == "o" || ext == "d") {
          continue;
        }
      }
      todelete.insert(fdest);
      continue;
    }

    // delete mismatched types so to copy over new ones
    struct stat sb1, sb2;
    stat(fsrc.c_str(), &sb1);
    stat(fdest.c_str(), &sb2);
    if ((sb1.st_mode & S_IFMT) != (sb2.st_mode & S_IFMT)) {
      todelete.insert(fdest.c_str());
      continue;
    }

    // updates
    if ((sb1.st_mode & S_IFMT) == S_IFDIR) {
      syncdir(fdest, fsrc);
    } else if (sb1.st_size != sb2.st_size ||
               !same(fsrc.c_str(), fdest.c_str())) {
      todelete.insert(fdest);
    }
  }

  // delete the ones to delete
  if (!todelete.empty()) {
    for (std::set<string>::const_iterator iter = todelete.begin();
         iter != todelete.end(); ++iter) {
      Logger::Info("sync: deleting %s", iter->c_str());
      fs::remove_all(*iter);
    }
  }

  // insert new ones
  while ((e = readdir(dsrc))) {
    string fdest = dest + e->d_name;
    if (access(fdest.c_str(), F_OK) < 0) {
      Logger::Info("sync: updating %s", fdest.c_str());
      if (keepSrc) {
        ssystem((string("cp -R ") + src + e->d_name + " " + dest).c_str());
      } else {
        rename((src + e->d_name).c_str(), (dest + e->d_name).c_str());
      }
    }
  }

  closedir(dsrc);
  closedir(ddest);
}

int FileUtil::copy(const char *srcfile, const char *dstfile) {
  int srcFd = open(srcfile, O_RDONLY);
  if (srcFd == -1) return -1;
  int dstFd = open(dstfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
  if (dstFd == -1) return -1;

  while (1) {
    char buf[8192];
    bool err = false;
    ssize_t rbytes = read(srcFd, buf, sizeof(buf));
    ssize_t wbytes;
    if (rbytes == 0) break;
    if (rbytes == -1) {
      err = true;
      Logger::Error("read failed: %s", folly::errnoStr(errno).c_str());
    } else if ((wbytes = write(dstFd, buf, rbytes)) != rbytes) {
      err = true;
      Logger::Error("write failed: %zd, %s", wbytes,
                    folly::errnoStr(errno).c_str());
    }
    if (err) {
      close(srcFd);
      close(dstFd);
      return -1;
    }
  }
  close(srcFd);
  close(dstFd);
  return 0;
}

static int force_sync(int fd) {
#if defined(__FreeBSD__) || defined(__APPLE__) || defined(_MSC_VER)
  return fsync(fd);
#else
  return fdatasync(fd);
#endif
}

int FileUtil::directCopy(const char *srcfile, const char *dstfile) {
  int srcFd = open(srcfile, O_RDONLY);
  if (srcFd == -1) return -1;
  int dstFd = open(dstfile, O_WRONLY | O_CREAT | O_TRUNC, 0666);
  if (dstFd == -1) return -1;

#if defined(__APPLE__)
  fcntl(srcFd, F_NOCACHE, 1);
  fcntl(dstFd, F_NOCACHE, 1);
#endif

  while (1) {
    char buf[1 << 20];
    bool err = false;
    ssize_t rbytes = read(srcFd, buf, sizeof(buf));
    ssize_t wbytes;
    if (rbytes == 0) break;

    if (rbytes == -1) {
      err = true;
      Logger::Error("read failed: %s", folly::errnoStr(errno).c_str());
    } else if (force_sync(srcFd) == -1) {
      err = true;
      Logger::Error("read sync failed: %s",
                    folly::errnoStr(errno).c_str());
    } else if (fadvise_dontneed(srcFd, 0) == -1) {
      err = true;
      Logger::Error("read cache drop failed: %s",
                    folly::errnoStr(errno).c_str());
    } else if ((wbytes = write(dstFd, buf, rbytes)) != rbytes) {
      err = true;
      Logger::Error("write failed: %zd, %s", wbytes,
                    folly::errnoStr(errno).c_str());
    } else if (force_sync(dstFd) == -1) {
      err = true;
      Logger::Error("write sync failed: %s",
                    folly::errnoStr(errno).c_str());
    } else if (fadvise_dontneed(dstFd, 0) == -1) {
      err = true;
      Logger::Error("write cache drop failed: %s",
                    folly::errnoStr(errno).c_str());
    }
    if (err) {
      close(srcFd);
      close(dstFd);
      return -1;
    }
  }
  close(srcFd);
  close(dstFd);
  return 0;
}

int FileUtil::rename(const char *oldname, const char *newname) {
  int ret = ::rename(oldname, newname);
  if (ret == 0) return 0;
  if (errno != EXDEV) return -1;

  copy(oldname, newname);
  unlink(oldname);
  return 0;
}

int FileUtil::directRename(const char *oldname, const char *newname) {
  int ret = ::rename(oldname, newname);
  if (ret == 0) return 0;
  if (errno != EXDEV) return -1;

  directCopy(oldname, newname);
  unlink(oldname);
  return 0;
}

int FileUtil::ssystem(const char* command) {
  int ret = system(command);
  if (ret == -1) {
    Logger::Error("system(\"%s\"): %s", command,
                  folly::errnoStr(errno).c_str());
  } else if (ret != 0) {
    Logger::Error("command failed: \"%s\"", command);
  }
  return ret;
}

size_t FileUtil::dirname_helper(char *path, int len) {
  if (len == 0) {
    /* Illegal use of this function */
    return 0;
  }

  /* Strip trailing slashes */
  register char *end = path + len - 1;
  while (end >= path && isDirSeparator(*end)) {
    end--;
  }
  if (end < path) {
    /* The path only contained slashes */
    path[0] = getDirSeparator();
    path[1] = '\0';
    return 1;
  }

  /* Strip filename */
  while (end >= path && !isDirSeparator(*end)) {
    end--;
  }
  if (end < path) {
    /* No slash found, therefore return '.' */
    path[0] = '.';
    path[1] = '\0';
    return 1;
  }

  /* Strip slashes which came before the file name */
  while (end >= path && isDirSeparator(*end)) {
    end--;
  }
  if (end < path) {
    path[0] = getDirSeparator();
    path[1] = '\0';
    return 1;
  }
  *(end+1) = '\0';

  return end + 1 - path;
}

std::string FileUtil::safe_dirname(const char *path, int len) {
  char* tmp_path = (char*)malloc(len+1);
  memcpy(tmp_path, path, len);
  tmp_path[len] = '\0';
  size_t newLen = dirname_helper(tmp_path, len);
  std::string ret;
  ret.assign(tmp_path, newLen);
  free((void *) tmp_path);
  return ret;
}

std::string FileUtil::safe_dirname(const char *path) {
  int len = strlen(path);
  return safe_dirname(path, len);
}

std::string FileUtil::safe_dirname(const std::string& path) {
  return safe_dirname(path.c_str(), path.size());
}

String FileUtil::relativePath(const std::string& fromDir,
                              const String& toFile) {

  size_t maxlen = (fromDir.size() + toFile.size()) * 3;

  // Sanity checks
  if (!isAbsolutePath(fromDir) || !isAbsolutePath(toFile.slice()) ||
      !isDirSeparator(fromDir[fromDir.size() - 1])) {
    return empty_string();
  }

  // Maybe we're lucky and this is an easy case
  int from_len = fromDir.size();
  if (strncmp(toFile.c_str(), fromDir.c_str(), from_len) == 0) {
    return toFile.substr(from_len);
  }

  String ret(maxlen, ReserveString);
  char* path = ret.mutableData();

  const char* from_dir = fromDir.c_str();
  const char* to_file = toFile.c_str();

  const char* from_start = from_dir + 1;
  const char* to_start = to_file + 1;

  while (true) {
    int seg_len = 0;
    char cur = from_start[0];
    while (cur && !isDirSeparator(cur)) {
      ++seg_len;
      cur = from_start[seg_len];
    }

    if (memcmp(from_start, to_start, seg_len + 1)) {
      break;
    }
    from_start += seg_len + 1;
    to_start += seg_len + 1;
  }

  // Now to build the path
  char cur = *from_start;
  char* path_end = path;
  while (cur) {
    if (isDirSeparator(cur)) {
      strcpy(path_end, "..");
      path_end[2] = getDirSeparator();
      maxlen -= 3;
      path_end += 3;
    }
    ++from_start;
    cur = *from_start;
  }

  if (!isDirSeparator(from_start[-1])) {
    strcpy(path_end, "..");
    path_end[2] = getDirSeparator();
    maxlen -= 3;
    path_end += 3;
  }

  // Ensure the result is null-terminated after the strcpy
  assert(to_start - to_file <= toFile.size());
  assert(path_end - path + strlen(to_start) <= ret.capacity());

  strcpy(path_end, to_start);
  return ret.setSize(strlen(path));
}

String FileUtil::canonicalize(const String &path) {
  return canonicalize(path.data(), path.size());
}

String FileUtil::canonicalize(const std::string &path) {
  return canonicalize(path.c_str(), path.size());
}

/* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

String FileUtil::canonicalize(const char *addpath, size_t addlen,
                              bool collapse_slashes /* = true */) {
  assert(strlen(addpath) <= addlen);
  // 4 for slashes at start, after root, and at end, plus trailing
  // null
  size_t maxlen = addlen + 4;
  size_t pathlen = 0; // is the length of the result path
  size_t seglen;  // is the end of the current segment
  auto pathend = addpath + addlen;

  /* Treat null as an empty path.
   */
  if (!addpath)
    addpath = "";

  String ret(maxlen-1, ReserveString);
  char *path = ret.mutableData();

#ifndef _MSC_VER
  if (addpath[0] == '/' && collapse_slashes) {
    /* Ignore the given root path, strip off leading
     * '/'s to a single leading '/' from the addpath,
     * and leave addpath at the first non-'/' character.
     */
    while (addpath[0] == '/')
      ++addpath;
    path[0] = '/';
    pathlen = 1;
  }
#endif

  while (*addpath) {
    /* Parse each segment, find the closing '/'
     */
    const char *next = addpath;
    while (*next && !isDirSeparator(*next)) {
      ++next;
    }
    seglen = next - addpath;

    if (seglen == 0) {
      /* / */
      if (!collapse_slashes) {
        path[pathlen++] = getDirSeparator();
      }
    } else if (seglen == 1 && addpath[0] == '.'
               && (pathlen > 0 || (*next && *(next+1)))) {
      /* ./ (safe to drop iff there is something before or after it) */
    } else if (seglen == 2 && addpath[0] == '.' && addpath[1] == '.') {
      /* backpath (../) */
      if (pathlen == 1 && isDirSeparator(path[0])) {
      } else if (pathlen == 0
                 || (pathlen == 3
                     && !memcmp(path + pathlen - 3, "..", 2)
                     && isDirSeparator(path[pathlen - 1]))
                 || (pathlen  > 3
                     && isDirSeparator(path[pathlen - 4])
                     && !memcmp(path + pathlen - 3, "..", 2)
                     && isDirSeparator(path[pathlen - 1]))) {
        /* Append another backpath, including
         * trailing slash if present.
         */
        memcpy(path + pathlen, "..", 2);
        if (*next) {
          path[pathlen + 2] = getDirSeparator();
        }
        pathlen += *next ? 3 : 2;
      } else {
        /* otherwise crop the prior segment */
        do {
          --pathlen;
        } while (pathlen && !isDirSeparator(path[pathlen - 1]));
      }
    } else {
      /* An actual segment, append it to the destination path */
      if (*next) {
        seglen++;
      }
      memcpy(path + pathlen, addpath, seglen);
      pathlen += seglen;
    }

    /* Skip over trailing slash to the next segment */
    if (*next) {
      ++next;
    }

    addpath = next;
  }

  // If there are null bytes in the path, treat it as the empty string
  if (addpath != pathend) pathlen = 0;

#ifdef _MSC_VER
  // Need to normalize to Windows directory separators, as the underlying
  // system calls don't like unix path separators.
  for (int i = 0; i < pathlen; i++) {
    if (path[i] == '/') {
      path[i] = '\\';
    }
  }
#endif
  ret.setSize(pathlen);
  return ret;
}

std::string FileUtil::normalizeDir(const std::string &dirname) {
  /*
   * normalizeDir may be called before very early one, such as
   * in Runtime option parsing, when MemoryManager may not have been
   * initialized
   */
  MemoryManager::TlsWrapper::getCheck();
  string ret = FileUtil::canonicalize(dirname).toCppString();
  if (!ret.empty() && !isDirSeparator(ret[ret.length() - 1])) {
    ret += getDirSeparator();
  }
  return ret;
}

void FileUtil::find(std::vector<std::string> &out,
                    const std::string &root, const char *path, bool php,
                    const std::set<std::string> *excludeDirs /* = NULL */,
                    const std::set<std::string> *excludeFiles /* = NULL */) {
  if (!path) path = "";
  if (isDirSeparator(*path)) path++;

  string spath = path;
  if (spath.length() && !isDirSeparator(spath[spath.length() - 1])) {
    spath += getDirSeparator();
  }
  if (excludeDirs && excludeDirs->find(spath) != excludeDirs->end()) {
    return;
  }

  string fullPath = root + path;
  if (fullPath.empty()) {
    return;
  }
  DIR *dir = opendir(fullPath.c_str());
  if (dir == nullptr) {
    Logger::Error("FileUtil::find(): unable to open directory %s",
                  fullPath.c_str());
    return;
  }
  if (!isDirSeparator(fullPath[fullPath.length() - 1])) {
    fullPath += getDirSeparator();
  }

  dirent *e;
  while ((e = readdir(dir))) {
    char *ename = e->d_name;

    // skipping .  .. hidden files
    if (ename[0] == '.' || !*ename) {
      continue;
    }
    string fe = fullPath + ename;
    struct stat se;
    if (stat(fe.c_str(), &se)) {
      Logger::Error("FileUtil::find(): unable to stat %s", fe.c_str());
      continue;
    }

    if ((se.st_mode & S_IFMT) == S_IFDIR) {
      string subdir = spath + ename;
      find(out, root, subdir.c_str(), php, excludeDirs, excludeFiles);
      continue;
    }

    // skipping "tags" files
    if (strcmp(ename, "tags") == 0) {
      continue;
    }

    // skipping emacs leftovers
    char last = ename[strlen(ename) - 1];
    if (last == '~' || last == '#') {
      continue;
    }

    bool isPHP = false;
    const char *p = strrchr(ename, '.');
    if (p) {
      isPHP = (strncmp(p + 1, "php", 3) == 0);
    } else {
      try {
        string line;
        std::ifstream fin(fe.c_str());
        if (std::getline(fin, line)) {
          if (line[0] == '#' && line[1] == '!' &&
              line.find("php") != string::npos) {
            isPHP = true;
          }
        }
      } catch (...) {
        Logger::Error("FileUtil::find(): unable to read %s", fe.c_str());
      }
    }

    if (isPHP == php &&
        (!excludeFiles ||
         excludeFiles->find(spath + ename) == excludeFiles->end())) {
      out.push_back(fe);
    }
  }

  closedir(dir);
}

bool FileUtil::isValidPath(const String& path) {
  return path.size() == strlen(path.data());
}

bool FileUtil::checkPathAndWarn(const String& path,
                                const char* func_name,
                                int param_pos) {
  if (!FileUtil::isValidPath(path)) {
    raise_warning(
      "%s() expects parameter %d to be a valid path, string given",
      func_name,
      param_pos
    );
    return false;
  }
  return true;
}

void FileUtil::checkPathAndError(const String& path,
                                 const char* func_name,
                                 int param_pos) {
  if (!FileUtil::isValidPath(path)) {
    raise_error(
      "%s() expects parameter %d to be a valid path, string given",
      func_name,
      param_pos
    );
  }
}

///////////////////////////////////////////////////////////////////////////////
}

max_stars_repo_name: lavagetto/hhvm

max_issues_repo_name: lavagetto/hhvm

max_forks_repo_name: lavagetto/hhvm


text:
/**
 * Swaggy Jenkins
 * Jenkins API clients generated from Swagger / Open API specification
 *
 * The version of the OpenAPI document: 1.1.2-pre.0
 * Contact: blah@cliffano.com
 *
 * NOTE: This class is auto generated by OpenAPI-Generator 5.4.0.
 * https://openapi-generator.tech
 * Do not edit the class manually.
 */

#include "ComputerSet.h"

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>

using boost::property_tree::ptree;
using boost::property_tree::read_json;
using boost::property_tree::write_json;

namespace org {
namespace openapitools {
namespace server {
namespace model {

ComputerSet::ComputerSet(boost::property_tree::ptree const& pt)
{
        fromPropertyTree(pt);
}

std::string ComputerSet::toJsonString(bool prettyJson /* = false */)
{
    return toJsonString_internal(prettyJson);
}

void ComputerSet::fromJsonString(std::string const& jsonString)
{
    fromJsonString_internal(jsonString);
}

boost::property_tree::ptree ComputerSet::toPropertyTree()
{
    return toPropertyTree_internal();
}

void ComputerSet::fromPropertyTree(boost::property_tree::ptree const& pt)
{
    fromPropertyTree_internal(pt);
}

std::string ComputerSet::toJsonString_internal(bool prettyJson)
{
	std::stringstream ss;
	write_json(ss, this->toPropertyTree(), prettyJson);
	return ss.str();
}

void ComputerSet::fromJsonString_internal(std::string const& jsonString)
{
	std::stringstream ss(jsonString);
	ptree pt;
	read_json(ss,pt);
	this->fromPropertyTree(pt);
}

ptree ComputerSet::toPropertyTree_internal()
{
	ptree pt;
	ptree tmp_node;
	pt.put("_class", m__class);
	pt.put("busyExecutors", m_BusyExecutors);
	// generate tree for Computer
	if (!m_Computer.empty()) {
		for (const auto &childEntry : m_Computer) {
            tmp_node.push_back(std::make_pair("", childEntry->toPropertyTree()));
		}
		pt.add_child("computer", tmp_node);
		tmp_node.clear();
	}
	pt.put("displayName", m_DisplayName);
	pt.put("totalExecutors", m_TotalExecutors);
	return pt;
}

void ComputerSet::fromPropertyTree_internal(ptree const &pt)
{
	ptree tmp_node;
	m__class = pt.get("_class", "");
	m_BusyExecutors = pt.get("busyExecutors", 0);
	// push all items of Computer into member vector
	if (pt.get_child_optional("computer")) {
		for (const auto &childTree : pt.get_child("computer")) {
            std::shared_ptr<HudsonMasterComputer> val =
                std::make_shared<HudsonMasterComputer>(childTree.second);
            m_Computer.emplace_back(std::move(val));
		}
	}
	m_DisplayName = pt.get("displayName", "");
	m_TotalExecutors = pt.get("totalExecutors", 0);
}

std::string ComputerSet::get_Class() const
{
    return m__class;
}

void ComputerSet::set_Class(std::string value)
{
	m__class = value;
}
int32_t ComputerSet::getBusyExecutors() const
{
    return m_BusyExecutors;
}

void ComputerSet::setBusyExecutors(int32_t value)
{
	m_BusyExecutors = value;
}
std::vector<std::shared_ptr<HudsonMasterComputer>> ComputerSet::getComputer() const
{
    return m_Computer;
}

void ComputerSet::setComputer(std::vector<std::shared_ptr<HudsonMasterComputer>> value)
{
	m_Computer = value;
}
std::string ComputerSet::getDisplayName() const
{
    return m_DisplayName;
}

void ComputerSet::setDisplayName(std::string value)
{
	m_DisplayName = value;
}
int32_t ComputerSet::getTotalExecutors() const
{
    return m_TotalExecutors;
}

void ComputerSet::setTotalExecutors(int32_t value)
{
	m_TotalExecutors = value;
}

std::vector<ComputerSet> createComputerSetVectorFromJsonString(const std::string& json)
{
    std::stringstream sstream(json);
    boost::property_tree::ptree pt;
    boost::property_tree::json_parser::read_json(sstream,pt);

    auto vec = std::vector<ComputerSet>();
    for (const auto& child: pt) {
        vec.emplace_back(ComputerSet(child.second));
    }

    return vec;
}

}
}
}
}

max_stars_repo_name: cliffano/jenkins-api-clients-generator

max_issues_repo_name: cliffano/jenkins-api-clients-generator

max_forks_repo_name: cliffano/jenkins-api-clients-generator


text:
#include <fstream>
#include <boost/filesystem.hpp>
#include <gtest/gtest.h>
#include <rime/resource.h>

using namespace rime;

static const ResourceType kMineralsType = ResourceType{
  "minerals",
  "not_",
  ".minerals",
};

TEST(RimeResourceResolverTest, ResolvePath) {
  ResourceResolver rr(kMineralsType);
  rr.set_root_path("/starcraft");
  auto actual = rr.ResolvePath("enough");
  boost::filesystem::path expected = boost::filesystem::system_complete(boost::filesystem::current_path()).root_name().string() + "/starcraft/not_enough.minerals";
  EXPECT_TRUE(actual.is_absolute());
  EXPECT_TRUE(expected == actual);
}

TEST(RimeResourceResolverTest, FallbackRootPath) {
  FallbackResourceResolver rr(kMineralsType);
  rr.set_fallback_root_path("fallback");
  boost::filesystem::create_directory("fallback");
  {
    boost::filesystem::path nonexistent_default = "not_present.minerals";
    boost::filesystem::remove(nonexistent_default);
    auto fallback = boost::filesystem::absolute("fallback/not_present.minerals");
    std::ofstream(fallback.string()).close();
    auto actual = rr.ResolvePath("present");
    EXPECT_TRUE(fallback == actual);
    boost::filesystem::remove(fallback);
  }
  {
    auto existent_default = boost::filesystem::absolute("not_falling_back.minerals");
    std::ofstream(existent_default.string()).close();
    auto actual = rr.ResolvePath("falling_back");
    EXPECT_TRUE(existent_default == actual);
    boost::filesystem::remove(existent_default);
  }
  {
    auto nonexistent_default = boost::filesystem::absolute("not_any.minerals");
    boost::filesystem::remove(nonexistent_default);
    auto nonexistent_fallback = boost::filesystem::absolute("fallback/not_any.minerals");
    boost::filesystem::remove(nonexistent_fallback);
    auto actual = rr.ResolvePath("any");
    EXPECT_TRUE(nonexistent_default == actual);
  }
  boost::filesystem::remove_all("fallback");
}

ext: cc