text:
#include "dynet/nodes.h"
#include "dynet/dynet.h"
#include "dynet/training.h"
#include "dynet/timing.h"
#include "dynet/rnn.h"
#include "dynet/gru.h"
#include "dynet/lstm.h"
#include "dynet/fast-lstm.h"
#include "dynet/expr.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <type_traits>
#include <time.h>

#include <boost/serialization/vector.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/program_options.hpp>

#include "s2s/dynet/dict.h"
#include "s2s/corpus/corpora.hpp"
#include "s2s/corpus/comp.hpp"
#include "s2s/corpus/options.hpp"
#include "s2s/corpus/batch.hpp"

#ifndef INCLUDE_GUARD_S2S_TEST_CORP_HPP
#define INCLUDE_GUARD_S2S_TEST_CORP_HPP

namespace s2s {

    void test_batch_random_sub(s2s_options &opts, dicts &dicts, const unsigned int alpha, unsigned int &cnt_unk, unsigned int &cnt_not_unk, parallel_corpus &para_corp){
        opts.drop_word_alpha = alpha;
        para_corp.reset_index();
        para_corp.set_para_batch_order(opts.max_batch_train, opts.src_tok_lim_train, opts.trg_tok_lim_train, opts.batch_type_train);
        batch batch_local(opts, dicts);
        unsigned int batch_cnt = 0;
        while(para_corp.next_batch_para(batch_local)){
            // word drop test
            batch_local.drop_word();
            // src feat
            for(unsigned int tid = 0; tid < batch_local.sent_word.size(); tid++){
                for(unsigned int sid = 0; sid < batch_local.sent_id.size(); sid++){
                    if(batch_local.sent_word.at(tid).at(sid) == dicts.dict_set_src.unk_id_word){
                        cnt_unk++;
                    }else{
                        cnt_not_unk++;
                    }
                }
            }
            batch_cnt++;
        }
        assert(batch_cnt > 0);
        std::cerr << "All: " << (cnt_unk + cnt_not_unk) << ", " << "Unk: " << cnt_unk << std::endl;
        para_corp.reset_index();
    }

    void test_batch_random(s2s_options &opts, dicts &dicts, parallel_corpus &para_corp){
        // alpha = 0.0
        {
            unsigned int cnt_unk = 0;
            unsigned int cnt_not_unk = 0;
            test_batch_random_sub(opts, dicts, 0.f, cnt_unk, cnt_not_unk, para_corp);
            assert(cnt_unk == 0);
            assert(cnt_not_unk > 0);
        }
        // alpha = 10000.0
        {
            unsigned int cnt_unk = 0;
            unsigned int cnt_not_unk = 0;
            test_batch_random_sub(opts, dicts, 10000.f, cnt_unk, cnt_not_unk, para_corp);
            assert(cnt_unk > 0);
        }
        // alpha = 1.0
        {
            unsigned int cnt_unk = 0;
            unsigned int cnt_not_unk = 0;
            test_batch_random_sub(opts, dicts, 1.f, cnt_unk, cnt_not_unk, para_corp);
            assert(cnt_unk > 0);
            assert(cnt_not_unk > 0);
        }
    }

    void test_corp_load_files(s2s_options &opts, dicts &dicts, parallel_corpus &para_corp){
        // load test
        para_corp.load_src(opts.srcfile);
        para_corp.load_trg(opts.trgfile);
        para_corp.load_check();
        if(opts.guided_alignment == true){
            para_corp.load_align(opts.alignfile);
            para_corp.load_check_with_align();
        }

        std::ifstream in_src(opts.srcfile);
        assert(in_src);
        std::ifstream in_trg(opts.trgfile);
        assert(in_trg);
        std::ifstream in_align(opts.alignfile);
        if(opts.guided_alignment == true){
            assert(in_align);
        }
        std::string line_src = "";
        std::string line_trg = "";
        std::string line_align = "";
        int sid = 0;
        while(getline(in_src, line_src) && getline(in_trg, line_trg) && (opts.guided_alignment == false || getline(in_align, line_align))) {
            // src
            {
                std::vector<std::string> str_tokens;
                boost::algorithm::split_regex(str_tokens, line_src, boost::regex(" "));
                std::vector<token > &sent = para_corp.src.at(sid);
                assert(sent.size() == str_tokens.size());
                for(unsigned int tok_id = 0; tok_id < sent.size(); tok_id++){
                    // word
                    std::string str_conv_word = dicts.dict_set_src.d_word.convert(sent.at(tok_id).word_id);
                    // feat
                    std::string str_conv = str_conv_word;
                    bool is_feat_unk = false;
                    for(unsigned int feat_type = 0; feat_type < sent.at(tok_id).feat_id_list.size(); feat_type++){
                        str_conv += "-|-";
                        std::string str_conv_local = dicts.dict_set_src.d_feat[feat_type].convert(sent.at(tok_id).feat_id_list.at(feat_type));
                        str_conv += str_conv_local;
                        if(str_conv_local == opts.unk_symbol){
                           is_feat_unk = true;
                        }
                    }
                    if(str_conv_word != opts.unk_symbol && is_feat_unk == false && str_conv != str_tokens.at(tok_id)){
                        std::cerr << str_conv << " " << str_tokens.at(tok_id) << std::endl;
                        assert(false);
                    }
                    // char
                    std::string str_char = "";
                    bool is_char_unk = false;
                    for(unsigned int char_id : sent.at(tok_id).char_id_seq){
                        if(char_id != dicts.dict_set_src.start_id_char && char_id != dicts.dict_set_src.end_id_char){
                            std::string str_char_local = dicts.dict_set_src.d_char.convert(char_id);
                            str_char += str_char_local;
                            if(str_char_local == opts.unk_symbol){
                                is_char_unk = true;
                            }
                        }
                    }
                    if(str_conv_word != opts.unk_symbol && is_char_unk == false && dicts.dict_set_src.d_word.convert(sent.at(tok_id).word_id) != str_char){
                        std::cerr << dicts.dict_set_src.d_word.convert(sent.at(tok_id).word_id) << " " << str_char << std::endl;
                        assert(false);
                    }
                }
            }
            // trg
            {
                std::vector<std::string> str_tokens;
                boost::algorithm::split_regex(str_tokens, line_trg, boost::regex(" "));
                std::vector<token > &sent = para_corp.trg.at(sid);
                assert(sent.size() == str_tokens.size());
                for(unsigned int tok_id = 0; tok_id < sent.size(); tok_id++){
                    // word
                    std::string str_conv = dicts.dict_set_trg.d_word.convert(sent.at(tok_id).word_id);
                    if(str_conv != opts.unk_symbol && str_conv != str_tokens.at(tok_id)){
                        std::cerr << str_conv << " " << str_tokens.at(tok_id) << std::endl;
                        assert(false);
                    }
                }
            }
            // align
            if(opts.guided_alignment == true){
                std::vector<std::string> str_tokens;
                boost::algorithm::split_regex(str_tokens, line_align, boost::regex(" "));
                std::vector<unsigned int> aligns(str_tokens.size());
                std::vector<unsigned int > &sent = para_corp.align.at(sid);
                for(unsigned int tok_id=0; tok_id < sent.size(); tok_id++){
                    std::string str_link = std::to_string(sent[tok_id]) + "-" + std::to_string(tok_id);
                    if(str_tokens[tok_id] != str_link){
                        std::cerr << str_tokens[tok_id] << ", " << str_link << std::endl;
                        assert(false);
                    }
                }
            }
            sid++;
        }
        in_src.close();
        in_trg.close();
        in_align.close();
        assert(sid > 0);
    }

    void test_corp_to_batch(s2s_options &opts, dicts &dicts, parallel_corpus &para_corp){
        // batch test
        unsigned int batch_cnt = 0;
        batch batch_local(opts, dicts);
        para_corp.reset_index();
        para_corp.set_para_batch_order(opts.max_batch_train, opts.src_tok_lim_train, opts.trg_tok_lim_train, opts.batch_type_train);
        while(para_corp.next_batch_para(batch_local)){
            // src word
            for(unsigned int tid = 0; tid < batch_local.sent_word.size(); tid++){
                unsigned int sid = 0;
                for(const unsigned int sent_id : batch_local.sent_id){
                    if(tid >= para_corp.src.at(sent_id).size()){
                        if(batch_local.sent_word.at(tid).at(sid) != dicts.dict_set_src.end_id_word){
                            std::cerr << batch_local.sent_word.at(tid).at(sid) << ", " << dicts.dict_set_src.end_id_word << std::endl;
                            assert(false);
                        }
                    }else{
                        if(batch_local.sent_word.at(tid).at(sid) != para_corp.src.at(sent_id).at(tid).word_id){
                            std::cerr << batch_local.sent_word.at(tid).at(sid) << ", " << para_corp.src.at(sent_id).at(tid).word_id << std::endl;
                            assert(false);
                        }
                    }
                    sid++;
                }
            }
            // src feat
            for(unsigned int tid = 0; tid < batch_local.sent_feat.size(); tid++){
                for(unsigned int fid = 0; fid < dicts.dict_set_src.d_feat.size(); fid++){
                    unsigned int sid = 0;
                    for(const unsigned int sent_id : batch_local.sent_id){
                        if(tid >= para_corp.src.at(sent_id).size()){
                            if(batch_local.sent_feat.at(fid).at(tid).at(sid) != dicts.dict_set_src.end_id_feat[fid]){
                                std::cerr << dicts.dict_set_src.d_feat[fid].convert(batch_local.sent_feat.at(fid).at(tid).at(sid)) << ", " << dicts.dict_set_src.d_feat[fid].convert(dicts.dict_set_src.end_id_feat[fid]) << std::endl;
                                assert(false);
                            }
                        }else{
                            if(batch_local.sent_feat.at(fid).at(tid).at(sid) != para_corp.src.at(sent_id).at(tid).feat_id_list.at(fid)){
                                std::cerr << dicts.dict_set_src.d_feat[fid].convert(batch_local.sent_feat.at(fid).at(tid).at(sid)) << ", " << dicts.dict_set_src.d_feat[fid].convert(para_corp.src.at(sent_id).at(tid).feat_id_list.at(fid)) << std::endl;
                                assert(false);
                            }
                        }
                        sid++;
                    }
                }
            }
            // src char
            for(unsigned int tid = 0; tid < batch_local.sent_char.size(); tid++){
                for(unsigned int cid = 0; cid < batch_local.sent_char.at(tid).size(); cid++){
                    unsigned int sid = 0;
                    for(const unsigned int sent_id : batch_local.sent_id){
                        if(tid >= para_corp.src.at(sent_id).size()){
                            if(batch_local.sent_char.at(tid).at(cid).at(sid) != dicts.dict_set_src.end_id_char){
                                std::cerr << batch_local.sent_char.at(tid).at(cid).at(sid) << ", " << dicts.dict_set_src.end_id_char << std::endl;
                                assert(false);
                            }
                        }else{
                            if(tid > 0 && tid < para_corp.src.at(sent_id).size() - 1 && batch_local.sent_char.at(tid).at(cid).at(sid) != dicts.dict_set_src.pad_id_char && batch_local.sent_char.at(tid).at(cid).at(sid) != para_corp.src.at(sent_id).at(tid).char_id_seq.at(cid)){
                                std::cerr << dicts.dict_set_src.d_char.convert(batch_local.sent_char.at(tid).at(cid).at(sid)) << ", " << dicts.dict_set_src.d_char.convert(para_corp.src.at(sent_id).at(tid).char_id_seq.at(cid)) << std::endl;
                                assert(false);
                            }
                        }
                        sid++;
                    }
                }
            }
            // trg word
            for(unsigned int tid = 0; tid < batch_local.sent_label.size(); tid++){
                unsigned int sid = 0;
                for(const unsigned int sent_id : batch_local.sent_id){
                    if(tid >= para_corp.trg.at(sent_id).size()){
                        if(batch_local.sent_label.at(tid).at(sid) != dicts.dict_set_trg.end_id_word){
                            std::cerr << batch_local.sent_label.at(tid).at(sid) << ", " << dicts.dict_set_trg.end_id_word << std::endl;
                            assert(false);
                        }
                    }else{
                        if(batch_local.sent_label.at(tid).at(sid) != para_corp.trg.at(sent_id).at(tid).word_id){
                            std::cerr << batch_local.sent_label.at(tid).at(sid) << ", " << para_corp.trg.at(sent_id).at(tid).word_id << std::endl;
                            assert(false);
                        }
                    }
                    sid++;
                }
            }
            batch_cnt++;
        }
        assert(batch_cnt > 0);
    }

    void test_corp_sort(s2s_options &opts, dicts &dicts, parallel_corpus &para_corp){
        // sort test
        para_corp.sort_para_sent(opts.sort_sent_type_train, opts.max_batch_train, opts.src_tok_lim_train, opts.trg_tok_lim_train);
        para_corp.set_para_batch_order(opts.max_batch_train, opts.src_tok_lim_train, opts.trg_tok_lim_train, opts.batch_type_train);
        para_corp.shuffle_batch(opts.shuffle_batch_type_train);
    }

    void test_corp(s2s_options &opts, dicts &dicts, parallel_corpus &para_corp){
        // file loading
        test_corp_load_files(opts, dicts, para_corp);
        // corpus to batch
        test_corp_to_batch(opts, dicts, para_corp);
        // word dropout
        test_batch_random(opts, dicts, para_corp);
        // sort sentences
        test_corp_sort(opts, dicts, para_corp);
    }

};

#endif

max_stars_repo_name: kamigaito/SLAHAN

max_issues_repo_name: kamigaito/SLAHAN

max_forks_repo_name: kamigaito/SLAHAN


text:
//
//  use_future.hpp
//  fibio
//
//  Created by Chen Xu on 14-3-31.
//  Copyright (c) 2014 0d0a.com. All rights reserved.
//

#ifndef fibio_asio_detail_use_future_hpp
#define fibio_asio_detail_use_future_hpp

#include <memory>

#include <boost/asio/async_result.hpp>
#include <boost/asio/detail/config.hpp>
#include <boost/asio/handler_type.hpp>
#include <boost/make_shared.hpp>
#include <boost/move/move.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/system/error_code.hpp>
#include <boost/system/system_error.hpp>
#include <boost/thread/detail/memory.hpp>
#include <boost/throw_exception.hpp>

#include <fibio/fibers/future/future.hpp>
#include <fibio/fibers/future/promise.hpp>

namespace fibio { namespace fibers { namespace asio { namespace detail {
    // Completion handler to adapt a promise as a completion handler.
    template<typename T>
    class promise_handler
    {
    public:
        // Construct from use_future special value.
        template<typename Allocator>
        promise_handler(asio::use_future_t<Allocator> uf)
        : promise_(std::allocate_shared<promise<T>>(uf.get_allocator(),
                                                    boost::allocator_arg,
                                                    uf.get_allocator()))
        , fiber_(fibio::fibers::detail::get_current_fiber_ptr())
        {}

        void operator()(T t)
        {
            promise_->set_value(t);
        }

        void operator()(const boost::system::error_code &ec, T t)
        {
            if (ec)
                promise_->set_exception(utility::copy_exception(boost::system::system_error(ec)));
            else
                promise_->set_value(t);
        }

        //private:
        std::shared_ptr<promise<T>> promise_;
        fibio::fibers::detail::fiber_base::ptr_t fiber_;
    };

    template<>
    class promise_handler<void>
    {
    public:
        // Construct from use_future special value. Used during rebinding.
        template<typename Allocator>
        promise_handler(use_future_t<Allocator> uf)
        : promise_(std::allocate_shared<promise<void>>(uf.get_allocator(),
                                                       boost::allocator_arg,
                                                       uf.get_allocator()))
        , fiber_(fibio::fibers::detail::get_current_fiber_ptr())
        {}

        void operator()()
        {
            promise_->set_value();
        }

        void operator()(const boost::system::error_code &ec)
        {
            if (ec)
                promise_->set_exception(utility::copy_exception(boost::system::system_error(ec)));
            else
                promise_->set_value();
        }

        //private:
        std::shared_ptr<promise<void>> promise_;
        fibio::fibers::detail::fiber_base::ptr_t fiber_;
    };

    // Ensure any exceptions thrown from the handler are propagated back to the
    // caller via the future.
    template<typename Function, typename T>
    void fibio_do_handler_invoke(Function f, fibio::fibers::asio::detail::promise_handler<T> h)
    {
        std::shared_ptr<fibio::fibers::promise<T>> p(h.promise_);
        try
        { f(); }
        catch (...)
        { p->set_exception(std::current_exception()); }
    }
}}}}    // End of namespace fibio::fibers::asio::detail

namespace boost { namespace asio {
    namespace detail {
        template<typename Function, typename T>
        void asio_handler_invoke(Function f, fibio::fibers::asio::detail::promise_handler<T> *h)
        {
            fibio::scheduler s(h->fiber_->get_scheduler());
            fibio::fiber(s,
                         fibio::fibers::asio::detail::fibio_do_handler_invoke<Function, T>,
                         f,
                         fibio::fibers::asio::detail::promise_handler<T>(std::move(*h))).detach();
        }
    }   // End of namespace boost::asio::detail

    // Handler traits specialisation for promise_handler.
    template<typename T>
    class async_result<fibio::fibers::asio::detail::promise_handler<T>>
    {
    public:
        // The initiating function will return a future.
        typedef fibio::fibers::future<T> type;

        // Constructor creates a new promise for the async operation, and obtains the
        // corresponding future.
        explicit async_result(fibio::fibers::asio::detail::promise_handler<T> &h)
        { value_ = h.promise_->get_future(); }

        // Obtain the future to be returned from the initiating function.
        type get()
        { return std::move(value_); }

    private:
        type value_;
    };

    // Handler type specialisation for use_future.
    template<typename Allocator, typename ReturnType>
    struct handler_type<fibio::fibers::asio::use_future_t<Allocator>, ReturnType()>
    { typedef fibio::fibers::asio::detail::promise_handler<void> type; };

    // Handler type specialisation for use_future.
    template<typename Allocator, typename ReturnType, typename Arg1>
    struct handler_type<fibio::fibers::asio::use_future_t<Allocator>, ReturnType(Arg1)>
    { typedef fibio::fibers::asio::detail::promise_handler<Arg1> type; };

    // Handler type specialisation for use_future.
    template<typename Allocator, typename ReturnType>
    struct handler_type<fibio::fibers::asio::use_future_t<Allocator>, ReturnType(boost::system::error_code)>
    { typedef fibio::fibers::asio::detail::promise_handler<void> type; };

    // Handler type specialisation for use_future.
    template<typename Allocator, typename ReturnType, typename Arg2>
    struct handler_type<fibio::fibers::asio::use_future_t<Allocator>, ReturnType(boost::system::error_code, Arg2)>
    { typedef fibio::fibers::asio::detail::promise_handler<Arg2> type; };
}}  // End of namespace boost::asio

#endif

ext: hpp

max_stars_repo_name: qicosmos/fibio

max_issues_repo_name: qicosmos/fibio

max_forks_repo_name: qicosmos/fibio


text:
/*
 * Copyright 2018 National Technology & Engineering Solutions of
 * Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS,
 * the U.S. Government retains certain rights in this software.
 *
 * The MIT License (MIT)
 *
 * Copyright (c) 2018 Sandia Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include <sstream>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <math.h>

#include <my_metadata_client.h>
#include <my_metadata_client_lua_functs.h>

using namespace std;

static bool testing_logging = false;
static bool extreme_debug_logging = false;
static bool zero_rank_logging = false;
static bool error_logging  = true;

static debugLog testing_log = debugLog(testing_logging, zero_rank_logging);
static debugLog error_log = debugLog(error_logging, zero_rank_logging);
static debugLog extreme_debug_log = debugLog(extreme_debug_logging, false);

// void make_real_attr_data (double test_real, string &serial_str) {
//     stringstream ss;
//     boost::archive::text_oarchive oa(ss);
//     oa << test_real;
//     serial_str = ss.str();
// }

// void make_int_attr_data (int test_int, string &serial_str) {
//     stringstream ss;
//     boost::archive::text_oarchive oa(ss);
//     oa << test_int;
//     serial_str = ss.str();
// }

// void make_string_attr_data (const string &test_str, string &serial_str) {
//     stringstream ss;
//     boost::archive::text_oarchive oa(ss);
//     oa << test_str;
//     serial_str = ss.str();
// }

// void make_blob_attr_data (const string &test_string, int test_int, string &serial_str) {
//     stringstream ss;
//     boost::archive::text_oarchive oa(ss);
//     // extreme_debug_log << "when making attr_data, string.size(): " << test_string.size() << endl;
//     oa << test_string;
//     oa << test_int;
//     serial_str = ss.str();
//     // extreme_debug_log << "serial_str: " << serial_str << endl;
// }

void print_var_attr_data(uint32_t count, const std::vector<md_catalog_var_attribute_entry> &attr_entries) {

    if(count != attr_entries.size()) {
        error_log << "Error trying to print var attr data. Count: " << count << " while attr_entries.size(): " << attr_entries.size() << endl;
    }

    for (int i=0; i<attr_entries.size(); i++) {

        switch(attr_entries.at(i).data_type) {
            case ATTR_DATA_TYPE_INT : {
                uint64_t deserialized_test_int;

                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                ia >> deserialized_test_int;

                cout << "data: int: " << deserialized_test_int << endl;
                break;
            }
            case ATTR_DATA_TYPE_REAL : {
                long double deserialized_test_real;

                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                ia >> deserialized_test_real;

                cout << "data: real: " << deserialized_test_real << endl;
                break;
            }
            case ATTR_DATA_TYPE_TEXT : {
                    // string deserialized_test_string;
                cout << "data: text: " << attr_entries.at(i).data << endl;
                break;
            }
            case ATTR_DATA_TYPE_BLOB : {
                vector<int> deserialized_vals(2);

                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                ia >> deserialized_vals;
                cout << "data: blob: val1: " << deserialized_vals.at(0) << " val2: " << deserialized_vals.at(1) << endl;
                break;
            }
            // case ATTR_DATA_TYPE_BLOB : {
            //     string deserialized_test_string;
            //     int deserialized_test_int;

            //     stringstream sso;
            //     sso << attr_entries.at(i).data;
            //     boost::archive::text_iarchive ia(sso);
            //     // extreme_debug_log << "sso: " << sso.str() << endl;
            //     ia >> deserialized_test_string;
            //     ia >> deserialized_test_int;

            //     testing_log << "data: blob: str: " << deserialized_test_string << " int: " << deserialized_test_int << endl;
            //     // testing_log << "str length: " << deserialized_test_string.size() << endl;

            //     break;
            // }
        }
        // extreme_debug_log << "serialized var data: " << attr_entries.at(i).data << endl;

        // testing_log << "data: string: " << deserialized_test_string << " int: " << deserialized_test_int << endl;
    }
 }

 void print_timestep_attr_data(uint32_t count, const std::vector<md_catalog_timestep_attribute_entry> &attr_entries) {

    if(count != attr_entries.size()) {
        error_log << "Error trying to print var attr data. Count: " << count << " while attr_entries.size(): " << attr_entries.size() << endl;
    }

    for (int i=0; i<attr_entries.size(); i++) {
        switch(attr_entries.at(i).data_type) {
            case ATTR_DATA_TYPE_INT : {
                // string deserialized_test_string;
                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                uint64_t deserialized_test_int;
                ia >> deserialized_test_int;
                cout << "data: int: " << deserialized_test_int << endl;
                break;
            }
            case ATTR_DATA_TYPE_REAL : {
                // string deserialized_test_string;
                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                long double deserialized_test_real;
                ia >> deserialized_test_real;
                cout << "data: real: " << deserialized_test_real << endl;
                break;
            }
            case ATTR_DATA_TYPE_TEXT : {
                    // string deserialized_test_string;
                cout << "data: text: " << attr_entries.at(i).data << endl;
                break;
            }
            case ATTR_DATA_TYPE_BLOB : {
                vector<int> deserialized_vals(2);

                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                ia >> deserialized_vals;
                cout << "data: blob: val1: " << deserialized_vals.at(0) << " val2: " << deserialized_vals.at(1) << endl;
                break;
            }
            // case ATTR_DATA_TYPE_BLOB : {
            //     // extreme_debug_log << "serialized data: " << attr_entries.at(i).data << endl;

            //     string deserialized_test_string;
            //     int deserialized_test_int;

            //     stringstream sso;
            //     sso << attr_entries.at(i).data;
            //     boost::archive::text_iarchive ia(sso);
            //     ia >> deserialized_test_string;
            //     ia >> deserialized_test_int;

            //     testing_log << "data: blob: str: " << deserialized_test_string << " int: " << deserialized_test_int << endl;
            //     break;
            // }
        }
    }
 }

  void print_run_attr_data(uint32_t count, const std::vector<md_catalog_run_attribute_entry> &attr_entries) {

    if(count != attr_entries.size()) {
        error_log << "Error trying to print var attr data. Count: " << count << " while attr_entries.size(): " << attr_entries.size() << endl;
    }

    for (int i=0; i<attr_entries.size(); i++) {
        switch(attr_entries.at(i).data_type) {
            case ATTR_DATA_TYPE_INT : {
                // string deserialized_test_string;
                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                uint64_t deserialized_test_int;
                ia >> deserialized_test_int;
                cout << "data: int: " << deserialized_test_int << endl;
                break;
            }
            case ATTR_DATA_TYPE_REAL : {
                // string deserialized_test_string;
                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                long double deserialized_test_real;
                ia >> deserialized_test_real;
                cout << "data: real: " << deserialized_test_real << endl;
                break;
            }
            case ATTR_DATA_TYPE_TEXT : {
                    // string deserialized_test_string;
                cout << "data: text: " << attr_entries.at(i).data << endl;
                break;
            }
            case ATTR_DATA_TYPE_BLOB : {
                vector<int> deserialized_vals(2);

                stringstream sso;
                sso << attr_entries.at(i).data;
                boost::archive::text_iarchive ia(sso);
                ia >> deserialized_vals;
                cout << "data: blob: val1: " << deserialized_vals.at(0) << " val2: " << deserialized_vals.at(1) << endl;
                break;
            }
            // case ATTR_DATA_TYPE_BLOB : {
            //     string deserialized_test_string;
            //     int deserialized_test_int;

            //     stringstream sso;
            //     sso << attr_entries.at(i).data;
            //     boost::archive::text_iarchive ia(sso);
            //     ia >> deserialized_test_string;
            //     ia >> deserialized_test_int;

            //     testing_log << "data: blob: str: " << deserialized_test_string << " int: " << deserialized_test_int << endl;
            //     break;
            // }

        }
    }
 }

void generate_data_for_proc(uint64_t ny, uint64_t nz,
                        int rank, const vector<md_dim_bounds> &bounding_box, vector<double> &data_vct,
                        uint64_t ndx, uint64_t ndy, uint64_t ndz, uint64_t chunk_vol)
{

    int number = ny;

    int y_digits = 0;
    while (number > 0) {
        number /= 10;
        y_digits++;
    }
    y_digits++;

    int z_digits = 0;
    int number2 = nz;

    while (number2 > 0) {
        number2 /= 10;
        z_digits++;
    }
    z_digits++;

    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy << " ndz: " << ndz << endl;

    // uint32_t offset = z + NZ(y + NY*x)
    // uint32_t offset = z1 + output.nz*(y1 + output.ny*x1);
    // extreme_debug_log << "starting offset: " << offset << " rank: " << rank << endl;

    extreme_debug_log << "z1: " << bounding_box.at(2).min << " y1: " << bounding_box.at(1).min << " x1: " << bounding_box.at(0).min << endl;

    uint64_t x1 = bounding_box.at(0).min;
    uint64_t y1 = bounding_box.at(1).min;
    uint64_t z1 = bounding_box.at(2).min;

    // extreme_debug_log << "chunk vol: " << chunk_vol << endl;

    for(int i=0; i<chunk_vol; i++) {

        uint64_t z = z1 + i % ndz;
        uint64_t y = y1 + (i / ndz)%ndy;
        // uint32_t x = (i+offset) / (ndz * ndy);
        uint64_t x = x1 + i / (ndz * ndy);

        double x_portion_of_value = (x+1) * pow(10.0, y_digits);
        // extreme_debug_log << "y: " << y << endl;
        double y_portion_of_value = (y+1);
        double z_portion_of_value = (z+1)/pow(10.0, z_digits-1);
        // extreme_debug_log << "x_portion_of_value: " << x_portion_of_value << " y_portion_of_value: " << y_portion_of_value << " z_portion_of_value: " << z_portion_of_value << endl;

        double val = (x+1) * pow(10.0, y_digits) + (y+1) +(z+1)/pow(10.0, z_digits-1);
        if(rank == 1 && i == chunk_vol-1){
            extreme_debug_log << "val: " << val << " x_portion_of_value: " << x_portion_of_value << " y_portion_of_value: " << y_portion_of_value << " z_portion_of_value: " << z_portion_of_value << endl;

            // extreme_debug_log << "z_portion_of_value: " << z_portion_of_value << " val: " << val << endl;
        }
        data_vct[i] = val;
    }
}

void generate_data_for_proc(const md_catalog_var_entry &var,
                        int rank, const vector<md_dim_bounds> &bounding_box, vector<double> &data_vct,
                        uint64_t ndx, uint64_t ndy, uint64_t ndz, uint64_t chunk_vol)
{

    // data_vct.reserve(chunk_vol);
    uint64_t ny = var.dims.at(1).max - var.dims.at(1).min + 1;
    uint64_t nz = var.dims.at(2).max - var.dims.at(2).min + 1;

   	generate_data_for_proc(ny, nz, rank, bounding_box, data_vct, ndx, ndy, ndz, chunk_vol);

}

void generate_data_for_proc2D(const md_catalog_var_entry &var,
                        int rank, const vector<md_dim_bounds> &bounding_box, vector<double> &data_vct,
                        uint64_t ndx, uint64_t ndy, uint64_t chunk_vol)
{
    uint64_t ny = var.dims.at(1).max - var.dims.at(1).min + 1;

    int number = ny;

    int y_digits = 0;
    while (number > 0) {
        number /= 10;
        y_digits++;
    }
    y_digits++;

    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy <<  endl;

    extreme_debug_log << "y1: " << bounding_box.at(1).min << " x1: " << bounding_box.at(0).min << endl;

    uint64_t x1 = bounding_box.at(0).min;
    uint64_t y1 = bounding_box.at(1).min;

    for(int i=0; i<chunk_vol; i++) {

        uint64_t y = y1 + i%ndy;
        uint64_t x = x1 + i / (ndy);

        double x_portion_of_value = (x+1) * pow(10.0, y_digits);
        double y_portion_of_value = (y+1);

        double val = x_portion_of_value + y_portion_of_value;
        if(rank == 1 && i == chunk_vol-1){
            extreme_debug_log << "val: " << val << " x_portion_of_value: " << x_portion_of_value << " y_portion_of_value: " << y_portion_of_value << endl;
        }
        data_vct[i] = val;
    }

    // extreme_debug_log << "data_vct.size(): " <<
}

static void get_obj_lengths(const md_catalog_var_entry &var,
                    uint64_t &x_width, uint64_t &last_x_width, uint64_t ndx, uint64_t chunk_size)
{

    uint64_t ceph_obj_size = 8000000; //todo

    extreme_debug_log << "chunk size: " << chunk_size << endl;
    uint32_t num_objs_per_chunk = round(chunk_size / ceph_obj_size);
    if(num_objs_per_chunk <= 0) {
        num_objs_per_chunk = 1;
    }
    extreme_debug_log << "num_objs_per_chunk: " << num_objs_per_chunk << endl;
    x_width = round(ndx / num_objs_per_chunk);
    if(x_width <= 0) {
        x_width = 1;
    }
    extreme_debug_log << "x_width: " << x_width << endl;
    last_x_width = ndx - x_width * (num_objs_per_chunk-1);
    if(last_x_width <= 0) {
        num_objs_per_chunk = num_objs_per_chunk + floor( (last_x_width-1) / x_width);
        last_x_width = ndx - x_width * (num_objs_per_chunk-1);
    }
}

int write_output(std::map <string, vector<double>> &data_outputs, int rank, const md_catalog_run_entry &run, const std::string &objector_funct_name,
                const md_catalog_var_entry &var, const vector<md_dim_bounds> &bounding_box, vector<double> &data_vct)
{
    int rc;

    uint64_t ndx = (var.dims[0].max - var.dims[0].min + 1) / run.npx; //ndx = nx / npx
    uint64_t ndy = 1;
    uint64_t ndz = 1;
    uint64_t chunk_vol;
    uint64_t chunk_size;

    int num_ranks;
    if(var.num_dims == 3) {
        num_ranks = run.npx*run.npy*run.npz;
        ndy = (var.dims[1].max - var.dims[1].min + 1) / run.npy; //ndx = nx / npx
        ndz = (var.dims[2].max - var.dims[2].min + 1) / run.npz; //ndx = nx / npx
    }
    else if(var.num_dims == 2) {
        num_ranks = run.npx*run.npy;
        ndy = (var.dims[1].max - var.dims[1].min + 1) / run.npy; //ndx = nx / npx
    }
    else if(var.num_dims == 1) {
        num_ranks = run.npx;
    }
    chunk_vol = ndx * ndy * ndz;
    chunk_size = chunk_vol * var.data_size;
    extreme_debug_log << "num_ranks: " << num_ranks << endl;
    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy << " ndz: " << ndz << endl;
    extreme_debug_log << "nx: " << (var.dims[0].max - var.dims[0].min) << " ny: " << (var.dims[1].max - var.dims[1].min) << " ndz: " << (var.dims[2].max - var.dims[2].min) << endl;

    std::vector<string> obj_names;

    uint64_t x_width;
    uint64_t last_x_width;

    if(var.num_dims == 3) {
        if (extreme_debug_logging) {
            printf("Bounding box 2: (%d, %d, %d),(%d, %d, %d)\n",bounding_box[0].min,bounding_box[1].min,bounding_box[2].min,bounding_box[0].max,bounding_box[1].max,bounding_box[2].max);
        }
    }

    //retrieve the object names for the proc's chunk
    rc = boundingBoxToObjNames(run, var, bounding_box, obj_names);
    if (rc != RC_OK) {
        error_log << "Error doing the bounding box to obj names, returning \n";
        return RC_ERR;
    }

    extreme_debug_log << "about to try generating data \n";
    data_vct.resize(chunk_vol);

    //generate the data for the process (which will later be read in from the data file)
    if(var.num_dims == 3) {
        generate_data_for_proc(var, rank, bounding_box, data_vct, ndx, ndy, ndz, chunk_vol);
    }
    else if (var.num_dims == 2) {
        generate_data_for_proc2D(var, rank, bounding_box, data_vct, ndx, ndy, chunk_vol);
    }
    extreme_debug_log << "data_vct.size(): " << data_vct.size() << endl;

    //figure out where to slice the data vector for each object
    get_obj_lengths(var, x_width, last_x_width, ndx, chunk_size);
    extreme_debug_log << "x_width: " << x_width << " last_x_width: " << last_x_width << endl;

    //slice the data array for each object
    for(int i=0; i<obj_names.size(); i++) {
        extreme_debug_log << "obj names.size(): " << obj_names.size() << endl;

        if(i != obj_names.size()-1) {

            vector<double>::const_iterator first = data_vct.begin() + x_width * ndy * ndz * i;
            vector<double>::const_iterator last = data_vct.begin() + x_width * ndy * ndz * (i+1);
            vector<double> temp(first, last);
            extreme_debug_log << "temp.size(): " << temp.size() << endl;

            extreme_debug_log << "obj name: " << obj_names[i] << " val 0: " << temp[0] << endl;
            data_outputs[obj_names[i]] = temp;
            extreme_debug_log << "storing data for " << obj_names[i] << endl;
        }
        else {
            vector<double>::const_iterator first = data_vct.end() - last_x_width * ndy * ndz;
            vector<double>::const_iterator last = data_vct.end();
            vector<double> temp(first, last);

            data_outputs[obj_names[i]] = temp;
            extreme_debug_log << "temp.size(): " << temp.size() << endl;

            extreme_debug_log << "storing data for " << obj_names[i] << endl;

        }
    }
    extreme_debug_log << "rank: " << rank << endl;
    return rc;
}

static void getFirstAndLastIndex(const md_catalog_run_entry &run, const md_catalog_var_entry &var, const vector<uint64_t> &offsets_and_counts,
                        uint64_t &first_index, uint64_t &last_index)
{

    uint64_t ndx = (var.dims[0].max - var.dims[0].min + 1) / run.npx; //ndx = nx / npx
    uint64_t ndy = (var.dims.at(1).max - var.dims.at(1).min + 1) / run.npy;
    uint64_t ndz = (var.dims.at(2).max - var.dims.at(2).min + 1) / run.npz;
    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy << " ndz: " << ndz << endl;

    uint64_t start_x = offsets_and_counts.at(0);
    uint64_t start_y = offsets_and_counts.at(2);
    uint64_t start_z = offsets_and_counts.at(4);
    uint64_t count_x = offsets_and_counts.at(1);
    uint64_t count_y = offsets_and_counts.at(3);
    uint64_t count_z = offsets_and_counts.at(5);

    if(extreme_debug_logging) {
        printf("Offsets and counts: x: (%d, %d), y: (%d, %d), z: (%d, %d)\n",offsets_and_counts[0],offsets_and_counts[1],offsets_and_counts[2],offsets_and_counts[3],offsets_and_counts[4],offsets_and_counts[5]);
    }

    uint64_t end_x = start_x + count_x - 1;
    uint64_t end_y = start_y + count_y - 1;
    uint64_t end_z = start_z + count_z - 1;

    first_index = start_z + ndz*(start_y + ndy*start_x);
    last_index = end_z + ndz*(end_y + ndy*end_x);
    extreme_debug_log << "last_index: " << last_index << endl;
    extreme_debug_log << "end_z: " << end_z << " end_y: " << end_y << " end_x: " << end_x << endl;

}

static void getFirstAndLastIndex2D(const md_catalog_run_entry &run, const md_catalog_var_entry &var, const vector<uint64_t> &offsets_and_counts,
                        uint64_t &first_index, uint64_t &last_index)
{
    extreme_debug_log << "var.dims.size(): " << var.dims.size() << endl;

    uint64_t ndx = (var.dims[0].max - var.dims[0].min + 1) / run.npx; //ndx = nx / npx
    uint64_t ndy = (var.dims.at(1).max - var.dims.at(1).min + 1) / run.npy;
    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy << endl;

    // extreme_debug_log << "got here \n";

    uint64_t start_x = offsets_and_counts.at(0);
    uint64_t start_y = offsets_and_counts.at(2);
    uint64_t count_x = offsets_and_counts.at(1);
    uint64_t count_y = offsets_and_counts.at(3);
    if(extreme_debug_logging) {
        printf("Offsets and counts: x: (%d, %d), y: (%d, %d)\n",offsets_and_counts[0],offsets_and_counts[1],offsets_and_counts[2],offsets_and_counts[3]);
    }

    uint64_t end_x = start_x + count_x - 1;
    uint64_t end_y = start_y + count_y - 1;

    first_index = start_y + ndy*start_x;
    last_index = end_y + ndy*end_x;
    extreme_debug_log << "end_y: " << end_y << " end_x: " << end_x << endl;
}

int write_output(std::map <string, vector<double>> &data_outputs, int rank, const md_catalog_run_entry &run, const std::string &objector_funct_name,
                const md_catalog_var_entry &var, const vector<md_dim_bounds> &bounding_box, const vector<double> &data_vct)
{
    int rc;

    uint64_t ndx = (var.dims[0].max - var.dims[0].min + 1) / run.npx; //ndx = nx / npx
    uint64_t ndy = 1;
    uint64_t ndz = 1;
    uint64_t chunk_vol;
    uint64_t chunk_size;

    int num_ranks;
    if(var.num_dims == 3) {
        num_ranks = run.npx*run.npy*run.npz;
        ndy = (var.dims[1].max - var.dims[1].min + 1) / run.npy; //ndx = nx / npx
        ndz = (var.dims[2].max - var.dims[2].min + 1) / run.npz; //ndx = nx / npx
    }
    else if(var.num_dims == 2) {
        num_ranks = run.npx*run.npy;
        ndy = (var.dims[1].max - var.dims[1].min + 1) / run.npy; //ndx = nx / npx
    }
    else if(var.num_dims == 1) {
        num_ranks = run.npx;
    }
    chunk_vol = ndx * ndy * ndz;
    chunk_size = chunk_vol * var.data_size;
    extreme_debug_log << "num_ranks: " << num_ranks << endl;
    extreme_debug_log << "ndx: " << ndx << " ndy: " << ndy << " ndz: " << ndz << endl;
    extreme_debug_log << "nx: " << (var.dims[0].max - var.dims[0].min) << " ny: " << (var.dims[1].max - var.dims[1].min) << " ndz: " << (var.dims[2].max - var.dims[2].min) << endl;

    std::vector<string> obj_names;

    uint64_t x_width;
    uint64_t last_x_width;

    if(var.num_dims == 3) {
        if (extreme_debug_logging) {
            printf("Bounding box 2: (%d, %d, %d),(%d, %d, %d)\n",bounding_box[0].min,bounding_box[1].min,bounding_box[2].min,bounding_box[0].max,bounding_box[1].max,bounding_box[2].max);
        }
    }

    //retrieve the object names for the proc's chunk
    rc = boundingBoxToObjNames(run, var, bounding_box, obj_names);
    if (rc != RC_OK) {
        error_log << "Error doing the bounding box to obj names, returning \n";
        return RC_ERR;
    }

    extreme_debug_log << "about to try generating data \n";

    //figure out where to slice the data vector for each object
    get_obj_lengths(var, x_width, last_x_width, ndx, chunk_size);
    extreme_debug_log << "x_width: " << x_width << " last_x_width: " << last_x_width << endl;

    //slice the data array for each object
    for(int i=0; i<obj_names.size(); i++) {
        extreme_debug_log << "obj names.size(): " << obj_names.size() << endl;

        if(i != obj_names.size()-1) {

            vector<double>::const_iterator first = data_vct.begin() + x_width * ndy * ndz * i;
            vector<double>::const_iterator last = data_vct.begin() + x_width * ndy * ndz * (i+1);
            vector<double> temp(first, last);
            extreme_debug_log << "temp.size(): " << temp.size() << endl;

            extreme_debug_log << "obj name: " << obj_names[i] << " val 0: " << temp[0] << endl;
            data_outputs[obj_names[i]] = temp;
            extreme_debug_log << "storing data for " << obj_names[i] << endl;
        }
        else {
            vector<double>::const_iterator first = data_vct.end() - last_x_width * ndy * ndz;
            vector<double>::const_iterator last = data_vct.end();
            vector<double> temp(first, last);

            data_outputs[obj_names[i]] = temp;
            extreme_debug_log << "temp.size(): " << temp.size() << endl;

            extreme_debug_log << "storing data for " << obj_names[i] << endl;

        }
    }
    extreme_debug_log << "rank: " << rank << endl;
    return rc;
}

// int retrieve_obj_names_and_data(std::map <string, vector<double>> data_outputs, const md_catalog_run_entry &run,
//                       const md_catalog_var_entry &var, const vector<md_dim_bounds> &bounding_box,
//                       vector<string> &obj_names, vector<uint64_t> &offsets_and_counts

// 					  )
int retrieve_obj_names_and_data(std::map <string, vector<double>> data_outputs, const md_catalog_run_entry &run,
                      const md_catalog_var_entry &var, const vector<md_dim_bounds> &bounding_box
					  )
{
    int rc;

    string errmsg;

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////this is just for the data/debugging purposes////////////////////////////////////////////////////////////////////////////////////
    uint64_t ny = 0;
    uint64_t nz = 0;

    if (var.num_dims > 1) {
        ny = (var.dims.at(1).max - var.dims.at(1).min + 1);
    }
    if (var.num_dims > 2) {
        nz = (var.dims.at(2).max - var.dims.at(2).min + 1);
    }

    // extreme_debug_log << "got here 2\n";

    int number = ny;
    int y_digits = 0;
    while (number > 0) {
        number /= 10;
        y_digits++;
    }
    y_digits++;

    int z_digits = 0;
    int number2 = nz;

    while (number2 > 0) {
        number2 /= 10;
        z_digits++;
    }
    z_digits++;

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

        // extreme_debug_log << "got here 4 \n";

	vector<string> obj_names;
	vector<uint64_t> offsets_and_counts;

    extreme_debug_log << "bounding box size: " << bounding_box.size() << endl;
    // printf("Bounding box 3: (%d, %d, %d),(%d, %d, %d)\n",bounding_box[0].min,bounding_box[1].min,bounding_box[2].min,bounding_box[0].max,bounding_box[1].max,bounding_box[2].max);

    // extreme_debug_log << "got here 3\n";

    rc = boundingBoxToObjNamesAndCounts(run, var, bounding_box, obj_names, offsets_and_counts);
    if (rc != RC_OK) {
        error_log << "error with boundingBoxToObjNamesAndCounts \n";
        return rc;
    }

    // extreme_debug_log << "got here 5 \n";

    // printf("Bounding box: (%d, %d, %d),(%d, %d, %d)\n",bounding_box[0].min,bounding_box[1].min,bounding_box[2].min,bounding_box[0].max,bounding_box[1].max,bounding_box[2].max);

    // extreme_debug_log << "obj_names.size(): " << obj_names.size() << endl;
    // extreme_debug_log << "offsets_and_counts.size(): " << offsets_and_counts.size() << endl;

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////this is just for the data/debugging purposes////////////////////////////////////////////////////////////////////////////////////
    for(int i=0; i<obj_names.size(); i++) {
        vector<uint64_t>::const_iterator first = offsets_and_counts.begin() + (2*var.num_dims)*i;
        vector<uint64_t>::const_iterator last = offsets_and_counts.begin() + (2*var.num_dims)*(i+1);
        vector<uint64_t> my_offsets_and_counts(first, last);

        if(extreme_debug_logging) {
            printf("Offsets and counts: x: (%d, %d), y: (%d, %d), z: (%d, %d)\n",my_offsets_and_counts[0],my_offsets_and_counts[1],my_offsets_and_counts[2],my_offsets_and_counts[3],my_offsets_and_counts[4],my_offsets_and_counts[5]);
        }

        vector<double> data_vct;
        extreme_debug_log << "obj name to find: " << obj_names[i] << endl;
        if(data_outputs.find(obj_names[i]) != data_outputs.end()) {
            data_vct = data_outputs.find(obj_names[i])->second;
            extreme_debug_log << "found the data vector \n";
        }
        else {
            error_log << "error. could not find given object name \n";
            error_log << "obj_name: " << obj_names[i] << endl;

            error_log << "map.size(): " << data_outputs.size() << " and map: \n";
            for (std::map<string,vector<double>>::iterator it=data_outputs.begin(); it!=data_outputs.end(); ++it) {
              string obj_name = it->first;
              vector<double> temp_ary = it->second;
              error_log << "obj_name: " << obj_name << endl;
            }
            return RC_ERR;
        }

        //note: this is just for debug purposes/////////////////////////////////////////////////////////////////////////////////////////////
        uint64_t first_index;
        uint64_t last_index;

        if(var.num_dims == 3) {
            getFirstAndLastIndex(run, var, my_offsets_and_counts, first_index, last_index);
        }
        else if(var.num_dims == 2) {
            getFirstAndLastIndex2D(run, var, my_offsets_and_counts, first_index, last_index);
        }
        extreme_debug_log << "first_index: " << first_index << endl;
        extreme_debug_log << "last_index: " << last_index << endl;
        extreme_debug_log << "data_vct.size(): " << data_vct.size() << endl;

        double first_val = data_vct.at(first_index);
        double last_val = data_vct.at(last_index);
        extreme_debug_log << "first val: " << first_val << " last val: " << last_val << endl;
        extreme_debug_log << "obj name:" << obj_names[i].c_str() << endl;

        uint64_t x1 = first_val / pow(10.0, y_digits) - 1;
        uint64_t x2 = last_val / pow(10.0, y_digits) - 1;
        uint64_t y1 = first_val - (x1+1)* pow(10.0, y_digits) -1;
        uint64_t y2 = last_val - (x2+1)* pow(10.0, y_digits) -1;
        // uint64_t y1 = (int)(first_val-1) % ny;
        // uint64_t y2 = (int)(last_val-1) % ny;
        extreme_debug_log << "ny: " << ny << " first_val-1: " << first_val-1 << " (first_val-1) mod ny: " << (int)(first_val-1) % ny << endl;

        if(var.num_dims == 3) {
            uint64_t z1 = round((first_val - (int)first_val)*pow(10.0, z_digits-1) -1);
            uint64_t z2 = round((last_val - (int)last_val)*pow(10.0, z_digits-1) -1);
            printf("data range for obj %d is: (%d, %d, %d),(%d, %d, %d)\n",i,x1,y1,z1,x2,y2,z2);
        }
        else if(var.num_dims == 2) {
            printf("data range for obj %d is: (%d, %d),(%d, %d)\n",i,x1,y1,x2,y2);
        }

    }
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

    return RC_OK;
}

int retrieveObjNamesAndDataForAttrCatalog(const std::map <string, vector<double>> &data_outputs, const md_catalog_run_entry &run,
										 const vector<md_catalog_var_entry> &var_entries,
                                         uint64_t run_id, uint64_t timestep_id,
                                         uint64_t txn_id,
                                         const vector<md_catalog_var_attribute_entry> &attr_entries )

// int retrieveObjNamesAndDataForAttrCatalog(const std::map <string, vector<double>> &data_outputs, const md_catalog_run_entry &run,
//                                          uint64_t run_id, uint64_t timestep_id,
//                                          uint64_t txn_id,
//                                          const vector<md_catalog_var_attribute_entry> &attr_entries,
//                                          vector<string> &obj_names, vector<uint64_t> &offsets_and_counts
//                                          )
{
    int rc;

    // vector<md_catalog_var_entry> var_entries;
    uint32_t count;

    // extreme_debug_log << "got here 1\n";

    // rc = metadata_catalog_var (server, run_id, timestep_id, txn_id, count, var_entries);
    // if (rc != RC_OK) {
    //     error_log << "Error cataloging the new set of var entries. Proceeding \n";
    //     return RC_ERR;
    // }

    for(int i = 0; i < attr_entries.size(); i++) {
        md_catalog_var_entry matching_var;

        md_catalog_var_attribute_entry attr = attr_entries.at(i);

        int index_var=0;
        while (index_var < var_entries.size() && var_entries.at(index_var).var_id != attr.var_id) {
            index_var++;
        }
        if (var_entries.size() <= index_var) {
            error_log << "error. couldn't find the var with id " << attr.var_id << endl;
            print_var_catalog (var_entries.size(), var_entries);
            return RC_ERR;
        }
        // extreme_debug_log << "got here 3 \n";

        matching_var = var_entries.at(index_var);

        // rc = retrieve_obj_names_and_data(data_outputs, run, matching_var, attr.dims, obj_names, offsets_and_counts);
        rc = retrieve_obj_names_and_data(data_outputs, run, matching_var, attr.dims );
        if (rc != RC_OK) {
            error_log << "error in retrieveObjNamesAndDataForAttrCatalog \n";
            return rc;
        }
    }
    return RC_OK;
}

ext: cpp

max_stars_repo_name: mlawsonca/empress

max_issues_repo_name: mlawsonca/empress

max_forks_repo_name: mlawsonca/empress


text:
#include "bamliquidator.h"
#include "bamliquidator_util.h"

#include <cmath>
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>

#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>

#include <hdf5.h>
#include <hdf5_hl.h>

#include <tbb/blocked_range.h>
#include <tbb/enumerable_thread_specific.h>
#include <tbb/parallel_for.h>
#include <tbb/task_scheduler_init.h>

//#define time_region_parsing
#ifdef time_region_parsing
#include <boost/timer/timer.hpp>
#endif

const size_t region_name_length = 64;

// this Region must match exactly the structure in HDF5
// -- see bamliquidator_batch.py function create_regions_table
struct Region
{
  uint32_t bam_file_key;
  char chromosome[64];
  char region_name[region_name_length];
  uint64_t start;
  uint64_t stop;
  char strand;
  uint64_t count;
  double normalized_count;

  bool is_valid(const std::map<std::string, size_t>& chromosome_to_length)
  {
    const auto it = chromosome_to_length.find(chromosome);
    if ( it != chromosome_to_length.end() )
    {
      if ( stop <= it->second )
      {
        return true;
      }
    }

    return false;
  }
};

std::ostream& operator<<(std::ostream& os, const Region& r)
{
  os << "bam file key " << r.bam_file_key << ' ' << r.chromosome << ' '
     << r.region_name << ' ' << r.start << " -> " << r.stop << ' '
     << r.strand << ' ' << r.normalized_count;
  return os;
}

// default_strand: optional argument, default _ indicates to use
//                 gff strand column or . (both) for .bed region file
std::vector<Region> parse_regions(const std::string& region_file_path,
                                  const std::string& region_format,
                                  const unsigned int bam_file_key,
                                  const std::map<std::string, size_t>& chromosome_to_length,
                                  const char default_strand = '_')
{
  int chromosome_column = 0;
  int name_column = 0;
  int start_column = 0;
  int stop_column = 0;
  int strand_column = 0;
  unsigned int min_columns = 0;

  if (region_format == "gff")
  {
    chromosome_column = 0;
    name_column = 1;
    start_column = 3;
    stop_column = 4;
    strand_column = 6;
    min_columns = 7;
  }
  else if (region_format == "bed")
  {
    chromosome_column = 0;
    name_column = 3;
    start_column = 1;
    stop_column = 2;
    strand_column = 5;
    min_columns = 3;
  }
  else
  {
    throw std::runtime_error("unsupported region file format (" + region_format + " for " + region_file_path
                             + "), please supply a .gff or .bed file");
  }

  std::ifstream region_file(region_file_path.c_str());
  if (!region_file.is_open())
  {
    throw std::runtime_error("failed to open region_file " + region_file_path);
  }

  std::vector<Region> regions;
  int line_number = 1;
  for(std::string line; std::getline(region_file, line); ++line_number)
  {
    std::vector<std::string> columns;
    boost::split(columns, line, boost::is_any_of("\t"));
    if (columns.size() < min_columns)
    {
      std::stringstream ss;
      ss << "Not enough columns parsing line " << line_number << " '" << line << "' of " << region_file_path;
      throw std::runtime_error(ss.str());
    }
    Region region;
    region.bam_file_key = bam_file_key;
    copy(region.chromosome,  columns[chromosome_column], sizeof(Region::chromosome));
    if (columns.size() > name_column)
    {
      copy(region.region_name, columns[name_column], sizeof(Region::region_name));
      if (columns[name_column].size() >= sizeof(Region::region_name))
      {
        Logger::warn() << "Truncated region on line " << line_number << " from '" << columns[name_column] << "' to '" << region.region_name << "'";
      }
    }
    else
    {
      copy(region.region_name, "", sizeof(Region::region_name));
    }
    region.start = boost::lexical_cast<uint64_t>(columns[start_column]);
    region.stop  = boost::lexical_cast<uint64_t>(columns[stop_column]);
    if (region.start > region.stop)
    {
      std::swap(region.start, region.stop);
    }

    if (columns.size() > strand_column)
    {
      if (columns[strand_column].size() != 1)
      {
        std::stringstream ss;
        ss << "error parsing strand: '" << columns[strand_column] << "' on line " << line_number;
        throw std::runtime_error(ss.str());
      }
      region.strand = default_strand == '_'
                    ? columns[strand_column][0]
                    : default_strand;
    }
    else
    {
      region.strand = default_strand == '_'
                    ? '.'
                    : default_strand;
    }
    region.count = 0;
    region.normalized_count = 0.0;

    if (region.is_valid(chromosome_to_length))
    {
      regions.push_back(region);
    }
    else
    {
      Logger::warn() << "Excluding invalid region on line " << line_number << ": " << region;
    }
  }

  return regions;
}

void write(hid_t& file, std::vector<Region>& regions)
{
  const size_t record_size = sizeof(Region);

  size_t record_offset[] = { HOFFSET(Region, bam_file_key),
                             HOFFSET(Region, chromosome),
                             HOFFSET(Region, region_name),
                             HOFFSET(Region, start),
                             HOFFSET(Region, stop),
                             HOFFSET(Region, strand),
                             HOFFSET(Region, count),
                             HOFFSET(Region, normalized_count) };

  size_t field_sizes[] = { sizeof(Region::bam_file_key),
                           sizeof(Region::chromosome),
                           sizeof(Region::region_name),
                           sizeof(Region::start),
                           sizeof(Region::stop),
                           sizeof(Region::strand),
                           sizeof(Region::count),
                           sizeof(Region::normalized_count) };

  herr_t status = H5TBappend_records(file, "region_counts", regions.size(), record_size, record_offset,
                                     field_sizes, regions.data());
  if (status != 0)
  {
    std::stringstream ss;
    ss << "Error appending record, status = " << status;
    throw std::runtime_error(ss.str());
  }
}

class Liquidator
{
public:
  Liquidator(const std::string& bam_file_path):
    bam_file_path(bam_file_path),
    fp(nullptr),
    bamidx(nullptr)
  {
    init();
  }

  Liquidator(const Liquidator& other):
    bam_file_path(other.bam_file_path),
    fp(nullptr),
    bamidx(nullptr)
  {
    init();
  }

  Liquidator& operator=(const Liquidator& other) = delete;

  ~Liquidator()
  {
    bam_index_destroy(bamidx);
    samclose(fp);
  }

  double liquidate(const std::string& chromosome, int start, int stop, char strand, unsigned int extension)
  {
    std::vector<double> counts = ::liquidate(fp, bamidx, chromosome, start, stop, strand, 1, extension);
    if (counts.size() != 1)
    {
      throw std::runtime_error("liquidate failed to provide exactly one count (count is " +
        boost::lexical_cast<std::string>(counts.size()) + ")");
    }
    return counts[0];
  }

private:
  std::string bam_file_path;
  samfile_t* fp;
  bam_index_t* bamidx;

  void init()
  {
    fp = samopen(bam_file_path.c_str(),"rb",0);
    if(fp == NULL)
    {
      throw std::runtime_error("samopen() error with " + bam_file_path);
    }

    bamidx = bam_index_load(bam_file_path.c_str());
    if (bamidx == NULL)
    {
      throw std::runtime_error("bam_index_load() error with " + bam_file_path);
    }
  }
};

typedef tbb::enumerable_thread_specific<Liquidator,
                                        tbb::cache_aligned_allocator<Liquidator>,
                                        tbb::ets_key_per_instance>
        Liquidators;

void liquidate_regions(std::vector<Region>& regions, const std::string& bam_file_path,
                       size_t region_begin, size_t region_end, unsigned int extension,
                       Liquidators& liquidators)
{
  Liquidator& liquidator = liquidators.local();

  for (size_t i=region_begin; i < region_end; ++i)
  {
    try
    {
      regions[i].count = liquidator.liquidate(regions[i].chromosome,
                                              regions[i].start,
                                              regions[i].stop,
                                              regions[i].strand,
                                              extension);
    } catch(const std::exception& e)
    {
      Logger::error() << "Aborting because failed to parse region " << i+1 << " (" << regions[i] << ") due to error: "
                      << e.what();
      throw;
    }
  }
}

void liquidate_and_write(hid_t& file, std::vector<Region>& regions,
                         unsigned int extension, const std::string& bam_file_path)
{
  Liquidators liquidators((Liquidator(bam_file_path)));

  tbb::parallel_for(
    tbb::blocked_range<int>(0, regions.size(), 1),
    [&](const tbb::blocked_range<int>& range)
    {
      liquidate_regions(regions, bam_file_path, range.begin(), range.end(), extension, liquidators);
    },
    tbb::auto_partitioner());

  write(file, regions);
}

int main(int argc, char* argv[])
{
  try
  {
    if (argc < 13 || argc % 2 != 1)
    {
      std::cerr << "usage: " << argv[0] << " number_of_threads region_file gff_or_bed_format extension bam_file bam_file_key hdf5_file "
                << "log_file write_warnings_to_stderr strand chr1 length1 ...\n"
        << "\ne.g. " << argv[0] << " /grail/annotations/HG19_SUM159_BRD4_-0_+0.gff gff"
        << "\n      /ifs/labs/bradner/bam/hg18/mm1s/04032013_D1L57ACXX_4.TTAGGC.hg18.bwt.sorted.bam 137 counts.hdf5 "
        << "\n      output/log.txt 1 _ chr1 247249719 chr2 242951149 chr3 199501827\n"
        << "\nstrand value of _ means use strand that is specified in region file (and use . if strand not specified in region file)."
        << "\nnumber of threads <= 0 means use a number of threads equal to the number of logical cpus."
        << "\nnote that this application is intended to be run from bamliquidator_batch.py -- see"
        << "\nhttps://github.com/BradnerLab/pipeline/wiki for more information"
        << std::endl;
      return 1;
    }

    const int number_of_threads = boost::lexical_cast<int>(argv[1]);
    const std::string region_file_path = argv[2];
    const std::string region_format = argv[3];
    const unsigned int extension = boost::lexical_cast<unsigned int>(argv[4]);
    const std::string bam_file_path = argv[5];
    const unsigned int bam_file_key = boost::lexical_cast<unsigned int>(argv[6]);
    const std::string hdf5_file_path = argv[7];
    const std::string log_file_path = argv[8];
    const bool write_warnings_to_stderr = boost::lexical_cast<bool>(argv[9]);
    const char strand = boost::lexical_cast<char>(argv[10]);
    const std::vector<std::pair<std::string, size_t>> chromosome_lengths = extract_chromosome_lengths(argc, argv, 11);

    tbb::task_scheduler_init init( number_of_threads <= 0
                                 ? tbb::task_scheduler_init::automatic
                                 : number_of_threads);

    Logger::configure(log_file_path, write_warnings_to_stderr);

    hid_t h5file = H5Fopen(hdf5_file_path.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
    if (h5file < 0)
    {
      Logger::error() << "Failed to open H5 file " << hdf5_file_path;
      return 3;
    }

    #ifdef time_region_parsing
    boost::timer::cpu_timer timer;
    #endif

    std::map<std::string, size_t> chromosome_to_length;
    for (auto& chr_length : chromosome_lengths)
    {
      chromosome_to_length[chr_length.first] = chr_length.second;
    }

    std::vector<Region> regions = parse_regions(region_file_path,
                                                region_format,
                                                bam_file_key,
                                                chromosome_to_length,
                                                strand);
    #ifdef time_region_parsing
    timer.stop();
    std::cout << "parsing regions took" << timer.format() << std::endl;
    #endif
    if (regions.size() == 0)
    {
      Logger::warn() << "No valid regions detected in " << region_file_path;
      return 0;
    }

    liquidate_and_write(h5file, regions, extension, bam_file_path);

    H5Fclose(h5file);

    return 0;
  }
  catch(const std::exception& e)
  {
    Logger::error() << "Unhandled exception: " << e.what();

    return 4;
  }
}

/* The MIT License (MIT)

   Copyright (c) 2013 John DiMatteo (jdimatteo@gmail.com)

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
 */

max_stars_repo_name: Drmirdeep/pipeline

max_issues_repo_name: GodloveD/pipeline

max_forks_repo_name: GodloveD/pipeline


text:
// (C) 2020 christian@schladetsch.com

#pragma once

#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>

#include "GoRepo/Repo.hpp"

namespace GoRepo {

namespace fs = boost::filesystem;

class RepoSet {
private:
    std::vector<Repo> _repos;

public:
    explicit RepoSet(const fs::path& root);

    const std::vector<Repo> &GetRepos() const { return _repos; }
};

}  // namespace GoRepo

max_stars_repo_name: cschladetsch/GoRepo

max_issues_repo_name: cschladetsch/GoRepo

max_forks_repo_name: cschladetsch/GoRepo


text:
//
//  hsapi.cpp
//  libhsapi
//
//  Created by horned-sungem on 2018/7/2.
//  Copyright © 2018年 Senscape. All rights reserved.
//

#include <algorithm>
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <cerrno>
#include <fstream>
#include <streambuf>
#include <string>
#include <utility>
#include <vector>

#include "hs_cpp.hpp"
#include "exception.hpp"
#include "exception_util.hpp"
#include "hsapi.hpp"
#include "log.hpp"
#include "tensor.hpp"

namespace hs {

HS::HS(int device_index, Device::LogLevel log_level,
       const std::string &cnn_type, const std::string &graph_file_path,
       const std::string &category_file_path, const int network_dimension,
       const std::vector<float> &mean, const float &scale, const int &top_n)
    : device_(nullptr), graph_(nullptr), tensor_(nullptr),
      device_index_(device_index), log_level_(log_level), cnn_type_(cnn_type),
      network_dimension_(network_dimension), mean_(mean), scale_(scale),
      top_n_(top_n), user_param_(nullptr) {
    initDevice();
    loadGraph(graph_file_path);
    loadCategories(category_file_path);
    tensor_ = std::make_shared<Tensor>(
        std::pair<int, int>(network_dimension_, network_dimension_), mean_,
        scale_);
    result_ = std::make_shared<Result>(cnn_type_);
}

HS::~HS() {}

void HS::classify() {
    assert(graph_->getHandle() != nullptr);
    try {
        uint16_t *probabilities;
        unsigned int length;
        int ret = hsGetResult(graph_->getHandle(),
                              reinterpret_cast<void **>(&probabilities),
                              &length, &user_param_);
        ExceptionUtil::tryToThrowHsException(ret);
        std::vector<uint16_t> result_vector(
            reinterpret_cast<uint16_t *>(probabilities),
            reinterpret_cast<uint16_t *>(probabilities) + length);
        ItemsPtr items = std::make_shared<Items>();
        for (size_t index = 0; index < length / 2; ++index) {
            float fp32;
#ifdef SUPPORT_F16C
            fp32 = _cvtsh_ss(probabilities[index]);
#else
            Tensor::fp16tofp32(&fp32, probabilities[index]);
#endif
            Item item;
            item.category = categories_[index];
            item.probability = fp32;
            items->push_back(item);
        }

        auto cmp = [](const Item &a, const Item &b) {
            return a.probability > b.probability;
        };
        std::sort(items->begin(), items->end(), cmp);

        if (!result_->getClassificationResult()->items.empty()) {
            result_->getClassificationResult()->items.clear();
        }

        for (auto i : *items) {
            result_->setClassificationResult(i);
            if (static_cast<int>(
                    result_->getClassificationResult()->items.size()) ==
                top_n_) {
                break;
            }
        }
        result_->setClassificationResult(graph_->getTimeTaken());
        device_->monitorThermal();
    } catch (HsMyriadError &e) {
        HS_ERROR_STREAM(e.what());
        std::string debug_info = graph_->getDebugInfo();
        HS_ERROR_STREAM("myriad debug info: " << debug_info);
    } catch (HSException &e) {
        HS_ERROR_STREAM(e.what());
    }
}

void HS::detect() {
    assert(graph_->getHandle() != nullptr);
    try {
        uint16_t *result;
        unsigned int length;
        int ret =
            hsGetResult(graph_->getHandle(), reinterpret_cast<void **>(&result),
                        &length, &user_param_);
        ExceptionUtil::tryToThrowHsException(ret);

        std::vector<uint16_t> result16_vector(
            reinterpret_cast<uint16_t *>(result),
            reinterpret_cast<uint16_t *>(result) + length / 2);
        std::vector<float> result32_vector;

        for (auto fp16 : result16_vector) {
            float fp32;
#ifdef SUPPORT_F16C
            fp32 = _cvtsh_ss(fp16);
#else
            Tensor::fp16tofp32(&fp32, fp16);
#endif
            result32_vector.push_back(fp32);
        }

        if (!cnn_type_.compare("tinyyolo_v1")) {
            result_->parseYoloResult(result32_vector, categories_,
                                     tensor_->getImageWidth(),
                                     tensor_->getImageHeight());
            result_->setDetectionResult(graph_->getTimeTaken());
        } else if (!cnn_type_.compare("mobilenetssd")) {
            result_->parseSSDResult(result32_vector, categories_,
                                    tensor_->getImageWidth(),
                                    tensor_->getImageHeight());
            result_->setDetectionResult(graph_->getTimeTaken());
        } else {
            HS_ERROR("unsupported cnn model");
        }
        device_->monitorThermal();
    } catch (HsMyriadError &e) {
        HS_ERROR_STREAM(e.what());
        std::string debug_info = graph_->getDebugInfo();
        HS_ERROR_STREAM("myriad debug info: " << debug_info);
    } catch (std::exception &e) {
        HS_ERROR_STREAM(e.what());
    }
}

void HS::loadTensor(const cv::Mat &image) {
    tensor_->clearTensor();
    tensor_->loadImageData(image);
    assert(graph_->getHandle() != nullptr);
    int ret = hsLoadTensor(graph_->getHandle(), tensor_->raw(),graph_->getId(), tensor_->size(),
                           user_param_);
    ExceptionUtil::tryToThrowHsException(ret);
}

ClassificationResultPtr HS::getClassificationResult() {
    return result_->getClassificationResult();
}

DetectionResultPtr HS::getDetectionResult() {
    return result_->getDetectionResult();
}

void HS::initDevice() {
    device_.reset(
        new Device(device_index_, static_cast<Device::LogLevel>(log_level_)));
}

void HS::loadGraph(const std::string &graph_file_path) {
    std::string graph = getFileContent(graph_file_path);
    graph_.reset(new Graph(device_, graph, network_dimension_));

    HS_INFO("graph is loaded");
}

void HS::loadCategories(const std::string &category_file_path) {
    try {
        std::string content = getFileContent(category_file_path);
        splitIntoLines(content, categories_);
        std::string first = categories_[0];
        boost::trim_right(first);

        if (boost::iequals(first, "classes")) {
            categories_.erase(categories_.begin());
        }
    } catch (int &e) {
        throw HSLoadCategoriesError();
    }
}

void HS::splitIntoLines(const std::string &content,
                        std::vector<std::string> &lines) {
    std::stringstream ss(content);
    std::string line;

    while (std::getline(ss, line, '\n')) {
        lines.push_back(line);
    }
}

std::string HS::getFileContent(const std::string &filename) {
    std::ifstream in(filename.c_str(), std::ios::in | std::ios::binary);

    if (!in) {
        throw(errno);
    }

    std::string content;
    in.seekg(0, std::ios::end);
    content.reserve(in.tellg());
    in.seekg(0, std::ios::beg);
    content.assign(std::istreambuf_iterator<char>(in),
                   std::istreambuf_iterator<char>());
    in.close();
    return content;
}

cv::Mat HS::getImage(bool truthy) {
    assert(graph_->getHandle() != nullptr);
    cv::Mat mat = graph_->getGraphImage(user_param_, scale_, mean_[0] * scale_, truthy);
    tensor_->clearTensor();
    tensor_->loadImageData(mat);
    return mat;
}

} // namespace hs

max_stars_repo_name: HornedSungem/SungemSDK-Cpp

max_issues_repo_name: HornedSungem/SungemSDK-Cpp

max_forks_repo_name: HornedSungem/SungemSDK-Cpp


text:
/**
 * @file clipper.cpp
 * @brief Core CLIPPER algorithm: find dense clusters w.r.t constraints
 * @author Parker Lusk <plusk@mit.edu>
 * @date 3 October 2020
 */

#include <chrono>
#include <iostream>
#include <functional>
#include <limits>
#include <queue>
#include <random>
#include <utility>
#include <vector>

#include <Eigen/Sparse>

#include "clipper/find_dense_cluster.h"
#include "clipper/utils.h"

namespace clipper {

inline void homotopy(Eigen::MatrixXd& Md, const Eigen::MatrixXd& M, const Eigen::MatrixXd& Cb, double d)
{
  // if (M.cols() < 2600) {
    Md = M - d*Cb;
  //   return;
  // }

// #pragma omp parallel for default(none) shared(Md, M, d, Cb)
//   for (size_t c=0; c<M.cols(); ++c) {
//     Md.col(c) = M.col(c) - d*Cb.col(c);
//   }
}

template <typename T, IsEigenBase<T>>
Solution findDenseCluster(const T& _M, const T& C,
                          const Eigen::VectorXd& u0, const Params& params)
{
  const auto t1 = std::chrono::high_resolution_clock::now();
  //
  // Initialization
  //

  const size_t n = _M.cols();

  // Zero out any entry corresponding to an active constraint
  const Eigen::MatrixXd M = _M.cwiseProduct(C);

  // Binary complement of constraint matrix
  const Eigen::MatrixXd Cb = Eigen::MatrixXd::Ones(n,n) - C;

  // one step of power method to have a good scaling of u
  Eigen::VectorXd u = M * u0;
  u /= u.norm();

  // initial value of d
  double d = 0; // zero if there are no active constraints
  Eigen::MatrixXd Cbu = Cb * u;
  const auto idxD = ((Cbu.array()>params.eps) && (u.array()>params.eps));
  if (idxD.sum() > 0) {
    Eigen::MatrixXd Mu = M * u;
    const Eigen::VectorXd num = idxD.select(Mu, std::numeric_limits<double>::infinity());
    const Eigen::VectorXd den = idxD.select(Cbu, 1);
    d = (num.array() / den.array()).minCoeff();
  }

  Eigen::MatrixXd Md = Eigen::MatrixXd(M.rows(), M.cols());
  homotopy(Md, M, Cb, d);

  // initialize memory
  Eigen::VectorXd gradF = Eigen::VectorXd(n);
  Eigen::VectorXd unew = Eigen::VectorXd(n);
  Eigen::VectorXd Mu = Eigen::VectorXd(n);
  Eigen::VectorXd num = Eigen::VectorXd(n);
  Eigen::VectorXd den = Eigen::VectorXd(n);

  //
  // Orthogonal projected gradient ascent with homotopy
  //

  double F = 0; // objective value

  size_t i, j, k; // iteration counters
  for (i=0; i<params.maxoliters; ++i) {
    F = u.transpose() * Md * u; // current objective value

    //
    // Orthogonal projected gradient ascent
    //

    for (j=0; j<params.maxiniters; ++j) {
      gradF = Md * u;

      // if (params.orthogonal) {
      //   // orthogonal projection of gradient onto tangent plane to S^n at u
      //   gradF = gradF - (gradF.transpose() * u) * u;

      //   if (gradF.norm() < params.tol_Fop) break;
      // }

      // double alpha = params.alpha;
      // if (alpha <= 0) {
      //   const auto idxA = ((gradF.array()<-params.eps) && (u.array()>params.eps));
      //   if (idxA.sum()) {
      //     const Eigen::VectorXd num = idxA.select(u, std::numeric_limits<double>::infinity());
      //     const Eigen::VectorXd den = idxA.select(gradF, 1);
      //     alpha = (num.array() / den.array()).abs().minCoeff();
      //   } else {
      //     alpha = std::pow(1.0/params.beta, 3) / gradF.norm();
      //   }
      // }

      double alpha = 1;

      //
      // Backtracking line search on gradient ascent
      //

      double Fnew = 0, deltaF = 0;
      for (k=0; k<params.maxlsiters; ++k) {
        unew = u + alpha * gradF;                     // gradient step
        unew = unew.cwiseMax(0);                      // project onto positive orthant
        unew.normalize();                             // project onto S^n
        Fnew = unew.transpose() * Md * unew;          // new objective value after step
        deltaF = Fnew - F;                            // change in objective value

        if (deltaF < -params.eps) {
          // objective value decreased---we need to backtrack, so reduce step size
          alpha = alpha * params.beta;
        } else {
          break; // obj value increased, stop line search
        }
      }
      const double deltau = (unew - u).norm();

      // update values
      F = Fnew;
      u = unew;

      // check if desired accuracy has been reached by gradient ascent
      if (deltau < params.tol_u || std::abs(deltaF) < params.tol_F) break;
    }

    //
    // Increase d
    //

    Cbu = Cb * u;
    const auto idxD = ((Cbu.array() > params.eps) && (u.array() > params.eps));
    if (idxD.sum() > 0) {
      Mu = M * u;
      num = idxD.select(Mu, std::numeric_limits<double>::infinity());
      den = idxD.select(Cbu, 1);
      const double deltad = (num.array() / den.array()).abs().minCoeff();

      d += deltad;
      homotopy(Md, M, Cb, d);

    } else {
      break;
    }
  }

  //
  // Generate output
  //

  // estimate cluster size using largest eigenvalue
  const int omega = std::round(F);

  // extract indices of nodes in identified dense cluster
  std::vector<int> I = utils::findIndicesOfkLargest(u, omega);

  const auto t2 = std::chrono::high_resolution_clock::now();
  const auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(t2 - t1);
  const double elapsed = static_cast<double>(duration.count()) / 1e9;

  Solution soln;
  soln.t = elapsed;
  soln.ifinal = i;
  std::swap(soln.nodes, I);
  soln.u.swap(u);
  soln.score = F;

  return soln;
}

template <typename T, IsEigenBase<T>>
Solution findDenseCluster(const T& M, const T& C,
                          const Params& params)
{
  return findDenseCluster(M, C, utils::randvec(M.cols()), params);
}

// template specializations for dense and sparse Eigen matrices
template Solution findDenseCluster<Eigen::MatrixXd>(const Eigen::MatrixXd&,
            const Eigen::MatrixXd&, const Eigen::VectorXd&, const Params&);
template Solution findDenseCluster<SpMat>(const SpMat&,
            const SpMat&, const Eigen::VectorXd&, const Params&);
template Solution findDenseCluster<Eigen::MatrixXd>(const Eigen::MatrixXd&,
            const Eigen::MatrixXd&, const Params&);
template Solution findDenseCluster<SpMat>(const SpMat&,
            const SpMat&, const Params&);

// methods for sparse matrices from scoreSparseConsistency

// sparse method similar to the above with the difference that M,C have zero
// elements along the diagonal
Solution findDenseClusterOfSparseGraph(const SpMat &M, const SpMat &C,
                                       const Eigen::VectorXd &u0,
                                       const Params &params) {
  // std::cout << "number of threads : " << Eigen::nbThreads() << std::endl;
  const auto t1 = std::chrono::high_resolution_clock::now();
  //
  // Initialization
  //

  const size_t n = M.cols();

  // un-necessary compute
  // Zero out any entry corresponding to an active constraint
  // const Eigen::MatrixXd M = _M.cwiseProduct(C);

  // this needs to be replaced
  // Binary complement of constraint matrix
  // const Eigen::MatrixXd Cb = Eigen::MatrixXd::Ones(n, n) - C;
  const Eigen::VectorXd ones = Eigen::VectorXd::Ones(n);

  // one step of power method to have a good scaling of u
  Eigen::VectorXd u = M * u0 + u0; // since M here is not diagonal
  // Eigen::VectorXd u = u0;
  u /= u.norm();

  // initial value of d
  double d = 0; // zero if there are no active constraints
  // Eigen::MatrixXd Cbu = Cb * u;
  Eigen::MatrixXd Cbu = ones * u.sum() - C * u - u;
  const auto idxD = ((Cbu.array() > params.eps) && (u.array() > params.eps));
  if (idxD.sum() > 0) {
    Eigen::MatrixXd Mu = M * u + u;
    const Eigen::VectorXd num =
        idxD.select(Mu, std::numeric_limits<double>::infinity());
    const Eigen::VectorXd den = idxD.select(Cbu, 1);
    d = (num.array() / den.array()).minCoeff();
  }

  // this should be replaced with an efficient representation
  // Md = M - d * Cb;
  // Eigen::MatrixXd Md = Eigen::MatrixXd(M.rows(), M.cols());
  // homotopy(Md, M, Cb, d);

  // initialize memory
  Eigen::VectorXd gradF = Eigen::VectorXd(n);
  Eigen::VectorXd gradFnew = Eigen::VectorXd(n);
  Eigen::VectorXd unew = Eigen::VectorXd(n);
  Eigen::VectorXd Mu = Eigen::VectorXd(n);
  Eigen::VectorXd num = Eigen::VectorXd(n);
  Eigen::VectorXd den = Eigen::VectorXd(n);

  //
  // Orthogonal projected gradient ascent with homotopy
  //

  double F = 0; // objective value

  size_t i, j, k; // iteration counters
  for (i = 0; i < params.maxoliters; ++i) {
    // F = u.transpose() * Md * u; // current objective value
    // gradF = (1 + d) * (M * u + u) - d * ones * u.sum();
    gradF = (1 + d) * u - d * ones * u.sum() + M * u + d * C * u;
    F = u.dot(gradF);
    // std::cout << std::endl;
    // std::cout << "obj F: " << F << " - dval: " << d << std::endl;
    //
    // Orthogonal projected gradient ascent
    //

    for (j = 0; j < params.maxiniters; ++j) {
      // gradF = Md * u;

      // if (params.orthogonal) {
      //   // orthogonal projection of gradient onto tangent plane to S^n at u
      //   gradF = gradF - (gradF.transpose() * u) * u;

      //   if (gradF.norm() < params.tol_Fop) break;
      // }

      // double alpha = params.alpha;
      // if (alpha <= 0) {
      //   const auto idxA = ((gradF.array()<-params.eps) &&
      //   (u.array()>params.eps)); if (idxA.sum()) {
      //     const Eigen::VectorXd num = idxA.select(u,
      //     std::numeric_limits<double>::infinity()); const Eigen::VectorXd den
      //     = idxA.select(gradF, 1); alpha = (num.array() /
      //     den.array()).abs().minCoeff();
      //   } else {
      //     alpha = std::pow(1.0/params.beta, 3) / gradF.norm();
      //   }
      // }

      double alpha = 1;

      //
      // Backtracking line search on gradient ascent
      //

      double Fnew = 0, deltaF = 0;
      for (k = 0; k < params.maxlsiters; ++k) {
        unew = u + alpha * gradF; // gradient step
        unew = unew.cwiseMax(0);  // project onto positive orthant
        unew.normalize();         // project onto S^n
        // Fnew = unew.transpose() * Md * unew; // new objective value after
        // step
        // gradFnew = (1 + d) * (M * unew + unew) - d * ones * unew.sum();
        gradFnew =
            (1 + d) * unew - d * ones * unew.sum() + M * unew + d * C * unew;
        Fnew = unew.dot(gradFnew);

        deltaF = Fnew - F; // change in objective value

        if (deltaF < -params.eps) {
          // objective value decreased---we need to backtrack, so reduce step
          // size
          alpha = alpha * params.beta;
        } else {
          // std::cout << "breaking at " << k << " out of ls " <<
          // params.maxlsiters
          //           << std::endl;
          break; // obj value increased, stop line search
        }
      }
      const double deltau = (unew - u).norm();

      // std::cout << "Fnew: " << Fnew << " Unew: " << unew.sum() << std::endl;
      // update values
      F = Fnew;
      u = unew;
      // gradF = gradFnew;

      // check if desired accuracy has been reached by gradient ascent
      if (deltau < params.tol_u || std::abs(deltaF) < params.tol_F) {
        // std::cout << "breaking at " << j << " out of in " <<
        // params.maxiniters
        //           << std::endl;
        break;
      }
    }

    //
    // Increase d
    //

    // Cbu = Cb * u;
    Cbu = ones * u.sum() - C * u - u;
    const auto idxD = ((Cbu.array() > params.eps) && (u.array() > params.eps));
    if (idxD.sum() > 0) {
      Mu = M * u + u;
      num = idxD.select(Mu, std::numeric_limits<double>::infinity());
      den = idxD.select(Cbu, 1);
      const double deltad = (num.array() / den.array()).abs().minCoeff();

      // std::cout << "delta d:" << deltad << std::endl;
      d += deltad;
      // homotopy(Md, M, Cb, d);

    } else {
      // std::cout << "breaking at " << i << " out of oo " << params.maxoliters
      //           << std::endl;
      break;
    }
  }
  //
  // Generate output
  //

  // estimate cluster size using largest eigenvalue
  const int omega = std::round(F);

  // extract indices of nodes in identified dense cluster
  std::vector<int> I = utils::findIndicesOfkLargest(u, omega);

  const auto t2 = std::chrono::high_resolution_clock::now();
  const auto duration =
      std::chrono::duration_cast<std::chrono::nanoseconds>(t2 - t1);
  const double elapsed = static_cast<double>(duration.count()) / 1e9;

  Solution soln;
  soln.t = elapsed;
  soln.ifinal = i;
  std::swap(soln.nodes, I);
  soln.u.swap(u);
  soln.score = F;

  return soln;
}

Solution findDenseClusterOfSparseGraph(const SpMat &M, const SpMat &C,
                                       const Params &params) {
  return findDenseClusterOfSparseGraph(M, C,
                                       C * Eigen::VectorXd::Ones(C.cols()) +
                                           Eigen::VectorXd::Ones(C.cols()),
                                       params);
}

} // ns clipper

max_stars_repo_name: ash-aldujaili/clipper

max_issues_repo_name: ash-aldujaili/clipper

max_forks_repo_name: ash-aldujaili/clipper


text:
// Copyright (c) 2015 The Bitcoin Core developers
// Copyright (c) 2017 The PIVX developers
// Copyright (c) 2018 The Betfint developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "reverselock.h"

#include <boost/bind.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>
#include <boost/thread.hpp>
#include <boost/test/unit_test.hpp>

BOOST_AUTO_TEST_SUITE(reverselock_tests)

BOOST_AUTO_TEST_CASE(reverselock_basics)
{
    boost::mutex mutex;
    boost::unique_lock<boost::mutex> lock(mutex);

    BOOST_CHECK(lock.owns_lock());
    {
        reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock);
        BOOST_CHECK(!lock.owns_lock());
    }
    BOOST_CHECK(lock.owns_lock());
}

BOOST_AUTO_TEST_CASE(reverselock_errors)
{
    boost::mutex mutex;
    boost::unique_lock<boost::mutex> lock(mutex);

    // Make sure trying to reverse lock an unlocked lock fails
    lock.unlock();

    BOOST_CHECK(!lock.owns_lock());

    bool failed = false;
    try {
        reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock);
    } catch(...) {
        failed = true;
    }

    BOOST_CHECK(failed);
    BOOST_CHECK(!lock.owns_lock());

    // Locking the original lock after it has been taken by a reverse lock
    // makes no sense. Ensure that the original lock no longer owns the lock
    // after giving it to a reverse one.

    lock.lock();
    BOOST_CHECK(lock.owns_lock());
    {
        reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock);
        BOOST_CHECK(!lock.owns_lock());
    }

    BOOST_CHECK(failed);
    BOOST_CHECK(lock.owns_lock());
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: betfintofficial/Betfint

max_issues_repo_name: betfintofficial/Betfint

max_forks_repo_name: betfintofficial/Betfint