text:
//
// Copyright (c) 2013-2017 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//

#ifndef BEAST_WEBSOCKET_IMPL_CLOSE_IPP
#define BEAST_WEBSOCKET_IMPL_CLOSE_IPP

#include <beast/core/handler_ptr.hpp>
#include <beast/core/static_buffer.hpp>
#include <beast/core/type_traits.hpp>
#include <boost/asio/handler_alloc_hook.hpp>
#include <boost/asio/handler_continuation_hook.hpp>
#include <boost/asio/handler_invoke_hook.hpp>
#include <boost/config.hpp>
#include <boost/throw_exception.hpp>
#include <memory>

namespace beast {
namespace websocket {

//------------------------------------------------------------------------------

// send the close message and wait for the response
//
template<class NextLayer>
template<class Handler>
class stream<NextLayer>::close_op
{
    using fb_type = detail::frame_streambuf;

    struct data : op
    {
        bool cont;
        stream<NextLayer>& ws;
        close_reason cr;
        fb_type fb;
        int state = 0;

        data(Handler& handler, stream<NextLayer>& ws_,
                close_reason const& cr_)
            : ws(ws_)
            , cr(cr_)
        {
            using boost::asio::asio_handler_is_continuation;
            cont = asio_handler_is_continuation(std::addressof(handler));
            ws.template write_close<
                static_buffer>(fb, cr);
        }
    };

    handler_ptr<data, Handler> d_;

public:
    close_op(close_op&&) = default;
    close_op(close_op const&) = default;

    template<class DeducedHandler, class... Args>
    close_op(DeducedHandler&& h,
            stream<NextLayer>& ws, Args&&... args)
        : d_(std::forward<DeducedHandler>(h),
            ws, std::forward<Args>(args)...)
    {
        (*this)(error_code{}, false);
    }

    void operator()()
    {
        (*this)(error_code{});
    }

    void
    operator()(error_code ec, std::size_t);

    void
    operator()(error_code ec, bool again = true);

    friend
    void* asio_handler_allocate(
        std::size_t size, close_op* op)
    {
        using boost::asio::asio_handler_allocate;
        return asio_handler_allocate(
            size, std::addressof(op->d_.handler()));
    }

    friend
    void asio_handler_deallocate(
        void* p, std::size_t size, close_op* op)
    {
        using boost::asio::asio_handler_deallocate;
        asio_handler_deallocate(
            p, size, std::addressof(op->d_.handler()));
    }

    friend
    bool asio_handler_is_continuation(close_op* op)
    {
        return op->d_->cont;
    }

    template<class Function>
    friend
    void asio_handler_invoke(Function&& f, close_op* op)
    {
        using boost::asio::asio_handler_invoke;
        asio_handler_invoke(
            f, std::addressof(op->d_.handler()));
    }
};

template<class NextLayer>
template<class Handler>
void
stream<NextLayer>::close_op<Handler>::
operator()(error_code ec, std::size_t)
{
    auto& d = *d_;
    if(ec)
        d.ws.failed_ = true;
    (*this)(ec);
}

template<class NextLayer>
template<class Handler>
void
stream<NextLayer>::close_op<Handler>::
operator()(error_code ec, bool again)
{
    auto& d = *d_;
    d.cont = d.cont || again;
    if(ec)
        goto upcall;
    for(;;)
    {
        switch(d.state)
        {
        case 0:
            if(d.ws.wr_block_)
            {
                // suspend
                d.state = 2;
                d.ws.wr_op_.emplace(std::move(*this));
                return;
            }
            if(d.ws.failed_ || d.ws.wr_close_)
            {
                // call handler
                d.ws.get_io_service().post(
                    bind_handler(std::move(*this),
                        boost::asio::error::operation_aborted));
                return;
            }
            d.ws.wr_block_ = &d;
            BOOST_FALLTHROUGH;

        case 1:
            // send close frame
            BOOST_ASSERT(d.ws.wr_block_ == &d);
            d.state = 99;
            d.ws.wr_close_ = true;
            boost::asio::async_write(d.ws.stream_,
                d.fb.data(), std::move(*this));
            return;

        case 2:
            BOOST_ASSERT(! d.ws.wr_block_);
            d.ws.wr_block_ = &d;
            d.state = 3;
            // The current context is safe but might not be
            // the same as the one for this operation (since
            // we are being called from a write operation).
            // Call post to make sure we are invoked the same
            // way as the final handler for this operation.
            d.ws.get_io_service().post(
                bind_handler(std::move(*this), ec));
            return;

        case 3:
            BOOST_ASSERT(d.ws.wr_block_ == &d);
            if(d.ws.failed_ || d.ws.wr_close_)
            {
                // call handler
                ec = boost::asio::error::operation_aborted;
                goto upcall;
            }
            d.state = 1;
            break;

        case 99:
            goto upcall;
        }
    }
upcall:
    if(d.ws.wr_block_ == &d)
        d.ws.wr_block_ = nullptr;
    d.ws.rd_op_.maybe_invoke() ||
        d.ws.ping_op_.maybe_invoke();
    d_.invoke(ec);
}

template<class NextLayer>
template<class CloseHandler>
async_return_type<
    CloseHandler, void(error_code)>
stream<NextLayer>::
async_close(close_reason const& cr, CloseHandler&& handler)
{
    static_assert(is_async_stream<next_layer_type>::value,
        "AsyncStream requirements not met");
    async_completion<CloseHandler,
        void(error_code)> init{handler};
    close_op<handler_type<
        CloseHandler, void(error_code)>>{
            init.completion_handler, *this, cr};
    return init.result.get();
}

template<class NextLayer>
void
stream<NextLayer>::
close(close_reason const& cr)
{
    static_assert(is_sync_stream<next_layer_type>::value,
        "SyncStream requirements not met");
    error_code ec;
    close(cr, ec);
    if(ec)
        BOOST_THROW_EXCEPTION(system_error{ec});
}

template<class NextLayer>
void
stream<NextLayer>::
close(close_reason const& cr, error_code& ec)
{
    static_assert(is_sync_stream<next_layer_type>::value,
        "SyncStream requirements not met");
    BOOST_ASSERT(! wr_close_);
    wr_close_ = true;
    detail::frame_streambuf fb;
    write_close<static_buffer>(fb, cr);
    boost::asio::write(stream_, fb.data(), ec);
    failed_ = ec != 0;
}

//------------------------------------------------------------------------------

} // websocket
} // beast

#endif

ext: ipp

max_stars_repo_name: vinniefalco/rippled

max_issues_repo_name: vinniefalco/rippled

max_forks_repo_name: vinniefalco/rippled


text:
/* vim:set ft=cpp ts=4 sw=4 sts=4 et cindent: */
/*
 * ***** BEGIN LICENSE BLOCK *****
 * Version: MIT
 *
 * Copyright (c) 2010-2013 Alan Antonuk
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy,
 * modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 * ***** END LICENSE BLOCK *****
 */

#include <SimpleAmqpClient.h>
#include <SimpleAmqpClient/SimpleRpcClient.h>
#include <SimpleAmqpClient/SimpleRpcServer.h>

#define BOOST_ALL_NO_LIB
#include <boost/bind.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/thread/thread.hpp>
#include <boost/utility.hpp>

#include <iostream>
#include <stdlib.h>

using namespace AmqpClient;

class thread_body : boost::noncopyable
{
public:
    thread_body()
    {
        char *szBroker = getenv("AMQP_BROKER");
        Channel::ptr_t channel;
        if (szBroker != NULL)
            channel = Channel::Create(szBroker);
        else
            channel = Channel::Create();

        server = SimpleRpcServer::Create(channel, "consume_timeout_test");

        m_thread =
            boost::make_shared<boost::thread>(boost::bind(&thread_body::run,
                                              this));
    }

    virtual ~thread_body()
    {
        std::cout << "Is joinable " << m_thread->joinable() << "inter reqd: " << m_thread->interruption_requested() << std::endl;
        m_thread->interrupt();
        std::cout << "Is joinable " << m_thread->joinable() << "inter reqd: " << m_thread->interruption_requested() << std::endl;
        m_thread->join();
    }

    void run()
    {
        std::cout << "Waiting for message... " << std::flush;
        while (!boost::this_thread::interruption_requested())
        {
            BasicMessage::ptr_t message;
            if (server->GetNextIncomingMessage(message, 1))
            {
                std::cout << "message received.\n";
                server->RespondToMessage(message, "this is a response");
                std::cout << "Body: " << message->Body() << std::endl;
            }
        }
    }

    SimpleRpcServer::ptr_t server;
    boost::shared_ptr<boost::thread> m_thread;

};
int main()
{
    try
    {
        boost::shared_ptr<thread_body> body = boost::make_shared<thread_body>();

        boost::this_thread::sleep(boost::posix_time::seconds(1));

        char *szBroker = getenv("AMQP_BROKER");
        Channel::ptr_t channel;
        if (szBroker != NULL)
            channel = Channel::Create(szBroker);
        else
            channel = Channel::Create();

        SimpleRpcClient::ptr_t client = SimpleRpcClient::Create(channel, "consume_timeout_test");
        std::string str = client->Call("Here is my message");
        std::cout << "Got response: " << str << std::endl;

        boost::this_thread::sleep(boost::posix_time::seconds(2));
    }
    catch (AmqpException &e)
    {
        std::cout << "error: " << e.what();
    }

    return 0;
}

ext: cpp

max_stars_repo_name: gini/SimpleAmqpClient

max_issues_repo_name: gini/SimpleAmqpClient

max_forks_repo_name: gini/SimpleAmqpClient


text:
//
// Copyright 2005-2007 Adobe Systems Incorporated
//
// Distributed under the Boost Software License, Version 1.0
// See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt
//
#ifndef BOOST_GIL_EXTENSION_NUMERIC_CHANNEL_NUMERIC_OPERATIONS_HPP
#define BOOST_GIL_EXTENSION_NUMERIC_CHANNEL_NUMERIC_OPERATIONS_HPP

#include <boost/gil/channel.hpp>

namespace boost { namespace gil {

// Function objects and utilities for channel-wise numeric operations.
//
// List of currently defined functors:
//    channel_plus_t (+)
//    channel_minus_t (-)
//    channel_multiplies_t (*)
//    channel_divides_t (/),
//    channel_plus_scalar_t (+s)
//    channel_minus_scalar_t (-s),
//    channel_multiplies_scalar_t (*s)
//    channel_divides_scalar_t (/s),
//    channel_halves_t (/=2)
//    channel_zeros_t (=0)
//    channel_assigns_t (=)

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of addition of two channel values.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel1, typename Channel2, typename ChannelResult>
struct channel_plus_t
{
    using ChannelRef1 = typename channel_traits<Channel1>::const_reference;
    using ChannelRef2 = typename channel_traits<Channel2>::const_reference;
    static_assert(std::is_convertible<ChannelRef1, ChannelResult>::value,
        "ChannelRef1 not convertible to ChannelResult");
    static_assert(std::is_convertible<ChannelRef2, ChannelResult>::value,
        "ChannelRef2 not convertible to ChannelResult");

    /// \param ch1 - first of the two addends (augend).
    /// \param ch2 - second of the two addends.
    auto operator()(ChannelRef1 ch1, ChannelRef2 ch2) const -> ChannelResult
    {
        return ChannelResult(ch1) + ChannelResult(ch2);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of subtraction of two channel values.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel1, typename Channel2, typename ChannelResult>
struct channel_minus_t
{
    using ChannelRef1 = typename channel_traits<Channel1>::const_reference;
    using ChannelRef2 = typename channel_traits<Channel2>::const_reference;
    static_assert(std::is_convertible<ChannelRef1, ChannelResult>::value,
        "ChannelRef1 not convertible to ChannelResult");
    static_assert(std::is_convertible<ChannelRef2, ChannelResult>::value,
        "ChannelRef2 not convertible to ChannelResult");

    /// \param ch1 - minuend operand of the subtraction.
    /// \param ch2 - subtrahend operand of the subtraction.
    auto operator()(ChannelRef1 ch1, ChannelRef2 ch2) const -> ChannelResult
    {
        return ChannelResult(ch1) - ChannelResult(ch2);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of multiplication of two channel values.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel1, typename Channel2, typename ChannelResult>
struct channel_multiplies_t
{
    using ChannelRef1 = typename channel_traits<Channel1>::const_reference;
    using ChannelRef2 = typename channel_traits<Channel2>::const_reference;
    static_assert(std::is_convertible<ChannelRef1, ChannelResult>::value,
        "ChannelRef1 not convertible to ChannelResult");
    static_assert(std::is_convertible<ChannelRef2, ChannelResult>::value,
        "ChannelRef2 not convertible to ChannelResult");

    /// \param ch1 - first of the two factors (multiplicand).
    /// \param ch2 - second of the two factors (multiplier).
    auto operator()(ChannelRef1 ch1, ChannelRef2 ch2) const -> ChannelResult
    {
        return ChannelResult(ch1) * ChannelResult(ch2);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of division of two channel values.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel1, typename Channel2, typename ChannelResult>
struct channel_divides_t
{
    using ChannelRef1 = typename channel_traits<Channel1>::const_reference;
    using ChannelRef2 = typename channel_traits<Channel2>::const_reference;
    static_assert(std::is_convertible<ChannelRef1, ChannelResult>::value,
        "ChannelRef1 not convertible to ChannelResult");
    static_assert(std::is_convertible<ChannelRef2, ChannelResult>::value,
        "ChannelRef2 not convertible to ChannelResult");

    /// \param ch1 - dividend operand of the two division operation.
    /// \param ch2 - divisor operand of the two division operation.
    auto operator()(ChannelRef1 ch1, ChannelRef2 ch2) const -> ChannelResult
    {
        return ChannelResult(ch1) / ChannelResult(ch2);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of adding scalar to channel value.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel, typename Scalar, typename ChannelResult>
struct channel_plus_scalar_t
{
    using ChannelRef = typename channel_traits<Channel>::const_reference;
    static_assert(std::is_convertible<ChannelRef, ChannelResult>::value,
        "ChannelRef not convertible to ChannelResult");
    static_assert(std::is_scalar<Scalar>::value, "Scalar not a scalar");
    static_assert(std::is_convertible<Scalar, ChannelResult>::value,
        "Scalar not convertible to ChannelResult");

    auto operator()(ChannelRef channel, Scalar const& scalar) const -> ChannelResult
    {
        return ChannelResult(channel) + ChannelResult(scalar);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of subtracting scalar from channel value.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel, typename Scalar, typename ChannelResult>
struct channel_minus_scalar_t
{
    using ChannelRef = typename channel_traits<Channel>::const_reference;
    static_assert(std::is_convertible<ChannelRef, ChannelResult>::value,
        "ChannelRef not convertible to ChannelResult");
    static_assert(std::is_scalar<Scalar>::value, "Scalar not a scalar");
    static_assert(std::is_convertible<Scalar, ChannelResult>::value,
        "Scalar not convertible to ChannelResult");

    /// \param channel - minuend operand of the subtraction.
    /// \param scalar - subtrahend operand of the subtraction.
    auto operator()(ChannelRef channel, Scalar const& scalar) const -> ChannelResult
    {
        // TODO: Convertion after subtraction vs conversion of operands in channel_minus_t?
        return ChannelResult(channel - scalar);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of channel value by a scalar.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel, typename Scalar, typename ChannelResult>
struct channel_multiplies_scalar_t
{
    using ChannelRef = typename channel_traits<Channel>::const_reference;
    static_assert(std::is_convertible<ChannelRef, ChannelResult>::value,
        "ChannelRef not convertible to ChannelResult");
    static_assert(std::is_scalar<Scalar>::value, "Scalar not a scalar");
    static_assert(std::is_convertible<Scalar, ChannelResult>::value,
        "Scalar not convertible to ChannelResult");

    /// \param channel - first of the two factors (multiplicand).
    /// \param scalar - second of the two factors (multiplier).
    auto operator()(ChannelRef channel, Scalar const& scalar) const -> ChannelResult
    {
        return ChannelResult(channel) * ChannelResult(scalar);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of dividing channel value by scalar.
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel, typename Scalar, typename ChannelResult>
struct channel_divides_scalar_t
{
    using ChannelRef = typename channel_traits<Channel>::const_reference;
    static_assert(std::is_convertible<ChannelRef, ChannelResult>::value,
        "ChannelRef not convertible to ChannelResult");
    static_assert(std::is_scalar<Scalar>::value, "Scalar not a scalar");
    static_assert(std::is_convertible<Scalar, ChannelResult>::value,
        "Scalar not convertible to ChannelResult");

    /// \param channel - dividend operand of the two division operation.
    /// \param scalar - divisor operand of the two division operation.
    auto operator()(ChannelRef channel, Scalar const& scalar) const -> ChannelResult
    {
        return ChannelResult(channel) / ChannelResult(scalar);
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Arithmetic operation of dividing channel value by 2
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel>
struct channel_halves_t
{
    using ChannelRef = typename channel_traits<Channel>::reference;

    auto operator()(ChannelRef channel) const -> ChannelRef
    {
        // TODO: Split into steps: extract with explicit conversion to double, divide and assign?
        //double const v = ch;
        //ch = static_cast<Channel>(v / 2.0);
        channel /= 2.0;
        return channel;
    }
};

/// \ingroup ChannelNumericOperations
/// \brief Operation of setting channel value to zero
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel>
struct channel_zeros_t
{
    using ChannelRef = typename channel_traits<Channel>::reference;

    auto operator()(ChannelRef channel) const -> ChannelRef
    {
        channel = Channel(0);
        return channel;
    }
};

/// \ingroup ChannelNumericOperations
/// structure for assigning one channel to another
/// \note This is a generic implementation; user should specialize it for better performance.
template <typename Channel1, typename Channel2>
struct channel_assigns_t
{
    using ChannelRef1 = typename channel_traits<Channel1>::const_reference;
    using ChannelRef2 = typename channel_traits<Channel2>::reference;
    static_assert(std::is_convertible<ChannelRef1, Channel2>::value,
        "ChannelRef1 not convertible to Channel2");

    /// \param ch1 - assignor side (input) of the assignment operation
    /// \param ch2 - assignee side (output) of the assignment operation
    auto operator()(ChannelRef1 ch1, ChannelRef2 ch2) const -> ChannelRef2
    {
        ch2 = Channel2(ch1);
        return ch2;
    }
};

}}  // namespace boost::gil

#endif

ext: hpp

max_stars_repo_name: kindlychung/mediasoup-sfu-cpp

max_issues_repo_name: kindlychung/mediasoup-sfu-cpp

max_forks_repo_name: kindlychung/mediasoup-sfu-cpp


text:
/*
 * MIT License
 *
 * Alexandria.org
 *
 * Copyright (c) 2021 Josef Cullhed, <info@alexandria.org>, et al.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <boost/test/unit_test.hpp>
#include "indexer/index_manager.h"
#include "indexer/sharded_index_builder.h"
#include "indexer/sharded_index.h"
#include "indexer/domain_level.h"
#include "indexer/merger.h"
#include "text/text.h"
#include "algorithm/hash.h"
#include "transfer/transfer.h"

BOOST_AUTO_TEST_SUITE(test_sharded_index_builder)

BOOST_AUTO_TEST_CASE(test_sharded_index_builder) {

	{
		indexer::sharded_index_builder<indexer::generic_record> idx("test_index", 10);

		idx.truncate();

		idx.add(101, indexer::generic_record(1000, 1.0f));
		idx.add(102, indexer::generic_record(1001, 1.0f));

		idx.append();
		idx.merge();
	}

	{
		indexer::sharded_index<indexer::generic_record> idx("test_index", 10);
		vector<indexer::generic_record> res = idx.find(101);

		BOOST_REQUIRE(res.size() == 1);
		BOOST_CHECK(res[0].m_value == 1000);
	}

}

BOOST_AUTO_TEST_CASE(test_group_by) {

	using indexer::domain_link_record;

	{
		indexer::sharded_index_builder<domain_link_record> idx("test_index", 1);

		idx.truncate();

		idx.add(101, domain_link_record(1000, 1.0f, 200));
		idx.add(101, domain_link_record(1004, 1.0f, 300));
		idx.add(101, domain_link_record(1001, 1.0f, 200));
		idx.add(101, domain_link_record(1003, 1.0f, 300));
		idx.add(101, domain_link_record(1002, 1.0f, 200));

		idx.add(102, domain_link_record(1000, 1.0f, 200));
		idx.add(102, domain_link_record(1001, 1.0f, 200));
		idx.add(102, domain_link_record(1005, 1.0f, 300));
		idx.add(102, domain_link_record(1002, 1.0f, 200));

		idx.add(103, domain_link_record(1000, 1.0f, 200));
		idx.add(103, domain_link_record(1001, 1.0f, 200));
		idx.add(103, domain_link_record(1004, 1.0f, 300));
		idx.add(103, domain_link_record(1002, 1.0f, 200));

		idx.append();
		idx.merge();
		idx.optimize();
	}

	{
		indexer::sharded_index<domain_link_record> idx("test_index", 1);

		auto identity = [](float score) {
			return score;
		};
		std::vector<size_t> counts;
		vector<domain_link_record> res = idx.find_group_by({101, 102}, identity, counts);

		BOOST_REQUIRE(res.size() == 1);
		BOOST_CHECK(res[0].m_score == 3.0f);
		BOOST_CHECK(counts[0] == 3);
	}

	{
		indexer::sharded_index<domain_link_record> idx("test_index", 1);
		auto times_two = [](float score) {
			return 2.0f * score;
		};
		std::vector<size_t> counts;
		vector<domain_link_record> res = idx.find_group_by({101, 103}, times_two, counts);

		BOOST_REQUIRE(res.size() == 2);

		sort(res.begin(), res.end(), domain_link_record::storage_order());
		BOOST_CHECK(res[0].m_score == 2.0f * (3.0f));
		BOOST_CHECK(res[1].m_score == 2.0f * (1.0f));
		BOOST_CHECK(counts[0] == 3);
		BOOST_CHECK(counts[1] == 1);
	}

}

BOOST_AUTO_TEST_CASE(test_score_mod) {

	using indexer::domain_record;

	{
		indexer::sharded_index_builder<domain_record> idx("test_index", 1);

		idx.truncate();

		idx.add(101, domain_record(1000, 1.0f));
		idx.add(101, domain_record(1004, 1.0f));
		idx.add(101, domain_record(1001, 1.0f));
		idx.add(101, domain_record(1003, 1.0f));
		idx.add(101, domain_record(1002, 1.0f));

		idx.add(102, domain_record(1000, 1.0f));
		idx.add(102, domain_record(1001, 1.0f));
		idx.add(102, domain_record(1005, 1.0f));
		idx.add(102, domain_record(1002, 1.0f));

		idx.append();
		idx.merge();
		idx.optimize();
	}

	{
		indexer::sharded_index<domain_record> idx("test_index", 1);
		uint64_t sum_id = 0;
		vector<domain_record> res = idx.find_top({101, 102},
				[&sum_id](uint64_t val) -> float {
					return (float)(sum_id++);
				}, 2);

		BOOST_REQUIRE(res.size() == 2);
		BOOST_CHECK(res[0].m_score == 3.0f);
		BOOST_CHECK(res[0].m_value == 1002);
		BOOST_CHECK(res[1].m_score == 2.0f);
		BOOST_CHECK(res[1].m_value == 1001);
	}

}

BOOST_AUTO_TEST_CASE(test_with_real_data) {

	const size_t mem_before = memory::allocated_memory();
	{
		vector<string> warc_paths = {"crawl-data/ALEXANDRIA-MANUAL-01/files/top_domains.txt.gz"};
		std::vector<std::string> local_files = transfer::download_gz_files_to_disk(warc_paths);

		indexer::index_manager idx_manager;

		indexer::domain_level domain_level;
		idx_manager.add_level(&domain_level);

		idx_manager.truncate();

		indexer::merger::start_merge_thread();
		idx_manager.add_index_files_threaded(local_files, 1);
		indexer::merger::stop_merge_thread();

		transfer::delete_downloaded_files(local_files);
	}
	const size_t mem_after = memory::allocated_memory();
	//BOOST_CHECK(mem_before == mem_after);
	cout << "diff: " << mem_after - mem_before << endl;

	{
		hash_table::hash_table ht("index_manager");
		indexer::index_manager idx_manager;
		indexer::domain_level domain_level;
		idx_manager.add_level(&domain_level);

		std::vector<indexer::return_record> res = idx_manager.find("microsoft corporation main site");

		BOOST_REQUIRE(res.size() == 1);
		const string host = ht.find(res[0].m_value);
		BOOST_CHECK(host == "microsoft.com");
	}
}

BOOST_AUTO_TEST_CASE(test_optimization) {
	{
		vector<string> warc_paths = {"crawl-data/ALEXANDRIA-MANUAL-01/files/50_top_domains.txt.gz"};
		std::vector<std::string> local_files = transfer::download_gz_files_to_disk(warc_paths);

		indexer::index_manager idx_manager;

		indexer::domain_level domain_level;
		idx_manager.add_level(&domain_level);

		idx_manager.truncate();

		indexer::merger::start_merge_thread();
		idx_manager.add_index_files_threaded(local_files, 1);
		indexer::merger::stop_merge_thread();

		transfer::delete_downloaded_files(local_files);

		idx_manager.merge();
	}

	{
		hash_table::hash_table ht("index_manager");
		indexer::index_manager idx_manager;
		indexer::domain_level domain_level;
		idx_manager.add_level(&domain_level);

		std::vector<indexer::return_record> res = idx_manager.find("the");

		BOOST_REQUIRE(res.size() > 0);

		// Check strict ordering of results.
		uint64_t prev_value = 0;
		for (auto &record : res) {
			BOOST_CHECK(record.m_value > prev_value);
			prev_value = record.m_value;
		}
	}

	{
		hash_table::hash_table ht("index_manager");
		indexer::index_manager idx_manager;
		indexer::domain_level domain_level;
		idx_manager.add_level(&domain_level);

		std::vector<indexer::return_record> res = idx_manager.find("the people");

		BOOST_REQUIRE(res.size() == 3);

		// Check strict ordering of results.
		uint64_t prev_value = 0;
		for (auto &record : res) {
			BOOST_CHECK(record.m_value > prev_value);
			prev_value = record.m_value;
		}
	}
}

BOOST_AUTO_TEST_CASE(test_optimization_with_save) {
	{

		indexer::sharded_index_builder<indexer::domain_link_record> idx("test_index", 10);

		idx.truncate();

		idx.add(1, indexer::domain_link_record(100, 0.1, 1001));
		idx.add(1, indexer::domain_link_record(101, 0.1, 1002));
		idx.add(1, indexer::domain_link_record(102, 0.1, 1001));
		idx.add(1, indexer::domain_link_record(103, 0.1, 1004));
		idx.add(1, indexer::domain_link_record(104, 0.1, 1001));

		idx.add(2, indexer::domain_link_record(100, 0.1, 1001));
		idx.add(2, indexer::domain_link_record(101, 0.1, 1002));
		idx.add(2, indexer::domain_link_record(102, 0.1, 1001));
		idx.add(2, indexer::domain_link_record(103, 0.1, 1004));
		idx.add(2, indexer::domain_link_record(104, 0.1, 1001));

		idx.append();
		idx.merge();
		idx.optimize();
	}

	{

		indexer::sharded_index_builder<indexer::domain_link_record> idx("test_index", 10);

		idx.add(1, indexer::domain_link_record(105, 0.1, 1001));
		idx.add(1, indexer::domain_link_record(106, 0.1, 1002));
		idx.add(1, indexer::domain_link_record(107, 0.1, 1001));
		idx.add(1, indexer::domain_link_record(108, 0.1, 1004));
		idx.add(1, indexer::domain_link_record(109, 0.1, 1001));

		idx.add(2, indexer::domain_link_record(100, 0.1, 1001));
		idx.add(2, indexer::domain_link_record(101, 0.1, 1002));
		idx.add(2, indexer::domain_link_record(102, 0.1, 1001));
		idx.add(2, indexer::domain_link_record(103, 0.1, 1004));
		idx.add(2, indexer::domain_link_record(104, 0.1, 1001));

		idx.append();
		idx.merge();
		idx.optimize();
	}

	{

		indexer::sharded_index<indexer::domain_link_record> idx("test_index", 10);

		vector<indexer::domain_link_record> res = idx.find(1);
		BOOST_CHECK(res.size() == 10);

		vector<indexer::domain_link_record> res2 = idx.find(2);
		BOOST_CHECK(res2.size() == 5);
	}

}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: alexandria-org/alexandria

max_issues_repo_name: alexandria-org/alexandria

max_forks_repo_name: alexandria-org/alexandria


text:
// Copyright DEWETRON GmbH 2017

#include "odkapi_data_set_xml.h"

#include <boost/test/unit_test.hpp>
#include <boost/version.hpp>

// MSVC warning https://lists.boost.org/boost-users/2014/11/83281.php
#if defined(_MSC_VER) && BOOST_VERSION == 105700
#pragma warning(disable:4003)
#endif

using namespace odk;

BOOST_AUTO_TEST_SUITE(data_set_parse)

BOOST_AUTO_TEST_CASE(parse_generate_parse)
{

    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <RegisterDataSet protocol_version="1.0" data_set_key="418" type="SCALED" mode="NORMAL">
                <Channels>
                  <Channel channel_id="1"/>
                  <Channel channel_id="2"/>
                </Channels>
              </RegisterDataSet>
            )xxx"
            ;

    PluginDataSet data_set;
    BOOST_CHECK(data_set.parse(xml_content));

    BOOST_CHECK(data_set.m_channels.size() == 2);

    BOOST_CHECK(data_set.parse(data_set.generate()));

    BOOST_CHECK(data_set.m_channels.size() == 2);
}
BOOST_AUTO_TEST_CASE(PluginDataSet_data_test)
{

    {
        const char* const xml_content =
            R"xxx(<?xml version='1.0' encoding='UTF-8'?>
                <RegisterDataSet protocol_version="1.0" data_set_key="418" type="SCALED" mode="NORMAL">
                    <Channels>
                      <Channel channel_id="1"/>
                      <Channel channel_id="2"/>
                    </Channels>
                  </RegisterDataSet>
                )xxx"
                ;

        PluginDataSet data_set;
        BOOST_CHECK(data_set.parse(xml_content));

        BOOST_CHECK(data_set.m_channels.size() == 2);
        BOOST_CHECK(data_set.m_id == 418);
        BOOST_CHECK(data_set.m_data_set_type == DataSetType::SCALED);
        BOOST_CHECK(data_set.m_data_mode == DataSetMode::NORMAL);
        BOOST_CHECK(data_set.m_policy == StreamPolicy::EXACT);
    }
    {
        const char* const xml_content =
            R"xxx(<?xml version='1.0' encoding='UTF-8'?>
                <RegisterDataSet protocol_version="1.0" data_set_key="418" type="RAW" mode="NORMAL">
                    <Channels>
                      <Channel channel_id="1"/>
                      <Channel channel_id="2"/>
                    </Channels>
                  </RegisterDataSet>
                )xxx"
                ;

        PluginDataSet data_set;
        BOOST_CHECK(data_set.parse(xml_content));

        BOOST_CHECK(data_set.m_channels.size() == 2);
        BOOST_CHECK(data_set.m_id == 418);
        BOOST_CHECK(data_set.m_data_set_type == DataSetType::RAW);
        BOOST_CHECK(data_set.m_data_mode == DataSetMode::NORMAL);
    }
    {
        const char* const xml_content =
            R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <RegisterDataSet protocol_version="1.0" data_set_key="418" type="RAW" mode="NORMAL" policy="EXACT">
                <Channels>
                    <Channel channel_id="1"/>
                    <Channel channel_id="2"/>
                </Channels>
                </RegisterDataSet>
            )xxx"
            ;

        PluginDataSet data_set;
        BOOST_CHECK(data_set.parse(xml_content));

        BOOST_CHECK(data_set.m_channels.size() == 2);
        BOOST_CHECK(data_set.m_id == 418);
        BOOST_CHECK(data_set.m_data_set_type == DataSetType::RAW);
        BOOST_CHECK(data_set.m_data_mode == DataSetMode::NORMAL);
        BOOST_CHECK(data_set.m_policy == StreamPolicy::EXACT);
    }
    {
        const char* const xml_content =
            R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <RegisterDataSet protocol_version="1.0" data_set_key="418" type="RAW" mode="NORMAL" policy="RELAXED">
                <Channels>
                    <Channel channel_id="1"/>
                    <Channel channel_id="2"/>
                </Channels>
                </RegisterDataSet>
            )xxx"
            ;

        PluginDataSet data_set;
        BOOST_CHECK(data_set.parse(xml_content));

        BOOST_CHECK(data_set.m_channels.size() == 2);
        BOOST_CHECK(data_set.m_id == 418);
        BOOST_CHECK(data_set.m_data_set_type == DataSetType::RAW);
        BOOST_CHECK(data_set.m_data_mode == DataSetMode::NORMAL);
        BOOST_CHECK(data_set.m_policy == StreamPolicy::RELAXED);
    }

}

BOOST_AUTO_TEST_CASE(parse_generate_parse_data_request)
{
    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <DataTransferRequest data_set_key="418">
                <Window start="0.0" end="12.34"/>
            </DataTransferRequest>
            )xxx"
    ;

    PluginDataRequest data_request;
    BOOST_CHECK(data_request.parse(xml_content));

    BOOST_CHECK_EQUAL(data_request.m_id, 418);
    BOOST_REQUIRE(data_request.m_data_window);
    BOOST_CHECK_EQUAL(data_request.m_data_window->m_start, 0.0);
    BOOST_CHECK_EQUAL(data_request.m_data_window->m_stop, 12.34);

    BOOST_CHECK(data_request.parse(data_request.generate()));

    BOOST_CHECK_EQUAL(data_request.m_id, 418);
    BOOST_REQUIRE(data_request.m_data_window);
    BOOST_CHECK_EQUAL(data_request.m_data_window->m_start, 0.0);
    BOOST_CHECK_EQUAL(data_request.m_data_window->m_stop, 12.34);
}

BOOST_AUTO_TEST_CASE(parse_generate_parse_data_request_single_value)
{
    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <DataTransferRequest data_set_key="418">
                <SingleValue timestamp="0.123"/>
            </DataTransferRequest>
            )xxx"
    ;

    PluginDataRequest data_request;
    BOOST_CHECK(data_request.parse(xml_content));

    BOOST_CHECK_EQUAL(data_request.m_id, 418);
    BOOST_REQUIRE(data_request.m_single_value);
    BOOST_CHECK_EQUAL(data_request.m_single_value->m_timestamp, 0.123);

    BOOST_CHECK(data_request.parse(data_request.generate()));

    BOOST_CHECK_EQUAL(data_request.m_id, 418);
    BOOST_REQUIRE(data_request.m_single_value);
    BOOST_CHECK_EQUAL(data_request.m_single_value->m_timestamp, 0.123);
}

BOOST_AUTO_TEST_CASE(parse_generate_parse_data_start_request)
{
    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <DataTransferRequest data_set_key="418">
                <Stream start="0.0" block_duration="0.1"/>
            </DataTransferRequest>
            )xxx"
    ;

    PluginDataStartRequest data_start_request;
    BOOST_CHECK(data_start_request.parse(xml_content));

    BOOST_CHECK_EQUAL(data_start_request.m_id, 418);
    BOOST_CHECK(data_start_request.m_start);
    BOOST_CHECK_EQUAL(data_start_request.m_start.get(), 0.0);
    BOOST_CHECK_EQUAL(data_start_request.m_block_duration.get(), 0.1);

    BOOST_CHECK(data_start_request.parse(data_start_request.generate().c_str()));

    BOOST_CHECK_EQUAL(data_start_request.m_id, 418);
    BOOST_CHECK(data_start_request.m_start);
    BOOST_CHECK_EQUAL(data_start_request.m_start.get(), 0.0);
    BOOST_CHECK_EQUAL(data_start_request.m_block_duration.get(), 0.1);
}

BOOST_AUTO_TEST_CASE(parse_generate_parse_data_start_request_optional)
{
    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <DataTransferRequest data_set_key="418">
                <Stream block_duration="0.1"/>
            </DataTransferRequest>
            )xxx"
    ;

    PluginDataStartRequest data_start_request;
    BOOST_CHECK(data_start_request.parse(xml_content));

    BOOST_CHECK_EQUAL(data_start_request.m_id, 418);
    BOOST_CHECK(!data_start_request.m_start);
    BOOST_CHECK_EQUAL(data_start_request.m_block_duration.get(), 0.1);

    BOOST_CHECK(data_start_request.parse(data_start_request.generate().c_str()));

    BOOST_CHECK_EQUAL(data_start_request.m_id, 418);
    BOOST_CHECK(!data_start_request.m_start);
    BOOST_CHECK_EQUAL(data_start_request.m_block_duration.get(), 0.1);
}

BOOST_AUTO_TEST_CASE(parse_generate_parse_data_stop_request)
{
    const char* const xml_content =
        R"xxx(<?xml version='1.0' encoding='UTF-8'?>
            <DataTransferRequest data_set_key="418"/>
            )xxx"
    ;

    PluginDataStopRequest data_stop_request;
    BOOST_CHECK(data_stop_request.parse(xml_content));
    BOOST_CHECK_EQUAL(data_stop_request.m_id, 418);

    BOOST_CHECK(data_stop_request.parse(data_stop_request.generate().c_str()));
    BOOST_CHECK_EQUAL(data_stop_request.m_id, 418);
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: HpLightcorner/OXYGEN-SDK

max_issues_repo_name: HpLightcorner/OXYGEN-SDK

max_forks_repo_name: HpLightcorner/OXYGEN-SDK


text:
/*ckwg +29
 * Copyright 2016-2017 by Kitware, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 *  * Neither name of Kitware, Inc. nor the names of any contributors may be used
 *    to endorse or promote products derived from this software without specific
 *    prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/**
 * \file
 * \brief Implementation of template process.
 */

#include "matlab_process.h"

#include <sprokit/processes/kwiver_type_traits.h>
#include <vital/types/timestamp.h>

#include <arrows/matlab/matlab_engine.h>
#include <arrows/matlab/matlab_util.h>
#include <arrows/ocv/image_container.h>
#include <kwiversys/SystemTools.hxx>

#include <opencv2/core/core.hpp>

#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>

#include <string>
#include <sstream>
#include <fstream>

namespace kwiver {
namespace matlab {

// ----------------------------------------------------------------
/**
 * \class matlab_process
 *
 * \brief Matlab interface template
 *
 * \iports
 *
 * \iport{timestamp} time stamp for incoming images.
 *
 * \iport{image} Input image to be processed.
 *
 * \oports
 *
 * \oport{image} Resulting image
 *
 * \configs
 *
 * \config{header} Header text. (string)
 *
 * \config{footer} Footer text (string)
 */

// config items
// <name>, <type>, <default string>, <description string>
create_config_trait( program_file, std::string, "", "Name of matlab process to interface to." );

//----------------------------------------------------------------
// Private implementation class
class matlab_process::priv
{
public:
  priv( matlab_process* parent );
  ~priv();

  void check_result();
  void eval( const std::string& expr );

  matlab_process* m_parent;

  // Configuration values
  std::string m_program_file;

  // MatLab support. The engine is allocated at the latest time.
  std::shared_ptr< kwiver::arrows::matlab::matlab_engine > m_matlab_engine;

}; // end priv class

// ================================================================
matlab_process
::matlab_process( kwiver::vital::config_block_sptr const& config )
  : process( config ),
    d( new matlab_process::priv( this ) )
{
  make_ports(); // create process ports
  make_config(); // declare process configuration
}

matlab_process
::~matlab_process()
{
}

// ----------------------------------------------------------------
/**
 * @brief Configure process
 *
 * This method is called prior to connecting ports to allow the
 * process to configure itself.
 */
void
matlab_process
::_configure()
{
  scoped_configure_instrumentation()

  // Need to delay creating engine because it is heavyweight
  d->m_matlab_engine = std::make_shared< kwiver::arrows::matlab::matlab_engine >();

  d->m_program_file = config_value_using_trait( program_file );

  // Create path to program file so we can do addpath('path');
  std::string full_path = kwiversys::SystemTools::CollapseFullPath( d->m_program_file );
  full_path = kwiversys::SystemTools::GetFilenamePath( full_path );

  d->eval( "addpath('" + full_path + "')" );

  // Get config values for this algorithm by extracting the subblock
  auto algo_config = this->get_config()->subblock( "matlab_config" );

  // Iterate over all values in this config block and pass the values
  // to the matlab as variable assignments.
  auto keys = algo_config->available_values();
  for( auto k : keys )
  {
    std::stringstream config_command;
    config_command <<  k << "=" << algo_config->get_value<std::string>( k ) << ";";
    LOG_DEBUG( logger(), "Sending config value: " << config_command.str() );

    d->eval( config_command.str() );
  }// end foreach

  // Call matlab function to complete the config
  d->eval( "configure_process()" );
}

// ----------------------------------------------------------------
void
matlab_process
::_step()
{
  kwiver::vital::timestamp frame_time;

  // See if optional input port has been connected.
  // Get input only if connected.
  if ( has_input_port_edge_using_trait( timestamp ) )
  {
    frame_time = grab_from_port_using_trait( timestamp );
  }

  kwiver::vital::image_container_sptr img = grab_from_port_using_trait( image );

  kwiver::vital::image_container_sptr out_image;
  {
    scoped_step_instrumentation();

    LOG_DEBUG( logger(), "Processing frame " << frame_time );

    // Convert inputs to matlab format and send to matlab engine
    // Need to establish an interface to the matlab function.
    // Could use parameters or well known names.

    // Sending an image to matlab
    kwiver::arrows::matlab::MxArraySptr mx_image = kwiver::arrows::matlab::convert_mx_image( img );
    d->m_matlab_engine->put_variable( "in_image", mx_image );

    // Call matlab step function
    d->eval( "step( in_image );" );

    kwiver::arrows::matlab::MxArraySptr mx_out_image = d->m_matlab_engine->get_variable( "out_image" );
    out_image = kwiver::arrows::matlab::convert_mx_image( mx_out_image );
  }

  push_to_port_using_trait( image, out_image );
}

// ------------------------------------------------------------------
void
matlab_process
::_init()
{
}

// ----------------------------------------------------------------
void
matlab_process
::make_ports()
{
  // Set up for required ports
  sprokit::process::port_flags_t required;
  sprokit::process::port_flags_t optional;
  required.insert( flag_required );

  // -- input --
  declare_input_port_using_trait( timestamp, optional );
  declare_input_port_using_trait( image, required );

  // -- output --
  declare_output_port_using_trait( image, optional );
}

// ----------------------------------------------------------------
void
matlab_process
::make_config()
{
  declare_config_using_trait( program_file );
}

// ================================================================
matlab_process::priv
::priv( matlab_process* parent)
  : m_parent( parent )
{
}

matlab_process::priv
::~priv()
{
}

// ------------------------------------------------------------------
void
matlab_process::priv
::check_result()
{
  const std::string& results( m_matlab_engine->output() );
  if ( results.size() > 0 )
  {
    LOG_INFO( m_parent->logger(), "Matlab output: " << results );
  }
}

// ------------------------------------------------------------------
void
matlab_process::priv
::eval( const std::string& expr )
{
  LOG_DEBUG( m_parent->logger(), "Matlab eval: " << expr );
  m_matlab_engine->eval( expr );
  check_result();
}

} } // end namespace

ext: cxx

max_stars_repo_name: judajake/kwiver

max_issues_repo_name: judajake/kwiver


text:
#include <boost/filesystem.hpp>
#include <sstream>
#include <iomanip>

#include "common/MainUtils.h"
#include "common/WriteTokenPool.h"
#include "core/File.h"
#include "core/MemoryUtils.h"
#include "core/NamedObjectCollection.h"
#include "core/Params.h"
#include "core/StatLogger.h"
#include "mapreduce/common/ChunkMap.h"
#include "mapreduce/common/CoordinatorClientFactory.h"
#include "mapreduce/common/CoordinatorClientInterface.h"
#include "mapreduce/common/JobInfo.h"
#include "mapreduce/common/URL.h"
#include "mapreduce/common/buffers/KVPairBuffer.h"
#include "mapreduce/workers/writer/BaseWriter.h"

BaseWriter* BaseWriter::newBaseWriter(
  const std::string& phaseName, const std::string& stageName, uint64_t id,
  Params& params, NamedObjectCollection& dependencies,
  File::AccessMode fileMode, StatLogger& logger) {

  std::string coordinatorPhaseName = phaseName;
  if (params.contains("FORCE_PHASE_TWO_OUTPUT_DIR") &&
      params.get<bool>("FORCE_PHASE_TWO_OUTPUT_DIR")) {
    // The second part of phase three needs to write to phase two output
    // directories.
    coordinatorPhaseName = "phase_two";
  }
  CoordinatorClientInterface* coordinatorClient =
    CoordinatorClientFactory::newCoordinatorClient(
      params, coordinatorPhaseName, stageName, id);

  // If this worker was spawned by a MultiProtocolWriter, want to make sure it
  // gets configured according to the parameters in use by its parent stage
  std::string parentStageName(stageName, 0, stageName.find_first_of(':'));

  bool directIO =
    params.get<bool>("DIRECT_IO." + phaseName + "." + parentStageName);

  std::string nodeIPAddress = params.get<std::string>("MYIPADDRESS");
  uint64_t logicalDiskSizeHint = 0;

  if (params.get<bool>("FILE_PREALLOCATION")) {
    // Preallocate to the partition size unless the user specifies a custom
    // preallocation size.
    if (params.contains("FILE_PREALLOCATION_SIZE")) {
      logicalDiskSizeHint = params.get<uint64_t>("FILE_PREALLOCATION_SIZE");
    } else {
      logicalDiskSizeHint = params.get<uint64_t>("PARTITION_SIZE");
    }
  }

  // Compute which disks belong to this writer.
  StringList* diskList = dependencies.get<StringList>("output_disk_list");
  uint64_t disksPerWriter = params.getv<uint64_t>(
    "DISKS_PER_WORKER.%s.%s", phaseName.c_str(), parentStageName.c_str());
  OutputDiskMap outputDisks;
  StringList::iterator iter = diskList->begin();
  for (uint64_t diskID = 0; iter != diskList->end(); diskID++, iter++) {
    // Populate output disk map with exactly the disks for this node
    if (diskID >= disksPerWriter * id && outputDisks.size() < disksPerWriter) {
      outputDisks[diskID] = *iter;
    }
  }

  TRITONSORT_ASSERT(outputDisks.size() == disksPerWriter,
         "Writer %llu should have %llu disks but has %llu", id, disksPerWriter,
         outputDisks.size());

  uint64_t bytesBeforeSimulatedFailure = 0;

  uint64_t peerID = params.get<uint64_t>("MYPEERID");
  if (params.containsv("SIMULATE_DISK_FAILURE.%llu.%llu", peerID, id)) {
    bytesBeforeSimulatedFailure = params.getv<uint64_t>(
      "SIMULATE_DISK_FAILURE.%llu.%llu", peerID, id);
  }

  WriteTokenPool* writeTokenPool = NULL;

  if (dependencies.contains<WriteTokenPool>("write_token_pool")) {
    writeTokenPool = dependencies.get<WriteTokenPool>("write_token_pool");
  }

  uint64_t numDisks =
    params.getv<uint64_t>("NUM_OUTPUT_DISKS.%s", phaseName.c_str());

  uint64_t largePartitionThreshold = 0;
  if (phaseName == "phase_one") {
    // Rename large partitions in phase one so we can process them in phase
    // three.
    largePartitionThreshold = params.get<uint64_t>("LARGE_PARTITION_THRESHOLD");
  }

  ChunkMap* chunkMap = NULL;
  if (dependencies.contains<ChunkMap>("chunk_map")) {
    chunkMap = dependencies.get<ChunkMap>("chunk_map");
  }

  BaseWriter* writer = new BaseWriter(
    id, nodeIPAddress, writeTokenPool, fileMode, directIO, logicalDiskSizeHint,
    outputDisks, *coordinatorClient, bytesBeforeSimulatedFailure, logger,
    params, numDisks, phaseName, largePartitionThreshold, chunkMap,
    peerID);

  return writer;
}

BaseWriter::BaseWriter(
  uint64_t _id, const std::string& _nodeIPAddress,
  WriteTokenPool* _writeTokenPool, File::AccessMode _fileMode, bool _directIO,
  uint64_t _logicalDiskSizeHint, OutputDiskMap& _outputDisks,
  CoordinatorClientInterface& _coordinatorClient,
  uint64_t _bytesBeforeSimulatedFailure, StatLogger& _logger,
  const Params& params, uint64_t _numDisks, const std::string& phaseName,
  uint64_t _largePartitionThreshold, ChunkMap* _chunkMap, uint64_t _nodeID)
  : id(_id),
    nodeIPAddress(_nodeIPAddress),
    fileMode(_fileMode),
    directIO(_directIO),
    logicalDiskSizeHint(_logicalDiskSizeHint),
    outputDisks(_outputDisks),
    bytesBeforeSimulatedFailure(_bytesBeforeSimulatedFailure),
    numDisks(_numDisks),
    largePartitionThreshold(_largePartitionThreshold),
    nodeID(_nodeID),
    coordinatorClient(_coordinatorClient),
    partitionMap(params, phaseName),
    writeTokenPool(_writeTokenPool),
    chunkMap(_chunkMap),
    logger(_logger),
    totalBytesWritten(0) {
  writeSizeStatID = logger.registerHistogramStat("write_size", 100);
}

BaseWriter::~BaseWriter() {
  delete &coordinatorClient;
}

File* BaseWriter::getFile(KVPairBuffer* writeBuffer) {
  // Get the job ID for this write.
  const std::set<uint64_t>& jobIDs = writeBuffer->getJobIDs();
  TRITONSORT_ASSERT(jobIDs.size() == 1, "Expected buffer being passed to writer to "
         "have exactly one job ID; this one has %llu", jobIDs.size());
  uint64_t jobID = *(jobIDs.begin());

  uint64_t partitionsPerDisk =
    partitionMap.getNumPartitionsPerOutputDisk(jobID);
  uint64_t partitionID = writeBuffer->getLogicalDiskID();
  uint64_t chunkID = writeBuffer->getChunkID();
  bool replica = false;

  // Determine which disk in the cluster this partition belongs to.
  uint64_t diskID = 0;
  if (chunkID < std::numeric_limits<uint64_t>::max()) {
    // This is a chunk buffer for a large partition in phase three.
    TRITONSORT_ASSERT(chunkMap != NULL, "Require chunk map for phase three.");
    diskID = chunkMap->getDiskID(partitionID, chunkID);
  } else {
    // This is a regular partition file.
    diskID = partitionID / partitionsPerDisk;

    // Adjust disk ID to be local to this node.
    uint64_t diskNode = diskID / numDisks;
    diskID = diskID % numDisks;

    if (diskNode != nodeID) {
      // This partition belongs to a different node, so assume that it's a
      // replica partition.
      replica = true;
    }
  }

  if (blackHoleWrites[diskID]) {
    return NULL;
  }

  // Verify that this writer handles the disk.
  OutputDiskMap::const_iterator outputDiskIter = outputDisks.find(diskID);
  TRITONSORT_ASSERT(outputDiskIter != outputDisks.end(),
         "Writer %llu got buffer for logical disk %llu, physical disk %llu, "
         "but is not responsible for this physical disk.", id, partitionID,
         diskID);

  // Get the corresponding file.
  if (chunkID < std::numeric_limits<uint64_t>::max()) {
    // Excuse the notation abuse...
    FilesForJobMap& chunkFilesForJob = chunkFiles[jobID];
    FileMap& chunkFileMap = chunkFilesForJob[partitionID];
    File*& chunkFile = chunkFileMap[chunkID];

    if (chunkFile == NULL) {
      chunkFile = newFile(
        partitionID, jobID, outputDiskIter->second, chunkID, replica);
    }

    return chunkFile;
  } else {
    FileMap& filesForJob = files[jobID];
    File*& file = filesForJob[partitionID];

    if (file == NULL) {
      file = newFile(
        partitionID, jobID, outputDiskIter->second, chunkID, replica);

      // Log an example file path for the first partition on each disk
      if (partitionID % partitionsPerDisk == 0) {
        logger.logDatum("example_output_filename", file->getFilename());
      }
    }

    return file;
  }
}

void BaseWriter::logBufferWritten(KVPairBuffer* writeBuffer) {
  totalBytesWritten += writeBuffer->getCurrentSize();
  logger.add(writeSizeStatID, writeBuffer->getCurrentSize());

  if (bytesBeforeSimulatedFailure != 0 &&
      totalBytesWritten >= bytesBeforeSimulatedFailure) {
    // We did all the error checking in getFile(), so no need to repeat it here.
    const std::set<uint64_t>& jobIDs = writeBuffer->getJobIDs();
    uint64_t jobID = *(jobIDs.begin());

    // Determine local disk ID for this buffer.
    uint64_t diskID = (writeBuffer->getLogicalDiskID() /
                       partitionMap.getNumPartitionsPerOutputDisk(jobID)) %
      numDisks;

    const std::string& outputDisk = outputDisks.at(diskID);

    // Tell the coordinator that this disk failed and stop handling writes.
    coordinatorClient.notifyDiskFailure(nodeIPAddress, outputDisk);
    blackHoleWrites[diskID] = true;
  }

  // Return write token if it exists.
  WriteToken* token = writeBuffer->getToken();
  if (token != NULL) {
    ABORT_IF(writeTokenPool == NULL, "You must have provided a write token "
             "pool, but this writer hasn't been given a write token pool");
    writeTokenPool->putToken(token);
  }
}

void BaseWriter::teardown() {
  uint64_t alignedBytesWritten = 0;

  // Close all partition files
  for (FilesForJobMap::iterator jobIter = files.begin();
       jobIter != files.end(); jobIter++) {
    FileMap& filesForJob = jobIter->second;

    for (FileMap::iterator fileIter = filesForJob.begin();
         fileIter != filesForJob.end(); fileIter++) {
      File*& file = fileIter->second;

      if (file != NULL) {
        alignedBytesWritten += file->getAlignedBytesWritten();

        file->close();

        // Rename large partition files.
        if (largePartitionThreshold > 0) {
          uint64_t fileSize = file->getCurrentSize();
          if (fileSize > largePartitionThreshold) {
            file->rename(file->getFilename() + ".large");
          }
        }

        delete file;
        file = NULL;
      }
    }
  }

  files.clear();

  // Close all chunk files.
  for (ChunkFilesForJobMap::iterator jobIter = chunkFiles.begin();
       jobIter != chunkFiles.end(); jobIter++) {
    // Excuse the notation abuse...
    FilesForJobMap& filesForJob = jobIter->second;

    for (FilesForJobMap::iterator partitionIter = filesForJob.begin();
         partitionIter != filesForJob.end(); partitionIter++) {
      FileMap& chunkFileMap = partitionIter->second;

      for (FileMap::iterator fileIter = chunkFileMap.begin();
           fileIter != chunkFileMap.end(); fileIter++) {
        File*& file = fileIter->second;

        if (file != NULL) {
          alignedBytesWritten += file->getAlignedBytesWritten();

          file->close();

          delete file;
          file = NULL;
        }
      }
    }
  }

  chunkFiles.clear();

  logger.logDatum("total_bytes_written", totalBytesWritten);
  logger.logDatum("direct_io_bytes_written", alignedBytesWritten);
}

File* BaseWriter::newFile(
  uint64_t logicalDiskUID, uint64_t jobID, const std::string& outputDisk,
  uint64_t chunkID, bool replica) {

  const themis::URL& outputDirectory = coordinatorClient.getOutputDirectory(jobID);

  std::ostringstream oss;
  oss << outputDisk << outputDirectory.path();

  if (replica) {
    // Write replica files to a separate directory.
    oss << "_replica";
  }

  // Create the directory if it doesn't exist.
  if (!boost::filesystem::exists(oss.str())) {
    ABORT_IF(!boost::filesystem::create_directories(oss.str()),
             "Could not create directory %s", oss.str().c_str());
  }

  oss << "/" << std::setfill('0') << std::setw(8) << logicalDiskUID
      << ".partition";
  if (chunkID < std::numeric_limits<uint64_t>::max()) {
    // This is a large chunk buffer.
    oss << ".chunk_" << std::setfill('0') << std::setw(8) << chunkID;
  }

  // Create the file and open it for writing
  File* logicalDiskFile = new (themis::memcheck) File(oss.str());
  TRITONSORT_ASSERT(fileMode == File::WRITE || fileMode == File::WRITE_POSIXAIO ||
         fileMode == File::WRITE_LIBAIO, "Unsupported write mode, must be "
         "WRITE, WRITE_POSIXAIO, or WRITE_LIBAIO");
  logicalDiskFile->open(fileMode, true);

  if (directIO) {
    // Write with O_DIRECT
    logicalDiskFile->enableDirectIO();
  }

  // Pre-allocate the file if we are given a hint as to how big it will be.
  if (logicalDiskSizeHint > 0) {
    logicalDiskFile->preallocate(logicalDiskSizeHint);
  }

  return logicalDiskFile;
}

ext: cc


text:
// Copyright (C) 2015, Pawel Tomulik <ptomulik@meil.pw.edu.pl>
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)

// txpl/vm/eval_ternary_op.hpp

/** // doc: txpl/vm/eval_ternary_op.hpp {{{
 * \file txpl/vm/eval_ternary_op.hpp
 * \brief Implements \ref txpl::vm::eval_ternary_op_visitor "vm::eval_ternary_op_visitor<>"
 */ // }}}
#ifndef TXPL_VM_EVAL_TERNARY_OP_HPP_INCLUDED
#define TXPL_VM_EVAL_TERNARY_OP_HPP_INCLUDED

#include <txpl/vm/can_apply_ternary_op.hpp>
#include <txpl/vm/apply_ternary_op.hpp>
#include <txpl/vm/disambiguate.hpp>
#include <txpl/vm/value_traits.hpp>
#include <txpl/vm/array.hpp>
#include <txpl/vm/object.hpp>
#include <txpl/vm/op_t.hpp>
#include <yaul/tml/bool.hpp>
#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>
#include <boost/variant/multivisitors.hpp>
#include <utility>

namespace txpl { namespace vm {
/** // doc: eval_ternary_op_visitor {{{
 * \ingroup VmGroup
 * \brief Applies ternary operator to its operands
 *
 * \tparam Op identifier of the ternary operator, see \ref txpl::vm::op_t "vm::op_t"
 * \tparam Value a universal (variant) value type used to store result of the operation.
 *
 * The purpose of this functor is to either invoke appropriate instantiation of
 * the \ref txpl::vm::apply_ternary_op "vm::apply_ternary_op<>" or to
 * indicate an error if the operands provided to the functor are not
 * compatible with the given operator \e Op. This visitor is intended to be
 * called by one of `boost::apply_visitor()` functions.
 */ // }}}
template<op_t Op, typename Value>
struct eval_ternary_op_visitor
  : boost::static_visitor<bool>
{
  /** // doc: Array {{{
   * \brief Array of values of \e Values
   */ //}}}
  using Array = array<Value>;
  /** // doc: Object {{{
   * \brief Object with \e Values
   */ // }}}
  using Object = object<Value>;
  /** // doc: eval_ternary_op_visitor() {{{
   * \brief Constructor
   * \param r reference to a variable where the operation result will be stored
   *        after of successful application of the operator \e Op
   *
   * The functor, when called successfully (see \ref operator()()), will store
   * its result in a variable referenced by \e r.
   */ // }}}
  eval_ternary_op_visitor(Value& r)
    : result(r)
  { }
  /** // doc operator() {{{
   * \brief Tries to apply operator \e Op to operands \e x1, \e x2
   * \tparam T1 type of the first operand,
   * \tparam T2 type of the second operand,
   * \tparam T3 type of the third operand,
   * \param x1 first operand,
   * \param x2 second operand,
   * \param x2 third operand.
   * \return `true` if the operator was successfully applied or `false`
   *         otherwise
   *
   * If succesfull, the method stores the result of operation under the
   * reference \e r provided as an argument to the
   * \ref eval_ternary_op_visitor(Value&) "constructor".
   */ // }}}
  template<typename T1, typename T2, typename T3>
  bool operator()(T1 const& x1, T2 const& x2, T3 const& x3) const
  { return apply(x1, x2, x3, typename can_apply_ternary_op<Op,T1,T2,T3>::type()); }
protected:
  /** // doc: apply() {{{
   * \brief Applies the operator \e Op to operands
   * \tparam T1 type of the first operand,
   * \tparam T2 type of the second operand,
   * \tparam T3 type of the third operand,
   * \param x1 first operand,
   * \param x2 second operand,
   * \param x3 third operand.
   * \returns `true`
   *
   * This statically dispatched method handles operands compatible with the
   * operator \e Op. It applies the operator \e Op to operands \e x1, \e x2,
   * \e x3, stores result under a reference provided as the argument \e r to
   * the \ref eval_ternary_op_visitor(Value&) "constructor", and returns `true`.
   */ // }}}
  template<typename T1, typename T2, typename T3>
  bool apply(T1 const& x1, T2 const& x2, T3 const& x3, yaul::tml::true_) const
  {
    result = disambiguate<value_traits<Value> >(apply_ternary_op<Op>::apply(x1, x2, x3));
    return true;
  }
  /** // doc: apply() {{{
   * \brief Specialization of ternary operations on arrays
   *
   * \param x1 first operand,
   * \param x2 second operand,
   * \param x3 third operand.
   * \returns `true` if arrays \e x1, \e x2, \e x3 are compatible with operator
   *          \e Op, or `false` otherwise
   *
   * Whether the \e x1, \e x2, \e x3 are compatible with \e Op is determined at
   * runtime (because this depends on the dimension, sizes and contents of
   * arrays which is know only at runtime).
   */ // }}}
  bool apply(Array const& x1, Array const& x2, Array const& x3, yaul::tml::true_) const
  {
    if(x1.size() != x2.size() || x1.size() != x3.size()) return false;
    array<Value> r(x1.size());
    for(size_t i = 0; i != r.size(); ++i)
      {
        auto op = eval_ternary_op_visitor<Op, Value>(r[i]);
        if(!boost::apply_visitor(op, x1[i], x2[i], x3[i]))
          return false;
      }
    result = std::forward<array<Value> >(r);
    return true;
  }
  /** // doc: apply() {{{
   * \brief Specialization of ternary operations on arrays
   *
   * \param x1 first operand,
   * \param x2 second operand,
   * \param x3 third operand.
   * \returns currently it does nothing and always returns `false`
   */ // }}}
  bool apply(Object const&, Object const&, Object const&, yaul::tml::true_) const
  {
    // TODO: elaborate if we're able to apply ternary ops to objects somehow??
    return false;
  }
  /** // doc: apply() {{{
   * \brief Does nothing and returns `false`
   * \tparam T1 type of the first operand,
   * \tparam T2 type of the second operand,
   * \tparam T3 type of the third operand,
   * \param x1 first operand,
   * \param x2 second operand,
   * \param x3 third operand.
   * \returns `false`
   *
   * This statically dispatched method handles operands incompatible with the
   * operator \e Op. It always  returns `false`. The data under the reference
   * provided as an argument to the \ref eval_ternary_op_visitor(Value&)
   * "constructor" remains unchanged.
   */ // }}}
  template<typename T1, typename T2, typename T3>
  bool apply(T1 const&, T2 const&, T3 const&, yaul::tml::false_) const
  { return false; }
private:
  Value& result;
};
/** // doc: eval_ternary_op() {{{
 * \ingroup VmGroup
 * \brief Creates \ref txpl::vm::eval_ternary_op_visitor "vm::eval_ternary_op_visitor<>"
 *
 * \tparam Op operator identifier, see \ref txpl::vm::op_t "vm::op_t",
 * \tparam Value a universal (variant) value type used to store the result of
 *         operation
 * \param result reference to a variable, where the result of operation
 *        shall be stored after successful use of the created functor.
 *
 * \b Example:
 * \snippet eval_ternary_op1.cpp Code
 * \b Output:
 * \snippet eval_ternary_op1.cpp Output
 */ // }}}
template<op_t Op, typename Value>
eval_ternary_op_visitor<Op, Value>
eval_ternary_op(Value& result)
{ return eval_ternary_op_visitor<Op, Value>(result); }
} } // end namespace txpl::vm

#endif /* TXPL_VM_EVAL_TERNARY_OP_HPP_INCLUDED */
// vim: set expandtab tabstop=2 shiftwidth=2:
// vim: set foldmethod=marker foldcolumn=4:

max_stars_repo_name: ptomulik/txpl

max_issues_repo_name: ptomulik/txpl

max_forks_repo_name: ptomulik/txpl