text:
//
// Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
//

//
// sandesh_statistics.cc
// Sandesh Statistics Implementation
//

#include <boost/foreach.hpp>

#include <sandesh/sandesh_types.h>
#include <sandesh/sandesh.h>
#include <sandesh/sandesh_statistics.h>

//
// SandeshMessageStatistics
//

// Detail stats
void SandeshMessageStatistics::Get(DetailStatsMap *m_detail_type_stats,
    SandeshMessageStats *detail_agg_stats) const {
    *m_detail_type_stats = detail_type_stats_map_;
    *detail_agg_stats = detail_agg_stats_;
}

void SandeshMessageStatistics::Get(DetailStatsList *v_detail_type_stats,
    SandeshMessageStats *detail_agg_stats) const {
    BOOST_FOREACH(DetailStatsMap::const_iterator::value_type it,
        detail_type_stats_map_) {
        v_detail_type_stats->push_back(*it.second);
    }
    *detail_agg_stats = detail_agg_stats_;
}

// Basic stats
static void PopulateBasicStats(const SandeshMessageStats &detail_stats,
    SandeshMessageBasicStats *basic_stats) {
    basic_stats->messages_sent = detail_stats.messages_sent;
    basic_stats->bytes_sent = detail_stats.bytes_sent;
    basic_stats->messages_received = detail_stats.messages_received;
    basic_stats->bytes_received = detail_stats.bytes_received;
    basic_stats->messages_sent_dropped = detail_stats.messages_sent_dropped;
    basic_stats->messages_received_dropped =
        detail_stats.messages_received_dropped;
}

static void PopulateBasicTypeStats(const SandeshMessageTypeStats &detail_stats,
    SandeshMessageTypeBasicStats *basic_stats) {
    basic_stats->message_type = detail_stats.message_type;
    PopulateBasicStats(detail_stats.stats, &basic_stats->stats);
}

void SandeshMessageStatistics::Get(BasicStatsList *v_basic_type_stats,
    SandeshMessageBasicStats *basic_agg_stats) const {
    BOOST_FOREACH(DetailStatsMap::const_iterator::value_type it,
        detail_type_stats_map_) {
        const SandeshMessageTypeStats *detail_stats(it.second);
        SandeshMessageTypeBasicStats basic_stats;
        PopulateBasicTypeStats(*detail_stats, &basic_stats);
        v_basic_type_stats->push_back(basic_stats);
    }
    PopulateBasicStats(detail_agg_stats_, basic_agg_stats);
}

static void UpdateDetailStatsDrops(SandeshMessageStats *smstats,
    bool sent, uint64_t bytes, SandeshTxDropReason::type send_dreason,
    SandeshRxDropReason::type recv_dreason) {
    if (sent) {
        switch (send_dreason) {
          case SandeshTxDropReason::ValidationFailed:
            smstats->messages_sent_dropped_validation_failed++;
            smstats->bytes_sent_dropped_validation_failed += bytes;
            break;
          case SandeshTxDropReason::RatelimitDrop:
            smstats->messages_sent_dropped_rate_limited++;
            smstats->bytes_sent_dropped_rate_limited += bytes;
            break;
          case SandeshTxDropReason::QueueLevel:
            smstats->messages_sent_dropped_queue_level++;
            smstats->bytes_sent_dropped_queue_level += bytes;
            break;
          case SandeshTxDropReason::NoClient:
            smstats->messages_sent_dropped_no_client++;
            smstats->bytes_sent_dropped_no_client += bytes;
            break;
          case SandeshTxDropReason::NoSession:
            smstats->messages_sent_dropped_no_session++;
            smstats->bytes_sent_dropped_no_session += bytes;
            break;
          case SandeshTxDropReason::NoQueue:
            smstats->messages_sent_dropped_no_queue++;
            smstats->bytes_sent_dropped_no_queue += bytes;
            break;
          case SandeshTxDropReason::ClientSendFailed:
            smstats->messages_sent_dropped_client_send_failed++;
            smstats->bytes_sent_dropped_client_send_failed += bytes;
            break;
          case SandeshTxDropReason::WrongClientSMState:
            smstats->messages_sent_dropped_wrong_client_sm_state++;
            smstats->bytes_sent_dropped_wrong_client_sm_state += bytes;
            break;
          case SandeshTxDropReason::WriteFailed:
            smstats->messages_sent_dropped_write_failed++;
            smstats->bytes_sent_dropped_write_failed += bytes;
            break;
          case SandeshTxDropReason::HeaderWriteFailed:
            smstats->messages_sent_dropped_header_write_failed++;
            smstats->bytes_sent_dropped_header_write_failed += bytes;
            break;
          case SandeshTxDropReason::SessionNotConnected:
            smstats->messages_sent_dropped_session_not_connected++;
            smstats->bytes_sent_dropped_session_not_connected += bytes;
            break;
          default:
            assert(0);
        }
        smstats->messages_sent_dropped++;
        smstats->bytes_sent_dropped += bytes;
    } else {
        switch (recv_dreason) {
          case SandeshRxDropReason::QueueLevel:
            smstats->messages_received_dropped_queue_level++;
            smstats->bytes_received_dropped_queue_level += bytes;
            break;
          case SandeshRxDropReason::NoQueue:
            smstats->messages_received_dropped_no_queue++;
            smstats->bytes_received_dropped_no_queue += bytes;
            break;
          case SandeshRxDropReason::DecodingFailed:
            smstats->messages_received_dropped_decoding_failed++;
            smstats->bytes_received_dropped_decoding_failed += bytes;
            break;
          case SandeshRxDropReason::ControlMsgFailed:
            smstats->messages_received_dropped_control_msg_failed++;
            smstats->bytes_received_dropped_control_msg_failed += bytes;
            break;
          case SandeshRxDropReason::CreateFailed:
            smstats->messages_received_dropped_create_failed++;
            smstats->bytes_received_dropped_create_failed += bytes;
            break;
          default:
            assert(0);
        }
        smstats->messages_received_dropped++;
        smstats->bytes_received_dropped += bytes;
    }
}

void SandeshMessageStatistics::UpdateSend(const std::string &msg_name,
    uint64_t bytes) {
    UpdateInternal(msg_name, bytes, true, false,
        SandeshTxDropReason::NoDrop, SandeshRxDropReason::NoDrop);
}

void SandeshMessageStatistics::UpdateSendFailed(const std::string &msg_name,
    uint64_t bytes, SandeshTxDropReason::type dreason) {
    UpdateInternal(msg_name, bytes, true, true, dreason,
        SandeshRxDropReason::NoDrop);
}

void SandeshMessageStatistics::UpdateRecv(const std::string &msg_name,
    uint64_t bytes) {
    UpdateInternal(msg_name, bytes, false, false,
        SandeshTxDropReason::NoDrop, SandeshRxDropReason::NoDrop);
}

void SandeshMessageStatistics::UpdateRecvFailed(const std::string &msg_name,
    uint64_t bytes, SandeshRxDropReason::type dreason) {
    UpdateInternal(msg_name, bytes, false, true,
        SandeshTxDropReason::NoDrop, dreason);
}

void SandeshMessageStatistics::UpdateInternal(const std::string &msg_name,
    uint64_t bytes, bool is_tx, bool dropped,
    SandeshTxDropReason::type send_dreason,
    SandeshRxDropReason::type recv_dreason) {
    // Update detail stats
    DetailStatsMap::iterator it = detail_type_stats_map_.find(msg_name);
    if (it == detail_type_stats_map_.end()) {
        std::string name(msg_name);
        SandeshMessageTypeStats *n_detail_mtstats(new SandeshMessageTypeStats);
        n_detail_mtstats->message_type = name;
        it = (detail_type_stats_map_.insert(name, n_detail_mtstats)).first;
    }
    SandeshMessageTypeStats *detail_mtstats = it->second;
    if (dropped) {
        UpdateDetailStatsDrops(&detail_mtstats->stats, is_tx,
            bytes, send_dreason, recv_dreason);
        UpdateDetailStatsDrops(&detail_agg_stats_, is_tx,
            bytes, send_dreason, recv_dreason);
    } else {
        if (is_tx) {
            detail_mtstats->stats.messages_sent++;
            detail_mtstats->stats.bytes_sent += bytes;
            detail_agg_stats_.messages_sent++;
            detail_agg_stats_.bytes_sent += bytes;
        } else {
            detail_mtstats->stats.messages_received++;
            detail_mtstats->stats.bytes_received += bytes;
            detail_agg_stats_.messages_received++;
            detail_agg_stats_.bytes_received += bytes;
        }
    }
}

//
// SandeshEventStatistics
//
void SandeshEventStatistics::Get(
    std::vector<SandeshStateMachineEvStats> *ev_stats) const {
    for (EventStatsMap::const_iterator it = event_stats.begin();
            it != event_stats.end(); ++it) {
        const EventStats *es = it->second;
        SandeshStateMachineEvStats ev_stat;
        ev_stat.event = it->first;
        ev_stat.enqueues = es->enqueues;
        ev_stat.dequeues = es->dequeues;
        ev_stat.enqueue_fails = es->enqueue_fails;
        ev_stat.dequeue_fails = es->dequeue_fails;
        ev_stats->push_back(ev_stat);
    }
    SandeshStateMachineEvStats ev_agg_stat;
    ev_agg_stat.enqueues = agg_stats.enqueues;
    ev_agg_stat.dequeues = agg_stats.dequeues;
    ev_agg_stat.enqueue_fails = agg_stats.enqueue_fails;
    ev_agg_stat.dequeue_fails = agg_stats.dequeue_fails;
    ev_stats->push_back(ev_agg_stat);
}

void SandeshEventStatistics::Update(std::string &event_name, bool enqueue,
    bool fail) {
    EventStatsMap::iterator it = event_stats.find(event_name);
    if (it == event_stats.end()) {
        it = (event_stats.insert(event_name, new EventStats)).first;
    }
    EventStats *es = it->second;
    if (enqueue) {
        if (fail) {
            es->enqueue_fails++;
            agg_stats.enqueue_fails++;
        } else {
            es->enqueues++;
            agg_stats.enqueues++;
        }
    } else {
        if (fail) {
            es->dequeue_fails++;
            agg_stats.dequeue_fails++;
        } else {
            es->dequeues++;
            agg_stats.dequeues++;
        }
    }
}

max_stars_repo_name: rombie/contrail-sandesh

max_issues_repo_name: rombie/contrail-sandesh

max_forks_repo_name: rombie/contrail-sandesh


text:
/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 Nathan Osman
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <boost/endian/conversion.hpp>

#include <win32pe/optionalheader.h>

#include "optionalheader_p.h"

using namespace win32pe;

OptionalHeaderPrivate::OptionalHeaderPrivate()
    : mMagic(0),
      mMajorLinkerVersion(0),
      mMinorLinkerVersion(0),
      mSizeOfCode(0),
      mSizeOfInitializedData(0),
      mSizeOfUninitializedData(0),
      mAddressOfEntryPoint(0),
      mBaseOfCode(0),
      mImageBaseLoBaseOfData(0),
      mImageBaseHi(0),
      mSectionAlignment(0),
      mFileAlignment(0),
      mMajorOperatingSystemVersion(0),
      mMinorOperatingSystemVersion(0),
      mMajorImageVersion(0),
      mMinorImageVersion(0),
      mMajorSubsystemVersion(0),
      mMinorSubsystemVersion(0),
      mWin32VersionValue(0),
      mSizeOfImage(0),
      mSizeOfHeaders(0),
      mCheckSum(0),
      mSubsystem(0),
      mDllCharacteristics(0),
      mSizeOfStackReserve(0),
      mSizeOfStackCommit(0),
      mSizeOfHeapReserve(0),
      mSizeOfHeapCommit(0),
      mLoaderFlags(0),
      mNumberOfRvaAndSizes(0),
      mDataDirectory{}
{
}

bool OptionalHeaderPrivate::read(std::istream &istream)
{
    if (!istream.read(
            reinterpret_cast<char*>(&mMagic),
            reinterpret_cast<char*>(&mSizeOfStackReserve) - reinterpret_cast<char*>(&mMagic)
        )) {
        return false;
    }

    boost::endian::little_to_native_inplace(mMagic);
    boost::endian::little_to_native_inplace(mSizeOfCode);
    boost::endian::little_to_native_inplace(mSizeOfInitializedData);
    boost::endian::little_to_native_inplace(mSizeOfUninitializedData);
    boost::endian::little_to_native_inplace(mAddressOfEntryPoint);
    boost::endian::little_to_native_inplace(mBaseOfCode);
    boost::endian::little_to_native_inplace(mImageBaseLoBaseOfData);
    boost::endian::little_to_native_inplace(mImageBaseHi);
    boost::endian::little_to_native_inplace(mSectionAlignment);
    boost::endian::little_to_native_inplace(mFileAlignment);
    boost::endian::little_to_native_inplace(mMajorOperatingSystemVersion);
    boost::endian::little_to_native_inplace(mMinorOperatingSystemVersion);
    boost::endian::little_to_native_inplace(mMajorImageVersion);
    boost::endian::little_to_native_inplace(mMinorImageVersion);
    boost::endian::little_to_native_inplace(mMajorSubsystemVersion);
    boost::endian::little_to_native_inplace(mMinorSubsystemVersion);
    boost::endian::little_to_native_inplace(mWin32VersionValue);
    boost::endian::little_to_native_inplace(mSizeOfImage);
    boost::endian::little_to_native_inplace(mSizeOfHeaders);
    boost::endian::little_to_native_inplace(mCheckSum);
    boost::endian::little_to_native_inplace(mSubsystem);
    boost::endian::little_to_native_inplace(mDllCharacteristics);

    std::streamsize size = mMagic == OptionalHeader::Win32 ? sizeof(uint32_t) : sizeof(uint64_t);
    if (!istream.read(reinterpret_cast<char*>(&mSizeOfStackReserve), size) ||
            !istream.read(reinterpret_cast<char*>(&mSizeOfStackCommit), size) ||
            !istream.read(reinterpret_cast<char*>(&mSizeOfHeapReserve), size) ||
            !istream.read(reinterpret_cast<char*>(&mSizeOfHeapCommit), size)) {
        return false;
    }

    boost::endian::little_to_native_inplace(mSizeOfStackReserve);
    boost::endian::little_to_native_inplace(mSizeOfStackCommit);
    boost::endian::little_to_native_inplace(mSizeOfHeapReserve);
    boost::endian::little_to_native_inplace(mSizeOfHeapCommit);

    if (!istream.read(
            reinterpret_cast<char*>(&mLoaderFlags),
            reinterpret_cast<char*>(&mDataDirectory) - reinterpret_cast<char*>(&mLoaderFlags)
        )) {
        return false;
    }

    boost::endian::little_to_native_inplace(mLoaderFlags);
    boost::endian::little_to_native_inplace(mNumberOfRvaAndSizes);

    if (!istream.read(reinterpret_cast<char*>(mDataDirectory), sizeof(mDataDirectory))) {
        return false;
    }

    for (int i = 0; i < OptionalHeader::DataDirectoryCount; ++i) {
        boost::endian::little_to_native_inplace(mDataDirectory[i].virtualAddress);
        boost::endian::little_to_native_inplace(mDataDirectory[i].size);
    }

    return true;
}

OptionalHeader::OptionalHeader()
    : d(new OptionalHeaderPrivate)
{
}

OptionalHeader::OptionalHeader(const OptionalHeader &other)
    : d(new OptionalHeaderPrivate(*other.d))
{
}

OptionalHeader::~OptionalHeader()
{
    delete d;
}

OptionalHeader &OptionalHeader::operator=(const OptionalHeader &other)
{
    *d = *other.d;
    return *this;
}

uint16_t OptionalHeader::magic() const
{
    return d->mMagic;
}

uint16_t OptionalHeader::majorOperatingSystemVersion() const
{
    return d->mMajorOperatingSystemVersion;
}

uint16_t OptionalHeader::minorOperatingSystemVersion() const
{
    return d->mMinorOperatingSystemVersion;
}

uint16_t OptionalHeader::subsystem() const
{
    return d->mSubsystem;
}

uint16_t OptionalHeader::dllCharacteristics() const
{
    return d->mDllCharacteristics;
}

const OptionalHeader::DataDirectoryItem (&OptionalHeader::dataDirectory() const)[OptionalHeader::DataDirectoryCount]
{
    return d->mDataDirectory;
}

ext: cpp


text:
#include "log4cplus_gelf/GelfAppender.h"

#include <log4cplus/spi/loggingevent.h>
#include <log4cplus/helpers/property.h>
#include <log4cplus/helpers/loglog.h>
#include <log4cplus/helpers/timehelper.h>
#include <gelfcpp/GelfMessage.hpp>
#include "gelfcpp/output/GelfJSONOutput.hpp"

#include "include/TransportFactory.h"

#include <iostream>
#include <sstream>
#include <boost/lexical_cast.hpp>

namespace log4cplus {

    /*
    * SYSLOG priorities
    */
    #define	LOG_EMERG	0	/* system is unusable */
    #define	LOG_ALERT	1	/* action must be taken immediately */
    #define	LOG_CRIT	2	/* critical conditions */
    #define	LOG_ERR		3	/* error conditions */
    #define	LOG_WARNING	4	/* warning conditions */
    #define	LOG_NOTICE	5	/* normal but significant condition */
    #define	LOG_INFO	6	/* informational */
    #define	LOG_DEBUG	7	/* debug-level messages */

    static const std::string fieldNameValidationExp = "^[\\w\\.\\-]*$";
    static const std::vector<std::string> deprecatedFieldNames = {"_id"};

    GelfAppender::GelfAppender() {

    }

    GelfAppender::GelfAppender(const helpers::Properties &properties)
            : Appender(properties) {
        properties.getString(mTransportKind, LOG4CPLUS_TEXT("transport"));
        properties.getString(mServerHost, LOG4CPLUS_TEXT("serverHost"));
        properties.getUInt(mServerPort, LOG4CPLUS_TEXT("serverPort"));

        properties.getString(mHost, LOG4CPLUS_TEXT("host"));
        properties.getString(mFacility, LOG4CPLUS_TEXT("facility"));

        properties.getBool(mIncludeLocationInformation, LOG4CPLUS_TEXT("mIncludeLocationInformation"));

        // Get additional properties
        auto additionalProps = properties.getPropertySubset(LOG4CPLUS_TEXT("additionalField."));
        auto propKeys = additionalProps.propertyNames();
        helpers::getLogLog().debug(
                LOG4CPLUS_TEXT("Additional fields:"));
        for (auto &propKey : propKeys) {
            auto propValue = additionalProps.getProperty(propKey);
            helpers::getLogLog().debug(
                    LOG4CPLUS_STRING_TO_TSTRING(propKey + "=" + propValue));
            mAdditionalFields.emplace(std::make_pair(propKey, propValue));
        }

        // Init transport
        try {
            int port = boost::lexical_cast<int>(mServerPort);
            mTransport = std::move(TransportFactory::create(mTransportKind, mServerHost, port));
        } catch (const std::exception &ex) {
            helpers::getLogLog().error(
                    LOG4CPLUS_TEXT("Cannot create transport: ") + LOG4CPLUS_C_STR_TO_TSTRING(ex.what()));
            throw;
        }
    }

    GelfAppender::~GelfAppender() {
        destructorImpl();
    }

    void GelfAppender::close() {
        helpers::getLogLog().debug(
                LOG4CPLUS_TEXT("Entering GelfAppender::close()..."));

        mTransport.reset();
        closed = true;
    }

    void GelfAppender::append(const log4cplus::spi::InternalLoggingEvent &event) {
        auto gelfJson = createGelfJsonFromEvent(event);

        // Send GELF payload via UDP socket
        sendGelfPayload(std::move(gelfJson));
    }

    int GelfAppender::getSysLogLevel(const LogLevel& ll) const {
        if(ll < INFO_LOG_LEVEL /* || ll < DEBUG_LOG_LEVEL*/) {
            return LOG_DEBUG;
        }
        else if(ll < WARN_LOG_LEVEL) {
            return LOG_INFO;
        }
        else if(ll < ERROR_LOG_LEVEL) {
            return LOG_WARNING;
        }
        else if(ll < FATAL_LOG_LEVEL) {
            return LOG_ERR;
        }
        else if(ll == FATAL_LOG_LEVEL) {
            return LOG_CRIT;
        }
        return LOG_ALERT;  // ll > FATAL_LOG_LEVEL
    }

    std::string GelfAppender::createGelfJsonFromEvent(const log4cplus::spi::InternalLoggingEvent &event) {

        gelfcpp::GelfMessage gelfMessage;

        // Prepare fields according to GELF Payload Specification
        gelfMessage.SetHost(mHost);
        gelfMessage.SetMessage(event.getMessage());

        auto &time = event.getTimestamp();
        auto sec = static_cast<unsigned int>(helpers::to_time_t(time));
        auto usec = static_cast<unsigned int>(helpers::microseconds_part(time));
        double timestamp = sec + (usec / 1000000.0);
        gelfMessage.SetTimestamp(timestamp);

        int level = getSysLogLevel(event.getLogLevel());
        gelfMessage.SetField("level", level);

        gelfMessage.SetField("facility", mFacility);
        gelfMessage.SetField("file", event.getFile());
        gelfMessage.SetField("line", event.getLine());

        // Insert additional fields
        for (const auto& kv: mAdditionalFields) {
            gelfMessage.SetField(kv.first, kv.second);
        }

        // Serialize GELF message
        std::ostringstream oss;
        gelfcpp::output::GelfJSONOutput json(oss);
        json.Write(gelfMessage);

        return oss.str();
    }

    void GelfAppender::sendGelfPayload(std::string payload) {
        mTransport->send(std::move(payload));
    }

}


text:
#include <jni.h>
#include <string>
#include <android/log.h>
#include <iostream>
#include <fstream>
#include <thread>
#include <boost/asio.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/steady_timer.hpp>
#include <namespaces.h>
#include <client.h>
#include <client_config.h>
#include <util/signal.h>
#include <util/crypto.h>
#include <condition_variable>

#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>

#include "debug.h"
#include "std_scoped_redirect.h"

using namespace std;
using ouinet::ClientConfig;

struct Defer {
    Defer(function<void()> f) : _f(move(f)) {}
    ~Defer() { _f(); }
    function<void()> _f;
};

// g_client is only accessed from the g_client_thread.
std::unique_ptr<ouinet::Client> g_client;
ouinet::asio::io_service g_ios;
thread g_client_thread;
bool g_crypto_initialized = false;

void start_client_thread(const vector<string>& args, const vector<string>& extra_path)
{
    if (!g_crypto_initialized) {
        ouinet::util::crypto_init();
        g_crypto_initialized = true;
    }

    char* old_path_c = getenv("PATH");
    if (old_path_c) {
        std::string old_path(old_path_c);
        std::set<std::string> old_path_entries;
        size_t index = 0;
        while (true) {
            size_t pos = old_path.find(':', index);
            if (pos == std::string::npos) {
                old_path_entries.insert(old_path.substr(index));
                break;
            } else {
                old_path_entries.insert(old_path.substr(index, pos - index));
                index = pos + 1;
            }
        }

        std::string new_path;
        for (size_t i = 0; i < extra_path.size(); i++) {
            if (!old_path_entries.count(extra_path[i])) {
                new_path += extra_path[i];
                new_path += ":";
            }
        }
        new_path += old_path;
        setenv("PATH", new_path.c_str(), 1);
    }

    if (g_client_thread.get_id() != thread::id()) return;

    g_client_thread = thread([=] {
            if (g_client) return;

            StdScopedRedirect redirect_guard;

            debug("Starting new ouinet client.");

            // In case we're restarting.
            g_ios.reset();

            vector<const char*> args_;
            args_.reserve(args.size());

            for (const auto& arg : args) {
                args_.push_back(arg.c_str());
            }

            try {
                ClientConfig cfg(args_.size(), (char**) args_.data());
                g_client = make_unique<ouinet::Client>(g_ios, move(cfg));
                g_client->start();
            }
            catch (const std::exception& e) {
                debug("Failed to start Ouinet client:");
                debug("%s", e.what());
                g_client.reset();
                return;
            }

            try {
                g_ios.run();
            }
            catch (const std::exception& e) {
                debug("Exception thrown from ouinet");
                debug("%s", e.what());
            }

            debug("Ouinet's main loop stopped.");
            g_client.reset();
        });
}

extern "C"
JNIEXPORT jint JNICALL
Java_ie_equalit_ouinet_Ouinet_nGetClientState(
        JNIEnv* env,
        jobject /* this */)
{
    // TODO: Avoid needing to keep this in sync by hand.
    if (!g_client)
        return g_ios.stopped() ? 6 /* stopped */ : 0 /* created */;
    switch (g_client->get_state()) {
    case ouinet::Client::RunningState::Created:  return 0;
    case ouinet::Client::RunningState::Failed:   return 1;
    case ouinet::Client::RunningState::Starting: return 2;
    case ouinet::Client::RunningState::Degraded: return 3;
    case ouinet::Client::RunningState::Started:  return 4;
    case ouinet::Client::RunningState::Stopping: return 5;
    case ouinet::Client::RunningState::Stopped:  return 6;
    }
    return -1;
}

extern "C"
JNIEXPORT void JNICALL
Java_ie_equalit_ouinet_Ouinet_nStartClient(
        JNIEnv* env,
        jobject /* this */,
        jobjectArray jargs,
        jobjectArray jpath)
{
    size_t argn = env->GetArrayLength(jargs);
    vector<string> args;
    args.reserve(argn);

    for (size_t i = 0; i < argn; ++i) {
        jstring jstr = (jstring) env->GetObjectArrayElement(jargs, i);
        const char* arg = env->GetStringUTFChars(jstr, 0);
        args.push_back(arg);
        env->ReleaseStringUTFChars(jstr, arg);
    }

    size_t pathn = env->GetArrayLength(jpath);
    vector<string> path;
    path.reserve(pathn);

    for (size_t i = 0; i < pathn; ++i) {
        jstring jstr = (jstring) env->GetObjectArrayElement(jpath, i);
        const char* dir = env->GetStringUTFChars(jstr, 0);
        path.push_back(dir);
        env->ReleaseStringUTFChars(jstr, dir);
    }

    start_client_thread(args, path);
}

extern "C"
JNIEXPORT void JNICALL
Java_ie_equalit_ouinet_Ouinet_nStopClient(
        JNIEnv *env,
        jobject /* this */)
{
    g_ios.post([] { if (g_client) g_client->stop(); });
    g_client_thread.join();
    g_client_thread = thread();
}

extern "C"
JNIEXPORT jstring JNICALL
Java_ie_equalit_ouinet_Ouinet_nGetCARootCert(
        JNIEnv* env,
        jclass /* class */,
        jstring j_repo_root)
{
    string repo_root = env->GetStringUTFChars(j_repo_root, NULL);
    return env->NewStringUTF(ouinet::Client::get_or_gen_ca_root_cert(repo_root).c_str());
}

extern "C"
JNIEXPORT void JNICALL
Java_ie_equalit_ouinet_Ouinet_nChargingStateChange(
        JNIEnv* env,
        jobject /* this */,
        jboolean j_is_charging) {
    g_ios.post([j_is_charging] {
        if (!g_client) return;
        g_client->charging_state_change(j_is_charging);
    });
}

extern "C"
JNIEXPORT void JNICALL
Java_ie_equalit_ouinet_Ouinet_nWifiStateChange(
        JNIEnv* env,
        jobject /* this */,
        jboolean j_is_wifi_connected) {
    g_ios.post([j_is_wifi_connected] {
        if (!g_client) return;
        g_client->wifi_state_change(j_is_wifi_connected);
    });
}

max_stars_repo_name: mhqz/ouinet

max_issues_repo_name: mhqz/ouinet

max_forks_repo_name: mhqz/ouinet


text:
#pragma once
#include <unordered_map>
#include <boost/get_pointer.hpp>
#include "type_traits/function_traits.hpp"
#include "type_traits/union_cast.hpp"
#include "mpl/mpl.hpp"
#include "util/console.hpp"

namespace stx
{
	template<class>
	class delegate{};
	template<class Ret, class ... Args>
	class delegate<Ret(Args...)>
	{
	public:
		using functor_type = typename to_function<Ret(Args...)>::type;
		using key_type = size_t;
		static constexpr key_type insert_failed = 0u;
	private:
		std::unordered_map<key_type,functor_type> functor;
	public:
		template<class origin_functor
				,class translator = stx::to_function<origin_functor>
				,class translated_functor_type = typename translator::type
				,typename std::enable_if<stx::mpl::and_<functor_type, translated_functor_type>::value>::type *& = stx::enabler>
		key_type operator+=(const origin_functor&& func)
		{
			//copy func to ref,when if func is rvalue reference
			auto ref = std::forward<origin_functor>(func);
			const auto key = stx::union_cast<key_type>(boost::get_pointer(ref));
			auto result = this->functor.emplace(key, ref);
			if (!result.second)
				return insert_failed;
			return key;
		}
		//can't remove rvalue reference functor
		template<class origin_functor
				,typename std::enable_if<!std::is_rvalue_reference<origin_functor>::value>::type *& = stx::enabler>
		void operator-=(const origin_functor&& func)
		{
			const auto key = stx::union_cast<key_type>(boost::get_pointer(func));
			auto it = this->functor.find(key);
			if (it != this->functor.end())
				this->functor.remove(it);
		}
		void operator()(Args... args) const
		{
			for (auto && item : functor) item(std::forward<Args>(args)...);
		}
	};
}

ext: hpp

max_stars_repo_name: Angeart/stx

max_issues_repo_name: Angeart/stx

max_forks_repo_name: Angeart/stx


text:
#ifndef TSA_VU8_UTIL_MAKE_ARG_STORAGE_HPP
#define TSA_VU8_UTIL_MAKE_ARG_STORAGE_HPP

// TODO: remove me

#include <boost/type_traits/remove_const.hpp>
#include <boost/type_traits/remove_reference.hpp>
#include <boost/mpl/identity.hpp>
#include <boost/mpl/transform.hpp>

#include <vu8/detail/ConvertibleString.hpp>

namespace vu8 { namespace detail {

namespace mpl = boost::mpl;

template <class T>
struct MakeArgStorageHelperRedirect : mpl::identity<T> {};

template <>
struct MakeArgStorageHelperRedirect<char const *>
  : mpl::identity<ConvertibleString> {};

template <class T>
struct MakeArgStorageHelper : MakeArgStorageHelperRedirect<
    typename boost::remove_const<
        typename boost::remove_reference<T>::type >::type > {};

template <class T>
struct MakeArgStorage
  : mpl::transform<T, detail::MakeArgStorageHelper<mpl::_1> > {};

} }
#endif


text:
//
// Created by realngnx on 11/06/2021.
//

#ifndef OPLB_PROXY_HPP
#define OPLB_PROXY_HPP

#include <memory>
#include <cstdlib>
#include <iostream>
#include <string>
#include <sstream>
#include <map>
#include <list>
#include <random>

#include <boost/bind.hpp>
#include <boost/asio.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/asio/io_service.hpp>

#include <curl/curl.h>

#if __has_include(<json/json.h>)
#include <json/json.h>
#else
#include <jsoncpp/json/json.h>
#endif

#include <wait.h>
#include <ctime>

#include "timer.hpp"
#include "util.hpp"
#include "sharding/node.hpp"
#include "sharding/sharder.hpp"

void get_available_nodes(const boost::system::error_code & /*e*/, timer *scheduler,
    std::map<std::string, std::string> &config, ConsistentHash& sharder);

namespace {
    std::size_t callback(
        const char *in,
        std::size_t size,
        std::size_t num,
        std::string *out
    );
}

namespace tcp_proxy {
    namespace ip = boost::asio::ip;

    class bridge : public boost::enable_shared_from_this<bridge> {
    public:
        typedef ip::tcp::socket socket_type;
        typedef boost::shared_ptr<bridge> ptr_type;

        explicit bridge(boost::asio::io_service &ios);

        socket_type downstream_socket_;
        socket_type upstream_socket_;

        socket_type &downstream_socket();
        socket_type &upstream_socket();
        void start(const std::string &upstream_host, unsigned short upstream_port);
        void handle_upstream_connect(const boost::system::error_code &error);

    private:

        // Read from remote server complete, send data to client
        void handle_upstream_read(const boost::system::error_code &error, const size_t &bytes_transferred);
        void handle_downstream_write(const boost::system::error_code &error);
        void handle_downstream_read(const boost::system::error_code &error, const size_t &bytes_transferred);
        void handle_upstream_write(const boost::system::error_code &error);
        void close();

        enum {
            max_data_length = 8196 // 8KB
        };
        unsigned char downstream_data_[max_data_length];
        unsigned char upstream_data_[max_data_length];

        boost::mutex mutex_;

    public:
        class acceptor {
        public:
            acceptor(boost::asio::io_service &io_service,
                     const std::string &local_host,
                     unsigned short local_port,
                     timer &t,
                     std::map<std::string, std::string> &config,
                     ConsistentHash& sharder);

            bool accept_connections();

        private:
            void handle_accept(const boost::system::error_code &error);
            void service_unavailable();

            boost::asio::io_service &io_service_;
            ip::address_v4 localhost_address;
            ip::tcp::acceptor acceptor_;
            ptr_type session_;
            ConsistentHash& sharder_;

        };
    };
}

#endif //OPLB_PROXY_HPP

max_stars_repo_name: oystr-foss/proxy-load-balancer

max_issues_repo_name: oystr-foss/proxy-load-balancer

max_forks_repo_name: oystr-foss/proxy-load-balancer


text:
/*
 * Copyright (c) 2019 Nobuyuki Umetani
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include <cmath>
#include <iostream>
#include <vector>
#include <chrono>
#include <Eigen/Core>
#if defined(_WIN32) // windows
#  define NOMINMAX   // to remove min,max macro
#  include <windows.h>  // should put before glfw3.h
#endif
#define GL_SILENCE_DEPRECATION
#include <GLFW/glfw3.h>

#include "delfem2/mshuni.h"
#include "delfem2/dtri2_v2dtri.h"
#include "delfem2/dtri.h"
#include "delfem2/eigen/ls_dense.h"
#include "delfem2/eigen/ls_sparse.h"
#include "delfem2/eigen/ls_ilu_sparse.h"
#include "delfem2/lsitrsol.h"
#include "delfem2/femsolidlinear.h"
#include "delfem2/glfw/viewer3.h"
#include "delfem2/glfw/util.h"
#include "delfem2/opengl/old/mshuni.h"

namespace dfm2 = delfem2;

void MakeMesh(
    std::vector<double>& aXY1,
    std::vector<unsigned int>& aTri1,
    std::vector<int>& aBCFlag,
    unsigned int ndim)
{
  std::vector< std::vector<double> > aaXY;
  const double len = 1.0;
  {
    aaXY.resize(1);
    aaXY[0].push_back(-len); aaXY[0].push_back(-len);
    aaXY[0].push_back(-len); aaXY[0].push_back(+len);
    aaXY[0].push_back(+len); aaXY[0].push_back(+len);
    aaXY[0].push_back(+len); aaXY[0].push_back(-len);
  }
  std::vector<delfem2::CDynPntSur> aPo2D;
  std::vector<delfem2::CDynTri> aETri;
  std::vector<delfem2::CVec2d> aVec2;
  delfem2::GenMesh(aPo2D,aETri,aVec2,
                   aaXY,0.05,0.05);
  MeshTri2D_Export(
      aXY1,aTri1,
      aVec2,aETri);
  const unsigned int np = aXY1.size()/2;
  aBCFlag.assign(np*ndim, 0);
  for(unsigned int ip=0;ip<np;++ip){
//    const double px = aXY1[ip*2+0];
    const double py = aXY1[ip*2+1];
    if( fabs(py-len) > 0.0001 ){ continue; }
    for(unsigned int idim=0;idim<ndim;++idim) {
      aBCFlag[ip * 2 + idim] = 1;
    }
  }
  std::cout<<"  ntri;"<<aTri1.size()/3<<"  nXY:"<<aXY1.size()/2<<std::endl;
}

void Solve1(
    std::vector<double>& aVal,
    const std::vector<double>& aXY1,
    const std::vector<unsigned int>& aTri1,
    const std::vector<int>& aBCFlag)
{
  const unsigned int np = aXY1.size()/2;
  const unsigned int nDoF = np*2;
  // -----------
  std::vector<unsigned int> psup_ind0, psup0;
  dfm2::JArray_PSuP_MeshElem(
      psup_ind0, psup0,
      aTri1.data(), aTri1.size()/3, 3,
      aXY1.size()/2);
  // -------------
  delfem2::CMatrixSparseBlock<Eigen::Matrix2d,Eigen::aligned_allocator<Eigen::Matrix2d>> mA;
  mA.Initialize(np);
  mA.SetPattern(psup_ind0.data(), psup_ind0.size(), psup0.data(), psup0.size());
  // ----------------------
  double myu = 10.0;
  double lambda = 10.0;
  double rho = 1.0;
  double g_x = 0.0;
  double g_y = -3.0;
  mA.setZero();
  Eigen::VectorXd vec_b(nDoF);
  vec_b.setZero();
  dfm2::MergeLinSys_SolidLinear_Static_MeshTri2D(
      mA,vec_b.data(),
      myu,lambda,rho,g_x,g_y,
      aXY1.data(), aXY1.size()/2,
      aTri1.data(), aTri1.size()/3,
      aVal.data());
  SetFixedBC_Dia(mA, aBCFlag.data(), 1.f);
  SetFixedBC_Col(mA, aBCFlag.data());
  SetFixedBC_Row(mA, aBCFlag.data());
  delfem2::setZero_Flag(vec_b, aBCFlag,0);
  // ---------------
  Eigen::VectorXd vec_x(vec_b.size());
  {
    double conv_ratio = 1.0e-6;
    int iteration = 1000;
    const std::size_t n = vec_b.size();
    Eigen::VectorXd tmp0(n), tmp1(n);
    std::vector<double> aConv = delfem2::Solve_CG(
        vec_b, vec_x, tmp0, tmp1,
        conv_ratio, iteration, mA);
    std::cout << aConv.size() << std::endl;
  }
//  SolveLinSys_PCG(mat_A,vec_b,vec_x,ilu_A, conv_ratio,iteration);
  // --------------
  {
    delfem2::CILU_SparseBlock<Eigen::Matrix2d,Eigen::aligned_allocator<Eigen::Matrix2d>> ilu;
    delfem2::ILU_SetPattern0(ilu,mA);
    delfem2::ILU_CopyValue(ilu,mA);
    delfem2::ILU_Decompose(ilu);
    Eigen::VectorXd vecX1(vec_b.size());
  }
  // --------------
  delfem2::XPlusAY(aVal,
      aBCFlag,
      1.0,vec_x);
}

int main()
{
  std::vector<unsigned int> aTri1;
  std::vector<double> aXY1;
  std::vector<int> aBCFlag; // master slave flag
  MakeMesh(
      aXY1, aTri1, aBCFlag,
      2);
  // ---
  std::vector<double> aVal;
  {
    const unsigned int np = aXY1.size()/2;
    aVal.assign(np * 2, 0.0);
    Solve1(aVal,aXY1,aTri1,aBCFlag);
  }
  // --------
  dfm2::glfw::CViewer3 viewer(1.5);
  dfm2::glfw::InitGLOld();
  viewer.OpenWindow();
  // ---------
  while(!::glfwWindowShouldClose(viewer.window)){
    viewer.DrawBegin_oldGL();
    delfem2::opengl::DrawMeshTri2D_FaceDisp2D(
        aXY1.data(), aXY1.size()/2,
        aTri1.data(), aTri1.size()/3,
        aVal.data(), 2);
    viewer.SwapBuffers();
    glfwPollEvents();
    viewer.ExitIfClosed();
  }
}


text:
/* training_data.cc
   Jeremy Barnes, 10 June 2003
   Copyright (c) 2003 Jeremy Barnes.  All rights reserved.
   $Source$

   Training data class.
*/

#include "config_impl.h"
#include "training_data.h"
#include "training_index.h"
#include "jml/utils/file_functions.h"
#include "jml/utils/pair_utils.h"
#include "jml/utils/filter_streams.h"
#include "jml/utils/vector_utils.h"
#include "jml/utils/sgi_numeric.h"
#include <boost/progress.hpp>
#include "jml/db/persistent.h"
#include "jml/utils/hash_map.h"
#include "jml/arch/demangle.h"

using namespace std;
using namespace DB;

namespace ML {

/*****************************************************************************/
/* TRAINING_DATA                                                             */
/*****************************************************************************/

Training_Data::Training_Data()
{
}

Training_Data::
Training_Data(std::shared_ptr<const Feature_Space> feature_space)
    : dirty_(false)
{
    init(feature_space);
}

Training_Data::Training_Data(const Training_Data & other)
    : data_(other.data_), index_(other.index_),
      feature_space_(other.feature_space_), dirty_(other.dirty_)
{
}

Training_Data::~Training_Data()
{
}

void Training_Data::
init(std::shared_ptr<const Feature_Space> feature_space)
{
    feature_space_ = feature_space;
    clear();
}

void Training_Data::clear()
{
    data_.clear();
    index_.reset();
    dirty_ = false;
}

void Training_Data::swap(Training_Data & other)
{
    std::swap(data_, other.data_);
    std::swap(index_, other.index_);
    std::swap(feature_space_, other.feature_space_);
    std::swap(dirty_, other.dirty_);
}

std::vector<Feature>
Training_Data::all_features() const
{
    return index().all_features();
}

void Training_Data::dump(const std::string & filename) const
{
    filter_ostream stream(filename);
    dump(stream);
}

void Training_Data::dump(std::ostream & stream) const
{
    stream << feature_space_->print() << endl;
    for (unsigned i = 0;  i < data_.size();  ++i) {
        stream << feature_space_->print(*data_[i])
               << endl;
    }
}

void Training_Data::serialize(DB::Store_Writer & store) const
{
    store << string("TRAINING_DATA");  // tag
    store << compact_size_t(1);  // version
    store << compact_size_t(data_.size());

    for (unsigned i = 0;  i < data_.size();  ++i)
        feature_space()->serialize(store, *data_[i]);

    store << compact_size_t(12345);  // ending marker
}

void Training_Data::save(const std::string & filename) const
{
    Store_Writer store(filename);
    serialize(store);
}

void Training_Data::reconstitute(DB::Store_Reader & store)
{
    string id;
    store >> id;
    if (id != "TRAINING_DATA")
        throw Exception("Training_Data::reconsitute(): object in store had ID "
                        + id  + ", expected TRAINING_DATA");

    compact_size_t version(store);

    switch (version) {
    case 1: {
        clear();
        compact_size_t size(store);
        for (unsigned i = 0;  i < size;  ++i) {
            std::shared_ptr<Feature_Set> ex;
            feature_space()->reconstitute(store, ex);
            add_example(ex);
        }

        compact_size_t marker(store);
        if (marker != 12345)
            throw Exception("Training_Data::reconstitue(): end marker invalid "
                            "or not found");
        break;
    }

    default:
        throw Exception("Training_Data::reconstitute(): unknown version");
    }
}

void Training_Data::load(const std::string & filename)
{
    Store_Reader store(filename);
    reconstitute(store);
}

std::shared_ptr<Feature_Set> & Training_Data::modify(int example)
{
    std::shared_ptr<Feature_Set> & fs = data_.at(example);
    dirty_ = true;
    if (!fs.unique())
        fs.reset(fs->make_copy());
    return fs;
}

void Training_Data::
add(const Training_Data & other, bool merge_index)
{
    bool can_merge = !(other.dirty_ || dirty_);

    for (unsigned x = 0;  x < other.example_count();  ++x)
        add_example(other.share(x));

    if (can_merge && merge_index) {
        // TODO: really merge them
        // Don't need to do now; the dirty flags will take care of it!
#if 0
        index_->merge(other.index_);
#endif
    }
}

int Training_Data::
add_example(const std::shared_ptr<Feature_Set> & example)
{
    example->sort();

    /* Add a new entry for the example. */
    int example_num = data_.size();
    data_.push_back(example);

    dirty_ = true;

    return example_num;
}

size_t Training_Data::
label_count(const Feature & predicted) const
{
    return feature_space()->info(predicted).value_count();
}

float
Training_Data::
modify_feature(int example_number,
               const Feature & feature,
               float new_val)
{
    std::shared_ptr<Feature_Set> & fs = data_[example_number];
    Mutable_Feature_Set * mut_fs = 0;

    float old_val = (*fs)[feature];

    if (old_val == new_val) return old_val;

    if (!mut_fs)
        mut_fs = dynamic_cast<Mutable_Feature_Set *>(fs.get());
    if (!mut_fs) {
        std::shared_ptr<Mutable_Feature_Set>
            mut(new Mutable_Feature_Set(fs->begin(), fs->end()));
        data_[example_number] = mut;
        mut_fs = mut.get();
    }

    notify_needs_reindex(feature);

    mut_fs->replace(feature, new_val);

    dirty_ = true;

    return old_val;
}

void
Training_Data::
fixup_grouping_features(const std::vector<Feature> & group_features,
                        std::vector<float> & offset)
{
    boost::timer t;

    size_t nf = group_features.size();

    bool ignore[nf];

    /* Categorical and string features can't be fixed up; we don't try to
       do it. */
    for (unsigned i = 0;  i < group_features.size();  ++i) {
        Feature_Info info = feature_space()->info(group_features[i]);
        if (info.value_count() != 0)
            ignore[i] = true;
        else ignore[i] = false;
    }

    //cerr << "fixup_grouping_features: features = " << group_features
    //     << " offsets = " << offset << endl;

    if (offset.empty())
        offset.resize(nf, 0.0);
    else if (offset.size() != group_features.size())
        throw Exception("fixup_grouping_features(): offsets don't match");

    vector<float> last(nf, 0.0);

    /* Get all features that have the grouping feature set. */
    for (unsigned x = 0;  x < data_.size();  ++x) {
        std::shared_ptr<Feature_Set> & fs = data_[x];

        //cerr << "example " << x << endl;

        for (unsigned f = 0;  f < nf;  ++f) {
            if (ignore[f]) continue;

            float group = (*fs)[group_features[f]];

            //cerr << "  group = " << group << " last = " << last[f] << endl;

            if (group < last[f])
                offset[f] += last[f] + 1.0;

            float new_group = group + offset[f];

            if (group != new_group)
                modify_feature(x, group_features[f], new_group);

            last[f] = group;
        }
    }

    for (unsigned f = 0;  f < nf;  ++f) {
        if (ignore[f]) continue;
        offset[f] += last[f] + 1.0;
    }

    //cerr << "fixup_grouping_features: " << t.elapsed() << "s" << endl;
}

void Training_Data::
preindex(const Feature & label, const std::vector<Feature> & features)
{
    Guard guard(index_lock);
    if (index_)
        throw Exception("preindex: already has index");

    //boost::timer timer;
    index_.reset(new Dataset_Index());
    index_->init(*this, label, features);
    dirty_ = false;
    //cerr << "preindex(): " << timer.elapsed() << "s for "
    //     << example_count() << " examples" << endl;
}

void Training_Data::preindex(const Feature & label)
{
    return;
    throw Exception("STUB", __PRETTY_FUNCTION__);
}

void Training_Data::preindex_features()
{
    return;
    throw Exception("STUB", __PRETTY_FUNCTION__);
}

vector<std::shared_ptr<Training_Data> >
Training_Data::
partition(const std::vector<float> & sizes_, bool random,
          const Feature & group_feature) const
{
    vector<std::shared_ptr<Training_Data> > output(sizes_.size());

    //cerr << "partitioning dataset" << endl;
    //dump(cerr);

    /* We do it differently depending upon whether or not we have a grouping
       feature. */
    if (group_feature == MISSING_FEATURE) {

        //cerr << "no group feature" << endl;

        /* This is a vector which tells us in which order we look at the
           examples. */
        vector<int> order(example_count());
        std::iota(order.begin(), order.end(), 0);
        if (random) std::random_shuffle(order.begin(), order.end());

        distribution<float> sizes(sizes_.begin(), sizes_.end());
        sizes.normalize();  sizes *= example_count();

        //cerr << "sizes = " << sizes << endl;

        unsigned ex = 0;
        for (unsigned i = 0;  i < sizes.size();  ++i) {
            /* Make an output training data object of the right type. */
            output[i].reset(make_type());
            output[i]->init(feature_space());

            /* Copy the data.  This is efficient since we use shared pointers to
               point to them; we are merely increasing the reference count of the
               pointer. */
            int n = (int)round(sizes[i]);

            /* Compensate for rounding errors. */
            while (ex + n > example_count()) --n;

            //cerr << "n = " << n << "  ex = " << ex << "  ex + n = " << ex + n
            //     << "  example_count = " << example_count() << endl;
            for (int x = ex;  x < ex + n;  ++x)
                output[i]->add_example(share(order[x]));

            ex += n;
        }
    }
    else {

        //cerr << "group feature" << endl;

        /* Get a list of groups. */
        Joint_Index group_dist
            = index().dist(group_feature, BY_EXAMPLE, IC_VALUE | IC_EXAMPLE);

        vector<float> groups;
        vector<vector<int> > group_examples;
        vector<int> group_numbers(example_count(), -1);

        float last_val = 0.0;
        for (Index_Iterator it = group_dist.begin();
             it != group_dist.end();  ++it) {
            float val = it->value();
            if (it == group_dist.begin() || val != last_val) {
                groups.push_back(val);
                group_examples.push_back(vector<int>());
                last_val = val;
            }
            group_examples.back().push_back(it->example());
            group_numbers[it->example()] = group_examples.size() - 1;
        }

        //cerr << "groups = " << groups << endl;

        int group_count = group_examples.size();

        //cerr << group_count << " groups" << endl;
        //cerr << example_count() << " examples" << endl;

        /* This is a vector which tells us in which order we look at the
           groups. */
        vector<int> order(group_count);
        std::iota(order.begin(), order.end(), 0);
        //cerr << "order = " << order << endl;
        if (random) std::random_shuffle(order.begin(), order.end());
        //cerr << "order = " << order << endl;

        distribution<float> sizes(sizes_.begin(), sizes_.end());
        sizes.normalize();  sizes *= group_count;

        //cerr << "sizes = " << sizes << endl;

        unsigned gr = 0;
        size_t total = 0;
        for (unsigned i = 0;  i < sizes.size();  ++i) {
            /* Make an output training data object of the right type. */
            output[i].reset(make_type());
            output[i]->init(feature_space());

            /* Copy the data.  This is efficient since we use shared pointers to
               point to them; we are merely increasing the reference count of the
               pointer. */
            int n = (int)round(sizes[i]);

            /* Compensate for rounding errors. */
            while (gr + n > group_count) --n;

            //cerr << "i = " << i << " gr = " << gr << " n = " << n << endl;

            //cerr << "adding " << n << " groups to dataset " << i << endl;
            for (int g = gr;  g < gr + n;  ++g) {
                //cerr << " " << groups[order[g]] << " ( ";
                for (int x = 0;  x < group_examples[order[g]].size();  ++x) {
                    output[i]->add_example(share(group_examples[order[g]][x]));
                    //cerr << group_examples[order[g]][x] << " ";
                }
                //cerr << " )  ";
            }
            cerr << endl;

            total += n;
            gr += n;
        }
    }

    return output;
}

Training_Data * Training_Data::make_copy() const
{
    return new Training_Data(*this);
}

Training_Data * Training_Data::make_type() const
{
    return new Training_Data();
}

const Dataset_Index & Training_Data::generate_index() const
{
    Guard guard(index_lock);
    if (!dirty_ && index_) return *index_;

    //boost::timer timer;
    index_.reset(new Dataset_Index());
    index_->init(*this);
    dirty_ = false;
    //cerr << "generate_index(): " << timer.elapsed() << "s for "
    //     << example_count() << " examples" << endl;
    return *index_;
}

size_t
Training_Data::
row_offset(size_t row) const
{
    throw Exception("Training_Data::row_offset(): class of type "
                    + demangle(typeid(*this).name())
                    + " doesn't support row offsets");
}

std::string
Training_Data::
row_comment(size_t row) const
{
    throw Exception("Training_Data::row_comment(): class of type "
                    + demangle(typeid(*this).name())
                    + " doesn't support row comments");
}

} // namespace ML

ext: cc

max_stars_repo_name: etnrlz/rtbkit

max_issues_repo_name: TuanTranEngineer/rtbkit

max_forks_repo_name: TuanTranEngineer/rtbkit


text:
#include "dwag.h"

#include <iostream> // for cout
#include <fstream>  // for ifstream
#include "zlib.h"  // for gzFile; presumes zstd library is installed
#include "gmm.h"

#define MIS_MATCH_CUT   5
#define MIN_LEVEL_CUT   30
#define MATCH(a,b,level) ((a) == (b) ? (level) : (level+1))
#define LENS    200

#define DEBUG   0   // 0

// use this to read part of the line
#define SHORT_RUN

#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>

#include <armadillo>

using namespace arma;

vector<vector<string>>& ReadTSV(const char* filename)
{
    using namespace std;

    ifstream infile(filename);
    if (!infile) {
        cout << "unable to load file" << endl;
    }
    string str;

    vector<vector<string>> vvStr;

    vector<string> vecTitle;
    int pos1, pos2;

    // title

    getline(infile, str);
    pos1 = 0;
    while((pos2 = str.find('\t',pos1))!= string::npos)
    {
        vecTitle.push_back(str.substr(pos1, pos2-pos1));
        pos1 = ++pos2;
    }

    // values
    while (getline(infile, str))
    {
        pos1 = 0;
        vector<string> vStr;
        while((pos2 = str.find('\t',pos1))!= string::npos)
        {
            vStr.push_back(str.substr(pos1, pos2-pos1));
            pos1 = ++pos2;
        }
        vStr.push_back(str.substr(pos1, string::npos));
        vvStr.push_back(vStr);
    }

    return vvStr;
}

int gmm_test()
{
    // create synthetic data containing
    // 2 clusters with normal distribution
    uword d = 1; // dimensionality
    uword N = 10000; // number of samples (vectors)
    mat data(d, N, fill::zeros);
    vec mean1 = linspace<vec>(1,d,d);
    vec mean2 = mean1 + 5;
    uword i = 0;
    while(i < N)
    {
        if(i < N) { data.col(i) = mean1 + randn<vec>(d); ++i; }
        if(i < N) { data.col(i) = mean1 + randn<vec>(d); ++i; }
        if(i < N) { data.col(i) = mean2 + randn<vec>(d); ++i; }
    }
    // model the data as a diagonal GMM with 2 Gaussians
    gmm_diag model;
    bool status = model.learn(data, 2, maha_dist, random_subset,10, 5, 1e-10, true);
    if(status == false) { cout << "learning failed" << endl; }

    model.means.print("means:");

    double overall_likelihood = model.avg_log_p(data);

    rowvec set_likelihood = model.log_p( data.cols(0,9) );
    double scalar_likelihood = model.log_p( data.col(0) );

    uword gaus_id = model.assign( data.col(0), eucl_dist );
    urowvec gaus_ids = model.assign( data.cols(0,9), prob_dist );

    urowvec histogram1 = model.raw_hist (data, prob_dist);
    rowvec histogram2 = model.norm_hist(data, eucl_dist);

    cout << "histogram2\t" << histogram2[0] << "\t" << histogram2[1] << endl;

    model.save("/Users/jyang/my_model.gmm");

    mat modified_dcovs = 2 * model.dcovs;

    model.set_dcovs(modified_dcovs);

    return 0;
}

vector<Record> read_fasta( char* fa_filename )
{
    gmm_test();
    /*
   GaussianMixture aGmm = GaussianMixture(4,4,0.1,0.1);
   //aGmm.setValue();

   Eigen::VectorXf ax;
   ax << 1,2,3,4;

   Eigen::VectorXf bx;
    bx << 2,3,4,5;

    aGmm.setValue(ax,bx);

    //aGmm.value();
    */
   vector<Record> records;

   ifstream fin( fa_filename );
   if(!fin)
   {
      cerr << "Couldn't open the input file!";
      return records;
   }
   string line;
   size_t counter = -1;

   getline(fin, line);
   while(fin)
   {
      if(line[0] == '>')
      {
         records.push_back(Record(&line[1]));
         counter++;
      }
      else
      {
         records[counter].field += line;
      }
      getline(fin, line);
   }

   return records;
}

string reverse_complement( string DNA ){
    string output;
    for( int i = DNA.size()-1; i >= 0; i -- ){
        switch ( DNA[i] ){
            case 'a':
            case 'A':
                output += 'T';
                break;
            case 'c':
            case 'C':
                output += 'G';
                break;
            case 'g':
            case 'G':
                output += 'C';
                break;
            case 't':
            case 'T':
                output += 'A';
                break;
            default:
                output += DNA[i];
                break;
        }
    }
    return output;
}

/*
 *    Building fake tree structure based on the last 60 seq. from fasta (reverse complement).
 *    DWAG starting pointer should be given.
 *
 *    infilename : fasta file
 */
dwag* build_tree_from_records(dwag* start, vector<Record>& records ){
    for ( int i = 0; i < records.size(); i++ ){
        string rc_seq = reverse_complement( records[i].field );
        //cout << records[i].header << endl << records[i].field << endl << rc_seq << endl;

        dwag* next = start;
        //cout << start->name << endl;
        for ( int j = 0; j < 100; j++ ){
            char aChar = rc_seq[j];
            //cout << aChar;

            switch( aChar ){
                case 'A':
                    if ( next->A == NULL ){
                        next->A = new dwag;
                        next->A->name = "A";
                        next->A->level = next->level + 1;
                        next->A->parent = next;
                        next = next->A;
                    }else{
                        next = next->A;
                    }
                break;
                case 'C':
                    if ( next->C == NULL ){
                        next->C = new dwag;
                        next->C->name = "C";
                        next->C->level = next->level + 1;
                        next->C->parent = next;
                        next = next->C;
                    }else{
                        next = next->C;
                    }
                break;
                case 'G':
                    if ( next->G == NULL ){
                        next->G = new dwag;
                        next->G->name = "G";
                        next->G->level = next->level + 1;
                        next->G->parent = next;
                        next = next->G;
                    }else{
                        next = next->G;
                    }
                break;
                case 'T':
                    if ( next->T == NULL ){
                        next->T = new dwag;
                        next->T->name = "T";
                        next->T->level = next->level + 1;
                        next->T->parent = next;
                        next = next->T;
                    }else{
                        next = next->T;
                    }
                break;
            }
            if ( next->index == -2 ) next->index = i; // assign gene index when it is first time
            else next->index = -1; // overlap with other gene
        }
        //cout << endl;
        //cout << get_sequence( next ) << endl;
    }
    //cout << "$$$$$$ " << start->name << endl;
    return start;
}

dwag* mismatch_search_tree_c( dwag* next, char* value, int level, int mismatch, int* global_mismatch, dwag* found ){
    if ( mismatch >= MIS_MATCH_CUT ){
        return found;
    }
    if (level >= MIN_LEVEL_CUT){
        if ( mismatch < *global_mismatch ){
            *global_mismatch = mismatch;
            found = next;
            //cout << "REF: " << get_sequence( found ) << endl;
            //cout << "QUR: " << value << endl;
        }
        return found;
    }

    if ( next->A != NULL ) found = mismatch_search_tree_c( next->A, value, level+1, MATCH(value[level],'A',mismatch), global_mismatch, found );
    if ( next->C != NULL ) found = mismatch_search_tree_c( next->C, value, level+1, MATCH(value[level],'C',mismatch), global_mismatch, found );
    if ( next->G != NULL ) found = mismatch_search_tree_c( next->G, value, level+1, MATCH(value[level],'G',mismatch), global_mismatch, found );
    if ( next->T != NULL ) found = mismatch_search_tree_c( next->T, value, level+1, MATCH(value[level],'T',mismatch), global_mismatch, found );

    return found;
}

/*
 *
 * This used whole fastq files
 *
 */
namespace io = boost::iostreams;

int pair_search_with_tree_normal(dwag* start, map< pair<int, int>, int >& output, char *fastq1, char *fastq2, bool bShortRun = false, int readlength = 151){
    //cout << "### " << start->name << endl;
    char buffer1[LENS];
    for (int xx = 0; xx<LENS; xx++) buffer1[xx] = NULL;
    char buffer2[LENS];
    for (int xx = 0; xx<LENS; xx++) buffer2[xx] = NULL;
    char gz1[LENS];
    char gz2[LENS];
    std::cout << fastq1 << std::endl;
    std::cout << fastq2 << std::endl;

    gzFile infile1 = gzopen(fastq1, "rb");
    if (!infile1) return -1;

    gzFile infile2 = gzopen(fastq2, "rb");
    if (!infile2) return -1;

    char buffer[LENS];
    int count = 0;
    int found_cnt1 = 0;
    int found_cnt2 = 0;
    int global_mismatch = 1000;
    int found_both_cnt = 0;
    dwag* found1 = NULL;
    dwag* found2 = NULL;

    int i = 0;
    while( 0 != gzgets(infile1, buffer1, LENS) ){
        if ( 0 == gzgets(infile2, buffer2, LENS) ){
            break;
        }

        i++;
        if ( i % 100000 == 0 ) {
            cout << count << "\t" << found_cnt1 << "\t" << found_cnt2 << "\t" << found_both_cnt << "\t" << found_both_cnt / (float) count * 100.0 << endl;
            if (  i > 1000000 && bShortRun == true ) break;
        }

        if ( i % 4 != 2 ) continue;
        count += 1;
        global_mismatch = 1000;

        found1 = NULL;
        if ( found1 == NULL ) {
            for ( int xx = 30; xx < 60; xx ++ ){
                found1 = mismatch_search_tree_c( start, &buffer1[xx], 0, 0, &global_mismatch, found1 ); // for read1
                if ( found1 != NULL ) break;
            }
        }
        if ( found1 != NULL ){
            found_cnt1 += 1;
        }

        global_mismatch = 1000;
        found2 = NULL;
        if ( found2 == NULL ) {
            for ( int xx = 30; xx < 60; xx ++ ){
                found2 = mismatch_search_tree_c( start, &buffer2[xx], 0, 0, &global_mismatch, found2 ); // for read1
                 if ( found2 != NULL ) break;
            }
        }
        if ( found2 != NULL ){
            /*if ( global_mismatch == MIS_MATCH_CUT-1 ){
                cout << "READ2: " << i << "\t" << global_mismatch << "\t" << found2->index << endl;
                cout << "REF: " << get_sequence( found2 ) << endl;
                cout << "QUR: " << string( &buffer2[54] ) << endl;
            }*/
            found_cnt2 += 1;
        }
        if ( found1 != NULL && found2 != NULL ){
            ++output[pair<int, int>(found1->index, found2->index)];
            found_both_cnt ++;
        }
    }
    cout << "Total: " << count << endl;
    cout << "Found 1: " << found_cnt1 << "\t" << found_cnt1 / (float) count * 100.0 << endl;
    cout << "Found 2: " << found_cnt2 << "\t" << found_cnt2 / (float) count * 100.0 << endl;
    cout << "Found Both: " << found_both_cnt << "\t" << found_both_cnt / (float) count * 100.0 << endl;
    gzclose( infile1 );
    gzclose( infile2 );

/*
    io::filtering_istream in;
    in.push(io::gzip_decompressor());
    in.push(io::file_source(fastq1));
    // read from in using std::istream interface

    std::string line;
    while (std::getline(in, line, '\n'))
    {
         std::cout << line ; // your code :)
    }
*/

    return 0;
}

/*
 *
 * This used only second line - seq files
 *
 */
int pair_search_with_tree(dwag* start, map< pair<int, int>, int >& output, char *fastq1, char *fastq2, bool bShortRun = false, int readlength = 151){
    //cout << "### " << start->name << endl;
    char buffer1[LENS];
    for (int xx = 0; xx<LENS; xx++) buffer1[xx] = NULL;
    char buffer2[LENS];
    for (int xx = 0; xx<LENS; xx++) buffer2[xx] = NULL;
    char gz1[LENS];
    char gz2[LENS];
    sprintf( gz1, "%s.2", fastq1 );
    sprintf( gz2, "%s.2", fastq2 );
    std::cout << gz1 << std::endl;
    std::cout << gz2 << std::endl;

    int size = 0;
    int i = 0;
    int line = 0;
    int global_mismatch = 1000;
    int found_cnt1 = 0;
    int found_cnt2 = 0;
    int found_both_cnt = 0;
    dwag* found1 = NULL;
    dwag* found2 = NULL;

    gzFile infile1 = gzopen(gz1, "rb");
    if (!infile1) return -1;
    gzFile infile2 = gzopen(gz2, "rb");
    if (!infile2) return -1;

    while( 0 != (size = gzread(infile1, buffer1, readlength+1) ) ){
        (size = gzread(infile2, buffer2, readlength+1) );
        //cout << buffer1 << buffer2 << endl;
        global_mismatch = 1000;
        i++;
        if ( i % 100000 == 0 ) {
            cout << i << "\t" << found_cnt1 << "\t" << found_cnt2 << "\t" << found_both_cnt << "\t" << found_both_cnt / (float) i << endl;
            if (  i > 1000000 && bShortRun == true ) break;
        }
        found1 = NULL;
        //found1 = mismatch_search_tree_c( start, &buffer1[52], 0, 0, &global_mismatch, found1 ); // for read1
        //if ( found1 == NULL ) found1 = mismatch_search_tree_c( start, &buffer1[50], 0, 0, &global_mismatch, found1 ); // for read1
        //if ( found1 == NULL ) found1 = mismatch_search_tree_c( start, &buffer1[51], 0, 0, &global_mismatch, found1 ); // for read1
        //if ( found1 == NULL ) found1 = mismatch_search_tree_c( start, &buffer1[53], 0, 0, &global_mismatch, found1 ); // for read1
        //if ( found1 == NULL ) found1 = mismatch_search_tree_c( start, &buffer1[54], 0, 0, &global_mismatch, found1 ); // for read1
        if ( found1 == NULL ) {
            for ( int xx = 0; xx < 100; xx ++ ){
                found1 = mismatch_search_tree_c( start, &buffer1[xx], 0, 0, &global_mismatch, found1 ); // for read1
                if ( found1 != NULL ) break;
            }
        }
        if ( found1 != NULL ){
            /*if ( global_mismatch == MIS_MATCH_CUT-1 ){
                cout << "READ1: " << i << "\t" << global_mismatch << "\t" << found1->index << endl;
                cout << "REF: " << get_sequence( found1 ) << endl;
                cout << "QUR: " << string( &buffer1[52] ) << endl;
            }*/
            found_cnt1 += 1;
        }
        global_mismatch = 1000;
        found2 = NULL;
        //found2 = mismatch_search_tree_c( start, &buffer2[54], 0, 0, &global_mismatch, found2 ); // for read2
        //if ( found2 == NULL ) found2 = mismatch_search_tree_c( start, &buffer2[52], 0, 0, &global_mismatch, found2 ); // for read2
        //if ( found2 == NULL ) found2 = mismatch_search_tree_c( start, &buffer2[53], 0, 0, &global_mismatch, found2 ); // for read2
        //if ( found2 == NULL ) found2 = mismatch_search_tree_c( start, &buffer2[55], 0, 0, &global_mismatch, found2 ); // for read2
        //if ( found2 == NULL ) found2 = mismatch_search_tree_c( start, &buffer2[56], 0, 0, &global_mismatch, found2 ); // for read2
        if ( found2 == NULL ) {
            for ( int xx = 0; xx < 100; xx ++ ){
                found2 = mismatch_search_tree_c( start, &buffer2[xx], 0, 0, &global_mismatch, found2 ); // for read1
                 if ( found2 != NULL ) break;
            }
        }
        if ( found2 != NULL ){
            /*if ( global_mismatch == MIS_MATCH_CUT-1 ){
                cout << "READ2: " << i << "\t" << global_mismatch << "\t" << found2->index << endl;
                cout << "REF: " << get_sequence( found2 ) << endl;
                cout << "QUR: " << string( &buffer2[54] ) << endl;
            }*/
            found_cnt2 += 1;
        }
        if ( found1 != NULL && found2 != NULL ){
            ++output[pair<int, int>(found1->index, found2->index)];
            found_both_cnt ++;
        }
    }
    cout << "Found 1: " << found_cnt1 << "\t" << found_cnt1 / (float) i << endl;
    cout << "Found 2: " << found_cnt2 << "\t" << found_cnt2 / (float) i << endl;
    cout << "Found Both: " << found_both_cnt << "\t" << found_both_cnt / (float) i << endl;
    gzclose( infile1 );
    gzclose( infile2 );
    return 0;
}

void print_map_pair_values( map<pair<int, int>,int>& output ){
    for ( map< pair<int,int>, int>::iterator it = output.begin(); it != output.end();  it++ ){
        cout << it->first.first << "," << it->first.second << "\t" << it->second << endl;
    }
}

void print_map_pair_map( map<pair<int, int>,int>& output, vector<Record>& records ){
    int total_cnt = 0;
    int zero_cnt = 0;
    for ( int i = 0; i < records.size(); i ++ ){
        for ( int j = 0; j < records.size(); j ++ ){
            int pair_cnt = output[ pair<int,int>(i,j) ];
            cout << i << "," << j << "\t" << records[i].header << "," << records[j].header << "\t" << pair_cnt << endl;
            if ( pair_cnt == 0 ) zero_cnt ++;
            total_cnt ++;
        }
    }
    cout << "Total: " << total_cnt << "\tZeros: " << zero_cnt << "\tFound: " << (total_cnt-zero_cnt) / (float)(total_cnt) * 100.0 << "%" << endl;
    /*for ( map< pair<int,int>, int>::iterator it = output.begin(); it != output.end();  it++ ){
        cout << it->first.first << "," << it->first.second << "\t" << it->second << endl;
    }*/
}

void write_comment_conditions( char * outfilename ){
    ofstream outFile( outfilename );
    outFile << "# ALLOWED_MISMATCH " << MIS_MATCH_CUT - 1 << endl;
    outFile << "# MIN_MATCH_LENGTH " << MIN_LEVEL_CUT << endl;
    outFile.close();
}

void write_map_pair_map( map<pair<int, int>,int>& output, vector<Record>& records, char * outfilename ){
    ofstream outFile;
    outFile.open( outfilename, ofstream::out | ofstream::app );

    int total_cnt = 0;
    int zero_cnt = 0;
    for ( int i = 0; i < records.size(); i ++ )
        outFile << "\t" << records[i].header;
    outFile << endl;
    for ( int i = 0; i < records.size(); i ++ ){
        outFile << records[i].header;
        for ( int j = 0; j < records.size(); j ++ ){
            int pair_cnt = output[ pair<int,int>(i,j) ];
            outFile << "\t" << pair_cnt;
            //outFile << i << "," << j << "\t" << records[i].header << "," << records[j].header << "\t" << pair_cnt << endl;
            if ( pair_cnt == 0 ) zero_cnt ++;
            total_cnt ++;
        }
        outFile << endl;
    }
    //outFile << "Total: " << total_cnt << "\tZeros: " << zero_cnt << "\tFound: " << (total_cnt-zero_cnt) / (float)(total_cnt) * 100.0 << "%" << endl;
    outFile.close();
    /*for ( map< pair<int,int>, int>::iterator it = output.begin(); it != output.end();  it++ ){
 *         cout << it->first.first << "," << it->first.second << "\t" << it->second << endl;
 *             }*/
}

/*
 * argv[1] : fasta
 * argv[2] : fastq1
 * argv[3] : fastq2
 */
int fusion_pair_search(int argc, char *argv[]){
    //std::cout << "fusion_pair_search" << std::endl;

    dwag start;
    map<pair<int, int>,int> output;

    //cout << "read_fasta: " << argv[2] << endl;
    vector<Record> records = read_fasta( argv[2] );

    //cout << "build_tree_from_records" << endl;
    build_tree_from_records( &start, records );

    //cout << "pair_search_with_tree: " << argv[3] << " , " << argv[4] << endl;
    pair_search_with_tree( &start, output, argv[3], argv[4], 151 );

    print_map_pair_values( output );
    print_map_pair_map( output, records );
    write_comment_conditions( argv[5] );
    write_map_pair_map( output, records, argv[5] );
    return 0;
}