text:
/* $Id: MultiMergeStream.cpp,v 1.32 2015/02/05 07:25:41 severin Exp $ */

/***

NAME - EEDB::SPStreams::MultiMergeStream

SYNOPSIS

DESCRIPTION

a variation on MergeStream, but designed to be a primary source made up of many
component databases (for example a collection of OSCFileDB). Allows one to
create virtual databases made up of component databases.

CONTACT

Jessica Severin <severin@gsc.riken.jp>

LICENSE

 * Software License Agreement (BSD License)
 * EdgeExpressDB [eeDB] system
 * copyright (c) 2007-2013 Jessica Severin RIKEN OSC
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Jessica Severin RIKEN OSC nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

APPENDIX

The rest of the documentation details each of the object methods. Internal methods are usually preceded with a _

***/

#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <string>
#include <stdarg.h>
#include <boost/algorithm/string.hpp>
#include <EEDB/Experiment.h>
#include <EEDB/FeatureSource.h>
#include <EEDB/EdgeSource.h>
#include <EEDB/Feature.h>
#include <EEDB/Expression.h>
#include <EEDB/Symbol.h>
#include <EEDB/SPStream.h>
#include <EEDB/Peer.h>
#include <EEDB/SPStreams/MultiMergeStream.h>
#include <EEDB/SPStreams/StreamBuffer.h>

using namespace std;
using namespace MQDB;

const char*  EEDB::SPStreams::MultiMergeStream::class_name = "EEDB::SPStreams::MultiMergeStream";

//function prototypes
MQDB::DBObject* _spstream_multimergestream_next_in_stream_func(EEDB::SPStream* node) {
  return ((EEDB::SPStreams::MultiMergeStream*)node)->_next_in_stream();
}

MQDB::DBObject* _spstream_multimergestream_fetch_object_by_id_func(EEDB::SPStream* node, string fid) {
  return ((EEDB::SPStreams::MultiMergeStream*)node)->_fetch_object_by_id(fid);
}

void _spstream_multimergestream_stream_clear_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_clear();
}

bool _spstream_multimergestream_stream_by_named_region_func(EEDB::SPStream* node, string assembly_name, string chrom_name, long int start, long int end) {
  return ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_by_named_region(assembly_name, chrom_name, start, end);
}

void _spstream_multimergestream_stream_features_by_metadata_search_func(EEDB::SPStream* node, string search_logic) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_features_by_metadata_search(search_logic);
}

void _spstream_multimergestream_stream_data_sources_func(EEDB::SPStream* node, string classname, string filter_logic) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_data_sources(classname, filter_logic);
}

void _spstream_multimergestream_get_dependent_datasource_ids_func(EEDB::SPStream* node, map<string,bool> &source_ids) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_get_dependent_datasource_ids(source_ids);
}

void _spstream_multimergestream_stream_chromosomes_func(EEDB::SPStream* node, string assembly_name, string chrom_name) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_chromosomes(assembly_name, chrom_name);
}

void _spstream_multimergestream_stream_peers_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_stream_peers();
}

void _spstream_multimergestream_reload_stream_data_sources_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_reload_stream_data_sources();
}

void _spstream_multimergestream_disconnect_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_disconnect();
}

void _spstream_multimergestream_reset_stream_node_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_reset_stream_node();
}

void _spstream_multimergestream_delete_func(MQDB::DBObject *obj) {
  delete (EEDB::SPStreams::MultiMergeStream*)obj;
}

void _spstream_multimergestream_get_proxies_by_name_func(EEDB::SPStream* node, string proxy_name, vector<EEDB::SPStream*> &proxies) {
  ((EEDB::SPStreams::MultiMergeStream*)node)->_get_proxies_by_name(proxy_name, proxies);
}

EEDB::SPStreams::MultiMergeStream::MultiMergeStream() {
  init();
}

EEDB::SPStreams::MultiMergeStream::MultiMergeStream(MQDB::Database* db) {
  init();
  database(db);
}

EEDB::SPStreams::MultiMergeStream::~MultiMergeStream() {
  for(unsigned i=0; i<_sourcestreams.size(); i++) {
    EEDB::SPStreams::MultistreamElement *mse = _sourcestreams[i];
    if(mse->stream != NULL) {
      mse->stream->disconnect();
      mse->stream->release();
      mse->stream = NULL;
    }
    delete mse;
  }
  _sourcestreams.clear();
}

void EEDB::SPStreams::MultiMergeStream::init() {
  EEDB::SPStream::init();
  _classname      = EEDB::SPStreams::MultiMergeStream::class_name;
  _module_name    = "MultiMergeStream";
  _funcptr_delete = _spstream_multimergestream_delete_func;

  //function pointer code
  _funcptr_next_in_stream                     = _spstream_multimergestream_next_in_stream_func;
  _funcptr_fetch_object_by_id                 = _spstream_multimergestream_fetch_object_by_id_func;
  _funcptr_disconnect                         = _spstream_multimergestream_disconnect_func;
  _funcptr_stream_clear                       = _spstream_multimergestream_stream_clear_func;
  _funcptr_stream_by_named_region             = _spstream_multimergestream_stream_by_named_region_func;
  _funcptr_stream_features_by_metadata_search = _spstream_multimergestream_stream_features_by_metadata_search_func;
  _funcptr_stream_data_sources                = _spstream_multimergestream_stream_data_sources_func;
  _funcptr_get_dependent_datasource_ids       = _spstream_multimergestream_get_dependent_datasource_ids_func;
  _funcptr_reload_stream_data_sources         = _spstream_multimergestream_reload_stream_data_sources_func;
  _funcptr_stream_chromosomes                 = _spstream_multimergestream_stream_chromosomes_func;
  _funcptr_stream_peers                       = _spstream_multimergestream_stream_peers_func;
  _funcptr_reset_stream_node                  = _spstream_multimergestream_reset_stream_node_func;
  _funcptr_get_proxies_by_name                = _spstream_multimergestream_get_proxies_by_name_func;

  //attribute variables
  _source_stream        = NULL;  //super class variable not used
  _active_sourcestreams = NULL;
  /*

  $self->{'_sourcestreams'} = [];
  #array of hashes hashes holding all information on each source

  $self->{'_sourcestream_output'} = "feature";
  $self->{'debug'} = 0;
  */
}

void EEDB::SPStreams::MultiMergeStream::display_info() {
  printf("%s\n", display_desc().c_str());
}

string EEDB::SPStreams::MultiMergeStream::display_desc() {
  string str = "MultiMergeStream [";
  return str;
}

string EEDB::SPStreams::MultiMergeStream::display_contents() {
  return display_desc();
}

/*
sub xml_start {
  my $self = shift;
  my $str = sprintf("<spstream module=\"%s\" >\n", $self->class);
  $str .= sprintf("<sourcestream_output value=\"%s\"/>\n", $self->sourcestream_output);

  foreach my $sshash (@{$self->{'_sourcestreams'}}) {
    $str .= "<stream_source>\n";
    $str .= $sshash->{'sourcestream'}->xml;
    $str .= "</stream_source>\n";
  }
  return $str;
}

sub _init_from_xmltree {
  my $self = shift;
  my $xmlTree = shift;  #a hash tree generated by XML::TreePP

  unless($xmlTree->{'-module'} eq "EEDB::SPStream::MultiMergeStream") { return undef; }

  if($xmlTree->{'sourcestream_output'}) {
    $self->sourcestream_output($xmlTree->{'sourcestream_output'}->{'-value'});
  }

  if($xmlTree->{'stream_source'}) {
    my $sources = $xmlTree->{'stream_source'};
    unless($sources =~ /ARRAY/) { $sources = [$sources]; }
    foreach my $sourceTree (@$sources) {
      my ($head,$tail) = EEDB::SPStream->create_stream_stack_from_xml($sourceTree);
      if($head) { $self->add_sourcestream($head); }
    }
  }
  return $self;
}
*/

////////////////////////////////////////////////////////////////////////////
//
// initialization and configuration methods
//
////////////////////////////////////////////////////////////////////////////

/***** clear_sourcestream_filters
  Description: resets directly connected source_streams to have no filters
*****/

void EEDB::SPStreams::MultiMergeStream::clear_sourcestream_filters()  {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    if((*it)->stream_is_source) {
      EEDB::SPStreams::SourceStream  *source = (EEDB::SPStreams::SourceStream*) (*it)->stream;
      source->clear_sourcestream_filters();
    }
  }
}

void EEDB::SPStreams::MultiMergeStream::add_source_id_filter(string fid)  {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    if((*it)->stream_is_source) {
      EEDB::SPStreams::SourceStream  *source = (EEDB::SPStreams::SourceStream*) (*it)->stream;
      source->add_source_id_filter(fid);
    }
  }
}

/***** add_expression_datatype_filter
  Description: sets filters for Expression data to only specified expression_datatypes
*****/

void EEDB::SPStreams::MultiMergeStream::add_expression_datatype_filter(EEDB::Datatype* datatype)  {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    if((*it)->stream_is_source) {
      EEDB::SPStreams::SourceStream  *source = (EEDB::SPStreams::SourceStream*) (*it)->stream;
      source->add_expression_datatype_filter(datatype);
    }
  }
}

/***** set_sourcestream_output
  Description: sets output object (Feature, Expression, Edge) when streaming
*****/

void  EEDB::SPStreams::MultiMergeStream::set_sourcestream_output(string value) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    if((*it)->stream_is_source) {
      EEDB::SPStreams::SourceStream  *source = (EEDB::SPStreams::SourceStream*) (*it)->stream;
      source->set_sourcestream_output(value);
    }
  }
}

/***** add_stream
  Description: add an EEDB::SPStream into the pool of streams to merge.
*****/

void EEDB::SPStreams::MultiMergeStream::add_stream(EEDB::SPStream* stream) {
  //add stream to streams to be merged
  if(stream == NULL) { return; }

  EEDB::SPStreams::MultistreamElement *mse = new EEDB::SPStreams::MultistreamElement;
  stream->retain();
  mse->stream = stream;
  _sourcestreams.push_back(mse);
}

/***** add_sourcestream
  Description: add an EEDB::SPStreams::SourceStream into the pool of streams to merge.
               SourceStream has additional API functionality which is enabled for these streams
*****/

void EEDB::SPStreams::MultiMergeStream::add_sourcestream(EEDB::SPStreams::SourceStream* stream) {
  //add stream to streams to be merged
  if(stream == NULL) { return; }

  EEDB::SPStreams::MultistreamElement *mse = new EEDB::SPStreams::MultistreamElement;
  stream->retain();
  mse->stream = stream;
  mse->stream_is_source = true;
  _sourcestreams.push_back(mse);
}

////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////

bool multistream_sort_func (EEDB::SPStreams::MultistreamElement *a, EEDB::SPStreams::MultistreamElement *b) {
  if(a == NULL) { return false; }
  if(b == NULL) { return true; }
  return (*a < *b);
}

void EEDB::SPStreams::MultiMergeStream::_resort_active_streams() {
  //head of active streams has been updated, rest of linked-list is already sorted.
  //just need to move the head element by insert-sort
  EEDB::SPStreams::MultistreamElement *st1, *st2, *st3;

  if(_active_sourcestreams == NULL) { return; }

  if(_active_sourcestreams->stream_is_empty) {
    //stream is empty so unlink and move _active_sourcestreams to next
    //printf("active head is empty, move to next stream\n");
    st1 = _active_sourcestreams;
    _active_sourcestreams = _active_sourcestreams->next;
    st1->next =NULL;
    st1->prev =NULL;
    if(_active_sourcestreams != NULL) { _active_sourcestreams->prev = NULL; }
    return;
  }
  if(_active_sourcestreams->next == NULL) { return; } //active stream is only one left

  //_active_sourcestreams maybe needs to move
  //move st1 pointer to location where need to insert
  st1 = _active_sourcestreams;
  while((st1->next !=NULL) and (*(st1->next) < *_active_sourcestreams)) {
    //next stream is less than active so need to move
    st1 = st1->next;
  }

  if(st1 != _active_sourcestreams) {
    //need to move active to between st1 and st1->next
    st3 = _active_sourcestreams->next;  //st3 will become new head active

    st2 = st1->next;
    _active_sourcestreams->prev = st1;
    _active_sourcestreams->next = st2;
    st1->next = _active_sourcestreams;
    if(st2!=NULL) {
      st2->prev = _active_sourcestreams;
    }
    _active_sourcestreams = st3;
    _active_sourcestreams->prev = NULL;
  }
}

void EEDB::SPStreams::MultiMergeStream::_init_active_streams() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->input_stream_count = 0;
    (*it)->stream_is_empty    = true;  //set to empty first
    (*it)->current_object     = (*it)->stream->next_in_stream();
    if((*it)->current_object != NULL) {
      (*it)->stream_is_empty = false;
      (*it)->input_stream_count = 1;
      (*it)->next = NULL;
      (*it)->prev = NULL;
      (*it)->object_class = MSE_OTHER;
      if((*it)->current_object->classname() == EEDB::Feature::class_name)    { (*it)->object_class = MSE_FEATURE; }
      if((*it)->current_object->classname() == EEDB::Expression::class_name) { (*it)->object_class = MSE_EXPRESS; }
    }
  }
  sort(_sourcestreams.begin(), _sourcestreams.end(), multistream_sort_func);

  _active_sourcestreams = NULL;  //linked list which will be sorted
  EEDB::SPStreams::MultistreamElement  *t_tail=NULL;

  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if(!(*it)->stream_is_empty) {
      if(_active_sourcestreams == NULL) {
        _active_sourcestreams = (*it);
      } else {
        t_tail->next = (*it);
        (*it)->prev  = t_tail;
      }
      t_tail = (*it);
    }
  }
}

//////////////////////////////////////////////////////////////////////////////////////////////////
//
// override method for subclasses which will
// do all the work
//
//////////////////////////////////////////////////////////////////////////////////////////////////

/***** reset_stream_node
  Description: re-initialize this stream node prior to reconfiguring stream
*****/

void EEDB::SPStreams::MultiMergeStream::_reset_stream_node() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->current_object     = NULL;
    (*it)->stream_is_empty    = true;  //reset to empty, _init_active_streams will change
    (*it)->object_class       = MSE_OTHER;
    (*it)->input_stream_count = 0;
    if((*it)->stream_is_source) {
      //EEDB::SPStreams::SourceStream  *source = (EEDB::SPStreams::SourceStream*) (*it)->stream;
      //source->sourcestream_output(_sourcestream_output);
    }
  }
}

/***** fetch_object_by_id
  Description: fetch single object from database
  Arg (1)    : $id (federatedID <uuid>::id:::<class>)
  Returntype : either an MQDB::DBObject* or NULL if not found
*****/

MQDB::DBObject* EEDB::SPStreams::MultiMergeStream::_fetch_object_by_id(string fid) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    MQDB::DBObject *obj = (*it)->stream->fetch_object_by_id(fid);
    if(obj!=NULL) { return obj; }
  }
  return NULL;
}

/***** next_in_stream
  Description: return the next object in the stream stack
               depending on the configuration this will either be an on-the-fly created object
               or an object passed through filters, or a primary object streamed out of database/peer
  Returntype : either an MQDB::DBObject* or NULL if end of stream
*****/

MQDB::DBObject* EEDB::SPStreams::MultiMergeStream::_next_in_stream() {
  //array of _sourcestreams is alrady sorted and preloaded so just
  //grab object from stream at head of array, then prefetch and resort

  if(_active_sourcestreams == NULL) { return NULL; }

  EEDB::SPStreams::MultistreamElement *mse = _active_sourcestreams;

  if(mse == NULL) { return NULL; }
  if(mse->stream_is_empty) { return NULL; }
  if(mse->stream == NULL) { return NULL; }

  MQDB::DBObject*  next_obj = mse->current_object;

  //now reload and resort
  mse->current_object = mse->stream->next_in_stream();
  if(mse->current_object == NULL) { mse->stream_is_empty = true; }
  else {
    mse->input_stream_count++;
    mse->object_class = MSE_OTHER;
    if(mse->current_object->classname() == EEDB::Feature::class_name)    { mse->object_class = MSE_FEATURE; }
    if(mse->current_object->classname() == EEDB::Expression::class_name) { mse->object_class = MSE_EXPRESS; }
  }

  _resort_active_streams();

  return next_obj;
}

/***** stream_clear
  Description: re-initialize the stream-stack back to a clear/empty state
*****/

void EEDB::SPStreams::MultiMergeStream::_stream_clear() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->stream_clear();
  }
  _reset_stream_node();
}

/***** stream_data_sources
  Description: stream all sources(FeatureSource, EdgeSource, and Experiment) out of database
*****/

void EEDB::SPStreams::MultiMergeStream::_stream_data_sources(string classname, string filter_logic) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->stream_data_sources(classname, filter_logic);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

void EEDB::SPStreams::MultiMergeStream::_reload_stream_data_sources() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->reload_stream_data_sources();
  }
}

void EEDB::SPStreams::MultiMergeStream::_get_dependent_datasource_ids(map<string,bool> &source_ids) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->get_dependent_datasource_ids(source_ids);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

/***** stream_by_named_region
  Description: configure/initialize the stream-stack to a stream from a named region.
               This will be passed down the stack until it reaches an SPStream::SourceStream instance
               which then knows how to create a new stream from a database.
*****/

bool EEDB::SPStreams::MultiMergeStream::_stream_by_named_region(string assembly_name, string chrom_name, long int start, long int end) {
  bool rtn = true;
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    rtn &= (*it)->stream->stream_by_named_region(assembly_name, chrom_name, start, end);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
  return rtn;
}

/***** stream_features_by_metadata_search
  Description: perform search of features through metadata filter logic
  Arg (1)    : a keyword/logic string which is applied to the metadata (optional)
*****/

void EEDB::SPStreams::MultiMergeStream::_stream_features_by_metadata_search(string search_logic) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->stream_features_by_metadata_search(search_logic);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

/***** stream_chromosomes
  Description: stream all EEDB::Chrom chromosomes from databases on the stream
*****/

void EEDB::SPStreams::MultiMergeStream::_stream_chromosomes(string assembly_name, string chrom_name) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->stream_chromosomes(assembly_name, chrom_name);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

void EEDB::SPStreams::MultiMergeStream::_get_proxies_by_name(string proxy_name, vector<EEDB::SPStream*> &proxies) {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->get_proxies_by_name(proxy_name, proxies);
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

/***** stream_peers
  Description: stream all known peers from database
*****/

void EEDB::SPStreams::MultiMergeStream::_stream_peers() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->stream_peers();
  }
  _init_active_streams();  //prepares for next_in_stream() calls
}

/***** disconnect
  Description: send "disconnect" message to all spstream modules
*****/

void  EEDB::SPStreams::MultiMergeStream::_disconnect() {
  vector<EEDB::SPStreams::MultistreamElement*>::iterator   it;
  for(it = _sourcestreams.begin(); it != _sourcestreams.end(); it++) {
    if((*it)->stream == NULL) { continue; }
    (*it)->stream->disconnect();
  }
}

////////////////////////////////////////////////////////////////////////////
//
// MultistreamElement section
//
////////////////////////////////////////////////////////////////////////////

EEDB::SPStreams::MultistreamElement::MultistreamElement() {
  //constructor
  stream = NULL;
  current_object = NULL;
  input_stream_count = 0;
  stream_is_empty = true;
  stream_is_source = false;
  object_class = MSE_OTHER;

}

bool EEDB::SPStreams::MultistreamElement::operator< (const EEDB::SPStreams::MultistreamElement& b) {
  MQDB::DBObject *obj1 = current_object;
  MQDB::DBObject *obj2 = b.current_object;

  if(obj1==obj2) { return false; }  //either same or both NULL (pointer compare)
  if(obj1==NULL) { return false; }  //NULL > real objects
  if(obj2==NULL) { return true; }   //obj1!=NULL but obj2==NULL (NULL > real objects)

  //if either a or b is an "other" object no sort order
  if((object_class == MSE_OTHER) or (b.object_class == MSE_OTHER)) { return false; }

  EEDB::Feature *feature1=NULL, *feature2=NULL;
  if(object_class == MSE_FEATURE) { feature1 = (EEDB::Feature*)obj1; }
  if(object_class == MSE_EXPRESS) { feature1 = ((EEDB::Expression*)obj1)->feature(); }

  if(b.object_class == MSE_FEATURE) { feature2 = (EEDB::Feature*)obj2; }
  if(b.object_class == MSE_EXPRESS) { feature2 = ((EEDB::Expression*)obj2)->feature(); }

  if(feature1==NULL or feature2==NULL) { return false; }

  return (*feature1) < (*feature2);
}


text:
// Copyright (c) 2014-2019 niXman (i dot nixman dog gmail dot com). All
// rights reserved.
//
// This file is part of CONFIG-CTOR(https://github.com/niXman/config-ctor) project.
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
//
//
// Boost Software License - Version 1.0 - August 17th, 2003
//
// Permission is hereby granted, free of charge, to any person or organization
// obtaining a copy of the software and accompanying documentation covered by
// this license (the "Software") to use, reproduce, display, distribute,
// execute, and transmit the Software, and to prepare derivative works of the
// Software, and to permit third-parties to whom the Software is furnished to
// do so, all subject to the following:
//
// The copyright notices in the Software and this entire statement, including
// the above license grant, this restriction and the following disclaimer,
// must be included in all copies of the Software, in whole or in part, and
// all derivative works of the Software, unless such copies or derivative
// works are solely in the form of machine-executable object code generated by
// a source language processor.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

#ifndef __config_ctor__config_ctor_hpp
#define __config_ctor__config_ctor_hpp

#include <config-ctor/flatjson.hpp>

#include <boost/tti/has_member_function.hpp>

#include <boost/preprocessor/cat.hpp>
#include <boost/preprocessor/comparison/greater_equal.hpp>
#include <boost/preprocessor/control/if.hpp>
#include <boost/preprocessor/stringize.hpp>
#include <boost/preprocessor/punctuation/comma_if.hpp>
#include <boost/preprocessor/seq/for_each_i.hpp>
#include <boost/preprocessor/tuple/size.hpp>
#include <boost/preprocessor/tuple/elem.hpp>

#include <string>
#include <iosfwd>
#include <fstream>
#include <stdexcept>

#include <cstdlib>
#include <cstdint>

#ifdef _WIN32
#   include <direct.h>
#else
#   include <unistd.h>
#endif // _WIN32

/***************************************************************************/

namespace config_ctor {
namespace details {

BOOST_TTI_HAS_MEMBER_FUNCTION(after_read)
BOOST_TTI_HAS_MEMBER_FUNCTION(before_write)

template<bool>
struct call_after_read_proxy {
    template<typename T>
    static void call(T&) {}
};

template<>
struct call_after_read_proxy<true> {
    template<typename T>
    static void call(T& o) { o.after_read(o); }
};

template<bool>
struct call_before_write_proxy {
    template<typename T>
    static void call(const T&) {}
};

template<>
struct call_before_write_proxy<true> {
    template<typename T>
    static void call(const T& o) { const_cast<T&>(o).before_write(const_cast<T&>(o)); }
};

std::string get_procpath();
std::string get_procname();

/***************************************************************************/

template<typename>
struct is_bool: std::false_type
{};

template<>
struct is_bool<bool>: std::true_type
{};

template<typename T>
struct is_integral: std::integral_constant<bool, std::is_integral<T>::value && !is_bool<T>::value>
{};

template<typename T>
struct is_floating: std::is_floating_point<T>
{};

template<typename T>
struct is_string: std::is_same<T, std::string>
{};

template<typename T>
struct is_null: std::is_null_pointer<T>
{};

template<typename T>
struct is_simple_type: std::integral_constant<
         bool
        ,is_bool<T>::value ||
         is_integral<T>::value ||
         is_floating<T>::value ||
         is_string<T>::value ||
         is_null<T>::value
    >
{};

template<typename T>
struct is_config_ctor_type: std::integral_constant<
         bool
        ,!is_simple_type<T>::value &&
         std::is_constructible<T, const flatjson::fjson &>::value
    >
{};

/***************************************************************************/

// bool
template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_bool<T>::value>::type* = nullptr)
{
    if ( !default_value ) {
        return cfg.at(key).to_bool();
    }

    return !cfg.contains(key) ? (std::strcmp(default_value, "true") == 0) : cfg.at(key).to_bool();
}

// integrals
template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_integral<T>::value>::type* = nullptr)
{
    std::string str;
    if ( !default_value ) {
        str = cfg.at(key).to_string();
    } else {
        str = !cfg.contains(key)
            ? default_value
            : cfg.at(key).to_string()
        ;
    }

    std::int64_t mult = 1;
    switch ( str.back() ) {
        case 't': case 'T': mult *= 1024; // fallthrough
        case 'g': case 'G': mult *= 1024; // fallthrough
        case 'm': case 'M': mult *= 1024; // fallthrough
        case 'k': case 'K': mult *= 1024; // fallthrough

        str.pop_back();
    }

    if ( std::is_unsigned<T>::value ) {
        return static_cast<T>(std::stoul(str) * mult);
    }

    return static_cast<T>(std::stol(str) * mult);
}

// floating
template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_floating<T>::value>::type* = nullptr)
{
    if ( !default_value ) {
        return cfg.at(key).to<T>();
    }

    return !cfg.contains(key)
        ? (flatjson::details::conv_to<T>(default_value, std::strlen(default_value)))
        : cfg.at(key).to<T>()
    ;
}

template<typename Iterator, typename Pred>
Iterator find_if(Iterator beg, Iterator end, Pred pred) {
    for ( ; beg != end ; ++beg ) {
        if ( pred(*beg) ) {
            return beg;
        }
    }

    return end;
}

// string
template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_string<T>::value>::type* = nullptr)
{
    std::string res;
    if ( !default_value ) {
        res = cfg.at(key).to_string();
    } else {
        res = !cfg.contains(key)
            ? default_value
            : cfg.at(key).to_string()
        ;
    }

    static auto get_home = []() -> std::string {
    #ifdef _WIN32
        return ::getenv("USERPROFILE");
    #elif defined(__linux__) || defined(__APPLE__)
        return ::getenv("HOME");
    #else
    #   error UNKNOWN HOST
    #endif
    };
    static auto get_user = []() -> std::string {
    #ifdef _WIN32
        return ::getenv("USERNAME");
    #elif defined(__linux__) || defined(__APPLE__)
        return ::getenv("USER");
    #else
    #   error UNKNOWN HOST
    #endif
    };
    static auto get_cwd  = []() -> std::string {
        char buf[1024];
        return ::getcwd(buf, sizeof(buf));
    };
    static auto get_temp = []() -> std::string {
        if (const char *temp = ::getenv("TMPDIR")) {
            return temp;
        } else if (const char *temp = ::getenv("TEMP")) {
            return temp;
        } else if (const char *temp = ::getenv("TMP")) {
            return temp;
        }
        return "/tmp";
    };
    static auto get_pid  = []() -> std::string {
        int pid = ::getpid();
        return std::to_string(pid);
    };
    static auto get_path = []() -> std::string { return ::getenv("PATH"); };
    static auto get_proc_name = []() -> std::string { return details::get_procname(); };
    static auto get_proc_path = []() -> std::string { return details::get_procpath(); };
    static auto replace = [](std::string &str, const std::string &ostr, const std::string &nstr) {
        std::string::size_type pos = 0u;
        while ( (pos = str.find(ostr, pos)) != std::string::npos ) {
            str.replace(pos, ostr.length(), nstr);
            pos += nstr.length();
        }
    };

    static auto trim = [](std::string &s) {
        if ( s.empty() ) return;

        static auto ltrim = [](std::string &s) {
            auto p = ::config_ctor::details::find_if(s.begin(), s.end(), [](int ch) { return !std::isspace(ch); });
            s.erase(s.begin(), p);
        };
        static auto rtrim = [](std::string &s) {
            auto p = ::config_ctor::details::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); });
            s.erase(p.base(), s.end());
        };

        ltrim(s);
        rtrim(s);
    };

    using pair_t = std::pair<const char *, std::string(*)()>;
    static const pair_t map[] = {
         {"{HOME}", get_home}
        ,{"{USER}", get_user}
        ,{"{CWD}", get_cwd}
        ,{"{TEMP}", get_temp}
        ,{"{PID}", get_pid}
        ,{"{PATH}", get_path}
        ,{"{PROC}", get_proc_name}
        ,{"{PROCPATH}", get_proc_path}
    };
    for ( const auto &it: map ) {
        replace(res, it.first, it.second());
    }

    // process {GETENV(var)} && {GETENV(var, default)}
    static const char getenv_pref[]   = "{GETENV(";
    static const auto getenv_pref_len = sizeof(getenv_pref)-1;
    static const char getenv_suff[]   = ")}";
    static const auto getenv_suff_len = sizeof(getenv_suff)-1;

    auto beg = res.find(getenv_pref, 0, getenv_pref_len);
    if ( beg != std::string::npos ) {
        auto end = res.find(getenv_suff, beg, getenv_suff_len);
        auto body = res.substr(beg+getenv_pref_len, end-(beg+getenv_pref_len));
        auto comma = body.find(',');
        std::string env, def;
        if ( comma != std::string::npos ) {
            env = body.substr(0, comma);
            def = body.substr(comma+1);
        } else {
            env = body;
        }
        trim(env);
        trim(def);

        const char *penv = ::getenv(env.c_str());
        if ( penv ) {
            res.replace(beg, end+getenv_suff_len-beg, penv);
        } else {
            if ( !def.empty() ) {
                res.replace(beg, end+getenv_suff_len-beg, def.c_str());
            } else {
                res.replace(beg, end+getenv_suff_len-beg, "<NULL>");
            }
        }
    }

    return res;
}

// any config-ctor type
template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_config_ctor_type<T>::value>::type* = nullptr)
{
    if ( !default_value ) {
        return T{cfg.at(key)};
    }

    const flatjson::fjson json = !cfg.contains(key)
        ? flatjson::fjson{default_value}
        : cfg.at(key)
    ;

    return T{json};
}

/***************************************************************************/

template<typename T>
struct is_stl_container_like {
    template<typename A>
    static constexpr bool test(
         A * pt
        ,A const * cpt = nullptr
        ,decltype(pt->begin()) * = nullptr
        ,decltype(pt->end()) * = nullptr
        ,decltype(cpt->begin()) * = nullptr
        ,decltype(cpt->end()) * = nullptr
        ,typename A::iterator * = nullptr
        ,typename A::const_iterator * = nullptr)
    {
        using iterator = typename A::iterator;
        using const_iterator = typename A::const_iterator;

        return
            std::is_same<decltype(pt->begin()),iterator>::value &&
            std::is_same<decltype(pt->end()),iterator>::value &&
            std::is_same<decltype(cpt->begin()),const_iterator>::value &&
            std::is_same<decltype(cpt->end()),const_iterator>::value
        ;
    }

    template<typename A>
    static constexpr bool test(...) { return false; }

    static const bool value = test<typename std::remove_const<T>::type>(nullptr);
};

template<
     typename T
    ,typename = typename std::enable_if<is_config_ctor_type<T>::value>::type
>
std::ostream& operator<< (std::ostream &os, const T &v) {
    return v.dump(os);
}

template<typename Iterator, std::size_t N>
void out_for_each(std::ostream &os, Iterator beg, Iterator end, const char(&delim)[N]) {
    for ( ; beg != end; ++beg ) {
        using value_type = typename std::iterator_traits<Iterator>::value_type;
        if ( is_string<value_type>::value ) {
            os << '\"';
        }

        os << *beg;

        if ( is_string<value_type>::value ) {
            os << '\"';
        }

        if ( std::next(beg) != end ) {
            os.write(delim, N - 1);
        }
    }
}

template<typename C>
void get_value_container_proxy(
     C &c
    ,const flatjson::fjson &json
    ,typename std::enable_if<is_simple_type<typename C::value_type>::value>::type* = nullptr)
{
    if ( !json.is_simple_type() ) {
        throw std::runtime_error("SIMPLE type expected!");
    }

    c.insert(c.end(), json.to<typename C::value_type>());
}

template<typename C>
void get_value_container_proxy(
     C &c
    ,const flatjson::fjson &json
    ,typename std::enable_if<is_config_ctor_type<typename C::value_type>::value>::type* = nullptr)
{
    c.insert(c.end(), typename C::value_type{json});
}

template<typename T>
T get_value(
     T *
    ,const char *key
    ,const flatjson::fjson &cfg
    ,const char *default_value
    ,typename std::enable_if<is_stl_container_like<T>::value && !is_string<T>::value>::type* = nullptr)
{
    flatjson::fjson json;

    if ( !default_value ) {
        json = cfg.at(key);
    } else {
        json = !cfg.contains(key)
            ? flatjson::fjson{default_value}
            : cfg.at(key)
        ;
    }

    if ( !json.is_array() ) {
        throw std::runtime_error("ARRAY type expected!");
    }

    T res;
    for ( std::size_t idx = 0u; idx < json.size(); ++idx ) {
        const auto it = json.at(idx);
        get_value_container_proxy(res, it);
    }

    return res;
}

/***************************************************************************/

template<typename T>
void write_value(
     std::ostream &os
    ,const T &v
    ,typename std::enable_if<is_bool<T>::value>::type* = nullptr)
{
    os << std::boolalpha << v;
}

template<typename T>
void write_value(
     std::ostream &os
    ,const T &v
    ,typename std::enable_if<is_integral<T>::value || is_floating<T>::value>::type* = nullptr)
{
    os << v;
}

template<typename T>
void write_value(
    std::ostream &os
    ,const T &v
    ,typename std::enable_if<is_string<T>::value>::type* = nullptr)
{
    os << '\"' << v.c_str() << '\"';
}

template<typename T>
void write_value(
     std::ostream &os
    ,const T &v
    ,typename std::enable_if<is_config_ctor_type<T>::value>::type* = nullptr)
{
    v.dump(os);
}

template<typename T>
void write_value(
     std::ostream &os
    ,const T &v
    ,typename std::enable_if<is_stl_container_like<T>::value && !is_string<T>::value>::type* = nullptr)
{
    os << '[';
    out_for_each(os, v.begin(), v.end(), ", ");
    os << ']';
}

/***************************************************************************/

inline void check_config_keys_for_object_keys(
     const char *configobj
    ,const flatjson::fjson &json
    ,const char **arr)
{
    auto keys = json.get_keys();
    for ( const auto &it: keys ) {
        // because I won't include algorithms in this header %)
        bool found = false;
        for ( const char **ait = arr; *ait; ++ait ) {
            if ( 0 == std::strncmp(it.data(), *ait, it.size()) ) {
                found = true;
                break;
            }
        }
        if ( !found ) {
            std::string msg = "config-ctor: config file has \"";
            msg.append(it.data(), it.size());
            msg += "\" key that doesn't exists in the \"";
            msg += configobj;
            msg += "\" object";
            throw std::runtime_error(msg);
        }
    }
}

} // ns details
} // ns config_ctor

/***************************************************************************/

#define __CONFIG_CTOR__WRAP_SEQUENCE_X(...) \
    ((__VA_ARGS__)) __CONFIG_CTOR__WRAP_SEQUENCE_Y
#define __CONFIG_CTOR__WRAP_SEQUENCE_Y(...) \
    ((__VA_ARGS__)) __CONFIG_CTOR__WRAP_SEQUENCE_X

#define __CONFIG_CTOR__WRAP_SEQUENCE_X0
#define __CONFIG_CTOR__WRAP_SEQUENCE_Y0

/***************************************************************************/

#define __CONFIG_CTOR__GENERATE_MEMBERS(unused, data, idx, elem) \
    BOOST_PP_TUPLE_ELEM(0, elem) /* type */ BOOST_PP_TUPLE_ELEM(1, elem) /* var name */ ;

#define __CONFIG_CTOR__INIT_MEMBERS_WITH_DEFAULT(...) \
    BOOST_PP_STRINGIZE(BOOST_PP_TUPLE_ELEM(2, __VA_ARGS__))

#define __CONFIG_CTOR__INIT_MEMBERS_WITHOUT_DEFAULT(...) \
    nullptr

#define __CONFIG_CTOR__INIT_MEMBERS__BODY(varname, withdefault, elem) \
    varname{ \
        ::config_ctor::details::get_value( \
             &varname \
            ,BOOST_PP_STRINGIZE(varname)  \
            ,json \
            ,BOOST_PP_IF( \
                 withdefault \
                ,__CONFIG_CTOR__INIT_MEMBERS_WITH_DEFAULT \
                ,__CONFIG_CTOR__INIT_MEMBERS_WITHOUT_DEFAULT \
            )(elem) \
        ) \
    }

#define __CONFIG_CTOR__INIT_MEMBERS__COLON_CASE(varname, withdefault, elem) \
    :__CONFIG_CTOR__INIT_MEMBERS__BODY(varname, withdefault, elem)

#define __CONFIG_CTOR__INIT_MEMBERS__COMMA_CASE(varname, withdefault, elem) \
    ,__CONFIG_CTOR__INIT_MEMBERS__BODY(varname, withdefault, elem)

#define __CONFIG_CTOR__INIT_MEMBERS(unused, data, idx, elem) \
    BOOST_PP_IF( \
         idx \
        ,__CONFIG_CTOR__INIT_MEMBERS__COMMA_CASE \
        ,__CONFIG_CTOR__INIT_MEMBERS__COLON_CASE \
    )( \
         BOOST_PP_TUPLE_ELEM(1, elem) \
        ,BOOST_PP_GREATER_EQUAL(BOOST_PP_TUPLE_SIZE(elem), 3)\
        ,elem \
    )

#define __CONFIG_CTOR__ENUM_MEMBERS(unused, data, idx, elem) \
    BOOST_PP_IF(idx, os << "   ,";, os << "    ";) \
    os << "\"" BOOST_PP_STRINGIZE(BOOST_PP_TUPLE_ELEM(1, elem)) "\":"; \
    ::config_ctor::details::write_value(os, BOOST_PP_TUPLE_ELEM(1, elem));

/***************************************************************************/

#define __CONFIG_CTOR__ENUM_WRITE_MEMBERS(unused, data, idx, elem) \
    BOOST_PP_IF(idx, os << "   ,";, os << "    ";) \
    os << "\"" BOOST_PP_STRINGIZE(BOOST_PP_TUPLE_ELEM(1, elem)) "\":"; \
    ::config_ctor::details::write_value(os, cfg.BOOST_PP_TUPLE_ELEM(1, elem)); \
    os << '\n';

#define __CONFIG_CTOR__GENERATE_MEMBERS_STR_NAMES(unused0, unused1, idx, elem) \
    BOOST_PP_COMMA_IF(idx) \
        BOOST_PP_STRINGIZE(BOOST_PP_TUPLE_ELEM(1, elem))

#define __CONFIG_CTOR__GENERATE_KEY_CHECKING(cfgname, keys, ptree, seq) { \
    static const char *keys[] = { \
        BOOST_PP_SEQ_FOR_EACH_I( \
             __CONFIG_CTOR__GENERATE_MEMBERS_STR_NAMES \
            ,~ \
            ,seq \
        ) \
        ,nullptr \
    }; \
    config_ctor::details::check_config_keys_for_object_keys( \
         cfgname \
        ,ptree \
        ,keys \
    ); \
}

#define __CONFIG_CTOR__GENERATE_CTOR_ARGS(unused, data, idx, elem) \
    BOOST_PP_COMMA_IF(idx) \
        BOOST_PP_TUPLE_ELEM(0, elem) arg##idx

#define __CONFIG_CTOR__GENERATE_INIT_LIST_COMMA_CASE(membername, argname) \
    ,membername{std::move(argname)}

#define __CONFIG_CTOR__GENERATE_INIT_LIST_COLON_CASE(membername, argname) \
    :membername{std::move(argname)}

#define __CONFIG_CTOR__GENERATE_INIT_LIST(unused, data, idx, elem) \
    BOOST_PP_IF( \
         idx \
        ,__CONFIG_CTOR__GENERATE_INIT_LIST_COMMA_CASE \
        ,__CONFIG_CTOR__GENERATE_INIT_LIST_COLON_CASE \
    )( \
         BOOST_PP_TUPLE_ELEM(1, elem) \
        ,arg##idx \
    )

/***************************************************************************/

#define __CONFIG_CTOR__GENERATE_STRUCT(name, seq, ...) \
    struct name { \
        __VA_ARGS__ /* user code will expanded here */ \
        \
        name() = default; \
        name(const name &) = default; \
        name(name &&) = default; \
        name& operator= (const name &) = default; \
        name& operator= (name &&) = default; \
        \
        name(const ::flatjson::fjson &json) \
            BOOST_PP_SEQ_FOR_EACH_I( \
                 __CONFIG_CTOR__INIT_MEMBERS \
                ,name \
                ,seq \
            ) \
        {} \
        \
        name( \
            BOOST_PP_SEQ_FOR_EACH_I( \
                 __CONFIG_CTOR__GENERATE_CTOR_ARGS \
                ,~ \
                ,seq \
            ) \
        ) \
            BOOST_PP_SEQ_FOR_EACH_I( \
                 __CONFIG_CTOR__GENERATE_INIT_LIST \
                ,~ \
                ,seq \
            ) \
        {} \
        \
        BOOST_PP_SEQ_FOR_EACH_I( \
             __CONFIG_CTOR__GENERATE_MEMBERS \
            ,~ \
            ,seq \
        ) \
        \
        static name read(std::istream &is) { \
            std::string str( \
                 (std::istreambuf_iterator<char>(is)) \
                ,std::istreambuf_iterator<char>()) \
            ; \
            const flatjson::fjson cfg{str.c_str(), str.c_str()+str.length()}; \
            if ( !cfg.is_valid() ) { \
                std::string msg{"config file parse error: "}; \
                msg += cfg.error_string(); \
                throw std::runtime_error{msg}; \
            } \
            \
            __CONFIG_CTOR__GENERATE_KEY_CHECKING(#name, keys, cfg, seq) \
            \
            name res{cfg}; \
            \
            ::config_ctor::details::call_after_read_proxy< \
                ::config_ctor::details::has_member_function_after_read<void (name::*)(name &)>::value \
            >::call(res); \
            \
            return res; \
        } \
        static name read(const std::string &fname) { \
            std::ifstream is{fname}; \
            assert(is.good()); \
            return read(is); \
        } \
        \
        static void write(std::ostream &os, const name &cfg) { \
            ::config_ctor::details::call_before_write_proxy< \
                ::config_ctor::details::has_member_function_before_write<void (name::*)(name &)>::value \
            >::call(cfg); \
            \
            os << "{\n"; \
            BOOST_PP_SEQ_FOR_EACH_I( \
                 __CONFIG_CTOR__ENUM_WRITE_MEMBERS \
                ,~ \
                ,seq \
            ) \
            os << "}\n"; \
        } \
        static void write(const std::string &fname, const name &cfg) { \
            std::ofstream os{fname}; \
            assert(os.good()); \
            return write(os, cfg); \
        } \
        \
        std::ostream& operator<< (std::ostream &os) { \
            return dump(os); \
        } \
        std::ostream& dump(std::ostream &os) const { \
            os << "{"; \
            BOOST_PP_SEQ_FOR_EACH_I( \
                 __CONFIG_CTOR__ENUM_MEMBERS \
                ,~ \
                ,seq \
            ) \
            os << "}"; \
            \
            return os; \
        } \
    };

/***************************************************************************/

#define CONSTRUCT_CONFIG(\
     name /* config struct name */ \
    ,seq  /* sequence of vars */ \
    ,...  /* user code*/ \
) \
    __CONFIG_CTOR__GENERATE_STRUCT( \
         name \
        ,BOOST_PP_CAT(__CONFIG_CTOR__WRAP_SEQUENCE_X seq, 0) \
        ,__VA_ARGS__ \
    )

/***************************************************************************/

#ifdef CONFIG_CTOR_HEADER_ONLY
#   include <config-ctor/config-ctor.cpp>
#endif // CONFIG_CTOR_HEADER_ONLY

#endif // __config_ctor__config_ctor_hpp

ext: hpp

max_stars_repo_name: niXman/config-ctor

max_issues_repo_name: niXman/config-ctor

max_forks_repo_name: niXman/config-ctor


text:
/*
 *   fixtures.hpp
 *
 *   Created: 04/16/2018
 *   Author: Bryant E. McDonnell
 *           EmNet LLC
 *
 *   Fixtures for Unit testing for SWMM-ToolkitAPI using Boost Test.

 Fixtures are used for building up and breaking down a simulation.
 They simplify the workflow and help reduce duplicative code.

* TABLE OF CONTENTS AND STRUCTURE

  ->  Defining the different testing fixtures
 */

#include <boost/test/included/unit_test.hpp>

#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <math.h>

#include "swmm5.h"
#include "toolkitAPI.h"
#include "../src/error.h"

// NOTE: Test Input File
#define DATA_PATH_INP "swmm_api_test.inp"
#define DATA_PATH_RPT "swmm_api_test.rpt"
#define DATA_PATH_OUT "swmm_api_test.out"

// NOTE: Test RTK Input File
#define DATA_PATH_INP_RTK "swmm_rtk_test.inp"
#define DATA_PATH_RPT_RTK "swmm_rtk_test.rpt"
#define DATA_PATH_OUT_RTK "swmm_rtk_test.out"

using namespace std;

// Defining Fixtures

/* Fixture Open Close
 1. Opens Model
 *. testing interactions
 2. Closes Model
*/
struct FixtureOpenClose{
    FixtureOpenClose() {
        swmm_open((char *)DATA_PATH_INP, (char *)DATA_PATH_RPT, (char *)DATA_PATH_OUT);
    }
    ~FixtureOpenClose() {
        swmm_close();
    }
};

/* Fixture Before Start
 1. Opens Model
 *. can choose to start simulation
 2. Starts Simulation
 3. Runs Simlation
 4. Ends simulation
 5. Closes Model
*/
struct FixtureBeforeStart{
    FixtureBeforeStart() {
        swmm_open((char *)DATA_PATH_INP, (char *)DATA_PATH_RPT, (char *)DATA_PATH_OUT);
    }
    ~FixtureBeforeStart() {
        swmm_start(0);
        int error;
        double elapsedTime = 0.0;
        do
        {
            error = swmm_step(&elapsedTime);
        }while (elapsedTime != 0 && !error);
        if (!error) swmm_end();
        if (!error) swmm_report();

        swmm_close();
    }
};

/* Fixture Before Step
 1. Opens Model
 2. Starts Simulation
 *. can choose iterate over simulation if simulation started,
    must call swmm_end()
 3. Closes Model
*/
struct FixtureBeforeStep{
    FixtureBeforeStep() {
        swmm_open((char *)DATA_PATH_INP, (char *)DATA_PATH_RPT, (char *)DATA_PATH_OUT);
        swmm_start(0);
    }
    ~FixtureBeforeStep() {
        swmm_close();
    }
};

/* Fixture Before Step
 1. Opens Model
 2. Starts Simulation
 *. can choose iterate over simulation if simulation started,
    must call swmm_end()
 3. Closes Model
*/
struct FixtureBeforeStep_RTK{
    FixtureBeforeStep_RTK() {
        swmm_open((char *)DATA_PATH_INP_RTK, (char *)DATA_PATH_RPT_RTK, (char *)DATA_PATH_OUT_RTK);
        swmm_start(0);
    }
    ~FixtureBeforeStep_RTK() {
        swmm_close();
    }
};

/* Fixture Before End
 1. Opens Model
 2. Starts Simulation
 3. Runs Simlation
 * can choose to interact after simulation end
 4. Ends simulation
 5. Closes Model
*/
struct FixtureBeforeEnd{
    FixtureBeforeEnd() {
        swmm_open((char *)DATA_PATH_INP, (char *)DATA_PATH_RPT, (char *)DATA_PATH_OUT);
        swmm_start(0);

        int error;
        double elapsedTime = 0.0;
        do
        {
            error = swmm_step(&elapsedTime);
        }while (elapsedTime != 0 && !error);
        BOOST_CHECK_EQUAL(0, error);
    }
    ~FixtureBeforeEnd() {
        swmm_end();
        swmm_close();
    }
};

/* Fixture Before Close
 1. Opens Model
 2. Starts Simulation
 3. Runs Simlation
 4. Ends simulation
 * can choose to interact after simulation end
 5. Closes Model
*/
struct FixtureBeforeClose{
    FixtureBeforeClose() {
        swmm_open((char *)DATA_PATH_INP, (char *)DATA_PATH_RPT, (char *)DATA_PATH_OUT);
        swmm_start(0);

        int error;
        double elapsedTime = 0.0;
        do
        {
            error = swmm_step(&elapsedTime);
        }while (elapsedTime != 0 && !error);
        BOOST_CHECK_EQUAL(0, error);
        swmm_end();
    }
    ~FixtureBeforeClose() {
        swmm_close();
    }
};

max_stars_repo_name: dleutnant/Stormwater-Management-Model

max_issues_repo_name: dleutnant/Stormwater-Management-Model

max_forks_repo_name: dleutnant/Stormwater-Management-Model


text:
#ifndef KNOCK_UTIL_HASH_HPP
#define KNOCK_UTIL_HASH_HPP

#include <Config/Config.hpp>

#include <vector>

#include <boost/filesystem.hpp>

namespace Knock {

namespace Hash { // Knock::Hash

constexpr size_t Sha256DigestLength = 32;

std::string fileSha256( const boost::filesystem::path& filePath );
std::string bufferSha256( const std::vector<unsigned char>& buf );
std::string bufferSha256( const void* pBuf, size_t bufSize );

} // Knock::Hash

} // Knock

#endif // End of include guard

max_stars_repo_name: WCChou/Knock

max_issues_repo_name: WCChou/Knock

max_forks_repo_name: WCChou/Knock


text:
/*
 * Copyright 2011, 2012 Esrille Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "HTMLAnchorElementImp.h"

#include <iostream>
#include <boost/bind.hpp>

#include <org/w3c/dom/events/MouseEvent.h>

#include "DOMTokenListImp.h"
#include "utf.h"

namespace org
{
namespace w3c
{
namespace dom
{
namespace bootstrap
{

HTMLAnchorElementImp::HTMLAnchorElementImp(HTMLAnchorElementImp* org, bool deep) :
    ObjectMixin(org, deep),
    clickListener(boost::bind(&HTMLAnchorElementImp::handleClick, this, _1))
{
    addEventListener(u"click", &clickListener, false, true);
}

HTMLAnchorElementImp::HTMLAnchorElementImp(DocumentImp* ownerDocument) :
    ObjectMixin(ownerDocument, u"a"),
    clickListener(boost::bind(&HTMLAnchorElementImp::handleClick, this, _1))
{
    addEventListener(u"click", &clickListener, false, true);
}

HTMLAnchorElementImp::~HTMLAnchorElementImp()
{
}

void HTMLAnchorElementImp::handleClick(events::Event event)
{
    events::MouseEvent mouse = interface_cast<events::MouseEvent>(event);
    std::u16string href = getHref();
    if (!href.empty() && mouse.getButton() == 0)
        getOwnerDocument().setLocation(href);
}

void HTMLAnchorElementImp::eval()
{
    HTMLElementImp::eval();

    std::u16string href = getHref();
    if (href.empty())
        return;
    setTabIndex(0);
}

std::u16string HTMLAnchorElementImp::getHref()
{
    return getAttribute(u"href");
}

void HTMLAnchorElementImp::setHref(const std::u16string& href)
{
    setAttribute(u"href", href);
}

std::u16string HTMLAnchorElementImp::getTarget()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setTarget(const std::u16string& target)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getPing()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setPing(const std::u16string& ping)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getRel()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setRel(const std::u16string& rel)
{
    // TODO: implement me!
}

DOMTokenList HTMLAnchorElementImp::getRelList()
{
    return new(std::nothrow) DOMTokenListImp(this, u"rel");
}

std::u16string HTMLAnchorElementImp::getMedia()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setMedia(const std::u16string& media)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getHreflang()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setHreflang(const std::u16string& hreflang)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getType()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setType(const std::u16string& type)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getText()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setText(const std::u16string& text)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getProtocol()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setProtocol(const std::u16string& protocol)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getHost()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setHost(const std::u16string& host)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getHostname()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setHostname(const std::u16string& hostname)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getPort()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setPort(const std::u16string& port)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getPathname()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setPathname(const std::u16string& pathname)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getSearch()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setSearch(const std::u16string& search)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getHash()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setHash(const std::u16string& hash)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getCoords()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setCoords(const std::u16string& coords)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getCharset()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setCharset(const std::u16string& charset)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getName()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setName(const std::u16string& name)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getRev()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setRev(const std::u16string& rev)
{
    // TODO: implement me!
}

std::u16string HTMLAnchorElementImp::getShape()
{
    // TODO: implement me!
    return u"";
}

void HTMLAnchorElementImp::setShape(const std::u16string& shape)
{
    // TODO: implement me!
}

}
}
}
}

ext: cpp

max_stars_repo_name: rvedam/es-operating-system

max_issues_repo_name: LambdaLord/es-operating-system

max_forks_repo_name: LambdaLord/es-operating-system


text:
#include <algorithm>
#include <unordered_map>

#include <Eigen/QR>
#include <descriptor-projection/build-projection-matrix.h>
#include <descriptor-projection/descriptor-projection.h>
#include <descriptor-projection/flags.h>

#include <vi-map/vertex.h>

namespace descriptor_projection {

// Compute the covariance of the descriptors, rows are states, columns are
// samples.
void ComputeCovariance(
    const Eigen::MatrixXf& data, Eigen::MatrixXf* covariance) {
  CHECK_NOTNULL(covariance);
  CHECK_GT(data.cols(), 0) << "Data must not be empty!";
  VLOG(4) << "Got " << data.cols()
          << " samples to compute the covariance from.";
  covariance->setZero(data.rows(), data.rows());
  constexpr int kBlockSize = 10000;
  const int num_blocks = data.cols() / kBlockSize + 1;
  for (int i = 0; i < num_blocks; ++i) {
    const int block_start = i * kBlockSize;
    const int block_size =
        std::min<int>((i + 1) * kBlockSize, data.cols()) - block_start;
    const Eigen::Block<const Eigen::MatrixXf>& data_block =
        data.block(0, block_start, data.rows(), block_size);

    const Eigen::MatrixXf centered =
        data_block.colwise() - data_block.rowwise().mean();
    double normalizer = std::max(static_cast<int>(data_block.cols() - 1), 1);
    covariance->noalias() += (centered * centered.adjoint()) / normalizer;
  }
  (*covariance) /= num_blocks;
}

void BuildListOfMatchesAndNonMatches(
    const Eigen::MatrixXf& all_descriptors, const std::vector<Track>& tracks,
    std::vector<descriptor_projection::DescriptorMatch>* matches,
    std::vector<descriptor_projection::DescriptorMatch>* non_matches) {
  CHECK_NOTNULL(matches);
  CHECK_NOTNULL(non_matches);
  for (const Track& track : tracks) {
    // Add pairs of matching descriptors to the list of matches.
    // TODO(slynen): Consider taking random pairs to not under-estimate the
    // variance.
    for (size_t i = 1; i < track.size(); ++i) {
      matches->emplace_back(track[i - 1], track[i]);
    }
  }

  std::random_device device;
  std::mt19937 generator(device());
  std::uniform_int_distribution<> distribution(0, all_descriptors.cols() - 1);

  for (size_t i = 1; i < static_cast<size_t>(all_descriptors.cols()) &&
                     i < matches->size() * 2;
       ++i) {
    unsigned int index_a = distribution(generator);
    unsigned int index_b = distribution(generator);
    if (index_a == index_b) {
      continue;
    }
    non_matches->emplace_back(index_a, index_b);
  }
}

void BuildCovarianceMatricesOfMatchesAndNonMatches(
    unsigned int descriptor_size, const Eigen::MatrixXf& all_descriptors,
    const std::vector<Track>& tracks, unsigned int* sample_size_matches,
    unsigned int* sample_size_non_matches, Eigen::MatrixXf* cov_matches,
    Eigen::MatrixXf* cov_non_matches) {
  CHECK_NOTNULL(sample_size_matches);
  CHECK_NOTNULL(sample_size_non_matches);
  CHECK_NOTNULL(cov_matches);
  CHECK_NOTNULL(cov_non_matches);

  {  // Scope to limit memory usage.
    unsigned int too_short_tracks = 0;
    unsigned int long_enough_tracks = 0;
    constexpr size_t kMinTrackLength = 5;
    size_t number_of_used_tracks = 0;
    Eigen::MatrixXf sumMuMu;
    sumMuMu.setZero(descriptor_size, descriptor_size);

    std::vector<size_t> descriptors_from_tracks;
    descriptors_from_tracks.reserve(500);

    // Centering.
    constexpr int kMaxNumSamples = 50000;
    for (const Track& track : tracks) {
      if (track.size() < kMinTrackLength) {
        ++too_short_tracks;
        continue;
      }
      if (number_of_used_tracks >= kMaxNumSamples) {
        LOG(WARNING) << "Truncated descriptors to " << kMaxNumSamples << ".";
        break;
      }
      ++long_enough_tracks;

      Eigen::Matrix<float, Eigen::Dynamic, 1> mean;
      mean.resize(descriptor_size, Eigen::NoChange);
      mean.setZero();

      for (const size_t& descriptor_idx : track) {
        descriptors_from_tracks.push_back(descriptor_idx);
        mean += all_descriptors.block(0, descriptor_idx, descriptor_size, 1);
      }

      mean /= track.size();
      CHECK_LE(mean.maxCoeff(), 1.0);
      CHECK_GE(mean.minCoeff(), 0.0);

      Eigen::MatrixXf mu_sq_current = (mean * mean.transpose()).eval();

      sumMuMu += mu_sq_current * track.size();
      ++number_of_used_tracks;
    }

    VLOG(3) << "Got " << long_enough_tracks << " tracks out of "
            << tracks.size() << " (dropped " << too_short_tracks
            << " tracks because they were too short)";

    CHECK(!descriptors_from_tracks.empty());

    VLOG(3) << "Computing matches covariance from "
            << descriptors_from_tracks.size()
            << " matches (descriptor size: " << descriptor_size << ")";

    *sample_size_matches = descriptors_from_tracks.size();

    Eigen::MatrixXf matches;
    matches.resize(descriptor_size, descriptors_from_tracks.size());
    matches.setZero();

    int matched_idx = 0;
    for (const size_t& descriptor_idx : descriptors_from_tracks) {
      matches.block(0, matched_idx, descriptor_size, 1) =
          all_descriptors.block(0, descriptor_idx, descriptor_size, 1);
      ++matched_idx;
    }

    CHECK_GT(descriptors_from_tracks.size(), number_of_used_tracks);

    // Covariance computation for matches.
    cov_matches->noalias() =
        (matches * matches.transpose() - sumMuMu) * 2.0 /
        static_cast<float>(
            descriptors_from_tracks.size() - number_of_used_tracks);
  }  // Scope to limit memory usage.

  // Use all descriptors to estimate the non-matching covariance.
  *sample_size_non_matches = all_descriptors.cols();
  // Compute sample covariances non matched descriptors.
  ComputeCovariance(all_descriptors, cov_non_matches);

  *cov_non_matches *= 2.0f;
}

void ComputeProjectionMatrix(
    const Eigen::MatrixXf& cov_matches, const Eigen::MatrixXf& cov_non_matches,
    Eigen::MatrixXf* A) {
  CHECK_NOTNULL(A);

  const int dimensionality = cov_matches.cols();
  A->resize(dimensionality, dimensionality);

  Eigen::JacobiSVD<Eigen::MatrixXf> svd(cov_matches, Eigen::ComputeFullV);

  CHECK_NE(svd.singularValues().minCoeff(), 0)
      << "Rank deficiency for matrix"
         " of samples detected. Probably too little matches.";

  Eigen::MatrixXf Av =
      svd.singularValues().cwiseSqrt().cwiseInverse().asDiagonal() *
      svd.matrixV().transpose();

  Eigen::JacobiSVD<Eigen::MatrixXf> svd_d(
      Av * cov_non_matches * Av.transpose(), Eigen::ComputeFullV);

  Eigen::MatrixXf singular_values_sqrt_inv =
      svd_d.singularValues().cwiseSqrt().cwiseInverse().asDiagonal();

  Eigen::MatrixXf eye;
  eye.resize(dimensionality, dimensionality);
  eye.setIdentity();

  *A = (eye - singular_values_sqrt_inv) * svd_d.matrixV().transpose() *
       svd.singularValues().cwiseInverse().asDiagonal() *
       svd.matrixV().transpose();
}
}  // namespace descriptor_projection

ext: cc

max_stars_repo_name: AdronTech/maplab

max_issues_repo_name: AdronTech/maplab

max_forks_repo_name: AdronTech/maplab