text:
/* boosting_tool_common.cc
   Jeremy Barnes, 6 June 2004
   Copyright (c) 2004 Jeremy Barnes.  All rights reserved.
   $Source$

   Implementation of our common code for the boosting tools.
*/

#include "jml/boosting/config_impl.h"
#include "boosting_tool_common.h"
#include "jml/utils/file_functions.h"
#include "jml/utils/parse_context.h"
#include "jml/boosting/training_data.h"
#include "jml/boosting/classifier.h"
#include "jml/boosting/probabilizer.h"
#include "jml/utils/vector_utils.h"
#include <boost/timer.hpp>
#include <boost/regex.hpp>
#include "jml/boosting/null_classifier.h"
#include "jml/boosting/null_feature_space.h"
#include "jml/boosting/evaluation.h"
#include "jml/boosting/feature_info.h"
#include "jml/boosting/training_index.h"
#include "jml/boosting/data_aliases.h"
#include "jml/boosting/weighted_training.h"
#include "jml/math/xdiv.h"
#include "jml/stats/distribution_ops.h"

using namespace std;

namespace ML {

namespace {

/** Graphic metrics, that don't put the extremities like the others. */

struct Graphic_Metrics {

    Graphic_Metrics()
        : counts(10)
    {
    }

    void record(float value, float weight = 1.0)
    {
        if (value < 0.0 || value > 1.0) {
            cerr << format("value = %.12f", value);
            throw Exception("Graphic_Metrics::record(): value out of range");
        }
        int bucket = (int)std::floor(value * 10.0);
        if (bucket == 10) bucket = 9;
        counts[bucket] += weight;
    }

    void histogram(const std::string & title,
                   std::vector<std::string> & lines,
                   int max_width,
                   std::string count_fmt = "detect") const
    {
        double max = counts.max();

        if (count_fmt == "detect") {
            bool all_integral = true;
            for (unsigned i = 0;  i < counts.size() && all_integral;  ++i)
                if (((int)counts[i]) != counts[i]) all_integral = false;
            count_fmt = (all_integral ? "%.0f" : "%0.1f");
        }

        /* Make sure we have enough lines. */
        if (lines.size() < 11) lines.resize(11);

        /* How wide is the widest line? */
        size_t max_line_width = 0;
        for (unsigned i = 0;  i < lines.size();  ++i)
            max_line_width = std::max(max_line_width, lines[i].size());

        /* Make them all this length. */
        for (unsigned i = 0;  i < lines.size();  ++i)
            lines[i].resize(max_line_width, ' ');

        /* Add the title */
        lines[0] += title;

        int stars = 15;  // how many stars we want

        /* Work out the width of the counts. */
        distribution<int> widths
            = round(counts / (max ? max : 1) * stars).cast<int>();

        /* Add the distributions. */
        for (unsigned i = 0;  i < 10;  ++i) {
            string & line = lines[i + 1];
            line += format("%d: ", i);
            line += string(widths[i], '*');
            line += ' ';
            line += format(count_fmt.c_str(), counts[i]);
            if (line.size() < max_width)
                line += string(max_width - line.size(), ' ');
        }
    }

    distribution<double> counts;
};

template<class It>
int idxmax(It begin, It end)
{
    It el = std::max_element(begin, end);
    if (el == end) return -1;
    return el - begin;
}

template<class It>
std::pair<int, int>
idxmax2(It begin, It end)
{
    typedef typename std::iterator_traits<It>::value_type value_type;
    if (begin == end) return make_pair(-1, -1);
    value_type best = *begin++, second = best;
    int ibest = 0, isecond = -1, i = 1;

    for (; begin != end;  ++i, ++begin) {
        value_type val = *begin;

        if (val > best) {
            isecond = ibest;
            second = best;
            ibest = i;
            best = val;
        }
        else if (val > second || isecond == -1) {
            isecond = i;
            second = val;
        }
    }

    return make_pair(ibest, isecond);
}

} // file scope

void calc_stats(const Classifier_Impl & current,
                const Optimization_Info & opt_info,
                const Decoder_Impl & prob,
                const Training_Data & data,
                //const distribution<float> & weights,
                int draw_graphs,
                bool dump_testing,
                int dump_confusion,
                bool by_group,
                const Feature & group_feature)
{
    cerr << "dump_testing = " << dump_testing << endl;
    cerr << "by_group = " << by_group << endl;

    size_t nl = current.label_count();
    size_t nx = data.example_count();

    distribution<float> weights;  // todo: pass this in

    if (weights.empty()) weights.resize(nx, 1.0);

    //cerr << "nl = " << nl << "  nx = " << nx << endl;

    //cerr << "draw_graphs = " << draw_graphs << "  dump_confusion = "
    //     << dump_confusion << endl;

    Feature predicted = current.predicted();

    if (nx == 0) { cout << "0 examples" << endl;  return;  } // nothing to do

    if (nl == 1) {
        /* Regression problem gets treated specially. */
        calc_stats_regression(current, opt_info, prob,
                              data, draw_graphs, dump_testing,
                              dump_confusion);
        return;
    }

    /* Test it out. */
    vector<distribution<float> > avg_label(nl, distribution<float>(nl, 0.0));
    distribution<float> avg_corr(nl, 0.0);
    distribution<float> avg_inc(nl, 0.0);
    double correct = 0.0, total = 0.0;
    vector<unsigned> num_label(nl);

    Graphic_Metrics metrics[2][nl];
    Graphic_Metrics margins[nl];
    Graphic_Metrics rankings[nl];
    Graphic_Metrics correct_graph, incorrect_graph;
    Graphic_Metrics ranking_graph, margin_graph;

    if (dump_testing) {
        cout << "examp lbl";
        for (unsigned i = 0;  i < current.label_count();  ++i)
            cout << format("%6s", format("in%d", i).c_str())
                 << format("%6s", format("out%d", i).c_str());
        cout << "  poss  corr  marg" << endl;
    }

    const vector<Label> & labels = data.index().labels(predicted);

    //cerr << "by_group = " << by_group << " group_feature = "
    //     << data.group_feature_ << "  nl = " << nl << endl;

    by_group = by_group && group_feature != MISSING_FEATURE && nl == 2;

    if (!by_group) {
        //boost::progress_timer timer;
        for (unsigned x = 0;  x < nx;  ++x) {
            distribution<float> input = current.predict(data[x], opt_info);

            distribution<float> result = prob.apply(input);
            //distribution<float> result = input;

            int highest, highest_2;
            boost::tie(highest, highest_2)
                = idxmax2(result.begin(), result.end());
            //int highest_np = idxmax(input.begin(), input.end());

            double w = weights[x];

            Correctness c = correctness(result, predicted, data[x]);
            correct += w * c.possible * c.correct;
            total += w * c.possible;

            int label = labels[x];  // todo: make it work again...

            // Value to record.  Add a tiny bias to make exact zero margin
            // sit on the positive side.
            float margin_to_record
                = std::max<float>
                (std::min<float>(1.0f, c.margin / 2.0f + 0.50000001f),
                 0.0f);

            if (c.margin >= -1.0 && c.margin <= 1 && isfinite(c.margin))
                margins[label].record(margin_to_record);

            distribution<float> result2 = result;
            int ranking = 1;
            while (result2.max() != result[label] && ranking < 10) {
                result2[idxmax(result2.begin(), result2.end())] = 0.0;
                ++ranking;
            }
            rankings[label].record(std::min<float>(ranking * 0.1f, 1.0f));

            ranking_graph.record(std::min<float>(ranking * 0.1f, 1.0f));

            if (c.margin >= -1.0 && c.margin <= 1.0 && isfinite(c.margin))
                margin_graph.record(margin_to_record);

            avg_label[label] += result;
            num_label[label] += 1;

            for (unsigned l = 0;  l < nl;  ++l) {
                if (result[l] >= 0.0 && result[l] <= 1.0 && isfinite(result[l])) {
                    metrics[label == l][l].record(result[l]);

                    if (label == l)
                        correct_graph.record(result[l]);
                    else incorrect_graph.record(result[l]);
                }

                if (label == l) avg_corr[l] += result[l];
                else avg_inc[l] += result[l];
            }

            if (dump_testing) {
                cout << format("%5d %3d", x, label);
                for (unsigned i = 0;  i < result.size();  ++i)
                    cout << format("%6.2f ", input[i])
                         << format("%5.2f", result[i]);
                cout <<  format(" %5.2f %5.2f %5.2f", c.possible, c.correct,
                                c.margin);
                //cout << data.row_comment(x);
                cout << endl;
            }
        }

        float recog_rate = xdiv(correct, total);

        distribution<float> label_freq = data.index().category_freqs(predicted);

        cerr << "label freq = " << label_freq << endl;

        cerr << format("recog rate = %9.1f/%7.0f = %8.3f%%",
                       correct, total,
                       recog_rate * 100.0) << endl;
        cerr << format("baseline   = %9.1f/%7.0f = %8.3f%%",
                       label_freq.max(), label_freq.total(),
                       xdiv<float>(label_freq.max(),
                                         label_freq.total()) * 100.0) << endl;

        float acc, rmse;
        boost::tie(acc, rmse) = current.accuracy(data);

        cerr << "non-prob accuracy              = "
             << format("%8.3f%% %8.3f%%",
                       acc * 100.0, rmse * 100.0) << endl;
    }
    else {
        /* Evaluate a group at a time */

        /* Get a list of groups. */
        Joint_Index group_index
            = data.index().dist(group_feature, BY_VALUE, IC_LABEL | IC_EXAMPLE);

        vector<float> groups;
        vector<vector<int> > group_examples;
        vector<int> group_numbers(nx, -1);

        float last_val = 0.0;
        for (Index_Iterator it = group_index.begin();
             it != group_index.end();  ++it) {
            float val = it->value();
            if (it == group_index.begin() || val != last_val) {
                groups.push_back(val);
                group_examples.push_back(vector<int>());
                last_val = val;
            }
            group_examples.back().push_back(it->example());
            group_numbers[it->example()] = group_examples.size() - 1;
        }

        int group_count = group_examples.size();
        double baseline = 0.0;

        //cerr << group_count << " groups" << endl;
        //cerr << nx << " examples" << endl;
        //cerr << "groups = " << groups << endl;

        /* Loop through a group at a time... */
        for (unsigned g = 0;  g < group_examples.size();  ++g) {
            int nxg = group_examples[g].size();

            distribution<float> correct_dist(nxg);
            distribution<float> values_dist(nxg);
            distribution<float> weights_dist(nxg);

            for (unsigned i = 0;  i < nxg;  ++i) {
                int x = group_examples[g][i];
                distribution<float> input = current.predict(data[x], opt_info);

                distribution<float> result = prob.apply(input);

                int label              = labels[x];
                correct_dist[i]        = label;
                values_dist[i]         = result[1];
                weights_dist[i]        = weights[x];
                avg_label[label] += result;
                num_label[label] += 1;
            }

            weights_dist /= weights_dist.size();
            double total_weight = weights_dist.total();

            float correct_value = 0.0;
            if (correct_dist.total() == 0)
                cerr << "warning: group " << groups[g] << " has no correct label"
                     << endl;
            else {
                correct_value = (values_dist * correct_dist).total()
                    / correct_dist.total();
                //if (correct_dist.total() > 1.0)
                //    cerr << "warning: group " << groups[g] << " has "
                //         << correct_dist.total() << " correct labels" << endl;
            }

            baseline += total_weight
                * (correct_dist.total() / correct_dist.size());

            int highest, highest_2;
            boost::tie(highest, highest_2)
                = idxmax2(values_dist.begin(), values_dist.end());

            bool is_correct = correct_dist[highest] != 0.0;

            for (unsigned i = 0;  i < nxg;  ++i) {
                if (values_dist[i] >= 0.0 && values_dist[i] <= 1.0 && isfinite(values_dist[i])) {
                    if (correct_dist[i] > 0.0)
                        correct_graph.record(values_dist[i], weights_dist[i]);
                    else incorrect_graph.record(values_dist[i], weights_dist[i]);
                }
            }

            float margin = (is_correct ? values_dist[highest]  - values_dist[highest_2]
                            : correct_value - values_dist[highest]);

            distribution<float> values_dist2 = values_dist;
            int ranking = 1;
            while (correct_dist[idxmax(values_dist2.begin(), values_dist2.end())] == 0.0
                   && ranking < 10) {
                values_dist2[idxmax(values_dist2.begin(), values_dist2.end())] = 0.0;
                ++ranking;
            }

            if (is_correct) correct += weights_dist.total();
            total += weights_dist.total();

            ranking_graph.record(ranking * 0.1);
            if (margin >= -1.0 && margin <= 1 && isfinite(margin))
                margin_graph.record((margin / 2) + 0.50000001, total_weight);
        }

        float recog_rate = xdiv<float>(correct, total);

        cerr << "recog rate = " << correct << "/" << total
             << " = " << recog_rate * 100.0 << "%" << endl;

        cerr << "baseline   = " << baseline << "/"
             << group_count << " = "
             << xdiv<float>(baseline, group_count) * 100.0
             << "%" << endl;
    }
    if (dump_confusion) {
        for (unsigned l = 0;  l < nl;  ++l) {
            avg_label[l] /= num_label[l];
            avg_corr[l] = xdiv(avg_corr[l], num_label[l]);
            avg_inc[l] = xdiv(avg_inc[l], nx - num_label[l]);
        }

        if (dump_confusion > 1) {
            cerr << "      ";
            for (unsigned lout = 0;  lout < nl;  ++lout)
                cerr << "  out  ";
            cerr << endl;

            cerr << " corr ";
            for (unsigned lout = 0;  lout < nl;  ++lout)
                cerr << format(" %3d   ", lout);
            cerr << endl;

            for (unsigned lcorr = 0;  lcorr < nl;  ++lcorr) {
                cerr << format(" %3d  ", lcorr);
                for (unsigned lout = 0;  lout < nl;  ++lout)
                    cerr << format("%6.3f ", avg_label[lcorr][lout]);
                cerr << endl;
            }

            cerr << "      ";
            for (unsigned l = 0;  l < nl;  ++l)
                cerr << "       ";
            cerr << endl;
        }

        cerr << " corr ";
        for (unsigned l = 0;  l < nl;  ++l)
            cerr << format("%6.3f ", avg_corr[l]);
        cerr << endl;

        cerr << " inc  ";
        for (unsigned l = 0;  l < nl;  ++l)
            cerr << format("%6.3f ", avg_inc[l]);
        cerr << endl;

        cerr << " gap  ";
        for (unsigned l = 0;  l < nl;  ++l)
            cerr << format("%6.3f ", avg_corr[l] - avg_inc[l]);
        cerr << endl;
        cerr << endl;
    }

    if (draw_graphs) {
        if (draw_graphs > 1 && !by_group) {
            for (unsigned l = 0;  l < nl;  ++l) {
                vector<string> lines(11);
                cerr << "graphs for label " << l << ":" << endl;
                metrics[1][l].histogram("correct:", lines, 20);
                metrics[0][l].histogram("incorrect:", lines, 40);
                margins[l].histogram("margins:", lines, 60);
                rankings[l].histogram("rankings:", lines, 80);
                std::copy(lines.begin(), lines.end(),
                          ostream_iterator<string>(cerr, "\n"));
                cerr << endl;
            }
        }

        {
            vector<string> lines(11);
            cerr << "overall graphs:" << endl;
            correct_graph.histogram("correct:", lines, 20);
            incorrect_graph.histogram("incorrect:", lines, 40);
            margin_graph.histogram("margins:", lines, 60);
            ranking_graph.histogram("rankings:", lines, 80);
            std::copy(lines.begin(), lines.end(),
                      ostream_iterator<string>(cerr, "\n"));
            cerr << endl;
        }
    }
}

void calc_stats_regression(const Classifier_Impl & current,
                           const Optimization_Info & opt_info,
                           const Decoder_Impl & prob,
                           const Training_Data & data, int draw_graphs,
                           bool dump_testing, int dump_confusion)
{
    size_t nl = current.label_count();
    size_t nx = data.example_count();

    Feature predicted = current.predicted();

    if (nl != 1)
        throw Exception("calc_stats_regression() called on classification "
                        "dataset");
    if (nl != current.label_count())
        throw Exception("calc_stats_regression(): nl doesn't match");

    /* Test it out. */
    if (dump_testing)
        cout << "examp       lbl     in     out     error";

    const vector<Label> & labels = data.index().labels(predicted);

    //double sum_r_squared = 0.0;

    /* Calculate the r-squared. */
    double sum_vsq = 0.0, sum_v = 0.0, sum_lsq = 0.0, sum_l = 0.0;
    double sum_vl = 0.0, n = nx;

    for (unsigned x = 0;  x < nx;  ++x) {
        distribution<float> input = current.predict(data[x], opt_info);

        distribution<float> result = prob.apply(input);

        float label = labels[x].value();

        float residual = label - result[0];

        double v = result[0], l = label;
        sum_vsq += v*v;  sum_v += v;
        sum_lsq += l*l;  sum_l += l;
        sum_vl  += v*l;

        if (dump_testing) {
            cout << format("%5d %10.4g %6.2f %6.2f %6.2f", x, label, input[0],
                           result[0], residual) << endl;
        }

        //sum_r_squared += residual * residual;
    }

    //cerr << "sum_r_squared = " << sum_r_squared / nx << endl;

    double svl = n * sum_vl - sum_v * sum_l;
    double svv = n * sum_vsq - sum_v * sum_v;
    double sll = n * sum_lsq - sum_l * sum_l;

    double r_squared = svl*svl / (svv * sll);
    double b = svl / svv;
    double bd = svl / sll;

#if 0
    cerr << "n = " << n << " sum_vsq = " << sum_vsq << " sum_v = "
         << sum_v << " sum_lsq = " << sum_lsq << " sum_l = " << sum_l
         << "  sum_vl = " << sum_vl << endl;

    cerr << "svl = " << svl << "  svv = " << svv << "  sll = " << sll << endl;
#endif

    cerr << "r^2 = " << r_squared << endl;
    cerr << "b = " << b << endl;
    cerr << "b' = " << bd << endl;
}

void remove_aliased_examples(Training_Data & data, const Feature & predicted,
                             int verbosity, bool profile)
{
    /** Clean the dataset by removing those examples which are aliased (where
        the features are the same). */
    boost::timer timer;

    /* Check for aliased rows in the training data. */
    vector<Alias> aliases = remove_aliases(data, predicted);
    int inhomogenous = 0;
    int aliased = 0;
    for (unsigned i = 0;  i < aliases.size();  ++i) {
        if (!aliases[i].homogenous) {
            aliased += aliases[i].examples.size();
            inhomogenous += 1;
        }
    }

    if (aliased > 0 && verbosity > 2) {
        cerr << aliased << " aliased examples in " << inhomogenous
             << " inhomogenous groups" << endl;

        if (verbosity > 4) {
            for (unsigned i = 0;  i < aliases.size();  ++i) {
                if (aliases[i].homogenous) continue;
                cerr << i << ": examples ";
                copy(aliases[i].examples.begin(),
                     aliases[i].examples.end(),
                     ostream_iterator<int>(cerr, " "));
                if (aliases[i].homogenous) cerr << "(homogenous)" << endl;
                else cerr << "(inhomogenous)" << endl;
            }
        }
    }

    if (profile)
        cerr << "[aliased: " << timer.elapsed() << "s]" << endl;
}

void do_features(const Training_Data & data,
                 std::shared_ptr<Mutable_Feature_Space> feature_space,
                 const string & predicted_name,
                 vector<string> ignore_features,
                 vector<string> optional_features,
                 int min_feature_count, int verbosity,
                 std::vector<Feature> & features,
                 Feature & predicted,
                 std::map<std::string, Feature> & feature_index,
                 vector<string> type_overrides)
{
    features.clear();
    vector<string> feature_names;

    features = data.all_features();
    std::sort(features.begin(), features.end());

    int total_features = features.size();

    for (unsigned i = 0;  i < features.size();  ++i) {
        feature_names.push_back(feature_space->print(features[i]));
    }

    /* Get an index of all of our variables. */
    feature_index.clear();
    for (unsigned i = 0;  i < features.size();  ++i)
        feature_index[feature_space->print(features[i])] = features[i];

    /* Find the predicted feature. */
    if (!feature_index.count(predicted_name))
        throw Exception("predicted variable " + predicted_name
                        + " not found in dataset");
    predicted = feature_index[predicted_name];

    Feature_Info predicted_info = feature_space->info(predicted);

    /* Ignore variables, if asked. */
    int ignored_features = 0;
    if (ignore_features.size()) {
        /* Get the names of all that are ignored.  These might be regexes, so
           we also have to search through with these regexes. */
        for (unsigned i = 0;  i < ignore_features.size();  ++i) {
            if (ignore_features[i].size() == 0) continue;  // blank line...
            else if (ignore_features[i][0] == '@') {
                /* If we get '@filename', we load them from a file. */
                string filename(ignore_features[i], 1);  // remove first char
                ifstream stream(filename.c_str());
                stream.exceptions(ios::badbit | ios::failbit);

                while (!stream.eof()) {
                    string s;
                    getline(stream, s);
                    ignore_features.push_back(s);
                }
            }
            else {
                string regex_str = ignore_features[i];
                bool negative = false;
                if (regex_str.length() && regex_str[0] == '!') {
                    regex_str = string(regex_str, 1);
                    negative = true;
                }

                /* Add it as a regular expression. */
                boost::regex regex(regex_str);

                /* Check out all of the variable names. */
                int matched = 0;
                for (unsigned j = 0;  j < features.size();  ++j) {
                    const string & name = feature_names[j];

                    bool match = boost::regex_match(name, regex);
                    if ((match && !negative) || (!match && negative)) {
                        std::swap(features[j], features.back());
                        std::swap(feature_names[j], feature_names.back());
                        features.pop_back();
                        feature_names.pop_back();
                        --j;
                        ++matched;
                    }
                }
                if (verbosity > 2 && matched > 0)
                    cerr << "regex '" << ignore_features[i] << "' matched "
                         << matched << " features" << endl;
                else if (verbosity > 0 && matched == 0)
                    cerr << "warning: regex '" << ignore_features[i]
                         << "' matched no features" << endl;
                ignored_features += matched;
            }
        }
    }

    /* Optional features, if asked. */
    if (optional_features.size()) {
        /* Get the names of all that are optional.  These might be regexes, so
           we also have to search through with these regexes. */
        for (unsigned i = 0;  i < optional_features.size();  ++i) {
            if (optional_features[i].size() == 0) continue;  // blank line...
            else if (optional_features[i][0] == '@') {
                /* If we get '@filename', we load them from a file. */
                string filename(optional_features[i], 1);  // remove first char
                ifstream stream(filename.c_str());
                stream.exceptions(ios::badbit | ios::failbit);

                while (!stream.eof()) {
                    string s;
                    getline(stream, s);
                    optional_features.push_back(s);
                }
            }
            else {
                /* Add it as a regular expression. */
                boost::regex regex(optional_features[i]);

                /* Check out all of the variable names. */
                int matched = 0;
                for (unsigned j = 0;  j < features.size();  ++j) {
                    const string & name = feature_names[j];

                    if (boost::regex_match(name, regex)) {
                        Mutable_Feature_Info info = feature_space->info(features[j]);
                        info.set_optional(true);
                        feature_space->set_info(features[j], info);
                        ++matched;
                    }
                }
                if (verbosity > 2 && matched > 0)
                    cerr << "regex '" << optional_features[i] << "' matched "
                         << matched << " features" << endl;
                else if (verbosity > 0 && matched == 0)
                    cerr << "warning: regex '" << optional_features[i]
                         << "' matched no features" << endl;
            }
        }
    }

    /* Ignore features with not enough examples. */
    int no_data_features = 0;
    for (unsigned i = 0;  i < features.size();  ++i) {
        int num_examples = data.index().count(features[i]);

        if (num_examples < min_feature_count) {
            std::swap(features[i], features.back());
            std::swap(feature_names[i], feature_names.back());
            features.pop_back();
            feature_names.pop_back();
            --i;
            no_data_features += 1;
        }
    }

    cerr << format("%d/%d/%d/%zd total/ignored/< %d examples/remaining "
                   "features",
                   total_features, ignored_features, no_data_features,
                   features.size(), min_feature_count) << endl;

    guess_all_info(data, *feature_space, true);

    /* Take any overrides in the feature info. */
    for (unsigned i = 0;  i < type_overrides.size();  ++i) {
        if (type_overrides[i].size() == 0) continue;  // blank line...
        else if (type_overrides[i][0] == '@') {
            /* If we get '@filename', we load them from a file. */
            string filename(type_overrides[i], 1);  // remove first char
            ifstream stream(filename.c_str());
            stream.exceptions(ios::badbit | ios::failbit);

            while (!stream.eof()) {
                string s;
                getline(stream, s);
                type_overrides.push_back(s);
            }
        }
        else {
            string::size_type pos = type_overrides[i].rfind('=');
            if (pos == string::npos)
                throw Exception("couldn't parse type override "
                                + type_overrides[i]
                                + ": format is REGEX=VALUE");

            string match(type_overrides[i], 0, pos);
            string result(type_overrides[i], pos + 1);

            /* Add it as a regular expression. */
            boost::regex regex(match);

            /* Check out all of the variable names. */
            int matched = 0;
            for (unsigned j = 0;  j < features.size();  ++j) {
                const string & name = feature_names[j];

                if (boost::regex_match(name, regex)) {
                    Mutable_Feature_Info info = feature_space->info(features[j]);

                    if (result == "CATEGORICAL")
                        info = data.index()
                            .guess_info_categorical(features[j]);
                    else if (result == "REAL")
                        info.set_type(REAL);
                    else throw Exception("unknown or unimplemented type "
                                         "override: " + result);

                    feature_space->set_info(features[j], info);
                    ++matched;
                }
            }
            if (verbosity > 2 && matched > 0)
                cerr << "regex '" << type_overrides[i] << "' matched "
                     << matched << " features" << endl;
            else if (verbosity > 0 && matched == 0)
                cerr << "warning: regex '" << type_overrides[i]
                     << "' matched no features" << endl;
        }
    }

    /* Print feature info if verbosity is high enough. */
    if (verbosity > 5) {
        cerr << "Feature info: " << endl;
        for (unsigned i = 0;  i < features.size();  ++i) {
            cerr << format("%-30s %s\n", feature_names[i].c_str(),
                           feature_space->info(features[i]).print().c_str());
        }
    }
}

void write_null_classifier(const std::string & filename,
                           const Feature & predicted,
                           std::shared_ptr<const Feature_Space> feature_space,
                           int verbosity)
{
    if (verbosity > 0)
        cerr << "warning: no data or less than 2 labels; can't do any "
             << "training" << endl;
    if (filename != "") {
        if (verbosity > 0)
            cerr << "writing null classifier to \'" << filename
                 << "\'... ";
        Classifier cl(Null_Classifier(feature_space, predicted));
        cl.save(filename);
        if (verbosity > 0) cerr << "done." << endl;
    }
}

void print_data_stats(const Training_Data & data)
{
}

void print_weight_spec(const std::vector<Weight_Spec> & weight_spec,
                       std::shared_ptr<Feature_Space> feature_space)
{
    for (unsigned i = 0;  i < weight_spec.size();  ++i) {
        const Feature & feature = weight_spec[i].feature;
        cerr << "weight spec " << i << ":" << endl;
        cerr << "  feature = " << feature_space->print(feature)
             << endl;
        if (weight_spec[i].weights.size() < 20) {
            for (map<float, float>::const_iterator it
                     = weight_spec[i].weights.begin();
                 it != weight_spec[i].weights.end();  ++it)
                cerr << "    " << feature_space->print(feature, it->first)
                     << " / " << it->second << endl;
        }
        else {
            cerr << "  " << weight_spec[i].weights.size() << " entries."
                 << endl;
        }
    }
}

} // namespace ML

max_stars_repo_name: etnrlz/rtbkit

max_issues_repo_name: TuanTranEngineer/rtbkit

max_forks_repo_name: TuanTranEngineer/rtbkit


text:
/*=============================================================================
    Copyright (c) 2003 Joel de Guzman
    Copyright (c) 2005 Eric Niebler

    Use, modification and distribution is subject to the Boost Software
    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
    http://www.boost.org/LICENSE_1_0.txt)
==============================================================================*/
#if !defined(FUSION_ITERATOR_DETAIL_CONS_ITERATOR_DEREF_TRAITS_HPP)
#define FUSION_ITERATOR_DETAIL_CONS_ITERATOR_DEREF_TRAITS_HPP

#include <boost/spirit/fusion/detail/config.hpp>
#include <boost/mpl/eval_if.hpp>
#include <boost/type_traits/is_const.hpp>
#include <boost/type_traits/add_const.hpp>
#include <boost/type_traits/add_reference.hpp>

namespace boost { namespace fusion
{
    struct cons_iterator_tag;

    namespace cons_detail
    {
        template <typename Iterator>
        struct deref_traits_impl
        {
            typedef typename Iterator::cons_type cons_type;
            typedef typename cons_type::car_type value_type;

            typedef typename mpl::eval_if<
                is_const<cons_type>
              , add_reference<typename add_const<value_type>::type>
              , add_reference<value_type> >::type
            type;

            static type
            call(Iterator const& i)
            {
                return detail::ref(i.cons.car);
            }
        };
    }

    namespace meta
    {
        template <typename Tag>
        struct deref_impl;

        template <>
        struct deref_impl<cons_iterator_tag>
        {
            template <typename Iterator>
            struct apply : cons_detail::deref_traits_impl<Iterator> {};
        };
    }
}}

#endif

ext: hpp

max_stars_repo_name: schinmayee/nimbus

max_issues_repo_name: schinmayee/nimbus

max_forks_repo_name: schinmayee/nimbus


text:
#pragma once
//! c/c++ headers
#include <functional>
#include <map>
#include <memory>
#include <tuple>
#include <unordered_map>
#include <utility>
//! dependency headers
#include <boost/functional/hash.hpp>
//! project headers

namespace correspondences {

/** @enum status_e
 * @brief return status for call to calc_correspondences
 *
 * @todo add more fields for better status resolution
 */
enum class status_e {
  failure,
  success
};

/** @typedef WeightKey_t
 * @brief key definition for weight associative container (an unordered map)
 * (i, j, k, l): i, k are indices for points in source distribution, and j, l
 * are points in target distribution
 */
using WeightKey_t = std::tuple<size_t, size_t, size_t, size_t>;

/** @struct key_hash : public std::unary_function<WeightKey_t, size_t>
 * @brief hash function for unordered_map with WeightKey_t as key
 * see https://stackoverflow.com/questions/3611951/building-an-unordered-map-with-tuples-as-keys
 */
struct key_hash : public std::unary_function<WeightKey_t, size_t> {
  /** key_hash::operator()
   * @brief operator overload for ()
   *
   * @param[in] k key to obtain hash for
   * @return seed hash value for key k
   */
  size_t operator()(const WeightKey_t& k) const {
    size_t seed = 0;
    boost::hash_combine(seed, std::get<0>(k));
    boost::hash_combine(seed, std::get<1>(k));
    boost::hash_combine(seed, std::get<2>(k));
    boost::hash_combine(seed, std::get<3>(k));
    return seed;
  }
};

/** @struct key_equal : public std::binary_function<WeightKey_t, WeightKey_t, bool>
 * @brief check if two WeightKey_t instances are equal
 */
struct key_equal : public std::binary_function<WeightKey_t, WeightKey_t, bool> {
  /** key_hash::operator()
   * @brief operator overload for ()
   *
   * @param[in] v0 first key for equality check
   * @param[in] v1 second key for equality check
   * @return flag indicating whether two keys are equal (=true) or not (=false)
   */
  bool operator()(const WeightKey_t& v0, const WeightKey_t& v1) const {
    return std::get<0>(v0) == std::get<0>(v1) &&
      std::get<1>(v0) == std::get<1>(v1) &&
      std::get<2>(v0) == std::get<2>(v1) &&
      std::get<3>(v0) == std::get<3>(v1);
  }
};

/** @typedef WeightTensor
 * @brief unordered map for storing weights associated with pairwise correspondences
 */
using WeightTensor = std::unordered_map<WeightKey_t, double, key_hash, key_equal>;

/** @struct correspondences::key_lthan
 * @brief less than comparator for correspondence map
 */
struct key_lthan :
  public std::binary_function<std::pair<size_t, size_t>, std::pair<size_t, size_t>, bool> {
  /** key_lthan::operator()
   * @brief operator overload for ()
   *
   * @param[in] left left-hand key for "<" comparison
   * @param[in] right right-hand key for "<" comparison
   * @return left < right
   *
   * @note assumes row-major ordering:  if two keys have the same row index, column index is used
   * for comparison, otherwise row index is used for comparison
   */
  bool operator()(const std::pair<size_t, size_t> & left,
      const std::pair<size_t, size_t> & right) const {
    if (left.first == right.first) {
      return left.second < right.second;
    } else {
      return left.first < right.first;
    }
  }
};

/** @typedef correspondences::correspondences_t
 * @brief container for correspondences - (src idx, tgt idx) -> correspondence weighting \in [0, 1]
 *
 * @note stores the strength of the correspondence as the value in the map container
 */
using correspondences_t = std::map<std::pair<size_t, size_t>, double, key_lthan>;
}  // namespace correspondences

max_stars_repo_name: jwdinius/nmsac

max_issues_repo_name: jwdinius/nmsac

max_forks_repo_name: jwdinius/nmsac


text:
/* demo_test.cpp                                 -*- C++ -*-
   Rémi Attab (remi.attab@gmail.com), 29 Apr 2014
   FreeBSD-style copyright and disclaimer apply

   Demo of the reflection system.
*/

#define BOOST_TEST_MAIN
#define BOOST_TEST_DYN_LINK

#include "reflect.h"
#include "dsl/all.h"
#include "types/std/map.h"
#include "types/std/smart_ptr.h"

#include <boost/test/unit_test.hpp>

using namespace reflect;

/******************************************************************************/
/* TEST                                                                       */
/******************************************************************************/

namespace test {

int counter = 0;
int inc(int i) { return counter += i; }

} // namespace test

reflectScope(test)
{
    reflectGlobalVar(test::counter);
    reflectGlobalFn(test::inc);
}

/******************************************************************************/
/* ZORISH                                                                     */
/******************************************************************************/

namespace test {

struct Zorish
{
    Zorish() : value(0) {}
    Zorish(int value) : value(value) {}

    int value;

    Zorish operator+ (const Zorish& other) const
    {
        return Zorish(value + other.value);
    }

    operator int() const { return value; }
};

} // namespace test

reflectType(test::Zorish)
{
    reflectPlumbing();
    reflectCons(int);

    reflectField(value);

    reflectOpCast(int);
    reflectOpArithmetic();
}

/******************************************************************************/
/* THINGY                                                                     */
/******************************************************************************/

namespace test {

struct Thingy : public Zorish
{
    std::map<int, int> bob;

    Thingy(int value) : Zorish(value) {}

    static std::shared_ptr<Thingy> create(int value)
    {
        auto thingy = std::make_shared<Thingy>(value);
        thingy->bob[10] = 100;
        thingy->bob[20] = 200;
        return thingy;
    }
};

} // namespace test

reflectType(test::Thingy)
{
    reflectPlumbing();
    reflectParent(test::Zorish);

    reflectCons(int);
    reflectField(bob);
    reflectFn(create);

    reflectCustom(weee) (test::Thingy& obj, int i) {
        return obj.bob[i] + i;
    };

    reflectTypeTrait(thing);
    reflectFnTrait(weee, interesting);
}

/******************************************************************************/
/* SCOPE                                                                      */
/******************************************************************************/

BOOST_AUTO_TEST_CASE(scope_)
{
    const Scope* sGlobal = scope();
    const Scope* sTest = scope("test");
    BOOST_CHECK_EQUAL(sGlobal->scope("test"), sTest);

    std::cerr << sGlobal->print() << std::endl;

    sTest->call<void>("inc", 10);
    BOOST_CHECK_EQUAL(test::counter, 10);
}

/******************************************************************************/
/* TYPE                                                                       */
/******************************************************************************/

BOOST_AUTO_TEST_CASE(type_)
{
    const Type* tThingy = type("test::Thingy");
    BOOST_CHECK_EQUAL(type<test::Thingy>(), tThingy);

    std::cerr << tThingy->print() << std::endl;
    BOOST_CHECK(tThingy->isChildOf<test::Zorish>());

    Value vThingy = tThingy->construct(10);
    BOOST_CHECK(tThingy->is("thing"));

    BOOST_CHECK(tThingy->hasField("value"));
    BOOST_CHECK_EQUAL(tThingy->field("value").type(), type<int>());
    BOOST_CHECK_EQUAL(vThingy.field<int>("value"), 10);

    BOOST_CHECK(tThingy->function("weee").is("interesting"));
    BOOST_CHECK(tThingy->hasFunction("weee"));
}

/******************************************************************************/
/* CONTAINERS                                                                 */
/******************************************************************************/

BOOST_AUTO_TEST_CASE(containers)
{
    const Type* tThingy = type<test::Thingy>();

    Value thingy = tThingy->call<Value>("create", 10);
    Value bob = (*thingy).field<Value>("bob");

    BOOST_CHECK(bob.is("map"));
    BOOST_CHECK_EQUAL(bob.type()->getValue<const Type*>("keyType"), type<int>());

    auto keys = bob.call< std::vector<int> >("keys");

    std::cerr << "bob = { ";
    for (const auto& key : keys) {
        std::cerr << key << ":" << bob[key].get<int>() << " ";
    }
    std::cerr << "}" << std::endl;
}

/******************************************************************************/
/* OPERATORS                                                                  */
/******************************************************************************/

BOOST_AUTO_TEST_CASE(operators)
{
    const Type* tThingy = type<test::Thingy>();
    const Type* tZorish = type<test::Zorish>();

    auto sum = Value(1) + tZorish->construct(10) + tThingy->construct(100);
    std::cerr << "sum is a " << sum.typeId() << std::endl;
    std::cerr << "sum = " << sum.get<int>() << std::endl;
}

ext: cpp

max_stars_repo_name: unbornchikken/reflect-cpp

max_issues_repo_name: unbornchikken/reflect-cpp

max_forks_repo_name: unbornchikken/reflect-cpp


text:
#define BOOST_TEST_MODULE pcraster model_engine maskpacking
#include <boost/test/unit_test.hpp>
#include "calc_globallibdefs.h"
#include "com_algorithm.h"
#include "com_csfcell.h"
#include "com_new.h"
#include "geo_csfmap.h"
#include "geo_rasterspace.h"
#include "geo_filecreatetester.h"
#include "calc_asispacking.h"
#include "calc_spatial.h"
#include "calc_nonspatial.h"
#include "calc_unpackedsrc.h"
#include "calc_unpackedcreation.h"
#include "calc_modelbuilder.h"

#define private public
#include "calc_maskpacking.h"

struct Fixture
{

    Fixture()
    {
        calc::globalInit();
    }

    ~Fixture()
    {
        calc::globalEnd();
    }

};

BOOST_FIXTURE_TEST_SUITE(maskpacking, Fixture)

BOOST_AUTO_TEST_CASE(testSpatialPacking)
{
  using namespace calc;

 geo::RasterDim rs(2,3);

 {
  UINT1 maskInit[6]= { 1, 1, 1, 1, 1, 1 };
  // Test both being equal with full mask
  std::vector<bool> mask(6);
  for(size_t i=0; i < 6; ++i)
    mask[i]=maskInit[i]==1;
  MaskPacking mp(rs,mask);
  AsIsPacking ap(rs);

  BOOST_CHECK(mp.d_evenIsValueRL);
  BOOST_CHECK(mp.d_rlIndex.size()==2);
  BOOST_CHECK(mp.d_rlIndex[0]==0);
  BOOST_CHECK(mp.d_rlIndex[1]==6);

  SpatialPacking *cs[2] = { &mp, &ap };

  for (size_t i=0; i < 2; i++) {
    BOOST_CHECK(cs[i]->rasterDim() == rs);

    float values[6];
    // 11,12,13,14,15,16
    std::generate_n(values,6,com::SeqInc<float>(11));

    Spatial in(VS_S,values,6);

    BOOST_CHECK(cs[i]->nrFieldCells()    == 6);
    BOOST_CHECK(cs[i]->toRasterId(0) == 0);
    BOOST_CHECK(cs[i]->toRasterId(2) == 2);
    BOOST_CHECK(cs[i]->toRasterId(5) == 5);
    BOOST_CHECK(cs[i]->toFieldId(0)  == 0);
    BOOST_CHECK(cs[i]->toFieldId(2)  == 2);
    BOOST_CHECK(cs[i]->toFieldId(5)  == 5);

    Field   *b=cs[i]->createSpatial(VS_B);
    BOOST_CHECK(dynamic_cast<Spatial *>(b));
    BOOST_CHECK(b->isSpatial());
    BOOST_CHECK(b->vs()==VS_B);
    BOOST_CHECK(b->nrValues()==6);
    delete b;

    // TEST UnpackedSrc

    const Field *f=cs[i]->unpack(&in);
    // unpacked src spatial
    UnpackedSrc usS(*cs[i],&in);

    NonSpatial  ns(VS_N, 4);
    // unpacked src nonspatial
    UnpackedSrc usN(*cs[i],&ns);

    BOOST_CHECK(usN.src() == &ns);
    BOOST_CHECK(usN.src()->vs() == VS_N);
    BOOST_CHECK(usN.src()->src_4()[0] == 4);

    BOOST_CHECK(f->nrValues() == 6);
    BOOST_CHECK(usS.src()->nrValues() == 6);
    // needed for MSVC release mode, equals fails otherwise:
    const float *src=f->src_f();
    BOOST_CHECK(std::equal(values,values+6,src));
    BOOST_CHECK(std::equal(values,values+6,usS.src()->src_f()));
    if (cs[i] == &mp) {
      BOOST_CHECK(f != &in);
      delete f;
      BOOST_CHECK(usS.src() != &in);
    } else {
      BOOST_CHECK(f == &in);
      BOOST_CHECK(usS.src() == &in);
    }

    { // test SpatialPacking::pack
      Field *packed = cs[i]->pack(&in);
      BOOST_CHECK(packed->nrValues() == 6);
      BOOST_CHECK(std::equal(values,values+6,packed->src_f()));
      if (cs[i] == &mp) {
        BOOST_CHECK(packed != &in);
        delete packed;
      } else
        BOOST_CHECK(packed == &in);
    }

    // TEST UnpackedCreation

    { // automatic clean up
      UnpackedCreation uc(*cs[i],VS_N);
      BOOST_CHECK(uc.unpacked()->nrValues()==6);
      BOOST_CHECK(uc.unpacked()->vs()==VS_N);
    }

    { //  clean up by delete
      UnpackedCreation uc(*cs[i],VS_N);
      BOOST_CHECK(uc.unpacked()->nrValues()==6);
      BOOST_CHECK(uc.unpacked()->vs()==VS_N);
      Field *rPacked = uc.releasePacked();
      BOOST_CHECK(rPacked->nrValues()==6);
      BOOST_CHECK(rPacked->vs()==VS_N);
      delete rPacked;
    }
   } // eofor
 }
 {
  UINT1 maskInit[6]   = { 1 , 1, MV_UINT1, 1,     0,     1};
  // rlIndex              0 -v-  2, -m-    3, -v- 4, -m- 5 -v- 6
  std::vector<bool> mask(6);
  for(size_t i=0; i < 6; ++i)
    mask[i]=maskInit[i]==1;
  MaskPacking mp(rs,mask);

  BOOST_CHECK(mp.d_evenIsValueRL);
  BOOST_CHECK(mp.d_rlIndex.size()==6);
  BOOST_CHECK(mp.d_rlIndex[0]==0);
  BOOST_CHECK(mp.d_rlIndex[1]==2);
  BOOST_CHECK(mp.d_rlIndex[2]==3);
  BOOST_CHECK(mp.d_rlIndex[3]==4);
  BOOST_CHECK(mp.d_rlIndex[4]==5);
  BOOST_CHECK(mp.d_rlIndex[5]==6);

  BOOST_CHECK(mp.nrFieldCells() == 4);
  BOOST_CHECK(mp.toRasterId(0) == 0);
  BOOST_CHECK(mp.toFieldId(0)  == 0);

  BOOST_CHECK(mp.toRasterId(2) == 3);
  BOOST_CHECK(mp.toFieldId(3)  == 2);

  BOOST_CHECK(mp.toRasterId(3) == 5);
  BOOST_CHECK(mp.toFieldId(5)  == 3);

  BOOST_CHECK(mp.toFieldId(2)  == IFieldRDConversion::invalidId());

  Field   *b=mp.createSpatial(VS_N);
  BOOST_CHECK(dynamic_cast<Spatial *>(b));
  BOOST_CHECK(b->isSpatial());
  BOOST_CHECK(b->vs()==VS_N);
  BOOST_CHECK(b->nrValues()==4);
  std::generate_n(b->dest_4(),4,com::SeqInc<INT4>(21));

  // TEST unpack/UnpackedSrc
  const Field *f=mp.unpack(b);
  // unpacked src spatial
  UnpackedSrc usS(mp,b);
  BOOST_CHECK(f->isSpatial());
  BOOST_CHECK(f->vs()==VS_N);
  BOOST_CHECK(f->nrValues()==6);
  INT4 bu[6]       = { 21 , 22, MV_INT4, 23, MV_INT4, 24};

  // needed for MSVC release mode, equals fails otherwise:
  const INT4 *src=f->src_4();
  BOOST_CHECK(std::equal(bu,bu+6,src));
  BOOST_CHECK(std::equal(bu,bu+6,usS.src()->src_4()));

  BOOST_CHECK(b!=f);
  delete f;
  delete b;

  REAL4 values[6]     = { 11,12,13       ,14,15,16};
  REAL4 packedVals[4] = { 11,12,          14,   16};
  Spatial in(VS_S,values,6);

  { // test SpatialPacking::pack
    Field *packed = mp.pack(&in);
    BOOST_CHECK(packed->nrValues() == 4);
    BOOST_CHECK(std::equal(packedVals,packedVals+4,packed->src_f()));
    BOOST_CHECK(packed != &in);
    delete packed;
  }

  // TEST UnpackedCreation

  { // automatic clean up
    UnpackedCreation uc(mp,VS_N);
    BOOST_CHECK(uc.unpacked()->nrValues()==6);
    BOOST_CHECK(uc.unpacked()->vs()==VS_N);
  }

  { //  clean up by delete
    UnpackedCreation uc(mp,VS_S);
    BOOST_CHECK(uc.unpacked()->nrValues()==6);
    BOOST_CHECK(uc.unpacked()->vs()==VS_S);
    std::generate_n(uc.unpacked()->dest_f(),6,com::SeqInc<float>(11));
    Field *rPacked = uc.releasePacked();
    BOOST_CHECK(rPacked->nrValues()==4);
    BOOST_CHECK(rPacked->vs()==VS_S);
    // needed for MSVC release mode, equals fails otherwise:
    const float* src=rPacked->src_f();
    BOOST_CHECK(std::equal(packedVals,packedVals+4,src));
    delete rPacked;
  }
 }
}

BOOST_AUTO_TEST_CASE(testScript)
{
  using namespace calc;

  geo::RasterSpace rs(80,50);
  /* make boolean mask with:
   *  - triangle of 1, where mask is defined
   *  - triangle of MV
   *  - lower part of 1
   * - set boolean mask als areamap
   * TODO fill assumes all equal on non-spatial need
   *      for masking?
   */

  UINT1 **mask = com::new2d<UINT1>(rs.nrRows(),rs.nrCols());
  REAL4 **result = com::new2d<REAL4>(rs.nrRows(),rs.nrCols());
  REAL4 **area = com::new2d<REAL4>(rs.nrRows(),rs.nrCols());
  REAL4 count=1;
  for(size_t r=0; r< rs.nrRows(); r++)
   for(size_t c=0; c< rs.nrCols(); c++) {
     mask[r][c] = (r < c ) ? MV_UINT1 : (r < rs.nrCols());
     pcr::setMV(result[r][c]);
     if (mask[r][c]==1)
       result[r][c]=count++;
   }
   std::fill_n(area[0],rs.nrCells(), count-1);

  {
   geo::CSFMap mFile(std::string("mvComprMask.map"),rs,VS_BOOLEAN);
   mFile.putCells(mask[0]);
  }
  {
   geo::CSFMap mFile(std::string("resultExpected.map"),rs,VS_SCALAR);
   mFile.putCells(result[0]);
  }
  {
   geo::CSFMap mFile(std::string("areaExpected.map"),rs,VS_SCALAR);
   mFile.putCells(area[0]);
  }

  {
    geo::FileCreateTester mt("resultComputed.map");
    geo::FileCreateTester ma("areaComputed.map");

    ModelBuilder mb;
    mb.setClone("mvComprMask.map");
    /* Will return other results on setMVCompression(false)
     * since the statements has no spatial input!:
     */
    mb.setMVCompression(true);

    mb.setModel("resultComputed.map = uniqueid(1);"
                "areaComputed.map = maparea(resultComputed.map)");

    mb.execute();

    BOOST_CHECK(mt.equalTo("resultExpected.map",true));
    BOOST_CHECK(ma.equalTo("areaExpected.map",true));

  }

  com::delete2d<UINT1>(mask);
  com::delete2d<REAL4>(result);
  com::delete2d<REAL4>(area);
}

BOOST_AUTO_TEST_SUITE_END()

ext: cc

max_stars_repo_name: quanpands/wflow

max_issues_repo_name: quanpands/wflow

max_forks_repo_name: quanpands/wflow


text: #include <boost/units/base_units/imperial/ounce.hpp>

max_stars_repo_name: miathedev/BoostForArduino

max_issues_repo_name: miathedev/BoostForArduino

max_forks_repo_name: miathedev/BoostForArduino


text:
/*
 * Firmament
 * Copyright (c) The Firmament Authors.
 * All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT
 * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR
 * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
 *
 * See the Apache Version 2.0 License for specific language governing
 * permissions and limitations under the License.
 */

// Naive implementation of a simple-minded queue-based scheduler.

#include "scheduling/simple/simple_scheduler.h"

#include <boost/timer/timer.hpp>

#include <deque>
#include <map>
#include <string>
#include <utility>
#include <vector>

#include "base/units.h"
#include "misc/map-util.h"
#include "misc/utils.h"
#include "storage/object_store_interface.h"

DEFINE_bool(randomly_place_tasks, false, "Place tasks randomly");

namespace firmament {
namespace scheduler {

using store::ObjectStoreInterface;

SimpleScheduler::SimpleScheduler(
    shared_ptr<JobMap_t> job_map,
    shared_ptr<ResourceMap_t> resource_map,
    ResourceTopologyNodeDescriptor* resource_topology,
    shared_ptr<ObjectStoreInterface> object_store,
    shared_ptr<TaskMap_t> task_map,
    shared_ptr<KnowledgeBase> knowledge_base,
    shared_ptr<TopologyManager> topo_mgr,
    MessagingAdapterInterface<BaseMessage>* m_adapter,
    SchedulingEventNotifierInterface* event_notifier,
    ResourceID_t coordinator_res_id,
    const string& coordinator_uri,
    TimeInterface* time_manager,
    TraceGenerator* trace_generator)
    : EventDrivenScheduler(job_map, resource_map, resource_topology,
                           object_store, task_map, knowledge_base, topo_mgr,
                           m_adapter, event_notifier, coordinator_res_id,
                           coordinator_uri, time_manager, trace_generator) {
  VLOG(1) << "SimpleScheduler initiated.";
}

SimpleScheduler::~SimpleScheduler() {
}

bool SimpleScheduler::FindResourceForTask(const TaskDescriptor& task_desc,
                                          ResourceID_t* best_resource) {
  // TODO(malte): This is an extremely simple-minded approach to resource
  // selection (i.e. the essence of scheduling). We will simply traverse the
  // resource map in some order, and grab the first resource available.
  VLOG(2) << "Trying to place task " << task_desc.uid() << "...";
  // Find the first idle resource in the resource map
  for (ResourceMap_t::iterator res_iter = resource_map_->begin();
       res_iter != resource_map_->end();
       ++res_iter) {
    VLOG(3) << "Considering resource " << res_iter->first
            << ", which is in state "
            << res_iter->second->descriptor().state();
    if (res_iter->second->descriptor().state() ==
        ResourceDescriptor::RESOURCE_IDLE) {
      *best_resource = res_iter->first;
      return true;
    }
  }
  // We have not found any idle resources in our local resource map. At this
  // point, we should start looking beyond the machine boundary and towards
  // remote resources.
  return false;
}

bool SimpleScheduler::FindRandomResourceForTask(const TaskDescriptor& task_desc,
                                                ResourceID_t* best_resource) {
  // TODO(malte): This is an extremely simple-minded approach to resource
  // selection (i.e. the essence of scheduling). We will simply traverse the
  // resource map in some order, and grab the first resource available.
  VLOG(2) << "Trying to place task " << task_desc.uid() << "...";
  vector<ResourceStatus*> resources;
  // Find the first idle resource in the resource map
  for (ResourceMap_t::iterator res_iter = resource_map_->begin();
       res_iter != resource_map_->end();
       ++res_iter) {
    resources.push_back(res_iter->second);
  }
  for (uint64_t max_attempts = 2000; max_attempts > 0; max_attempts--) {
    uint32_t resource_index =
      static_cast<uint32_t>(rand_r(&rand_seed_)) % resources.size();
    if (resources[resource_index]->descriptor().state() ==
        ResourceDescriptor::RESOURCE_IDLE) {
      *best_resource = ResourceIDFromString(
          resources[resource_index]->descriptor().uuid());
      return true;
    }
  }
  // We have not found any idle resources in our local resource map. At this
  // point, we should start looking beyond the machine boundary and towards
  // remote resources.
  return false;
}

void SimpleScheduler::HandleTaskCompletion(TaskDescriptor* td_ptr,
                                           TaskFinalReport* report) {
  ResourceID_t res_id = ResourceIDFromString(td_ptr->scheduled_to_resource());
  ResourceStatus* rs = FindPtrOrNull(*resource_map_, res_id);
  CHECK_NOTNULL(rs);
  ResourceDescriptor* rd_ptr = rs->mutable_descriptor();
  // TODO(ionel): This assumes no PU sharing.
  rd_ptr->clear_current_running_tasks();
  EventDrivenScheduler::HandleTaskCompletion(td_ptr, report);
}

void SimpleScheduler::HandleTaskEviction(TaskDescriptor* td_ptr,
                                         ResourceDescriptor* rd_ptr) {
  // TODO(ionel): This assumes no PU sharing.
  rd_ptr->clear_current_running_tasks();
  EventDrivenScheduler::HandleTaskEviction(td_ptr, rd_ptr);
}

void SimpleScheduler::HandleTaskFailure(TaskDescriptor* td_ptr) {
  ResourceID_t res_id = ResourceIDFromString(td_ptr->scheduled_to_resource());
  ResourceStatus* rs = FindPtrOrNull(*resource_map_, res_id);
  CHECK_NOTNULL(rs);
  ResourceDescriptor* rd_ptr = rs->mutable_descriptor();
  // TODO(ionel): This assumes no PU sharing.
  rd_ptr->clear_current_running_tasks();
  EventDrivenScheduler::HandleTaskFailure(td_ptr);
}

void SimpleScheduler::KillRunningTask(
    TaskID_t task_id,
    TaskKillMessage::TaskKillReason reason) {
  // TODO(ionel): Make sure the task is removed from current_running_tasks
  // when it is killed.
}

void SimpleScheduler::HandleTaskFinalReport(const TaskFinalReport& report,
                                            TaskDescriptor* td_ptr) {
  boost::lock_guard<boost::recursive_mutex> lock(scheduling_lock_);
  EventDrivenScheduler::HandleTaskFinalReport(report, td_ptr);
  TaskID_t task_id = td_ptr->uid();
  vector<EquivClass_t> equiv_classes;
  // We create two equivalence class IDs:
  // 1) an equivalence class ID per task_id
  // 2) an equivalence class ID per program
  // We create these equivalence class IDs in order to make the EC
  // statistics view on the web UI work.
  EquivClass_t task_agg =
    static_cast<EquivClass_t>(HashCommandLine(*td_ptr));
  equiv_classes.push_back(task_agg);
  equiv_classes.push_back(task_id);
  knowledge_base_->ProcessTaskFinalReport(equiv_classes, report);
}

void SimpleScheduler::PopulateSchedulerResourceUI(
    ResourceID_t res_id,
    TemplateDictionary* dict) const {
  // At the moment to we do not show any resource-specific information
  // for the simple scheduler.
}

void SimpleScheduler::PopulateSchedulerTaskUI(TaskID_t task_id,
                                              TemplateDictionary* dict) const {
  // At the moment to we do not show any task-specific information
  // for the simple scheduler.
}

uint64_t SimpleScheduler::ScheduleAllJobs(SchedulerStats* scheduler_stats) {
  return ScheduleAllJobs(scheduler_stats, NULL);
}

uint64_t SimpleScheduler::ScheduleAllJobs(SchedulerStats* scheduler_stats,
                                          vector<SchedulingDelta>* deltas) {
  boost::lock_guard<boost::recursive_mutex> lock(scheduling_lock_);
  vector<JobDescriptor*> jobs;
  for (auto& job_id_jd : jobs_to_schedule_) {
    jobs.push_back(job_id_jd.second);
  }
  uint64_t num_scheduled_tasks = ScheduleJobs(jobs, scheduler_stats, deltas);
  return num_scheduled_tasks;
}

uint64_t SimpleScheduler::ScheduleJob(JobDescriptor* jd_ptr,
                                      SchedulerStats* scheduler_stats) {
  uint64_t num_scheduled_tasks = 0;
  VLOG(2) << "Preparing to schedule job " << jd_ptr->uuid();
  boost::lock_guard<boost::recursive_mutex> lock(scheduling_lock_);
  LOG(INFO) << "START SCHEDULING " << jd_ptr->uuid();
  boost::timer::cpu_timer scheduler_timer;
  // Get the set of runnable tasks for this job
  unordered_set<TaskID_t> runnable_tasks = ComputeRunnableTasksForJob(jd_ptr);
  VLOG(2) << "Scheduling job " << jd_ptr->uuid() << ", which has "
          << runnable_tasks.size() << " runnable tasks.";
  JobID_t job_id = JobIDFromString(jd_ptr->uuid());
  for (unordered_set<TaskID_t>::const_iterator task_iter =
       runnable_tasks.begin();
       task_iter != runnable_tasks.end();
       ++task_iter) {
    TaskDescriptor* td = FindPtrOrNull(*task_map_, *task_iter);
    CHECK(td);
    trace_generator_->TaskSubmitted(td);
    VLOG(2) << "Considering task " << td->uid() << ":\n"
            << td->DebugString();

    ResourceID_t best_resource;
    bool success = false;
    if (FLAGS_randomly_place_tasks) {
      success = FindRandomResourceForTask(*td, &best_resource);
    } else {
      success = FindResourceForTask(*td, &best_resource);
    }
    if (!success) {
      VLOG(2) << "No suitable resource found, will need to try again.";
    } else {
      ResourceStatus* rp = FindPtrOrNull(*resource_map_, best_resource);
      CHECK(rp);
      VLOG(1) << "Scheduling task " << td->uid() << " on resource "
              << rp->descriptor().uuid();
      // Remove the task from the runnable set.
      runnable_tasks_[job_id].erase(td->uid());
      HandleTaskPlacement(td, rp->mutable_descriptor());
      num_scheduled_tasks++;
    }
  }
  if (num_scheduled_tasks > 0)
    jd_ptr->set_state(JobDescriptor::RUNNING);
  if (scheduler_stats != NULL) {
    scheduler_stats->scheduler_runtime_ = scheduler_timer.elapsed().wall /
      NANOSECONDS_IN_MICROSECOND;
  }
  LOG(INFO) << "STOP SCHEDULING " << jd_ptr->uuid();
  return num_scheduled_tasks;
}

uint64_t SimpleScheduler::ScheduleJobs(const vector<JobDescriptor*>& jds_ptr,
                                       SchedulerStats* scheduler_stats,
                                       vector<SchedulingDelta>* deltas) {
  boost::lock_guard<boost::recursive_mutex> lock(scheduling_lock_);
  uint64_t num_scheduled_tasks = 0;
  boost::timer::cpu_timer scheduler_timer;
  // TODO(ionel): Populate scheduling deltas!
  for (auto& jd_ptr : jds_ptr) {
    num_scheduled_tasks += ScheduleJob(jd_ptr, scheduler_stats);
  }
  if (scheduler_stats != NULL) {
    scheduler_stats->scheduler_runtime_ =
      static_cast<uint64_t>(scheduler_timer.elapsed().wall) /
      NANOSECONDS_IN_MICROSECOND;
  }
  return num_scheduled_tasks;
}

}  // namespace scheduler
}  // namespace firmament

max_stars_repo_name: Container-Projects/firmament

max_issues_repo_name: Container-Projects/firmament

max_forks_repo_name: Container-Projects/firmament


text:
// Copyright John Maddock 2006.
// Copyright Paul A. Bristow 2007, 2009

//  Use, modification and distribution are subject to the
//  Boost Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#include <pch.hpp>

#ifdef _MSC_VER
# pragma warning (disable : 4996) // POSIX name for this item is deprecated
# pragma warning (disable : 4224) // nonstandard extension used : formal parameter 'arg' was previously defined as a type
# pragma warning (disable : 4180) // qualifier applied to function type has no meaning; ignored
#endif

#include <boost/math/concepts/real_concept.hpp>
#include <boost/test/test_exec_monitor.hpp>
#include <boost/test/floating_point_comparison.hpp>
#include <boost/math/special_functions/beta.hpp>
#include <boost/math/tools/stats.hpp>
#include <boost/math/tools/test.hpp>
#include <boost/math/constants/constants.hpp>
#include <boost/type_traits/is_floating_point.hpp>
#include <boost/array.hpp>
#include "functor.hpp"

#include "test_beta_hooks.hpp"
#include "handle_test_result.hpp"

//
// DESCRIPTION:
// ~~~~~~~~~~~~
//
// This file tests the function beta.  There are two sets of tests, spot
// tests which compare our results with selected values computed
// using the online special function calculator at
// functions.wolfram.com, while the bulk of the accuracy tests
// use values generated with NTL::RR at 1000-bit precision
// and our generic versions of these functions.
//
// Note that when this file is first run on a new platform many of
// these tests will fail: the default accuracy is 1 epsilon which
// is too tight for most platforms.  In this situation you will
// need to cast a human eye over the error rates reported and make
// a judgement as to whether they are acceptable.  Either way please
// report the results to the Boost mailing list.  Acceptable rates of
// error are marked up below as a series of regular expressions that
// identify the compiler/stdlib/platform/data-type/test-data/test-function
// along with the maximum expected peek and RMS mean errors for that
// test.
//

void expected_results()
{
   //
   // Define the max and mean errors expected for
   // various compilers and platforms.
   //
#if LDBL_MANT_DIG == 106
   // Darwin:
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      "Mac OS.*",                    // platform
      "(long\\s+)?double",           // test type(s)
      "Beta Function: Medium.*",     // test data group
      "boost::math::beta", 200, 35); // test function
#endif

   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "(long\\s+)?double",           // test type(s)
      "Beta Function: Small.*",      // test data group
      "boost::math::beta", 8, 5);    // test function
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "(long\\s+)?double",           // test type(s)
      "Beta Function: Medium.*",     // test data group
      "boost::math::beta", 160, 35); // test function
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "(long\\s+)?double",           // test type(s)
      "Beta Function: Divergent.*",  // test data group
      "boost::math::beta", 30, 6);   // test function
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "real_concept",                // test type(s)
      "Beta Function: Small.*",      // test data group
      "boost::math::beta", 15, 15);   // test function
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "real_concept",                // test type(s)
      "Beta Function: Medium.*",     // test data group
      "boost::math::beta", 150, 40); // test function
   add_expected_result(
      ".*",                          // compiler
      ".*",                          // stdlib
      ".*",                          // platform
      "real_concept",                // test type(s)
      "Beta Function: Divergent.*",  // test data group
      "boost::math::beta", 25, 8);   // test function

   //
   // Finish off by printing out the compiler/stdlib/platform names,
   // we do this to make it easier to mark up expected error rates.
   //
   std::cout << "Tests run with " << BOOST_COMPILER << ", "
      << BOOST_STDLIB << ", " << BOOST_PLATFORM << std::endl;
}

template <class T>
void do_test_beta(const T& data, const char* type_name, const char* test_name)
{
   typedef typename T::value_type row_type;
   typedef typename row_type::value_type value_type;

   typedef value_type (*pg)(value_type, value_type);
#if defined(BOOST_MATH_NO_DEDUCED_FUNCTION_POINTERS)
   pg funcp = boost::math::beta<value_type, value_type>;
#else
   pg funcp = boost::math::beta;
#endif

   boost::math::tools::test_result<value_type> result;

   std::cout << "Testing " << test_name << " with type " << type_name
      << "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n";

   //
   // test beta against data:
   //
   result = boost::math::tools::test(
      data,
      bind_func(funcp, 0, 1),
      extract_result(2));
   handle_test_result(result, data[result.worst()], result.worst(), type_name, "boost::math::beta", test_name);
#ifdef TEST_OTHER
   if(::boost::is_floating_point<value_type>::value){
      funcp = other::beta;
      result = boost::math::tools::test(
         data,
         bind_func(funcp, 0, 1),
         extract_result(2));
      print_test_result(result, data[result.worst()], result.worst(), type_name, "other::beta");
   }
#endif
   std::cout << std::endl;
}
template <class T>
void test_beta(T, const char* name)
{
   //
   // The actual test data is rather verbose, so it's in a separate file
   //
   // The contents are as follows, each row of data contains
   // three items, input value a, input value b and beta(a, b):
   //
#  include "beta_small_data.ipp"

   do_test_beta(beta_small_data, name, "Beta Function: Small Values");

#  include "beta_med_data.ipp"

   do_test_beta(beta_med_data, name, "Beta Function: Medium Values");

#  include "beta_exp_data.ipp"

   do_test_beta(beta_exp_data, name, "Beta Function: Divergent Values");
}

#undef small // VC++ #defines small char !!!!!!
template <class T>
void test_spots(T)
{
   //
   // Basic sanity checks, tolerance is 20 epsilon expressed as a percentage:
   //
   T tolerance = boost::math::tools::epsilon<T>() * 20 * 100;
   T small = boost::math::tools::epsilon<T>() / 1024;
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(1), static_cast<T>(1)), static_cast<T>(1), tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(1), static_cast<T>(4)), static_cast<T>(0.25), tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(4), static_cast<T>(1)), static_cast<T>(0.25), tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(small, static_cast<T>(4)), 1/small, tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(4), small), 1/small, tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(4), static_cast<T>(20)), static_cast<T>(0.00002823263692828910220214568040654997176736L), tolerance);
   BOOST_CHECK_CLOSE(::boost::math::beta(static_cast<T>(0.0125L), static_cast<T>(0.000023L)), static_cast<T>(43558.24045647538375006349016083320744662L), tolerance);
}

int test_main(int, char* [])
{
   expected_results();
   BOOST_MATH_CONTROL_FP;
   test_spots(0.0F);
   test_spots(0.0);
#ifndef BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
   test_spots(0.0L);
#if !BOOST_WORKAROUND(__BORLANDC__, BOOST_TESTED_AT(0x582))
   test_spots(boost::math::concepts::real_concept(0.1));
#endif
#else
   std::cout << "<note>The long double tests have been disabled on this platform "
      "either because the long double overloads of the usual math functions are "
      "not available at all, or because they are too inaccurate for these tests "
      "to pass.</note>" << std::cout;
#endif

   test_beta(0.1F, "float");
   test_beta(0.1, "double");
#ifndef BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
   test_beta(0.1L, "long double");
#ifndef BOOST_MATH_NO_REAL_CONCEPT_TESTS
#if !BOOST_WORKAROUND(__BORLANDC__, BOOST_TESTED_AT(0x582))
   test_beta(boost::math::concepts::real_concept(0.1), "real_concept");
#endif
#endif
#else
   std::cout << "<note>The long double tests have been disabled on this platform "
      "either because the long double overloads of the usual math functions are "
      "not available at all, or because they are too inaccurate for these tests "
      "to pass.</note>" << std::cout;
#endif
   return 0;
}

max_issues_repo_name: boost-cmake/vintage

max_forks_repo_name: boost-cmake/vintage


text:
//test_miniz
#include <miniz.h>
#include <string>
#include <sstream>
#include <boost/log/trivial.hpp>
#include <iostream>
#define ifThenElse(a,b,c) (a ? b : c)
#define min(a,b) ( a < b ? a : b)
#define max(a,b) ( a > b ? a : b)
#define Z_BUF_SIZE (1024*1024)
typedef unsigned int uint;
const std::string deflate(const std::string& data, const int level){
  z_stream strm = {0};
  std::string outbuf(Z_BUF_SIZE, 0);
  std::string inbuf(min(Z_BUF_SIZE, data.length()), 0);
  std::stringstream buf;
  strm.next_out = reinterpret_cast<unsigned char*>(&outbuf[0]);
  strm.avail_out = Z_BUF_SIZE;
  strm.next_in = reinterpret_cast<unsigned char*>(&inbuf[0]);
  strm.avail_in = 0;

  uint remaining = data.length();

  if(deflateInit(&strm, level) != Z_OK){
    BOOST_LOG_TRIVIAL(error) << "Unable to init deflate";
    return data;
  }
  while(1){
    if(!strm.avail_in){
      uint bytesToRead = min(Z_BUF_SIZE, remaining);
      inbuf = data.substr(data.length() - remaining, bytesToRead);
      strm.next_in = reinterpret_cast<unsigned char*>(&inbuf[0]);
      strm.avail_in = bytesToRead;
      remaining -= bytesToRead;
    }
    int status = deflate(&strm, ifThenElse(remaining, Z_NO_FLUSH, Z_FINISH));
    if(status == Z_STREAM_END || (!strm.avail_out)){
      // Output buffer is full, or compression is done.
      uint n = Z_BUF_SIZE - strm.avail_out;
      buf << outbuf.substr(0,n);
      std::fill(outbuf.begin(), outbuf.end(), 0);
      strm.next_out = reinterpret_cast<unsigned char*>(&outbuf[0]);
      strm.avail_out = Z_BUF_SIZE;
    }
    if(status == Z_STREAM_END)
      break;
    else if(status != Z_OK)
      BOOST_LOG_TRIVIAL(error) << "Deflate status not 'OK'";
      return data;
  }
  if(deflateEnd(&strm) != Z_OK){
    BOOST_LOG_TRIVIAL(warning) << "zlib unable to cleanup deflate stream";
  }
  std::string ret = buf.str();
  if(ret.length() != strm.total_out){
    BOOST_LOG_TRIVIAL(error) << "Output size mismatch - Expected: " << strm.total_out << " Got: " << ret.length();
  }
  // BOOST_LOG_TRIVIAL(trace) << "DEFLATE: " << ((ret.length() * 100) / data.length()) << "%" << ret;
  return ret;
}

const std::string inflate(const std::string& data){
  z_stream strm = {0};
  std::string outbuf(Z_BUF_SIZE, 0);
  std::string inbuf(Z_BUF_SIZE, 0);
  std::stringstream buf;
  strm.next_out = reinterpret_cast<unsigned char*>(&outbuf[0]);
  strm.avail_out = Z_BUF_SIZE;
  strm.next_in = reinterpret_cast<unsigned char*>(&inbuf[0]);
  strm.avail_in = 0;
  uint remaining = data.length();
  if(inflateInit(&strm) != Z_OK){
    throw std::runtime_error("Unable to inflate command data: Init failed.\n");
  }
  while(1){
    if(!strm.avail_in){
      uint bytesToRead = min(Z_BUF_SIZE, remaining);
      inbuf = data.substr(data.length() - remaining, bytesToRead);
      strm.next_in = reinterpret_cast<unsigned char*>(&inbuf[0]);
      strm.avail_in = bytesToRead;
      remaining -= bytesToRead;
    }
    int status = inflate(&strm, Z_SYNC_FLUSH);
    if ((status == Z_STREAM_END) || (!strm.avail_out)){
        // Output buffer is full, or decompression is done
      uint n = Z_BUF_SIZE - strm.avail_out;
      buf << outbuf.substr(0,n);
      std::fill(outbuf.begin(), outbuf.end(), 0);
      strm.next_out = reinterpret_cast<unsigned char*>(&outbuf[0]);
      strm.avail_out = Z_BUF_SIZE;
    }
    if(status == Z_STREAM_END)
      break;
    else if(status != Z_OK)
      throw std::runtime_error("Unable to inflate");
  }
  if(inflateEnd(&strm) != Z_OK){
    BOOST_LOG_TRIVIAL(warning) << "zlib unable to cleanup inflate stream";
  }
  std::string ret = buf.str();
  if(ret.length() != strm.total_out){
    BOOST_LOG_TRIVIAL(error) << "Output size mismatch - Expected: " << strm.total_out << " Got: " << ret.length();
  }
  return ret;
}

int main(int argc, char** argv){
	std::string data = "A001 CAPABILITY\nA001 CAPABILITY\nA001 CAPABILITY\nA001 CAPABILITY\nA001 CAPABILITY\n";
	std::cout << "Data:" <<data << std::endl;
	std::string cdata = deflate(data, 4);
	std::cout << "Deflated:" <<cdata << std::endl;
	std::string udata = inflate(cdata);
	std::cout << "Inflated:" <<udata << std::endl;
	return 0;
}

max_stars_repo_name: ztipnis/imaplw

max_stars_repo_licenses: BSL-1.0 OpenSSL MIT

max_issues_repo_name: ztipnis/imaplw

max_issues_repo_licenses: BSL-1.0 OpenSSL MIT

max_forks_repo_name: ztipnis/imaplw

max_forks_repo_licenses: BSL-1.0 OpenSSL MIT


text:
// Copyright 2010-2015 RethinkDB, all rights reserved.
#ifndef RDB_PROTOCOL_FUNC_HPP_
#define RDB_PROTOCOL_FUNC_HPP_

#include <map>
#include <string>
#include <utility>
#include <vector>

#include "errors.hpp"
#include <boost/variant/static_visitor.hpp>

#include "containers/counted.hpp"
#include "containers/uuid.hpp"
#include "rdb_protocol/datum.hpp"
#include "rdb_protocol/env.hpp"
#include "rdb_protocol/op.hpp"
#include "rdb_protocol/sym.hpp"
#include "rdb_protocol/term.hpp"
#include "rdb_protocol/term_storage.hpp"
#include "rpc/serialize_macros.hpp"

class js_runner_t;

namespace ql {

class func_visitor_t;

class func_t : public slow_atomic_countable_t<func_t>, public bt_rcheckable_t {
public:
    virtual ~func_t();

    virtual scoped_ptr_t<val_t> call(
        env_t *env,
        const std::vector<datum_t> &args,
        eval_flags_t eval_flags = NO_FLAGS) const = 0;

    virtual optional<size_t> arity() const = 0;

    virtual deterministic_t is_deterministic() const = 0;

    // Used by info_term_t.
    virtual std::string print_source() const = 0;

    // Similar to `print_source()`, but prints a JS expression of the form
    // `function(arg1, arg2, ...) { return body; }`. Fails if the func_t has a non-empty
    // captured scope.
    virtual std::string print_js_function() const = 0;

    virtual void visit(func_visitor_t *visitor) const = 0;

    void assert_deterministic(const char *extra_msg) const;

    bool filter_call(env_t *env,
                     datum_t arg,
                     counted_t<const func_t> default_filter_val) const;

    // These are simple, they call the vector version of call.
    scoped_ptr_t<val_t> call(env_t *env, eval_flags_t eval_flags = NO_FLAGS) const;
    scoped_ptr_t<val_t> call(env_t *env,
                             datum_t arg,
                             eval_flags_t eval_flags = NO_FLAGS) const;
    scoped_ptr_t<val_t> call(env_t *env,
                             datum_t arg1,
                             datum_t arg2,
                             eval_flags_t eval_flags = NO_FLAGS) const;

    virtual bool is_simple_selector() const {
        return false;
    }

protected:
    explicit func_t(backtrace_id_t bt);

private:
    virtual bool filter_helper(env_t *env, datum_t arg) const = 0;

    DISABLE_COPYING(func_t);
};

class reql_func_t : public func_t {
public:
    // Used when constructing in an existing environment - reusing another term storage
    reql_func_t(const var_scope_t &captured_scope,
                std::vector<sym_t> arg_names,
                counted_t<const term_t> body);

    // Used when constructing from a function read off the wire
    reql_func_t(scoped_ptr_t<term_storage_t> &&_storage,
                const var_scope_t &captured_scope,
                std::vector<sym_t> arg_names,
                counted_t<const term_t> body);

    ~reql_func_t();

    scoped_ptr_t<val_t> call(
        env_t *env,
        const std::vector<datum_t> &args,
        eval_flags_t eval_flags) const;

    optional<size_t> arity() const;

    deterministic_t is_deterministic() const;

    std::string print_source() const;
    std::string print_js_function() const;

    void visit(func_visitor_t *visitor) const;

    bool is_simple_selector() const final;

private:
    template <cluster_version_t> friend class wire_func_serialization_visitor_t;
    bool filter_helper(env_t *env, datum_t arg) const;

    // Only contains the parts of the scope that `body` uses.
    var_scope_t captured_scope;

    // The argument names, for the corresponding positional argument number.
    std::vector<sym_t> arg_names;

    // Term storage if this was deserialized off the wire to ensure the lifetime of
    // the raw term tree.
    scoped_ptr_t<term_storage_t> term_storage;

    // The body of the function, which gets ->eval(...) called when call(...) is called.
    counted_t<const term_t> body;

    DISABLE_COPYING(reql_func_t);
};

class js_func_t : public func_t {
public:
    js_func_t(const std::string &_js_source,
              uint64_t timeout_ms,
              backtrace_id_t backtrace);
    ~js_func_t();

    // Some queries, like filter, can take a shortcut object instead of a
    // function as their argument.
    scoped_ptr_t<val_t> call(env_t *env,
                             const std::vector<datum_t> &args,
                             eval_flags_t eval_flags) const;

    optional<size_t> arity() const;

    deterministic_t is_deterministic() const;

    std::string print_source() const;
    std::string print_js_function() const;

    void visit(func_visitor_t *visitor) const;

private:
    template <cluster_version_t> friend class wire_func_serialization_visitor_t;
    bool filter_helper(env_t *env, datum_t arg) const;

    std::string js_source;
    uint64_t js_timeout_ms;

    DISABLE_COPYING(js_func_t);
};

class func_visitor_t {
public:
    virtual void on_reql_func(const reql_func_t *reql_func) = 0;
    virtual void on_js_func(const js_func_t *js_func) = 0;
protected:
    func_visitor_t() { }
    virtual ~func_visitor_t() { }
    DISABLE_COPYING(func_visitor_t);
};

// Some queries, like filter, can take a shortcut object instead of a
// function as their argument.
counted_t<const func_t> new_constant_func(datum_t obj, backtrace_id_t bt);
counted_t<const func_t> new_pluck_func(datum_t obj, backtrace_id_t bt);
counted_t<const func_t> new_get_field_func(datum_t obj, backtrace_id_t bt);
counted_t<const func_t> new_eq_comparison_func(datum_t obj, backtrace_id_t bt);
counted_t<const func_t> new_page_func(datum_t method, backtrace_id_t bt);

class js_result_visitor_t : public boost::static_visitor<val_t *> {
public:
    js_result_visitor_t(const std::string &_code,
                        uint64_t _timeout_ms,
                        const bt_rcheckable_t *_parent)
        : code(_code),
          timeout_ms(_timeout_ms),
          parent(_parent) { }

    // The caller needs to take ownership of the return value.  Boost static_visitor
    // can't handle movable types.

    // This JS evaluation resulted in an error
    val_t *operator()(const std::string &err_val) const;
    // This JS call resulted in a JSON value
    val_t *operator()(const datum_t &json_val) const;
    // This JS evaluation resulted in an id for a js function
    val_t *operator()(const js_id_t id_val) const;

private:
    std::string code;
    uint64_t timeout_ms;
    const bt_rcheckable_t *parent;
};

// Evaluating this returns a `func_t` wrapped in a `val_t`.
class func_term_t : public term_t {
public:
    func_term_t(compile_env_t *env, const raw_term_t &term);

    // eval(scoped_env_t *env) is a dumb wrapper for this.  Evaluates the func_t without
    // going by way of val_t, and without requiring a full-blown env.
    counted_t<const func_t> eval_to_func(const var_scope_t &env_scope) const;

private:
    virtual void accumulate_captures(var_captures_t *captures) const;
    virtual deterministic_t is_deterministic() const;
    virtual scoped_ptr_t<val_t> term_eval(scope_env_t *env, eval_flags_t flags) const;
    virtual const char *name() const { return "func"; }

    std::vector<sym_t> arg_names;
    counted_t<const term_t> body;

    var_captures_t external_captures;
};

}  // namespace ql

#endif // RDB_PROTOCOL_FUNC_HPP_

max_stars_repo_name: NicoHood/rethinkdb

max_issues_repo_name: NicoHood/rethinkdb

max_forks_repo_name: NicoHood/rethinkdb


text:
/*
 * Stream-Processing Architecture for Real-time Cyber-physical Security (SPARCS)
 */

/*
 * File:   C37_DataChannel.cpp
 * Author: mcp
 *
 * Created on April 12, 2017, 12:35 AM
 */
#define BOOST_BIND_NO_PLACEHOLDERS //prevent error on docker

#include <time.h>
#include <cstdlib>
#include <vector>
#include <boost/bind.hpp>
#include <boost/asio.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/asio/async_result.hpp>
#include <boost/asio/write.hpp>
#include <boost/asio/buffer.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/crc.hpp>      // for boost::crc_basic, boost::crc_optimal
#include <boost/cstdint.hpp>  // for boost::uint16_t
#include <boost/circular_buffer.hpp>
#include <boost/circular_buffer/base.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>

/* Json library */
//#include <jsoncpp/json/json.h>

using namespace boost::asio;
using namespace boost::asio::ip;
using namespace boost;

#include "C37_inputDesc.h"
#include "C37_MsgFormats.h"
#include "C37_Buffer.h"
#include "C37_CfgDescriptor.h"
#include "C37_Reformat.h"
#include "C37_DataChannel.h"

C37_DataChannel::C37_DataChannel (io_service& IO,
        boost::asio::ip::tcp::endpoint& ENDPT) : io(IO),
        sock(std::make_shared<boost::asio::ip::tcp::socket>(io)),
        endpoint(ENDPT) {
    //sock = std::make_shared<boost::asio::ip::tcp::socket>(io);
}

C37_DataChannel::~C37_DataChannel() {
}

void C37_DataChannel::connect() {
    sock->async_connect(endpoint,
        boost::bind(&C37_DataChannel::connect_handler, this,
        boost::asio::placeholders::error));
}

/* not implemented yet
void C37_DataChannel::sendHDRcmd() {
    sendCommand(HDR_CMD_VALUE, nullptr, 0);
}
*/

void C37_DataChannel::sendCFG_1cmd() {
    sendCommand(CFG_1_CMD_VALUE, nullptr, 0);
}

void C37_DataChannel::sendCFG_2cmd() {
    sendCommand(CFG_2_CMD_VALUE, nullptr, 0);
}

/* not implemented yet
void C37_DataChannel::sendCFG_3cmd() {
    sendCommand(CFG_3_CMD_VALUE, nullptr, 0);
}
/
 */

/* not implemented yet
void C37_DataChannel::sendEXTcmd(unsigned char * data, unsigned int dataLen) {
    sendCommand(EXTFR_CMD_VALUE, nullptr, 0);
}
*/

void C37_DataChannel::connect_handler(const boost::system::error_code &ec)
{
  std::cout<< "Connect status: " << ec.message() << std::endl;
  if (!ec)
  {
      NumConnects++;
      boost::asio::async_read(*sock, C37_buf.getBufferDesc(),
          boost::bind(&C37_DataChannel::read_handler, this,
           placeholders::error, placeholders::bytes_transferred));
  }
  else {
      sock->close();
      sock.reset(new boost::asio::ip::tcp::socket(io));
      connect();
  }
}

void C37_DataChannel::read_handler(const boost::system::error_code &ec,
  std::size_t bytes_transferred)
{
    if(bytes_transferred!=0) //Changed by Reinhard
  //if (!ec)
  {
    C37_buf.setBytesReceived(bytes_transferred);
    while (C37_buf.examineBuffer() == 0) {
        BytesRecv += C37_buf.msgLenBytes;
        switch (C37_buf.msgType) {
            case DATA_MSG_TYPE_VALUE: {
                uint32_t fracTime = (C37_buf.buffer[FRACSEC_INDX + 1] << 16) +
                    (C37_buf.buffer[FRACSEC_INDX + 2] << 8) + C37_buf.buffer[FRACSEC_INDX + 3];
                uint32_t time = (C37_buf.buffer[SOC_INDX] << 24) + (C37_buf.buffer[SOC_INDX + 1] << 16) +
                    (C37_buf.buffer[SOC_INDX + 2] << 8) + C37_buf.buffer[SOC_INDX + 3]; //Reinhard: this is the same as an endianswap
                C37eventsRecv++;
                c37reformat.processC37event(fracTime, time, &C37_buf);
                C37eventsQueued++;
                C37_buf.removeMsgFromBuffer();
                break;
            }
            case CFG_1_MSG_TYPE_VALUE: {
                this->cfg_1_Descriptor =
                        C37_desc.parseCfg_msg(C37_buf.buffer,
                        C37_buf.msgLenBytes);
                C37_buf.removeMsgFromBuffer();
                break;
            }
            case CFG_2_MSG_TYPE_VALUE: {
                this->cfg_2_Descriptor =
                        C37_desc.parseCfg_msg(C37_buf.buffer,
                        C37_buf.msgLenBytes);
                C37_buf.removeMsgFromBuffer();
                break;
            }
            default: {
                C37_buf.removeMsgFromBuffer();
                break;
            }
        }
    }

    boost::asio::async_read(*sock, C37_buf.getBufferDesc(),
          boost::bind(&C37_DataChannel::read_handler, this,
           placeholders::error, placeholders::bytes_transferred));
  }
  else {
//      if (ec.message()=="End of file"){
//           boost::asio::
//          boost::asio::async_read(*sock, C37_buf.getBufferDesc(),
//          boost::bind(&C37_DataChannel::read_handler, this,
//           placeholders::error, placeholders::bytes_transferred));
//          std::cout<< "END of file error with " << bytes_transferred <<"bytes_transferred" << std::endl;
//
//      }
//      else{
        C37_buf.init();
        sock->close();
        std::cout<< "DISCONNECTED reason: "<< ec.message() << std::endl;
        sock.reset(new boost::asio::ip::tcp::socket(io));
        connect();
     // }
  }
}

void C37_DataChannel::sendCommand(unsigned int commandType,
        unsigned char * data, unsigned int dataLen)
{
    printf("sending command. \n");
    if(data == nullptr) {
        dataLen = 0;
    }

    auto sendData = std::make_shared<std::vector<unsigned char> >(EXTFRAME_INDX + dataLen + 2);
    unsigned char x = (*sendData).size();
    printf("Send buffer sizeL %u \n", x);

    (*sendData)[SYNC_INDX] = SYNC_VALUE;
    (*sendData)[SYNC_INDX + 1] = CMD_SYNC_VALUE;
    unsigned int frameLen = EXTFRAME_INDX + dataLen + 2;
    (*sendData)[FRAMESIZE_INDX] = (frameLen >> 8) & 0xff;
    (*sendData)[FRAMESIZE_INDX + 1] = frameLen & 0xff;
    (*sendData)[IDCODE_INDX] = 0;
    (*sendData)[IDCODE_INDX + 1] = 1;

    std::time_t t = std::time(nullptr);

    (*sendData)[SOC_INDX] = (t >> 24) & 0xff;
    (*sendData)[SOC_INDX + 1] = (t >> 16) & 0xff;
    (*sendData)[SOC_INDX + 2] = (t >> 8) & 0xff;
    (*sendData)[SOC_INDX + 3] = t & 0xff;
    (*sendData)[FRACSEC_INDX] = 0;
    (*sendData)[FRACSEC_INDX + 1] = 0;
    (*sendData)[FRACSEC_INDX + 2] = 0;
    (*sendData)[FRACSEC_INDX + 3] = 0;
    (*sendData)[CMD_INDX] = 0;
    (*sendData)[CMD_INDX + 1] = commandType & 0xff;
    this->crc_ccitt.reset();
    this->crc_ccitt.process_bytes(&((*sendData)[0]), (*sendData).size());
    //printf("%x \n", crc_ccitt.checksum());
    (*sendData)[dataLen + CMD_INDX + 2] = (crc_ccitt.checksum() >> 8) & 0xff;
    ((*sendData)[dataLen + CMD_INDX + 3]) = crc_ccitt.checksum() & 0xff;
    for(int i = 0; i < (*sendData).size(); ++i) {
        printf("%x\n ", (*sendData)[i]);
    }
    std::cout << std::endl;

    boost::asio::async_write(*sock, asio::buffer(*sendData),
        boost::bind(&C37_DataChannel::send_handler, this,
         placeholders::error));
}

void C37_DataChannel::send_handler(const boost::system::error_code &ec)
{
  std::cout<< "Send status: " << ec.message() << std::endl;
  if (!ec)
  {
      //boost::asio::async_read(sock, C37_buf.getBufferDesc(),
          //boost::bind(&C37_DataChannel::read_handler, this,
           //placeholders::error, placeholders::bytes_transferred));
  }
}

//}

max_stars_repo_name: lbnl-cybersecurity/sparcs

max_issues_repo_name: lbnl-cybersecurity/sparcs

max_forks_repo_name: lbnl-cybersecurity/sparcs


text:
// Copyright (C) 2019 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#define BOOST_STL_INTERFACES_DISABLE_CMCSTL2
#include <boost/stl_interfaces/iterator_interface.hpp>

#include <algorithm>
#include <array>
#include <iostream>

#include <cassert>

//[ node_defn
template<typename T>
struct node
{
    T value_;
    node * next_; // == nullptr in the tail node
};
//]

//[ node_iterator_class_head
template<typename T>
struct node_iterator
    : boost::stl_interfaces::
          iterator_interface<node_iterator<T>, std::forward_iterator_tag, T>
//]
{
    //[ node_iterator_ctors
    constexpr node_iterator() noexcept : it_(nullptr) {}
    constexpr node_iterator(node<T> * it) noexcept : it_(it) {}
    //]

    //[ node_iterator_user_ops
    constexpr T & operator*() const noexcept { return it_->value_; }
    constexpr node_iterator & operator++() noexcept
    {
        it_ = it_->next_;
        return *this;
    }
    friend constexpr bool
    operator==(node_iterator lhs, node_iterator rhs) noexcept
    {
        return lhs.it_ == rhs.it_;
    }
    //]

    //[ node_iterator_using_declaration
    using base_type = boost::stl_interfaces::
        iterator_interface<node_iterator<T>, std::forward_iterator_tag, T>;
    using base_type::operator++;
    //]

private:
    node<T> * it_;
};

//[ node_iterator_concept_check Equivalent to
// static_assert(std::forward_iterator<node_iterator>, ""), or nothing in
// C++17 and earlier.
BOOST_STL_INTERFACES_STATIC_ASSERT_CONCEPT(node_iterator, std::forward_iterator)
//]

int main()
{
    std::array<node<int>, 5> nodes;
    std::generate(nodes.begin(), nodes.end(), [] {
        static int i = 0;
        return node<int>{i++};
    });
    std::adjacent_find(
        nodes.begin(), nodes.end(), [](node<int> & a, node<int> & b) {
            a.next_ = &b;
            return false;
        });
    nodes.back().next_ = nullptr;

    //[ node_iterator_usage
    node_iterator<int> const first(&nodes[0]);
    node_iterator<int> const last;
    for (auto it = first; it != last; it++) {
        std::cout << *it << " "; // Prints 0 1 2 3 4
    }
    std::cout << "\n";
    //]
}