text:
#define GrB_USE_SEQUENTIAL
#define private public

#include <vector>
#include <iostream>
#include <string>
#include <numeric>
#include <algorithm>
#include <map>

#include <cstdio>
#include <cstdlib>

#include "graphblas/graphblas.hpp"
#include "graphblas/backend/sequential/sequential.hpp"
#include "test/test.hpp"

#define BOOST_TEST_MAIN
#define BOOST_TEST_MODULE triple_suite

#include <boost/test/included/unit_test.hpp>
#include <boost/program_options.hpp>

void testMxm( char const* tsv, std::vector<int>& correct,
              std::vector<int>& correct_t )
{
  std::vector<graphblas::Index> row_indices;
  std::vector<graphblas::Index> col_indices;
  std::vector<float> values;
  graphblas::Index nrows, ncols, nvals;
  graphblas::Info err;

  int argc = 3;
  char** argv = (char**) malloc(3*sizeof(char*));
  argv[1] = (char *) "--verbose=false";
  argv[2] = (char *) "--merge.r=0";

  // Read in sparse matrix
  err = readTsv( tsv, row_indices, col_indices, values, nrows, ncols,
    nvals );

  graphblas::BlockState<graphblas::T> state( nrows, ncols );
  err = state.parseArgs( argc, argv );
  err = state.build( row_indices, col_indices, values, nvals );

  std::vector<int> k_out;
  std::vector<int> k_in;
  int temp;

  printArray( "curr_part", state.block_state_.curr_partition_, nrows );
  err = graphblas::mxm( state.block_state_.blk_,
                                  state.block_state_.curr_partition_,
                                  state.block_state_.adj_ );
  /*for( int i=0; i<nrows; i++ )
  {
    err = state.block_state_.adj_.getRowLength( temp, i );
    k_out.push_back( temp );
    err = state.block_state_.adj_t_.getRowLength( temp, i );
    k_in.push_back( temp );
  }

  BOOST_ASSERT_LIST( k_out, correct, std::min(nrows,10) );
  BOOST_ASSERT_LIST( k_in, correct_t, std::min(nrows,10) );

  BOOST_ASSERT( err==0 );*/
}

struct TestMatrix
{
  TestMatrix() :
    DEBUG(true) {}

  bool DEBUG;
};

BOOST_AUTO_TEST_SUITE(triple_suite)

BOOST_FIXTURE_TEST_CASE( triple1, TestMatrix )
{
  std::vector<int> adj_deg   = {1, 1, 3, 2, 2, 3, 3, 0, 1, 2};
  std::vector<int> adj_t_deg = {3, 3, 3, 2, 3, 1, 0, 3, 2, 0};
  testMxm( "data/small/test_cc.tsv", adj_deg, adj_t_deg );
}

BOOST_FIXTURE_TEST_CASE( triple2, TestMatrix )
{
  std::vector<int> adj_deg   = {1, 1, 3, 2, 2, 3, 3};
  std::vector<int> adj_t_deg = {3, 3, 3, 2, 3, 1, 0};
  testMxm( "data/small/test_bc.tsv", adj_deg, adj_t_deg );
}

BOOST_FIXTURE_TEST_CASE( triple3, TestMatrix )
{
  std::vector<int> adj_deg   = {4, 3, 3, 1, 1, 2, 4, 4, 1, 3};
  std::vector<int> adj_t_deg = {4, 4, 2, 1, 2, 1, 3, 3, 4, 1};
  testMxm( "data/small/chesapeake.tsv", adj_deg, adj_t_deg );
}

/*BOOST_FIXTURE_TEST_CASE( triple4, TestMatrix )
{
  testDup( "data/small/test_cc.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple5, TestMatrix )
{
  testDup( "data/small/test_bc.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple6, TestMatrix )
{
  testDup( "data/small/chesapeake.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple7, TestMatrix )
{
  testShiftRight( "data/small/test_cc.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple8, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert({{2, 1}, {3, 0}, {5,1}});
  std::vector<graphblas::Index> correct = {3, 4, -1};
  testEwiseSub( "data/small/test_cc.tsv", out_edge, correct);
}

BOOST_FIXTURE_TEST_CASE( triple9, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert({{2,0}, {4,1}});
  std::vector<graphblas::Index> correct_ind = {2, 4};
  std::vector<graphblas::T> correct_val = {1, 1};
  testEwiseAdd( "data/small/test_cc.tsv", out_edge, 0, correct_ind,
      correct_val, 3);
}

BOOST_FIXTURE_TEST_CASE( triple10, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert({{1,1}, {2,0}, {4,1}, {5,2}, {6,1}});
  std::vector<graphblas::Index> correct_ind = {1, 2, 3, 4, 5, 6};
  std::vector<graphblas::T> correct_val = {1, 1, 1, 2, 2, 1};
  testEwiseAdd( "data/small/test_cc.tsv", out_edge, 4, correct_ind,
      correct_val, 2);
}

BOOST_FIXTURE_TEST_CASE( triple11, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert( {{1,1}, {2,0}, {4,1}, {5,2}, {6,1}, {7,1}, {8,10}} );
  std::vector<graphblas::Index> correct_ind = {1, 2, 3, 4, 5, 6, 7, 8};
  std::vector<graphblas::T> correct_val = {1, 1, 1, 2, 2, 1, 1, 10};
  testEwiseAdd( "data/small/test_cc.tsv", out_edge, 7, correct_ind,
      correct_val, 1);
}

BOOST_FIXTURE_TEST_CASE( triple12, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert({{2, 1}, {3, 0}, {4,1}});
  std::vector<graphblas::Index> correct = {1, 2, 3, 0, 2, 4, 3, 4, -1, 5, 6, 2, 6, -1, 6, 8, 9, 10, 9, 10};
  testEwiseSubCol( "data/small/test_cc.tsv", out_edge, correct);
}

BOOST_FIXTURE_TEST_CASE( triple13, TestMatrix )
{
  testResize( "data/small/test_cc.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple14, TestMatrix )
{
  testResize( "data/small/chesapeake.tsv" );
}

BOOST_FIXTURE_TEST_CASE( triple15, TestMatrix )
{
  testRequest( "data/small/test_cc.tsv", 5 );
}

BOOST_FIXTURE_TEST_CASE( triple16, TestMatrix )
{
  testRequest( "data/small/chesapeake.tsv", 5 );
}

BOOST_FIXTURE_TEST_CASE( triple17, TestMatrix )
{
  testRequest( "data/static/simulated_blockmodel_graph_1000_nodes.tsv", 5 );
}

BOOST_FIXTURE_TEST_CASE( triple18, TestMatrix )
{
  std::map<graphblas::Index,graphblas::Index> out_edge;
  out_edge.insert( {{1,1}, {2,0}, {4,1}, {5,2}, {6,1}, {7,1}, {8,10}} );
  std::vector<graphblas::Index> correct_ind = {1, 2, 3, 0, 2, 4, 5, 3, 4, 5, 5, 6, 2, 5, 6, 5, 6, 5, 5, 8, 9, 10, 5, 9, 10 };
  std::vector<graphblas::T>     correct_val = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 10, 1, 1 };
  testEwiseAddCol( "data/small/test_cc.tsv", out_edge, correct_ind, correct_val);
}*/

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: BenBrock/graphblast

max_issues_repo_name: BenBrock/graphblast

max_forks_repo_name: BenBrock/graphblast


text:
#include <boost/test/unit_test.hpp>
#include "Buffer/CircularBuffer.h"
#include "Buffer/LinearBuffer.h"

BOOST_AUTO_TEST_SUITE(CircularBufferTest)

struct CircularBufferGeneralTestFixture
{
    CircularBuffer m_bufContent;

    CircularBufferGeneralTestFixture()
    {
        m_bufContent.reserve(512);
        m_bufContent.inc_size(127);
        for (size_t i = 0; i < 127; ++i)
        {
            m_bufContent[i] = static_cast<us8>(i + 1);
        }
    }

    ~CircularBufferGeneralTestFixture()
    {
    }
};

BOOST_FIXTURE_TEST_CASE(GeneralTest, CircularBufferGeneralTestFixture)
{
    CircularBuffer buf(512);
    const CircularBuffer &constBuf = buf;
    BOOST_TEST((constBuf.size() == 0 && constBuf.capacity() == 512));
    BufDescriptor bufs[2];
    size_t bufSize = buf.free_buffers(bufs);
    BOOST_TEST(bufSize == 1);

    for (size_t i = 0; i < bufs[0].m_size; ++i)
    {
        bufs[0].m_beg[i] = static_cast<us8>(i);
    }

    buf.inc_size(512);
    buf.pop_front(256);
    buf.inc_size(128);
    bufSize = buf.free_buffers(bufs);
    BOOST_TEST(bufSize == 1);
    BOOST_TEST(bufs[0].m_size == 128);
    buf.pop_front(257);
    bufSize = buf.free_buffers(bufs);
    BOOST_TEST(bufSize == 2);
    BOOST_TEST(bufs[0].m_size == 384);
    BOOST_TEST(bufs[1].m_size == 1);
    BOOST_TEST((buf == m_bufContent), boost::test_tools::per_element());
    buf.reserve(513);
    BOOST_TEST((buf == m_bufContent), boost::test_tools::per_element());
    buf.pop_front(126);
    buf.inc_size(387);
    buf.reserve(514);
    BOOST_TEST(buf[0] == 127);
    buf.pop_front(buf.size());
    buf.inc_size(1);
    buf.clear();
    BOOST_TEST(buf.size() == 0);

    buf.inc_size(127);
    LinearBuffer linearBuf(513);
    buf.copy_to(linearBuf, 0, 127);
    for (LinearBuffer::size_type i = 0; i < linearBuf.size(); ++i)
    {
        BOOST_TEST(buf[i] == linearBuf[i]);
    }
    buf.pop_front(126);
    buf.inc_size(387);
    buf.copy_to(linearBuf, 0, 388);
    for (LinearBuffer::size_type i = 0; i < linearBuf.size(); ++i)
    {
        BOOST_TEST(buf[i] == linearBuf[i]);
        BOOST_TEST(constBuf[i] == linearBuf[i]);
    }
}

template<typename T> void CircularBufferIteratorTestCore(T &&iter)
{
    BOOST_TEST(*(++iter) == 2);
    BOOST_TEST(*(--iter) == 1);
    BOOST_TEST(*(iter += 2) == 3);
    BOOST_TEST(*(iter -= 2) == 1);
    auto iter2 = iter + 2;
    BOOST_TEST(*iter2 == 3);
    iter2 = iter2 - 2;
    BOOST_TEST(*iter2 == 1);
    bool result = iter2 - iter2 == 0;
    BOOST_TEST(result, "iter2 - iter2 == 0");
    ++iter2;
    result = iter < iter2;
    BOOST_TEST(result, "iter < iter2");
}

template<typename T> void CircularBufferReverseIteratorTestCore(T &&iter)
{
    BOOST_TEST(*(++iter) == 126);
    BOOST_TEST(*(--iter) == 127);
    BOOST_TEST(*(iter += 2) == 125);
    BOOST_TEST(*(iter -= 2) == 127);
    auto iter2 = iter + 2;
    BOOST_TEST(*iter2 == 125);
    iter2 = iter2 - 2;
    BOOST_TEST(*iter2 == 127);
    bool result = iter2 - iter2 == 0;
    BOOST_TEST(result, "iter2 - iter2 == 0");
    ++iter2;
    result = iter < iter2;
    BOOST_TEST(result, "iter < iter2");
}

BOOST_FIXTURE_TEST_CASE(IteratorTest, CircularBufferGeneralTestFixture)
{
    const CircularBuffer &constBuf = m_bufContent;
    CircularBufferIteratorTestCore(m_bufContent.begin());
    CircularBufferIteratorTestCore(constBuf.begin());
    CircularBufferIteratorTestCore(constBuf.cbegin());
    CircularBufferReverseIteratorTestCore(m_bufContent.rbegin());
    CircularBufferReverseIteratorTestCore(constBuf.rbegin());
    CircularBufferReverseIteratorTestCore(constBuf.crbegin());
    BOOST_TEST(*--m_bufContent.end() == 127);
    BOOST_TEST(*--constBuf.end() == 127);
    BOOST_TEST(*--constBuf.cend() == 127);
    BOOST_TEST(*--m_bufContent.rend() == 1);
    BOOST_TEST(*--constBuf.rend() == 1);
    BOOST_TEST(*--constBuf.crend() == 1);
}

struct CircularBufferCopyCtrlTestFixture
{
    CircularBuffer m_bufContent;

    CircularBufferCopyCtrlTestFixture()
    {
        m_bufContent.reserve(129);
        m_bufContent.inc_size(129);
        for (size_t i = 0; i < 129; ++i)
        {
            m_bufContent[i] = static_cast<us8>(i + 127);
        }
    }

    ~CircularBufferCopyCtrlTestFixture()
    {
    }
};

BOOST_FIXTURE_TEST_CASE(CopyCtrlTest, CircularBufferCopyCtrlTestFixture)
{
    CircularBuffer buf(512);
    BufDescriptor bufs[2];
    buf.free_buffers(bufs);
    for (size_t i = 0; i < bufs[0].m_size; ++i)
    {
        bufs[0].m_beg[i] = static_cast<us8>(i);
    }
    buf.inc_size(256);
    buf.pop_front(127);

    CircularBuffer buf2(buf);
    BOOST_TEST((buf2 == m_bufContent), boost::test_tools::per_element());
    CircularBuffer buf3(std::move(buf2));
    BOOST_TEST((buf3 == m_bufContent), boost::test_tools::per_element());

    buf2 = buf3;
    BOOST_TEST((buf2 == m_bufContent), boost::test_tools::per_element());
    buf2 = CircularBuffer(256);
    buf2 = std::move(buf3);
    BOOST_TEST((buf2 == m_bufContent), boost::test_tools::per_element());

    buf2 = buf2;
    BOOST_TEST(buf2.size() == 129);
    BOOST_TEST((buf2 == m_bufContent), boost::test_tools::per_element());
    buf2 = std::move(buf2);
    BOOST_TEST(buf2.size() == 129);
    BOOST_TEST((buf2 == m_bufContent), boost::test_tools::per_element());
}

struct CircularBufferLogicOperatorTestFixture
{
    CircularBuffer m_buf1;

    CircularBuffer m_buf2;

    CircularBufferLogicOperatorTestFixture()
    {
        m_buf1.reserve(128);
        m_buf1.inc_size(128);
        m_buf2.reserve(128);
        m_buf2.inc_size(128);
        for (size_t i = 0; i < m_buf1.size(); ++i)
        {
            m_buf1[i] = static_cast<us8>(i + 127);
            m_buf2[i] = m_buf1[i];
        }
    }

    ~CircularBufferLogicOperatorTestFixture()
    {
    }
};

BOOST_FIXTURE_TEST_CASE(LogicOperatorTest, CircularBufferLogicOperatorTestFixture)
{
    BOOST_TEST(m_buf1 == m_buf2);
    BOOST_TEST(!(m_buf1 != m_buf2));
    m_buf1.reserve(m_buf1.size() + 1);
    m_buf1.inc_size(1);
    m_buf1[m_buf1.size() - 1] = 255;
    BOOST_TEST(m_buf1 > m_buf2);
    BOOST_TEST(m_buf2 < m_buf1);
    m_buf1.pop_front(1);
    BOOST_TEST(m_buf2 < m_buf1);
    BOOST_TEST(!(m_buf1 < m_buf2));
    BOOST_TEST(!(m_buf1 < m_buf1));
}

BOOST_AUTO_TEST_SUITE_END()

ext: cpp

max_stars_repo_name: xvyvx/Utils

max_issues_repo_name: xvyvx/Utils

max_forks_repo_name: xvyvx/Utils


text:
//
//  Copyright (C) 2021 Greg Landrum
//
//   @@ All Rights Reserved @@
//  This file is part of the RDKit.
//  The contents are covered by the terms of the BSD license
//  which is included in the file license.txt, found at the root
//  of the RDKit source tree.
//
#include "Embedder.h"

#include <RDGeneral/BoostStartInclude.h>
#include <boost/lexical_cast.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <RDGeneral/BoostEndInclude.h>

namespace RDKit {
namespace DGeomHelpers {

#define PT_OPT_GET(opt) params.opt = pt.get(#opt, params.opt)

void updateEmbedParametersFromJSON(EmbedParameters &params,
                                   const std::string &json) {
  if (json.empty()) {
    return;
  }
  std::istringstream ss;
  ss.str(json);
  boost::property_tree::ptree pt;
  boost::property_tree::read_json(ss, pt);
  PT_OPT_GET(maxIterations);
  PT_OPT_GET(numThreads);
  PT_OPT_GET(randomSeed);
  PT_OPT_GET(clearConfs);
  PT_OPT_GET(useRandomCoords);
  PT_OPT_GET(ignoreSmoothingFailures);
  PT_OPT_GET(enforceChirality);
  PT_OPT_GET(useExpTorsionAnglePrefs);
  PT_OPT_GET(useBasicKnowledge);
  PT_OPT_GET(pruneRmsThresh);
  PT_OPT_GET(onlyHeavyAtomsForRMS);
  PT_OPT_GET(ETversion);
  PT_OPT_GET(embedFragmentsSeparately);
  PT_OPT_GET(useSmallRingTorsions);
  PT_OPT_GET(useMacrocycleTorsions);
  PT_OPT_GET(useMacrocycle14config);
  PT_OPT_GET(forceTransAmides);
  PT_OPT_GET(useSymmetryForPruning);
  std::map<int, RDGeom::Point3D> *cmap = nullptr;
  const auto coordMap = pt.get_child_optional("coordMap");
  if (coordMap) {
    // NOTE: this leaks since EmbedParameters uses a naked pointer and we don't
    // have any way to tie the lifetime of the memory we allocate here to the
    // EmbedParameters object itself.
    cmap = new std::map<int, RDGeom::Point3D>();
    for (const auto &entry : *coordMap) {
      RDGeom::Point3D pt;

      auto itm = entry.second.begin();
      pt.x = itm->second.get_value<float>();
      ++itm;
      pt.y = itm->second.get_value<float>();
      ++itm;
      pt.z = itm->second.get_value<float>();

      (*cmap)[boost::lexical_cast<int>(entry.first)] = pt;
    }
    params.coordMap = cmap;
  }
}
}  // namespace DGeomHelpers
}  // namespace RDKit

max_stars_repo_name: jungb-basf/rdkit

max_issues_repo_name: jungb-basf/rdkit

max_forks_repo_name: bp-kelley/rdkit


text:
#ifndef BOOST_PYTHON_UTIL_HPP_
#define BOOST_PYTHON_UTIL_HPP_

/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
// Name        :
// Author      : Avi
// Revision    : $Revision: #5 $
//
// Copyright 2009-2020 ECMWF.
// This software is licensed under the terms of the Apache Licence version 2.0
// which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
// In applying this licence, ECMWF does not waive the privileges and immunities
// granted to it by virtue of its status as an intergovernmental organisation
// nor does it submit to any jurisdiction.
//
// Description :
//
/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8

#include <vector>
#include <boost/core/noncopyable.hpp>
#include <boost/python.hpp>      // IWYU pragma: keep

class Variable;

// See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects

class BoostPythonUtil : private boost::noncopyable {
public:

   /// Convert python list to a vector of integers. raises a type error if integer extraction fails
   static void list_to_int_vec(const boost::python::list& list, std::vector<int>& int_vec);
   static void list_to_str_vec(const boost::python::list& list, std::vector<std::string>& int_vec);
   static void list_to_str_vec(const boost::python::list& list, std::vector<Variable>& vec );
   static void dict_to_str_vec(const boost::python::dict& dict, std::vector<std::pair<std::string,std::string> >& str_pair);
   static void dict_to_str_vec(const boost::python::dict& dict, std::vector<Variable>& vec);
};

template<typename T> const T copyObject(const T& v) { return v; }

#endif

ext: hpp

max_stars_repo_name: ecmwf/ecflow

max_issues_repo_name: CoollRock/ecflow

max_forks_repo_name: CoollRock/ecflow


text:
#include<iostream>
#include<fstream>
#include<vector>
#include <deal.II/lac/sparse_matrix.h>
#include <deal.II/lac/sparsity_pattern.h>

using namespace dealii;

// Computing matrix A times vector b, and using the A[0] to store the result and return it.
std::vector<double> multiply(std::vector<std::vector<double>> A, std::vector<double> b)
{
  int m=A.size();
  int n=b.size();
  for (int i=0;i<m;i++)
    {
      double temp=0;
      for(int j=0;j<n;j++)
	{
	  temp+=A[i][j]*b[j];
	}
      A[0][i]=temp;
      //std::cout<<temp;
      //std::cout<<"The result of "<<i+1<<"th iteration\n";

    }

  return A[0];
}

// Computing u'*v;
double multiply(std::vector<double> u, std::vector<double> v)
{
  if(u.size()!=v.size())
    {
      std::cout<<"Multiply() ERROR: u'*v error! Please Check the size of u and v!";
    }
  else
    {
      double sum=0;
      for(int i=0;i<u.size();i++)
	{
	  sum+=u[i]*v[i];
	}

      return sum;
    }
}

// Design for return transpose matrix of A;
std::vector<std::vector<double>> transpose(std::vector<std::vector<double>> A)
{
  int n=A.size();
  int m=A[0].size();
    if (n < m)
      {
        for (int i = 0; i < n; i++)
	  {
            for (int j = 0; j < i; j++)
	      {
                double temp = A[j][i];
                A[j][i] = A[i][j];
                A[i][j] = temp;
	      }
	  }
        for (int i = n; i < m; i++)
	  {
            std::vector<double> temp;
            for (int j = 0; j < n; j++)
	      {
                temp.push_back(A[j][i]);
	      }
            A.push_back(temp);
	  }
        for (int i = 0; i < n; i++)
	  {
            for (int j = n; j < m; j++)
	      {
                A[i].pop_back();
	      }
	  }
      }
    else if(m<n)
      {
        for (int i = 0; i < m; i++)
	  {
            for (int j = 0; j < i; j++)
	      {
                double temp = A[j][i];
                A[j][i] = A[i][j];
		A[i][j] = temp;
	      }
	  }
        for (int i = 0; i < m; i++)
	  {
            for (int j = m; j < n; j++)
	      {
                A[i].push_back(A[j][i]);
	      }
	  }

        for (int i = m; i < n; i++)
	  {
            A.pop_back();
	  }

      }
    else
      {
	for(int i=0;i<n;i++)
	  {
	    for(int j=0;j<i;j++)
	      {
		double temp=A[j][i];
		A[j][i]=A[i][j];
		A[i][j]=temp;
	      }
	  }
      }

    return A;

}

//This function computes the matrix A times matrix B, and its result will be stored in A.
// It can be improved if reducing the use of the temp matrix to store the original values which
// is used in computing.
void multiply(std::vector<std::vector<double>> &A, std::vector<std::vector<double>> B)
{
    int n = A.size();
    int m = B.size();
    int col = B[0].size();
    std::vector<double> temp(m);
    std::vector<std::vector<double>> A0(n, temp);

    if (col < n)
      {
        for (int i = 0; i < n; i++)
	  {
            for (int j = 0; j < col; j++)
	      {
                double tempValue = 0;
                for (int t = 0; t < m; t++)
		  {
                    tempValue += A[i][t] * B[t][j];
		  }
                A0[i][j] = tempValue;
	      }
            for (int j = col; j < m; j++)
	      {
                A0[i].pop_back();
	      }
	  }

        A = A0;
      }
    else
      {
        for (int i = 0; i <n; i++)
	  {
            for (int j = 0; j < m; j++)
	      {
                double tempValue = 0;
                for (int t = 0; t < m; t++)
		  {
                    tempValue += A[i][t] * B[t][j];
		  }
                A0[i][j] = tempValue;
	      }
            for (int j = m; j < col; j++)
	      {
                double tempValue = 0;
                for (int t = 0; t < m; t++)
		  {
                    tempValue += A[i][t] * B[t][j];
		  }
                A0[i].push_back(tempValue);
	      }
	  }
        A = A0;
      }
}

// this function will turn the matrix Q into the nxn identity matrix;
void identitymatrix(std::vector<std::vector<double>> &Q, int n)
{
  Q.clear();
  std::vector<double> temp(n,0);
  for (int i=0;i<n;i++)
    {
      Q.push_back(temp);
      Q[i][i]=1;
    }
}

// This function computes the infi-norm of vector x;
double infi_Norm(std::vector<double> x)
{
  double norm=0;
  for (int i=0;i<x.size();i++)
    {
      if (norm<abs(x[i]))
	{
	  norm=abs(x[i]);
	}
    }
  return norm;
}

//////
// Multiplication of SparseMatrix A and vector x0;
std::vector<double> multiply(const SparseMatrix<double> &A, std::vector<double> x0)
{
  std::vector<double> x(x0.size(),0);
  //  std::cout<<"Flag 1!!\n";
  //std::cout<<A.n()<<"\n";

  if (A.n()!=x0.size())
    {
      std::cout<<"Function multiply() ERROR: The sizes of matrix and vector are not same! Please check it!\n";
    }
  else
    {
      // std::cout<<"TEST POINT 2 \n";
      for(int k=0;k<A.m();k++)
	{
	  //std::cout<<"this is the "<<k<<"th iterations \n";
	  SparseMatrix<double>::const_iterator i=A.begin(k);

	  while(i!=A.end(k))
	    {
	      x[k]+=i->value()*x0[i->column()];
	      ++i;
	    }
	}
      return x;
    }
}

//void Householder(std::vector<std::vector<double>> *A, std::vector<std::vector<double>> *P)
//void Householder(std::vector<std::vector<double>> *A, std::vector<std::vector<double>> *P)

// This function computes the householder process of the symmetric definite matrix, it return
// two matrices. The tridiagonal matrix coordinating with A and the hermitian unitary matrix P.
void Householder(std::vector<std::vector<double>> &A, std::vector<std::vector<double>> &P)
{
  int n=A.size();
  std::vector<std::vector<double>> Pk;

  for(int k=0;k<n-2;k++)
    {
      identitymatrix(Pk, n);
      double q=0;
      double alpha=0;
      // Computing the value of q in the Householder function;
      for(int i=k+1;i<n;i++)
	{
	  q+=A[i][k]*(A[i][k]);
	}

      // Computing the value of alpha
      if(A[k+1][k]==0)
	{
	  alpha=-sqrt(q);
	}
      else
	{
	  alpha=-sqrt(q)*A[k+1][k]/abs(A[k+1][k]);
	  // std::cout<<"the abs A[k+1][k] is "<<A[k+1][k]<<std::endl;
	}

      double RSQ=alpha*alpha-alpha*A[k+1][k]; // RSQ = 2r^2;

      //std::cout<<"Check Point 2, This is RSQ:::"<<RSQ<<"\n";

      std::vector<double> v(n-k,0), u(n-k,0), z(n-k,0);
      v[0]=0;
      v[1]=A[k+1][k]-alpha;

      //std::cout<<"Check Point 3::: "<<v[1]<<" \n";
      // w=(1/sqrt(2*RSQ)*v=1/2r*v);
      for(int j=2;j<v.size();j++)
	{
	  v[j]=A[k+j][k];
	}

      // std::cout<<"CheckPPPPPoint555 \n";

      for(int j=0;j<n-k;j++)
	{
	  for(int i=1;i<n-k;i++)
	    {
	      u[j]+=A[k+j][k+i]*v[i];
	    }
	  u[j]/=RSQ;
	}

      //std::cout<<"Check POOOIONIOHIHO\n";

      double PROD=0;
      for(int i=1;i<n-k;i++)
	{
	  PROD+=v[i]*u[i];
	}

      //std::cout<<"CheckPoint 4!!!!!!!!!\n";

      for(int j=0;j<n-k;j++)
	{
	  z[j]=u[j]-PROD/(2*RSQ)*v[j];
	}

      for(int l=k+1;l<n-1;l++)
	{
	  for(int j=l+1;j<n;j++)
	    {
	      A[j][l]=A[j][l]-v[l-k]*z[j-k]-v[j-k]*z[l-k];
	      A[l][j]=A[j][l];
	    }
	  A[l][l]=A[l][l]-2*v[l-k]*z[l-k];
	}

      A[n-1][n-1]=A[n-1][n-1]-2*v[n-k-1]*z[n-k-1];

      for(int j=k+2;j<n;j++)
	{
	  A[k][j]=0;
	  A[j][k]=0;
	}

      A[k+1][k]=A[k+1][k]-v[1]*z[0];
      A[k][k+1]=A[k+1][k];

      for (int j=k+1;j<n;j++)
	{
	  std::vector<double> temp(j-k,0);
	  for(int i=k+1;i<=j;i++)
	    {
	      for(int t=k+1;t<n;t++)
		{
		  temp[i-k-1]+=v[i-k]*v[t-k]*Pk[t][j]/(2*RSQ);
		}
	    }

	  //temp=multiply(W,Pj);// to get the jth column of the W*P;
	  for (int i=k+1;i<=j;i++)
	    {
	      Pk[i][j]-=2*temp[i-k-1];
	      Pk[j][i]=Pk[i][j];
	    }
	}
      multiply(P,Pk);

    }

}

///////
// This function computes the QR factorization of the tridiagonal matrix A.
// Input: tridiagonal matrix A, identity matrix Q
// Output: uptriangle matrix A_, the orthonormal matrix Q; A=Q*A_;
//
// tips: Maybe I can improve the function by replace the matrix A by two vectors an bn-1;
// an contains the diagonal entries of A; bn-1 contains the sub-diagonal entries of A;

void QR(std::vector<std::vector<double>> &A, std::vector<std::vector<double>> &Q)
{
  int n=A.size();

  // obtain the diagonal and subdiagonal entries of matrix A;
  /*
  std::vector<double>an(n),bn(n-1);
  for (int i=0;i<A.size()-1;i++)
    {
      an[i]=A[i][i];
      bn[i]=A[i+1][i];
    }
  an[n-1]=A[n-1][n-1];
  */

  for (int i=0;i<A.size()-1;i++)
    {
      double theta=0;
      double tempa1=A[i][i],tempa2=A[i+1][i+1],tempb=A[i+1][i],tempc=A[i][i+1];
      theta=atan(A[i+1][i]/A[i][i]);
      double c=cos(theta), s=sin(theta);

      A[i][i]=c*tempa1+s*tempb;
      A[i][i+1]=c*tempc+s*tempa2;
      A[i+1][i+1]=-s*tempc+c*tempa2;
      A[i+1][i]=-s*tempa1+c*tempb;
      if (i<A.size()-2)
	{
	  A[i][i+2]=s*A[i+1][i+2];
	  A[i+1][i+2]=c*A[i+1][i+2];
	}

      // update the orthogonal matrix Q by Q_k=Q_k-1*Q;

      for(int k=0;k<n;k++)
	{
	  double tmp1=Q[k][i],tmp2=Q[k][i+1];
	  Q[k][i]=Q[k][i]*c+Q[k][i+1]*s;
	  Q[k][i+1]=tmp1*(-s)+tmp2*c;
	}

    }

}

// This function computes the eigenvalue of the matrix A by QR methods, and it will also return the
// matrix Q during the process;
void QRSolver(std::vector<std::vector<double>> &A, std::vector<std::vector<double>> &Q, double tol=0.001)
{
  int n=A.size();
  // using householder transform matrix A into the tridiagonal matrix
  // store it in A and store the Householder unitary matrix in Q; A=QA_Q;
  //Householder(A,Q);

  // Qk as the initial matrix for every QR factorization, multiplying it together to get the
  // final unitary matrix;
  std::vector<std::vector<double>> Qk;

  // Get the subdiagonal entries of the matrix A and verify if its norm smaller than the tolerance;
  // If it is small enough that means we diagonalize the matrix A and the diagonal entries are
  // eigenvalues of A.
  std::vector<double> b(n-1,1);
  for(int i=0;i<n-1;i++)
    {
      b.push_back(A[i+1][i]);
    }

  int num=0;
  while (infi_Norm(b)>tol&&num<1000)
    {
      num++;

      b.clear();

      Householder(A,Q);

      identitymatrix(Qk,n);

      QR(A,Qk);

      // Compute the num-th iteration, get the Qk in this step;
      multiply(Q,Qk);
      // compute R*Q beacuse I store the R(computed above) into A, So I directly use multiply
      // function to get A*Q into A=RQ.
      multiply(A,Qk);

      //update the entries of b, i.e. the subdiagonal entries of matrix A=RQ;
        for(int i=0;i<n-1;i++)
	  {
	    b.push_back(A[i+1][i]);
	  }
    }
}

class MinTrace
{
 public:
  MinTrace();
  MinTrace(SparseMatrix<double> A, SparseMatrix<double> M, int p=1);

  // get the random initial matrix V which satisifies that: V'*M*V=I_p;
  std::vector<std::vector<double>>rand_V();

  // compute the orthonormal vectors corresponding to M from the original vectors U={u_1,u_2,...,u_p};
  std::vector<std::vector<double>> M_GS(std::vector<std::vector<double>> U);

  // Compute the solution vector of the equation of X_k[i] and PAP*d_i=PA*X_k[i];
  std::vector<double>CG(std::vector<double> X_ki);
  // get the p smallest eigenvalues of A correponding with M;
  std::vector<double> min_trace(int max_iter, double eps);

 private:
  SparseMatrix<double> A;
  SparseMatrix<double> M;
  int p; // The number of the eigenvalues to be solved;
};

MinTrace::MinTrace()
{
  int p=3;
  SparsityPattern sparsity_pattern(10,10,{2,3,3,3,3,3,3,3,3,2});
  SparsityPattern sp2(10,10,1);
  //sparsity_pattern.add(1,2);
  sparsity_pattern.add(0,1);
  sparsity_pattern.add(9,8);
  for (int i=1;i<9;i++)
    {
      sparsity_pattern.add(i,i+1);
      sparsity_pattern.add(i,i-1);
    }

  A.reinit(sparsity_pattern);
  M.reinit(sp2);

  for (int k=0;k<A.m();k++)
    {
      SparseMatrix<double>::iterator i=A.begin(k);
      i->value()=2;
      while(i!=A.end(k))
	{
	  i->value()+=1;
	  ++i;
	}
    }

  SparseMatrix<double>::iterator i=M.begin();
  double num=0;
  while(i!=M.end())
    {
      num=num+1;
      i->value()=num;
      ++i;
    }
}

// ATTENTION!!! There is a difficulty that SparseMatrix can not be copied directly!
MinTrace::MinTrace(SparseMatrix<double> A0, SparseMatrix<double> M0, int p0)
{

};

std::vector<std::vector<double>> MinTrace::M_GS(std::vector<std::vector<double>> U)
{
  std::vector<std::vector<double>> V;
  V=transpose(U);
}

std::vector<std::vector<double>> MinTrace::rand_V()
{
  std::cout<<"CheckPoint1\n";
  std::vector<double> x(M.n(),0);
  for (int i=0;i<x.size();i++)
    {
      std::cout<<x[i]<<" ";
    }
  std::cout<<"\n";
  std::cout<<"CheckPoint666 \n";
  std::vector<std::vector<double>> V(p,x);
  std::cout<<"CheckPoint12314113\n";

  V[0][0]=1;
  V[1][1]=sqrt(2.0)/2;
  V[2][2]=sqrt(3.0)/3;
  std::cout<<"231wqrdasd24121321321321adsadq2q1312321sada\n";
  return V;
}

std::vector<double> MinTrace::min_trace(int max_iter=10, double eps=1e-03)
{
  std::cout<<M.n()<<"   it is a test!\n";

  std::vector<double> x(p); // define the eigenvector;
  std::vector<std::vector<double>> V; // V is an n x p matrix and stored as V[p][n];

  V=rand_V();
  std::cout<<"wewqewqewqewqeqw\n";
  //  for(int k=0;k<max_iter;k++)
    for(int k=0;k<1;k++)
    {
      std::cout<<"CheckPoint222\n";
      std::vector<std::vector<double>> VMV(p,x),tmpVMV(p); // VMV is a p x p matrix;
      std::cout<<"AAAAAAAAAAAAAAAAAAAAAAAA\n";
      for(int i=0;i<p;i++)
	{
	  tmpVMV[i]=multiply(M,V[i]);
	  std::cout<<"CheckPoint111111111\n";
	}

      std::cout<<"CheckPoint2\n";
      for(int i=0;i<p;i++)
	{
	  for(int j=0;j<p;j++)
	    {
	      double tmp=0;
	      for(int l=0;l<M.n();l++)
		{
		  tmp+=V[i][l]*tmpVMV[j][l];
		}
	      VMV[i][j]=tmp;
	    }
	}

      for(int i=0;i<p;i++)
	{
	  for(int j=0;j<p;j++)
	    {
	      std::cout<<VMV[i][j]<<" ";
	    }
	  std::cout<<"\n";
	}
    }

   return x;
}

int main()
{
  SparseMatrix<double> A, M;
  // std::ofstream sparsematrix1 ("original_matrix.1");
  //A.print(sparsematrix1);
  std::vector<unsigned int> row_length(10,10);
  unsigned int t=3;
  SparsityPattern sparsity_pattern(10,10,{2,3,3,3,3,3,3,3,3,2});
  SparsityPattern sp2(10,10,1);
  //sparsity_pattern.add(1,2);
  sparsity_pattern.add(0,1);
  sparsity_pattern.add(9,8);
  for (int i=1;i<9;i++)
    {
      sparsity_pattern.add(i,i+1);
      sparsity_pattern.add(i,i-1);
    }

  A.reinit(sparsity_pattern);
  M.reinit(sp2);

  // In this way, I can construct a SparseMatrix to be the test data for the algorithm.
  for (int k=0;k<A.m();k++)
    {
      SparseMatrix<double>::iterator i=A.begin(k);
      i->value()=2;
      while(i!=A.end(k))
	{
	  i->value()+=1;
	  ++i;
	}
    }

  SparseMatrix<double>::iterator i=M.begin();
  double num=0;
  while(i!=M.end())
    {
      num=num+1;
      i->value()=num;
      ++i;
    }

  std::ofstream out ("sparsity_pattern.1");
  sparsity_pattern.print_gnuplot(out);

  std::ofstream sparsematrix ("sparse_matrix.1");
  A.print(sparsematrix);

  std::ofstream sparsematrix2 ("sparse_matrix.2");
  M.print(sparsematrix2);

  std::vector<double> x(10,1);
  x=multiply(M,x);
  for(int i=0;i<x.size();i++)
    {
      std::cout<<x[i]<<" ";
    }
 // std::cout<<"\n"<<1e-03<<"\n";

  MinTrace solve;

//  solve.min_trace();
  //sparsity_pattern.print();
  return 0;
}


text:
#define BOOST_TEST_MODULE pichi endpoint test

#include "utils.hpp"
#include <algorithm>
#include <array>
#include <boost/asio/buffers_iterator.hpp>
#include <boost/beast/core/flat_static_buffer.hpp>
#include <boost/test/unit_test.hpp>
#include <initializer_list>
#include <pichi/common.hpp>
#include <pichi/net/helpers.hpp>

using namespace std;
using namespace pichi;
namespace asio = boost::asio;
namespace beast = boost::beast;
namespace sys = boost::system;

using EndpointType = net::Endpoint::Type;
using net::makeEndpoint;
using net::parseEndpoint;
using net::serializeEndpoint;

class StubSocket {
public:
  explicit StubSocket() {}

  void write(initializer_list<uint8_t> buf)
  {
    copy_n(cbegin(buf), buf.size(), asio::buffers_begin(buffer_.prepare(buf.size())));
    buffer_.commit(buf.size());
  }

  void write(uint8_t c)
  {
    copy_n(&c, 1, asio::buffers_begin(buffer_.prepare(1)));
    buffer_.commit(1);
  }

  void read(MutableBuffer<uint8_t> buf)
  {
    assertTrue(buffer_.size() >= buf.size());
    copy_n(asio::buffers_begin(buffer_.data()), buf.size(), begin(buf));
    buffer_.consume(buf.size());
    transfered_ += buf.size();
  }

  size_t transfered() const { return transfered_; }

private:
  beast::flat_static_buffer<1024> buffer_;
  size_t transfered_ = 0;
};

BOOST_AUTO_TEST_SUITE(ENDPOINT)

BOOST_AUTO_TEST_CASE(serialize_Empty_Host)
{
  auto buf = array<uint8_t, 64>{};
  BOOST_CHECK_EXCEPTION(serializeEndpoint({EndpointType::DOMAIN_NAME, ""s, "1"s}, buf), Exception,
                        verifyException<PichiError::MISC>);
  BOOST_CHECK_EXCEPTION(serializeEndpoint({EndpointType::IPV4, ""s, "1"s}, buf), Exception,
                        verifyException<PichiError::MISC>);
  BOOST_CHECK_EXCEPTION(serializeEndpoint({EndpointType::IPV6, ""s, "1"s}, buf), Exception,
                        verifyException<PichiError::MISC>);
}

BOOST_AUTO_TEST_CASE(serialize_Empty_Port)
{
  auto buf = array<uint8_t, 64>{};
  for (auto host : {"localhost"sv, "127.0.0.1"sv, "::1"sv}) {
    BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(host, ""sv), buf), Exception,
                          verifyException<PichiError::MISC>);
  }
}

BOOST_AUTO_TEST_CASE(serialize_Alphabetic_Port)
{
  auto buf = array<uint8_t, 64>{};
  for (auto host : {"localhost"sv, "127.0.0.1"sv, "::1"sv}) {
    BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(host, "http"sv), buf), invalid_argument,
                          [](auto&&) { return true; });
  }
}

BOOST_AUTO_TEST_CASE(serialize_Port_Out_Of_Range)
{
  auto buf = array<uint8_t, 64>{};
  for (auto host : {"localhost"sv, "127.0.0.1"sv, "::1"sv}) {
    BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(host, 0), buf), Exception,
                          verifyException<PichiError::MISC>);
    BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(host, "65536"sv), buf), Exception,
                          verifyException<PichiError::MISC>);
  }
}

BOOST_AUTO_TEST_CASE(serialize_Invalid_IP_Address)
{
  auto buf = array<uint8_t, 64>{};

  for (auto type : {EndpointType::IPV4, EndpointType::IPV6}) {
    BOOST_CHECK_EXCEPTION(serializeEndpoint({type, "localhost"s, "1"s}, buf), sys::system_error,
                          verifyException<asio::error::invalid_argument>);
  }
}

BOOST_AUTO_TEST_CASE(serialize_Domain_Name_Too_Long)
{
  auto buf = array<uint8_t, 1024>{};
  BOOST_CHECK_EXCEPTION(
      serializeEndpoint({EndpointType::DOMAIN_NAME, string(0x100, 'a'), "1"s}, buf), Exception,
      verifyException<PichiError::MISC>);
}

BOOST_AUTO_TEST_CASE(serialize_Domain_Lack_Of_Buffer)
{
  auto host = "localhost"sv;

  auto enough = array<uint8_t, 13>{};
  BOOST_CHECK_EQUAL(enough.size(), serializeEndpoint(makeEndpoint(host, 1), enough));

  auto lack = array<uint8_t, 12>{};
  BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(host, 1), lack), Exception,
                        verifyException<PichiError::MISC>);
}

BOOST_AUTO_TEST_CASE(serialize_Domain)
{
  auto host = "localhost"sv;
  auto port = 443_u16;
  auto expect =
      array<uint8_t, 13>{0x03, 0x09, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', 0x01, 0xbb};

  auto fact = array<uint8_t, 13>{};
  auto len = serializeEndpoint(makeEndpoint(host, port), fact);

  BOOST_CHECK_EQUAL(expect.size(), len);
  BOOST_CHECK_EQUAL_COLLECTIONS(cbegin(expect), cend(expect), cbegin(fact), cend(fact));
}

BOOST_AUTO_TEST_CASE(serialize_IPv4_Lack_Of_Buffer)
{
  auto address = "127.0.0.1"sv;

  auto enough = array<uint8_t, 7>{};
  BOOST_CHECK_EQUAL(enough.size(), serializeEndpoint(makeEndpoint(address, 1), enough));

  auto lack = array<uint8_t, 6>{};
  BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(address, 1), lack), Exception,
                        verifyException<PichiError::MISC>);
}

BOOST_AUTO_TEST_CASE(serialize_IPv4)
{
  auto address = "127.0.0.1"sv;
  auto port = 443_u16;
  auto expect = array<uint8_t, 7>{0x01, 0x7f, 0x00, 0x00, 0x01, 0x01, 0xbb};

  auto fact = array<uint8_t, 7>{};
  BOOST_CHECK_EQUAL(expect.size(), serializeEndpoint(makeEndpoint(address, port), fact));
  BOOST_CHECK_EQUAL_COLLECTIONS(cbegin(expect), cend(expect), cbegin(fact), cend(fact));
}

BOOST_AUTO_TEST_CASE(serialize_IPv6_Lack_Of_Buffer)
{
  auto address = "::1"sv;

  auto enough = array<uint8_t, 19>{};
  BOOST_CHECK_EQUAL(enough.size(), serializeEndpoint(makeEndpoint(address, 1), enough));

  auto lack = array<uint8_t, 18>{};
  BOOST_CHECK_EXCEPTION(serializeEndpoint(makeEndpoint(address, 1), lack), Exception,
                        verifyException<PichiError::MISC>);
}

BOOST_AUTO_TEST_CASE(serialize_IPv6)
{
  auto address = "::1"sv;
  auto port = 443_u16;
  auto expect = array<uint8_t, 19>{0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                                   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0xbb};

  auto fact = array<uint8_t, 19>{};
  BOOST_CHECK_EQUAL(expect.size(), serializeEndpoint(makeEndpoint(address, port), fact));
  BOOST_CHECK_EQUAL_COLLECTIONS(cbegin(expect), cend(expect), cbegin(fact), cend(fact));
}

BOOST_AUTO_TEST_CASE(parse_Invalid_Type)
{
  for (auto i = 0; i < 0x100; ++i) {
    if (i == 0x01 || i == 0x03 || i == 0x04) continue;
    auto t = static_cast<uint8_t>(i);
    auto stub = StubSocket{};
    stub.write(t);
    BOOST_CHECK_EXCEPTION(parseEndpoint([&stub](auto buf) { stub.read(buf); }), Exception,
                          verifyException<PichiError::BAD_PROTO>);
  }
}

BOOST_AUTO_TEST_CASE(parse_Empty_Domain)
{
  auto stub = StubSocket{};
  stub.write({0x03, 0x00, 0x01, 0xbb});
  BOOST_CHECK_EXCEPTION(parseEndpoint([&stub](auto buf) { stub.read(buf); }), Exception,
                        verifyException<PichiError::BAD_PROTO>);
}

BOOST_AUTO_TEST_CASE(parse_Domain)
{
  auto stub = StubSocket{};
  stub.write({0x03, 0x09, 'l', 'o', 'c', 'a', 'l', 'h', 'o', 's', 't', 0x01, 0xbb});

  auto ep = parseEndpoint([&stub](auto buf) { stub.read(buf); });
  BOOST_CHECK(EndpointType::DOMAIN_NAME == ep.type_);
  BOOST_CHECK_EQUAL("localhost"sv, ep.host_);
  BOOST_CHECK_EQUAL("443"sv, ep.port_);
  BOOST_CHECK_EQUAL(13_sz, stub.transfered());
}

BOOST_AUTO_TEST_CASE(parse_IPv4)
{
  auto stub = StubSocket{};
  stub.write({0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0xbb});

  auto ep = parseEndpoint([&stub](auto buf) { stub.read(buf); });
  BOOST_CHECK(EndpointType::IPV4 == ep.type_);
  BOOST_CHECK_EQUAL("0.0.0.0"sv, ep.host_);
  BOOST_CHECK_EQUAL("443"sv, ep.port_);
  BOOST_CHECK_EQUAL(7_sz, stub.transfered());
}

BOOST_AUTO_TEST_CASE(parse_IPv6)
{
  auto stub = StubSocket{};
  stub.write({0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
              0x00, 0x00, 0x00, 0x01, 0xbb});

  auto ep = parseEndpoint([&stub](auto buf) { stub.read(buf); });
  BOOST_CHECK(EndpointType::IPV6 == ep.type_);
  BOOST_CHECK_EQUAL("::"sv, ep.host_);
  BOOST_CHECK_EQUAL("443"sv, ep.port_);
  BOOST_CHECK_EQUAL(19_sz, stub.transfered());
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: rosorio/pichi

max_issues_repo_name: rosorio/pichi

max_forks_repo_name: rosorio/pichi


text:
#include "Utils.h"
#include "Keywords.h"
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/date_time/posix_time/time_serialize.hpp>
#include <boost/filesystem.hpp>
#include <boost/serialization/vector.hpp>
#include <macgyver/Astronomy.h>
#include <macgyver/Exception.h>
#include <macgyver/StringConversion.h>
#include <macgyver/TypeName.h>
#include <spine/Convenience.h>
#include <fstream>

namespace SmartMet
{
namespace Engine
{
namespace Observation
{
bool removePrefix(std::string& parameter, const std::string& prefix)
{
  try
  {
    std::size_t prefixLen = prefix.length();
    if ((parameter.length() > prefixLen) && parameter.compare(0, prefixLen, prefix) == 0)
    {
      parameter.erase(0, prefixLen);
      return true;
    }

    return false;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

/** *\brief Return true of a parameter looks to be normal enough to be an observation
 */

bool not_special(const Spine::Parameter& theParam)
{
  try
  {
    switch (theParam.type())
    {
      case Spine::Parameter::Type::Data:
      case Spine::Parameter::Type::Landscaped:
        return true;
      case Spine::Parameter::Type::DataDerived:
      case Spine::Parameter::Type::DataIndependent:
        return false;
    }
    // NOT REACHED
    return false;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

std::string trimCommasFromEnd(const std::string& what)
{
  try
  {
    size_t end = what.find_last_not_of(",");
    return what.substr(0, end + 1);
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

/* Translates parameter names to match the parameter name in the database.
 * If the name is not found in parameter map, return the given name.
 */

std::string translateParameter(const std::string& paramname,
                               const std::string& stationType,
                               ParameterMap& parameterMap)
{
  try
  {
    // All parameters are in lower case in parametermap
    std::string p = Fmi::ascii_tolower_copy(paramname);
    std::string ret = parameterMap.getParameter(p, stationType);
    if (!ret.empty())
      return ret;
    return p;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

// Calculates station direction in degrees from given coordinates
void calculateStationDirection(Spine::Station& station)
{
  try
  {
    double lon1 = deg2rad(station.requestedLon);
    double lat1 = deg2rad(station.requestedLat);
    double lon2 = deg2rad(station.longitude_out);
    double lat2 = deg2rad(station.latitude_out);

    double dlon = lon2 - lon1;

    double direction = rad2deg(
        atan2(sin(dlon) * cos(lat2), cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(dlon)));

    if (direction < 0)
      direction += 360.0;

    station.stationDirection = std::round(10.0 * direction) / 10.0;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

// Utility method to convert degrees to radians
double deg2rad(double deg)
{
  return (deg * PI / 180);
}

// Utility method to convert radians to degrees
double rad2deg(double rad)
{
  return (rad * 180 / PI);
}

std::string windCompass8(double direction, const std::string& missingValue)
{
  if (direction < 0)
    return missingValue;
  std::array<std::string, 8> names{"N", "NE", "E", "SE", "S", "SW", "W", "NW"};

  int i = static_cast<int>((direction + 22.5) / 45) % 8;
  return names[i];
}

std::string windCompass16(double direction, const std::string& missingValue)
{
  if (direction < 0)
    return missingValue;
  std::array<std::string, 16> names{"N",
                                    "NNE",
                                    "NE",
                                    "ENE",
                                    "E",
                                    "ESE",
                                    "SE",
                                    "SSE",
                                    "S",
                                    "SSW",
                                    "SW",
                                    "WSW",
                                    "W",
                                    "WNW",
                                    "NW",
                                    "NNW"};

  int i = static_cast<int>((direction + 11.25) / 22.5) % 16;
  return names[i];
}

std::string windCompass32(double direction, const std::string& missingValue)
{
  if (direction < 0)
    return missingValue;
  std::array<std::string, 32> names{"N", "NbE", "NNE", "NEbN", "NE", "NEbE", "ENE", "EbN",
                                    "E", "EbS", "ESE", "SEbE", "SE", "SEbS", "SSE", "SbE",
                                    "S", "SbW", "SSW", "SWbS", "SW", "SWbW", "WSW", "WbS",
                                    "W", "WbN", "WNW", "NWbW", "NW", "NWbN", "NNW", "NbW"};

  int i = static_cast<int>((direction + 5.625) / 11.25) % 32;
  return names[i];
}

std::string parseParameterName(const std::string& parameter)
{
  try
  {
    std::string name = Fmi::ascii_tolower_copy(parameter);

    removePrefix(name, "qc_");

    // No understrike
    if (name.find_last_of("_") == std::string::npos)
      return name;

    size_t startpos = name.find_last_of("_");
    size_t endpos = name.length();

    int length = boost::numeric_cast<int>(endpos - startpos - 1);

    // Test appearance of the parameter between TRS_10MIN_DIF and TRS_10MIN_DIF_1
    std::string sensorNumber = name.substr(startpos + 1, length);

    if (Fmi::looks_unsigned_int(sensorNumber))
      return name.substr(0, startpos);

    return name;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

/*
 * The sensor number is given after an underscore, for example KELI_1
 */

int parseSensorNumber(const std::string& parameter)
{
  try
  {
    int defaultSensorNumber = 1;
    std::string sensorNumber;
    std::size_t startpos = parameter.find_last_of("_");
    std::size_t endpos = parameter.length();
    int length = boost::numeric_cast<int>(endpos - startpos - 1);

    // If sensor number is given, for example KELI_1, return requested number
    if (startpos != std::string::npos && endpos != std::string::npos)
    {
      sensorNumber = parameter.substr(startpos + 1, length);
      try
      {
        return Fmi::stoi(sensorNumber);
      }
      catch (...)
      {
        return defaultSensorNumber;
      }
    }

    return defaultSensorNumber;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

Spine::Stations removeDuplicateStations(Spine::Stations& stations)
{
  try
  {
    std::vector<int> ids;
    Spine::Stations noDuplicates;
    for (const Spine::Station& s : stations)
    {
      if (std::find(ids.begin(), ids.end(), s.station_id) == ids.end())
      {
        noDuplicates.push_back(s);
        // BUG? Why is station_id double?
        ids.push_back(boost::numeric_cast<int>(s.station_id));
      }
    }
    return noDuplicates;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

// ----------------------------------------------------------------------
/*!
 * \brief Round down the given time to start of day
 */
// ----------------------------------------------------------------------

boost::posix_time::ptime day_start(const boost::posix_time::ptime& t)
{
  if (t.is_not_a_date_time() || t.is_special())
    return t;
  return boost::posix_time::ptime(t.date(), boost::posix_time::hours(0));
}

// ----------------------------------------------------------------------
/*!
 * \brief Round up the given time to end of day
 */
// ----------------------------------------------------------------------

boost::posix_time::ptime day_end(const boost::posix_time::ptime& t)
{
  if (t.is_not_a_date_time() || t.is_special())
    return t;
  auto tmp = boost::posix_time::ptime(t.date(), boost::posix_time::hours(0));
  tmp += boost::gregorian::days(1);
  return tmp;
}

void logMessage(const std::string& message, bool quiet)
{
  try
  {
    if (!quiet)
      std::cout << Spine::log_time_str() << ' ' << message << std::endl;
  }
  catch (...)
  {
    throw Fmi::Exception::Trace(BCP, "Operation failed!");
  }
}

boost::optional<int> calcSmartsymbolNumber(int wawa,
                                           int cloudiness,
                                           double temperature,
                                           const boost::local_time::local_date_time& ldt,
                                           double lat,
                                           double lon)
{
  boost::optional<int> smartsymbol = {};

  const std::array<int, 10> wawa_group1{0, 4, 5, 10, 20, 21, 22, 23, 24, 25};
  const std::array<int, 5> wawa_group2{30, 31, 32, 33, 34};

  const int cloudiness_limit1 = 0;
  const int cloudiness_limit2 = 1;
  const int cloudiness_limit3 = 5;
  const int cloudiness_limit4 = 7;
  const int cloudiness_limit5 = 9;

  if (std::find(std::begin(wawa_group1), std::end(wawa_group1), wawa) != std::end(wawa_group1))
  {
    if (cloudiness <= cloudiness_limit1)
      smartsymbol = 1;
    else if (cloudiness <= cloudiness_limit2)
      smartsymbol = 2;
    else if (cloudiness <= cloudiness_limit3)
      smartsymbol = 4;
    else if (cloudiness <= cloudiness_limit4)
      smartsymbol = 6;
    else if (cloudiness <= cloudiness_limit5)
      smartsymbol = 7;
  }
  else if (std::find(std::begin(wawa_group2), std::end(wawa_group2), wawa) != std::end(wawa_group2))
  {
    if (cloudiness <= cloudiness_limit1)
      smartsymbol = 1;
    else if (cloudiness <= cloudiness_limit2)
      smartsymbol = 2;
    else if (cloudiness <= cloudiness_limit3)
      smartsymbol = 4;
    else if (cloudiness <= cloudiness_limit4)
      smartsymbol = 6;
    else if (cloudiness <= cloudiness_limit5)
      smartsymbol = 9;
  }

  else if (wawa == 40 || wawa == 41)
  {
    if (temperature <= 0)
    {
      if (cloudiness <= 5)
        smartsymbol = 51;
      else if (cloudiness <= 7)
        smartsymbol = 54;
      else if (cloudiness <= 9)
        smartsymbol = 57;
    }
    else
    {
      if (cloudiness <= 5)
        smartsymbol = 31;
      else if (cloudiness <= 7)
        smartsymbol = 34;
      else if (cloudiness <= 9)
        smartsymbol = 37;
    }
  }
  else if (wawa == 42)
  {
    if (temperature <= 0)
    {
      if (cloudiness <= 5)
        smartsymbol = 53;
      else if (cloudiness <= 7)
        smartsymbol = 56;
      else if (cloudiness <= 9)
        smartsymbol = 59;
    }
    else
    {
      if (cloudiness <= 5)
        smartsymbol = 33;
      else if (cloudiness <= 7)
        smartsymbol = 36;
      else if (cloudiness <= 9)
        smartsymbol = 39;
    }
  }
  else if (wawa >= 50 && wawa <= 53)
  {
    if (cloudiness <= 9)
      smartsymbol = 11;
  }
  else if (wawa >= 54 && wawa <= 56)
  {
    if (cloudiness <= 9)
      smartsymbol = 14;
  }
  else if (wawa == 60)
  {
    if (cloudiness <= 5)
      smartsymbol = 31;
    else if (cloudiness <= 7)
      smartsymbol = 34;
    else if (cloudiness <= 9)
      smartsymbol = 37;
  }
  else if (wawa == 61)
  {
    if (cloudiness <= 5)
      smartsymbol = 31;
    else if (cloudiness <= 7)
      smartsymbol = 34;
    else if (cloudiness <= 9)
      smartsymbol = 37;
  }
  else if (wawa == 62)
  {
    if (cloudiness <= 5)
      smartsymbol = 32;
    else if (cloudiness <= 7)
      smartsymbol = 35;
    else if (cloudiness <= 9)
      smartsymbol = 38;
  }
  else if (wawa == 63)
  {
    if (cloudiness <= 5)
      smartsymbol = 33;
    else if (cloudiness <= 7)
      smartsymbol = 36;
    else if (cloudiness <= 9)
      smartsymbol = 39;
  }
  else if (wawa >= 64 && wawa <= 66)
  {
    if (cloudiness <= 9)
      smartsymbol = 17;
  }
  else if (wawa == 67)
  {
    if (cloudiness <= 5)
      smartsymbol = 41;
    else if (cloudiness <= 7)
      smartsymbol = 44;
    else if (cloudiness <= 9)
      smartsymbol = 47;
  }
  else if (wawa == 68)
  {
    if (cloudiness <= 5)
      smartsymbol = 42;
    else if (cloudiness <= 7)
      smartsymbol = 45;
    else if (cloudiness <= 9)
      smartsymbol = 48;
  }
  else if (wawa == 70)
  {
    if (cloudiness <= 5)
      smartsymbol = 51;
    else if (cloudiness <= 7)
      smartsymbol = 54;
    else if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 71)
  {
    if (cloudiness <= 5)
      smartsymbol = 51;
    else if (cloudiness <= 7)
      smartsymbol = 54;
    else if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 72)
  {
    if (cloudiness <= 5)
      smartsymbol = 52;
    else if (cloudiness <= 7)
      smartsymbol = 55;
    else if (cloudiness <= 9)
      smartsymbol = 58;
  }
  else if (wawa == 73)
  {
    if (cloudiness <= 5)
      smartsymbol = 53;
    else if (cloudiness <= 7)
      smartsymbol = 56;
    else if (cloudiness <= 9)
      smartsymbol = 59;
  }
  else if (wawa == 74)
  {
    if (cloudiness <= 5)
      smartsymbol = 51;
    else if (cloudiness <= 7)
      smartsymbol = 54;
    else if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 75)
  {
    if (cloudiness <= 5)
      smartsymbol = 52;
    else if (cloudiness <= 7)
      smartsymbol = 55;
    else if (cloudiness <= 9)
      smartsymbol = 58;
  }
  else if (wawa == 76)
  {
    if (cloudiness <= 5)
      smartsymbol = 53;
    else if (cloudiness <= 7)
      smartsymbol = 56;
    else if (cloudiness <= 9)
      smartsymbol = 59;
  }
  else if (wawa == 77)
  {
    if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 78)
  {
    if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 80)
  {
    if (temperature <= 0)
    {
      if (cloudiness <= 5)
        smartsymbol = 51;
      else if (cloudiness <= 7)
        smartsymbol = 54;
      else if (cloudiness <= 9)
        smartsymbol = 57;
    }
    else
    {
      if (cloudiness <= 5)
        smartsymbol = 21;
      else if (cloudiness <= 7)
        smartsymbol = 24;
      else if (cloudiness <= 9)
        smartsymbol = 27;
    }
  }
  else if (wawa >= 81 && wawa <= 84)
  {
    if (cloudiness <= 5)
      smartsymbol = 21;
    else if (cloudiness <= 7)
      smartsymbol = 24;
    else if (cloudiness <= 9)
      smartsymbol = 27;
  }
  else if (wawa == 85)
  {
    if (cloudiness <= 5)
      smartsymbol = 51;
    else if (cloudiness <= 7)
      smartsymbol = 54;
    else if (cloudiness <= 9)
      smartsymbol = 57;
  }
  else if (wawa == 86)
  {
    if (cloudiness <= 5)
      smartsymbol = 52;
    else if (cloudiness <= 7)
      smartsymbol = 55;
    else if (cloudiness <= 9)
      smartsymbol = 58;
  }
  else if (wawa == 87)
  {
    if (cloudiness <= 5)
      smartsymbol = 53;
    else if (cloudiness <= 7)
      smartsymbol = 56;
    else if (cloudiness <= 9)
      smartsymbol = 59;
  }
  else if (wawa == 89)
  {
    if (cloudiness <= 5)
      smartsymbol = 61;
    else if (cloudiness <= 7)
      smartsymbol = 64;
    else if (cloudiness <= 9)
      smartsymbol = 67;
  }

  // Add day/night information
  Fmi::Astronomy::solar_position_t sp = Fmi::Astronomy::solar_position(ldt, lon, lat);
  if (smartsymbol)
  {
    if (sp.dark())
      return 100 + *smartsymbol;
    return *smartsymbol;
  }

  // No valid combination found, return empty value
  return {};
}

Spine::TimeSeries::TimeSeriesVectorPtr initializeResultVector(const Settings& settings)
{
  Spine::TimeSeries::TimeSeriesVectorPtr ret =
      boost::make_shared<Spine::TimeSeries::TimeSeriesVector>();

  // Set timeseries objects for each requested parameter
  for (unsigned int i = 0; i < settings.parameters.size(); i++)
    ret->emplace_back(Spine::TimeSeries::TimeSeries(settings.localTimePool));

  return ret;
}

boost::posix_time::ptime epoch2ptime(double epoch)
{
  boost::posix_time::ptime ret =
      boost::posix_time::from_time_t(static_cast<std::time_t>(floor(epoch)));
  ret += boost::posix_time::microseconds(static_cast<long>((epoch - floor(epoch)) * 1000000));

  return ret;
}

std::string getStringValue(const Spine::TimeSeries::Value& tv)
{
  // For some reason different databases/drivers don't simply use int for FMISID.
  // This is workaround code, FMISID should always be int.

  if (const double* dvalue = boost::get<double>(&tv))
    return Fmi::to_string(*dvalue);

  if (const int* ivalue = boost::get<int>(&tv))
    return Fmi::to_string(*ivalue);

  if (const std::string* svalue = boost::get<std::string>(&tv))
    return *svalue;

  // These are just for getting more informative error messages:

  if (boost::get<SmartMet::Spine::TimeSeries::None>(&tv))
  {
    throw Fmi::Exception(BCP, "Encountered NULL FMISID");
  }

  if (boost::get<SmartMet::Spine::TimeSeries::LonLat>(&tv))
  {
    throw Fmi::Exception(BCP, "Encountered LonLat FMISID");
  }

  if (boost::get<boost::local_time::local_date_time>(&tv))
  {
    throw Fmi::Exception(BCP, "Encountered date FMISID");
  }

  // All should be handled above, but if someone extends the variant, this handles it too:

  throw Fmi::Exception(BCP, "Failed to extract FMISID (double/int/string) from variant");
}

}  // namespace Observation
}  // namespace Engine
}  // namespace SmartMet

max_stars_repo_name: fmidev/smartmet-engine-observation

max_issues_repo_name: fmidev/smartmet-engine-observation

max_forks_repo_name: fmidev/smartmet-engine-observation


text:
/*
 *             Copyright Andrey Semashev 2016.
 * Distributed under the Boost Software License, Version 1.0.
 *    (See accompanying file LICENSE_1_0.txt or copy at
 *          http://www.boost.org/LICENSE_1_0.txt)
 */
/*!
 * \file   formatters/max_size_decorator.hpp
 * \author Andrey Semashev
 * \date   06.07.2016
 *
 * The header contains implementation of a string length limiting decorator.
 */

#ifndef BOOST_LOG_EXPRESSIONS_FORMATTERS_MAX_SIZE_DECORATOR_HPP_INCLUDED_
#define BOOST_LOG_EXPRESSIONS_FORMATTERS_MAX_SIZE_DECORATOR_HPP_INCLUDED_

#include <cstddef>
#include <string>
#include <boost/assert.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/move/core.hpp>
#include <boost/move/utility_core.hpp>
#include <boost/core/addressof.hpp>
#include <boost/phoenix/core/actor.hpp>
#include <boost/phoenix/core/meta_grammar.hpp>
#include <boost/phoenix/core/terminal_fwd.hpp>
#include <boost/phoenix/core/is_nullary.hpp>
#include <boost/phoenix/core/environment.hpp>
#include <boost/phoenix/support/vector.hpp>
#include <boost/fusion/sequence/intrinsic/at_c.hpp>
#include <boost/type_traits/remove_cv.hpp>
#include <boost/type_traits/remove_reference.hpp>
#include <boost/log/detail/config.hpp>
#include <boost/log/detail/custom_terminal_spec.hpp>
#include <boost/log/utility/formatting_ostream.hpp>
#include <boost/log/detail/header.hpp>

#ifdef BOOST_HAS_PRAGMA_ONCE
#pragma once
#endif

namespace boost {

BOOST_LOG_OPEN_NAMESPACE

namespace expressions {

namespace aux {

//! String size limiting decorator stream output terminal
template< typename LeftT, typename SubactorT, typename CharT >
class max_size_decorator_output_terminal
{
private:
    //! Self type
    typedef max_size_decorator_output_terminal< LeftT, SubactorT, CharT > this_type;

public:
#ifndef BOOST_LOG_DOXYGEN_PASS
    //! Internal typedef for type categorization
    typedef void _is_boost_log_terminal;
#endif

    //! Character type
    typedef CharT char_type;
    //! String type
    typedef std::basic_string< char_type > string_type;
    //! String size type
    typedef std::size_t size_type;
    //! Adopted actor type
    typedef SubactorT subactor_type;

    //! Result type definition
    template< typename >
    struct result;

    template< typename ThisT, typename ContextT >
    struct result< ThisT(ContextT) >
    {
        typedef typename remove_cv< typename remove_reference< ContextT >::type >::type context_type;
        typedef typename phoenix::evaluator::impl<
            typename LeftT::proto_base_expr&,
            context_type,
            phoenix::unused
        >::result_type type;
    };

private:
    //! Left argument actor
    LeftT m_left;
    //! Adopted formatter actor
    subactor_type m_subactor;
    //! Max size of the formatted string produced by the adopted formatter
    size_type m_max_size;
    //! Overflow marker
    string_type m_overflow_marker;

public:
    /*!
     * Initializing constructor. Creates decorator of the \a fmt formatter with the specified \a decorations.
     */
    max_size_decorator_output_terminal(LeftT const& left, subactor_type const& sub, size_type max_size, string_type const& overflow_marker = string_type()) :
        m_left(left), m_subactor(sub), m_max_size(max_size), m_overflow_marker(overflow_marker)
    {
        BOOST_ASSERT(overflow_marker.size() <= max_size);
    }
    /*!
     * Copy constructor
     */
    max_size_decorator_output_terminal(max_size_decorator_output_terminal const& that) :
        m_left(that.m_left), m_subactor(that.m_subactor), m_max_size(that.m_max_size), m_overflow_marker(that.m_overflow_marker)
    {
    }

    /*!
     * Invokation operator
     */
    template< typename ContextT >
    typename result< this_type(ContextT const&) >::type operator() (ContextT const& ctx)
    {
        // Flush the stream and keep the current write position in the target string
        typedef typename result< this_type(ContextT const&) >::type result_type;
        result_type strm = phoenix::eval(m_left, ctx);
        strm.flush();

        if (!strm.rdbuf()->storage_overflow())
        {
            const size_type old_max_size = strm.rdbuf()->max_size();
            const size_type start_pos = strm.rdbuf()->storage()->size(), max_size_left = strm.rdbuf()->storage()->max_size() - start_pos;
            strm.rdbuf()->max_size(start_pos + (m_max_size <= max_size_left ? m_max_size : max_size_left));

            try
            {
                // Invoke the adopted formatter
                phoenix::eval(m_subactor, ctx);

                // Flush the buffered characters and apply decorations
                strm.flush();

                if (strm.rdbuf()->storage_overflow())
                {
                    if (!m_overflow_marker.empty())
                    {
                        // Free up space for the overflow marker
                        strm.rdbuf()->max_size(strm.rdbuf()->max_size() - m_overflow_marker.size());

                        // Append the marker
                        strm.rdbuf()->max_size(strm.rdbuf()->storage()->max_size());
                        strm.rdbuf()->storage_overflow(false);
                        strm.rdbuf()->append(m_overflow_marker.data(), m_overflow_marker.size());
                    }
                    else
                    {
                        strm.rdbuf()->storage_overflow(false);
                    }
                }

                // Restore the original size limit
                strm.rdbuf()->max_size(old_max_size);
            }
            catch (...)
            {
                strm.rdbuf()->storage_overflow(false);
                strm.rdbuf()->max_size(old_max_size);
                throw;
            }
        }

        return strm;
    }

    /*!
     * Invokation operator
     */
    template< typename ContextT >
    typename result< const this_type(ContextT const&) >::type operator() (ContextT const& ctx) const
    {
        // Flush the stream and keep the current write position in the target string
        typedef typename result< const this_type(ContextT const&) >::type result_type;
        result_type strm = phoenix::eval(m_left, ctx);
        strm.flush();

        if (!strm.rdbuf()->storage_overflow())
        {
            const size_type old_max_size = strm.rdbuf()->max_size();
            const size_type start_pos = strm.rdbuf()->storage()->size(), max_size_left = strm.rdbuf()->storage()->max_size() - start_pos;
            strm.rdbuf()->max_size(start_pos + (m_max_size <= max_size_left ? m_max_size : max_size_left));

            try
            {
                // Invoke the adopted formatter
                phoenix::eval(m_subactor, ctx);

                // Flush the buffered characters and apply decorations
                strm.flush();

                if (strm.rdbuf()->storage_overflow())
                {
                    if (!m_overflow_marker.empty())
                    {
                        // Free up space for the overflow marker
                        strm.rdbuf()->max_size(strm.rdbuf()->max_size() - m_overflow_marker.size());

                        // Append the marker
                        strm.rdbuf()->max_size(strm.rdbuf()->storage()->max_size());
                        strm.rdbuf()->storage_overflow(false);
                        strm.rdbuf()->append(m_overflow_marker.data(), m_overflow_marker.size());
                    }
                    else
                    {
                        strm.rdbuf()->storage_overflow(false);
                    }
                }

                // Restore the original size limit
                strm.rdbuf()->max_size(old_max_size);
            }
            catch (...)
            {
                strm.rdbuf()->storage_overflow(false);
                strm.rdbuf()->max_size(old_max_size);
                throw;
            }
        }

        return strm;
    }

    BOOST_DELETED_FUNCTION(max_size_decorator_output_terminal())
};

} // namespace aux

/*!
 * String size limiting decorator terminal class. This formatter allows to limit the maximum total length
 * of the strings generated by other formatters.
 *
 * The \c max_size_decorator_terminal class aggregates the formatter being decorated, the maximum string length
 * it can produce and an optional truncation marker string, which will be put at the end of the output if the limit is exceeded. Note that
 * the marker length is included in the limit and as such must not exceed it.
 * The \c max_size_decorator_terminal class is a formatter itself, so it can be used to construct
 * more complex formatters, including nesting decorators.
 */
template< typename SubactorT, typename CharT >
class max_size_decorator_terminal
{
private:
    //! Self type
    typedef max_size_decorator_terminal< SubactorT, CharT > this_type;

public:
#ifndef BOOST_LOG_DOXYGEN_PASS
    //! Internal typedef for type categorization
    typedef void _is_boost_log_terminal;
#endif

    //! Character type
    typedef CharT char_type;
    //! String type
    typedef std::basic_string< char_type > string_type;
    //! String size type
    typedef std::size_t size_type;
    //! Stream type
    typedef basic_formatting_ostream< char_type > stream_type;
    //! Adopted actor type
    typedef SubactorT subactor_type;

    //! Result type definition
    typedef string_type result_type;

private:
    //! Adopted formatter actor
    subactor_type m_subactor;
    //! Max size of the formatted string produced by the adopted formatter
    size_type m_max_size;
    //! Overflow marker
    string_type m_overflow_marker;

public:
    /*!
     * Initializing constructor.
     */
    max_size_decorator_terminal(subactor_type const& sub, size_type max_size, string_type const& overflow_marker = string_type()) :
        m_subactor(sub), m_max_size(max_size), m_overflow_marker(overflow_marker)
    {
        BOOST_ASSERT(overflow_marker.size() <= max_size);
    }
    /*!
     * Copy constructor
     */
    max_size_decorator_terminal(max_size_decorator_terminal const& that) :
        m_subactor(that.m_subactor), m_max_size(that.m_max_size), m_overflow_marker(that.m_overflow_marker)
    {
    }

    /*!
     * \returns Adopted subactor
     */
    subactor_type const& get_subactor() const
    {
        return m_subactor;
    }

    /*!
     * \returns Max string size limit
     */
    size_type get_max_size() const
    {
        return m_max_size;
    }

    /*!
     * \returns Max string size limit
     */
    string_type const& get_overflow_marker() const
    {
        return m_overflow_marker;
    }

    /*!
     * Invokation operator
     */
    template< typename ContextT >
    result_type operator() (ContextT const& ctx)
    {
        string_type str;
        stream_type strm(str);
        strm.rdbuf()->max_size(m_max_size);

        // Invoke the adopted formatter
        typedef phoenix::vector3<
            subactor_type*,
            typename fusion::result_of::at_c<
                typename remove_cv<
                    typename remove_reference<
                        typename phoenix::result_of::env< ContextT const& >::type
                    >::type
                >::type::args_type,
                0
            >::type,
            stream_type&
        > env_type;
        env_type env = { boost::addressof(m_subactor), fusion::at_c< 0 >(phoenix::env(ctx).args()), strm };
        phoenix::eval(m_subactor, phoenix::make_context(env, phoenix::actions(ctx)));

        // Flush the buffered characters and see of overflow happened
        strm.flush();

        if (strm.rdbuf()->storage_overflow() && !m_overflow_marker.empty())
        {
            strm.rdbuf()->max_size(strm.rdbuf()->max_size() - m_overflow_marker.size());
            strm.rdbuf()->max_size(str.max_size());
            strm.rdbuf()->storage_overflow(false);
            strm.rdbuf()->append(m_overflow_marker.data(), m_overflow_marker.size());
            strm.flush();
        }

        return BOOST_LOG_NRVO_RESULT(str);
    }

    /*!
     * Invokation operator
     */
    template< typename ContextT >
    result_type operator() (ContextT const& ctx) const
    {
        string_type str;
        stream_type strm(str);
        strm.rdbuf()->max_size(m_max_size);

        // Invoke the adopted formatter
        typedef phoenix::vector3<
            const subactor_type*,
            typename fusion::result_of::at_c<
                typename remove_cv<
                    typename remove_reference<
                        typename phoenix::result_of::env< ContextT const& >::type
                    >::type
                >::type::args_type,
                0
            >::type,
            stream_type&
        > env_type;
        env_type env = { boost::addressof(m_subactor), fusion::at_c< 0 >(phoenix::env(ctx).args()), strm };
        phoenix::eval(m_subactor, phoenix::make_context(env, phoenix::actions(ctx)));

        // Flush the buffered characters and see of overflow happened
        strm.flush();

        if (strm.rdbuf()->storage_overflow())
        {
            strm.rdbuf()->max_size(strm.rdbuf()->max_size() - m_overflow_marker.size());
            strm.rdbuf()->max_size(str.max_size());
            strm.rdbuf()->storage_overflow(false);
            strm.rdbuf()->append(m_overflow_marker.data(), m_overflow_marker.size());
            strm.flush();
        }

        return BOOST_LOG_NRVO_RESULT(str);
    }

    BOOST_DELETED_FUNCTION(max_size_decorator_terminal())
};

/*!
 * Character decorator actor
 */
template< typename SubactorT, typename CharT, template< typename > class ActorT = phoenix::actor >
class max_size_decorator_actor :
    public ActorT< max_size_decorator_terminal< SubactorT, CharT > >
{
public:
    //! Base terminal type
    typedef max_size_decorator_terminal< SubactorT, CharT > terminal_type;
    //! Character type
    typedef typename terminal_type::char_type char_type;

    //! Base actor type
    typedef ActorT< terminal_type > base_type;

public:
    //! Initializing constructor
    explicit max_size_decorator_actor(base_type const& act) : base_type(act)
    {
    }

    //! Returns reference to the terminal
    terminal_type const& get_terminal() const
    {
        return this->proto_expr_.child0;
    }
};

#ifndef BOOST_LOG_DOXYGEN_PASS

#define BOOST_LOG_AUX_OVERLOAD(left_ref, right_ref)\
    template< typename LeftExprT, typename SubactorT, typename CharT, template< typename > class ActorT >\
    BOOST_FORCEINLINE phoenix::actor< aux::max_size_decorator_output_terminal< phoenix::actor< LeftExprT >, SubactorT, CharT > >\
    operator<< (phoenix::actor< LeftExprT > left_ref left, max_size_decorator_actor< SubactorT, CharT, ActorT > right_ref right)\
    {\
        typedef aux::max_size_decorator_output_terminal< phoenix::actor< LeftExprT >, SubactorT, CharT > terminal_type;\
        phoenix::actor< terminal_type > actor = {{ terminal_type(left, right.get_terminal().get_subactor(), right.get_terminal().get_max_size(), right.get_terminal().get_overflow_marker()) }};\
        return actor;\
    }

#include <boost/log/detail/generate_overloads.hpp>

#undef BOOST_LOG_AUX_OVERLOAD

#endif // BOOST_LOG_DOXYGEN_PASS

namespace aux {

template< typename CharT >
class max_size_decorator_gen
{
private:
    typedef CharT char_type;
    typedef std::basic_string< char_type > string_type;
    typedef std::size_t size_type;

private:
    size_type m_max_size;
    string_type m_overflow_marker;

public:
    explicit max_size_decorator_gen(size_type max_size, string_type const& overflow_marker = string_type()) :
        m_max_size(max_size), m_overflow_marker(overflow_marker)
    {
        BOOST_ASSERT(overflow_marker.size() <= max_size);
    }

    template< typename SubactorT >
    BOOST_FORCEINLINE max_size_decorator_actor< SubactorT, char_type > operator[] (SubactorT const& subactor) const
    {
        typedef max_size_decorator_actor< SubactorT, char_type > result_type;
        typedef typename result_type::terminal_type terminal_type;
        typename result_type::base_type act = {{ terminal_type(subactor, m_max_size, m_overflow_marker) }};
        return result_type(act);
    }
};

} // namespace aux

/*!
 * The function returns a decorator generator object. The generator provides <tt>operator[]</tt> that can be used
 * to construct the actual decorator.
 *
 * \param max_size The maximum number of characters (i.e. string element objects) that the decorated formatter can produce.
 */
template< typename CharT >
BOOST_FORCEINLINE aux::max_size_decorator_gen< CharT > max_size_decor(std::size_t max_size)
{
    return aux::max_size_decorator_gen< CharT >(max_size);
}

/*!
 * The function returns a decorator generator object. The generator provides <tt>operator[]</tt> that can be used
 * to construct the actual decorator.
 *
 * \param max_size The maximum number of characters (i.e. string element objects) that the decorated formatter can produce.
 * \param overflow_marker The marker string which is appended to the output if the \a max_size limit is exceeded. Must be
 *                        a non-null pointer to a zero-terminated string.
 *
 * \pre The \a overflow_marker length must not exceed the \a max_size limit.
 */
template< typename CharT >
BOOST_FORCEINLINE aux::max_size_decorator_gen< CharT > max_size_decor(std::size_t max_size, const CharT* overflow_marker)
{
    return aux::max_size_decorator_gen< CharT >(max_size, overflow_marker);
}

/*!
 * The function returns a decorator generator object. The generator provides <tt>operator[]</tt> that can be used
 * to construct the actual decorator.
 *
 * \param max_size The maximum number of characters (i.e. string element objects) that the decorated formatter can produce.
 * \param overflow_marker The marker string which is appended to the output if the \a max_size limit is exceeded.
 *
 * \pre The \a overflow_marker length must not exceed the \a max_size limit.
 */
template< typename CharT >
BOOST_FORCEINLINE aux::max_size_decorator_gen< CharT > max_size_decor(std::size_t max_size, std::basic_string< CharT > const& overflow_marker)
{
    return aux::max_size_decorator_gen< CharT >(max_size, overflow_marker);
}

} // namespace expressions

BOOST_LOG_CLOSE_NAMESPACE // namespace log

#ifndef BOOST_LOG_DOXYGEN_PASS

namespace phoenix {

namespace result_of {

template< typename SubactorT, typename CharT >
struct is_nullary< custom_terminal< boost::log::expressions::max_size_decorator_terminal< SubactorT, CharT > > > :
    public mpl::false_
{
};

template< typename LeftT, typename SubactorT, typename CharT >
struct is_nullary< custom_terminal< boost::log::expressions::aux::max_size_decorator_output_terminal< LeftT, SubactorT, CharT > > > :
    public mpl::false_
{
};

} // namespace result_of

} // namespace phoenix

#endif

} // namespace boost

#include <boost/log/detail/footer.hpp>

#endif // BOOST_LOG_EXPRESSIONS_FORMATTERS_MAX_SIZE_DECORATOR_HPP_INCLUDED_

max_stars_repo_name: rudylee/expo

max_issues_repo_name: rudylee/expo

max_forks_repo_name: rudylee/expo


text:
#include <string>
#include <iostream>
#include <fstream>
#include <thread>
#include <csignal>

#include <arpa/inet.h>

#include <boost/program_options.hpp>

#include <dmtr/libos.h>
#include <dmtr/libos/persephone.hh>

#include "PspWorker.hh"
#include "common.hh"
#include "logging.h"

template <typename T>
static void as_sga(T &from, dmtr_sgarray_t &sga) {
    sga.sga_buf = nullptr;
    sga.sga_numsegs = 1;
    sga.sga_segs[0].sgaseg_buf = &from;
    sga.sga_segs[0].sgaseg_len = sizeof(from);
}

struct KvRequest {
    int req_qfd;
    dmtr_sgarray_t sga;

    const char *data;
    size_t data_len;
    const char *data_end;
    KvRequest(int qfd, dmtr_sgarray_t &sga) :
            req_qfd(qfd), sga(sga),
            data((char*)sga.sga_segs[0].sgaseg_buf),
            data_len(sga.sga_segs[0].sgaseg_len),
            data_end((char*)data + data_len) {}

    KvRequest(std::string &str) :
        data(str.c_str()),
        data_len(str.size()),
        data_end(str.c_str() + str.size()) {}

    // FIXME: is_reqtype must be called (with the proper reqtype) before the key is used
    // This isn't necessarily bad. Should be made more clear though
    const char *key = NULL;
    const char *key_end = NULL;

    bool is_reqtype(const char *type, size_t qlen) {
        if (strncmp(data, type, qlen) == 0) {
            key = data+qlen;
            key_end = (const char*)memchr(key, ' ', data_len - (qlen));
            if (key_end == NULL) {
                key_end = data + data_len;
            }
            return true;
        }
        return false;
    }

};

class KvResponse {
private:
    void *data;
    size_t data_size;
    bool data_valid = false;

public:
    int req_qfd;
    KvResponse(int req_qfd) :
            data_valid(false), req_qfd(req_qfd)
    {}

    void set_data(const std::string &resp) {
        data_valid = true;
        data = malloc(resp.size());
        data_size = resp.size();
        memcpy(data, resp.c_str(), data_size);
    }

    ~KvResponse() {
        if (data_valid) {
             free(data);
        }
    }

    int move_to_sga(dmtr_sgarray_t &sga) {
        if (!data_valid) {
            return -1;
        }
        sga.sga_buf = data;
        sga.sga_numsegs = 1;
        sga.sga_segs[0].sgaseg_buf = data;
        sga.sga_segs[0].sgaseg_len = data_size;
        data_valid = false;
        return 0;
    }

    // This may be more confusing than using set_data() explicitely,
    // but setting a KvResponse equal to a string just copies the data in
    KvResponse & operator=(const std::string &resp) {
        set_data(resp);
        return *this;
    }
};

class NetWorker : public PspWorker {
public:

    enum worker_choice {
        RR, KEY
    };

private:

    struct sockaddr_in bind_addr;

    worker_choice choice_fn;

    using hr_clock = std::chrono::high_resolution_clock;
    std::vector<hr_clock::time_point> entry_times;
    std::vector<hr_clock::time_point> exit_times;
    std::string log_filename;

    bool record_lat;
    int lqd;
    std::vector<dmtr_qtoken_t> tokens;

    int start_offset = 0;

    int worker_offset = 0;

    int round_robin_choice(dmtr_qresult_t &dequeued) {
        int n_peers = peer_ids.size();
        log_debug("Choosing from %d peers", n_peers);
        if (n_peers == 0) {
            return -1;
        }
        if (++worker_offset >= n_peers) {
            worker_offset = 0;
        }

        return peer_ids[worker_offset];
    }

    static inline int fast_atoi( const char * str, const char *end)
    {
        int val = 0;
        while( *str != ' ' && str < end) {
            val = val*10 + (*str++ - '0');
        }
        return val;
    }

    int key_based_routing(dmtr_qresult_t &dequeued) {
        void *buf = dequeued.qr_value.sga.sga_segs[0].sgaseg_buf;
        char *req = static_cast<char*>(buf);
        char *req_end = req + dequeued.qr_value.sga.sga_segs[0].sgaseg_len;
        char *space;
        for (space=req; *space != ' ' && space < req_end; ++space) {};
        if (space == req_end) {
            // FIXME: This means all PUT requests go to the same server
            return peer_ids[0];
        }
        int idx = fast_atoi(space + 1, req + dequeued.qr_value.sga.sga_segs[0].sgaseg_len);
        return peer_ids[idx % peer_ids.size()];
    }

    int choose_worker(dmtr_qresult_t &dequeued) {
        switch (choice_fn) {
            case KEY:
                return key_based_routing(dequeued);
            case RR:
            default:
                return round_robin_choice(dequeued);
        }
    }

    long int ns_since_start(hr_clock::time_point &tp) {
        return std::chrono::duration_cast<std::chrono::nanoseconds>(tp - entry_times[0]).count();
    }

public:

    int dump_times() {
        if (log_filename.size() == 0) {
            return 0;
        }
        std::ofstream logfile(log_filename);
        if (logfile.is_open()) {
            logfile << "entry\texit" << std::endl;
            for (unsigned int i=0; i < exit_times.size(); i++) {
                logfile << ns_since_start(entry_times[i]) << "\t"
                        << ns_since_start(exit_times[i]) << std::endl;
            }
            logfile.close();
            log_info("Wrote net logs to %s", log_filename.c_str());
            return 0;
        } else {
            log_error("Coult not open logfile %s", log_filename.c_str());
            return -1;
        }
    }

public:

    ~NetWorker() {
        psu.ioqapi.close(lqd);
    }

    NetWorker(struct sockaddr_in &addr,
              worker_choice choice = RR,
              std::string log_filename = "",
              int argc = 0, char **argv = NULL) :
        PspWorker(0, dmtr::io_queue::NETWORK_Q, argc, argv),
        bind_addr(addr), choice_fn(choice),
        log_filename(log_filename), record_lat(log_filename.size() > 0)
    {
        entry_times.reserve(10000000);
        exit_times.reserve(10000000);
    }

    int setup() {
        pin_thread(pthread_self(), 4);
        DMTR_OK(psu.ioqapi.socket(lqd, AF_INET, SOCK_STREAM, 0));
        DMTR_OK(psu.ioqapi.bind(lqd,
                                reinterpret_cast<struct sockaddr*>(&bind_addr),
                                sizeof(bind_addr)));
        dmtr_qtoken_t token;
        DMTR_OK(psu.ioqapi.listen(lqd, 100));
        DMTR_OK(psu.ioqapi.accept(token, lqd));
        tokens.push_back(token);

        for (int peer_id : peer_ids) {
            DMTR_OK(pop_from_peer(peer_id, token));
            tokens.push_back(token);
        }
        return 0;
    }

    int dequeue(dmtr_qresult_t &dequeued) {
        int idx;
        int status = psu.wait_any(&dequeued, &start_offset, &idx, tokens.data(), tokens.size());
        if (status == EAGAIN) {
            return EAGAIN;
        }
        tokens.erase(tokens.begin() + idx);
        log_debug("wait_any returned %d", status);
        if (status == ECONNABORTED) {
            return EAGAIN;
        }
        return status;
    }

    int work(int status, dmtr_qresult_t &dequeued) {
        hr_clock::time_point entry_time = hr_clock::now();
        if (status != 0) {
            log_error("NetWorker work() received non-0 status %d", status);
            return -1;
        }
        if (dequeued.qr_qd == lqd) {
            assert(dequeued.qr_opcode == DMTR_OPC_ACCEPT);
            dmtr_qtoken_t token;
            DMTR_OK(psu.ioqapi.pop(token, dequeued.qr_value.ares.qd));
            tokens.push_back(token);

            DMTR_OK(psu.ioqapi.accept(token, lqd));
            tokens.push_back(token);
            log_debug("Accepted a new connection");
            return 0;
        }
        if (dequeued.qr_opcode == DMTR_OPC_PUSH) {
            // sga segment must be freed after pushing to the client
            free(dequeued.qr_value.sga.sga_segs[0].sgaseg_buf);
            return 0;
        }
        log_debug("Received POP code");
        assert(DMTR_OPC_POP == dequeued.qr_opcode);
        int dequeued_id = get_peer_id(dequeued.qr_qd);
        if (dequeued_id == -1) {
            if (record_lat) {
                entry_times.push_back(entry_time);
            }
            // New request
            int new_worker_id = choose_worker(dequeued);
            KvRequest *kvr = new KvRequest(dequeued.qr_qd, dequeued.qr_value.sga);
            dmtr_sgarray_t sga_req;
            as_sga(*kvr, sga_req);
            if (blocking_push_to_peer(new_worker_id, sga_req) == -1) {
                log_warn("Could not push to worker %d", new_worker_id);
            } else {
                log_debug("NetWorker pushed to peer %d", new_worker_id);
            }

            dmtr_qtoken_t token;
            DMTR_OK(psu.ioqapi.pop(token, dequeued.qr_qd));
            tokens.push_back(token);
        } else {
            // Returned from peer
            dmtr_sgarray_t &sga = dequeued.qr_value.sga;
            assert(sga.sga_numsegs == 1 && sga.sga_segs[0].sgaseg_len == sizeof(KvResponse));

            auto resp = static_cast<KvResponse*>(sga.sga_segs[0].sgaseg_buf);
            dmtr_qtoken_t token;

            dmtr_sgarray_t resp_sga;
            resp->move_to_sga(resp_sga);
            DMTR_OK(psu.ioqapi.push(token, resp->req_qfd, resp_sga));
            int status = psu.wait(NULL, token);
            if (status == EAGAIN) {
                tokens.push_back(token);
            }
            if (record_lat) {
                exit_times.push_back(hr_clock::now());
            }
            DMTR_OK(status);

            DMTR_OK(psu.ioqapi.pop(token, dequeued.qr_qd));
            tokens.push_back(token);
            delete resp;
        }
        return 0;
    }
};

class KvStore {
private:
    bool writeable;
    bool readable;
    std::unordered_map<std::string, std::string> store;

    // Pre-calculating the length of the strings for the sake of optimization
    // (I don't think it made a huge difference)
    const char *PUT_STR = "PUT ";
    const size_t PUT_LEN = strlen(PUT_STR);
    const char *GET_STR = "GET ";
    const size_t GET_LEN = strlen(GET_STR);
    const char *SZOF_STR = "SZOF ";
    const size_t SZOF_LEN = strlen(SZOF_STR);
    const char *NNZ_STR = "NNZ ";
    const size_t NNZ_LEN = strlen(NNZ_STR);

    int process_put(KvRequest &input, KvResponse &output) {
        if (!writeable) {
            output = "ERR: Not writeable";
            return -1;
        }
        if (input.key_end == NULL) {
            output = "ERR: No KEY";
            return 0;
        }
        if (input.key_end >= input.data_end) {
            output = "ERR: No value";
            return 0;
        }

        // Make a copy of the key and value to store in the map.
        // Don't think this is avoidable
        std::string key(input.key, input.key_end);
        std::string value(input.key_end+1, input.data_end);
        store[key] = value;
        output = "SUCCESS";
        return 0;
    }

    int process_get(KvRequest &input, KvResponse &output) {
        if (!readable) {
            output = "ERR: Not readable";
            return -1;
        }
        std::string key(input.key, input.key_end);
        auto it = store.find(key);
        if (it == store.end()) {
            output = "ERR: Bad key " + key;
            return -1;
        }
        output = it->second;
        return 0;
    }

    int process_szof(KvRequest &input, KvResponse &output) {
        if (!readable) {
            output = "ERR: Not readable";
            return -1;
        }

        std::string key(input.key, input.key_end);
        auto it = store.find(key);
        if (it == store.end()) {
            output = "ERR: Bad key";
            return -1;
        }
        // Using strlen() rather than str::size so that it requires accessing the full string
        output = std::to_string(strlen(it->second.c_str()));
        return 0;
    }

    // This is really slow. Much slower than strlen
    // (though in theory both are iterating over the string)
    int process_nnz(KvRequest &input, KvResponse &output) {
        if (!readable) {
            output = "ERR: Not readable";
            return -1;
        }

        std::string key(input.key, input.key_end);
        auto it = store.find(key);
        if (it == store.end()) {
            output = "ERR: Bad key";
            return -1;
        }
        int count = 0;
        for (const char c: it->second) {
            if (c != '0') {
                count++;
            }
        }
        output = std::to_string(count);
        return 0;
    }

public:

    int process_req(KvRequest &req, KvResponse &output) {
        if (req.is_reqtype(PUT_STR, PUT_LEN)) {
            return process_put(req, output);
        } else if (req.is_reqtype(GET_STR, GET_LEN)) {
            return process_get(req, output);
        } else if (req.is_reqtype(SZOF_STR, SZOF_LEN)) {
            return process_szof(req, output);
        } else if (req.is_reqtype(NNZ_STR, NNZ_LEN)) {
            return process_nnz(req, output);
        }
        output = "ERR: Unknown reqtype";
        return -1;
    }

    KvStore(const std::string filename) : writeable(true), readable(false) {
        std::ifstream input_file(filename);
        if (input_file.is_open()) {
            std::string line;
            while (std::getline(input_file, line)) {
                KvRequest req(line);
                KvResponse output(0);
                if (process_req(req, output)) {
                    log_warn("Could not process line %s", line.c_str());
                }
            }
            input_file.close();
        } else {
            log_warn("Could not open input file %s", filename.c_str());
            log_warn("KV store will be writeable! May have concurrency issues");
            writeable = true;
            readable = true;
            return;
        }
        writeable = false;
        readable = true;
    }
};

class StoreWorker : public PspWorker {

    int networker_qd;
    dmtr_qtoken_t pop_token;

    int n_accesses = 0;

private:
    using hr_clock = std::chrono::high_resolution_clock;
    KvStore &store;
    bool record_lat;
    std::string log_filename_base;
    std::vector<hr_clock::time_point> entry_times;
    std::vector<hr_clock::time_point> exit_times;

    long int ns_since_start(hr_clock::time_point &tp) {
        return std::chrono::duration_cast<std::chrono::nanoseconds>(tp - entry_times[0]).count();
    }

public:

    int dump_times() {
        if (log_filename_base.size() == 0) {
            return 0;
        }
        std::string log_filename = log_filename_base + "_s" + std::to_string(worker_id);
        std::ofstream logfile(log_filename);
        if (logfile.is_open()) {
            logfile << "entry\texit" << std::endl;
            for (unsigned int i=0; i < exit_times.size(); i++) {
                logfile << ns_since_start(entry_times[i]) << "\t"
                        << ns_since_start(exit_times[i]) << std::endl;
            }
            logfile.close();
            log_info("Wrote net logs to %s", log_filename.c_str());
            return 0;
        } else {
            log_error("Coult not open logfile %s", log_filename.c_str());
            return -1;
        }
    }

    StoreWorker(int id, KvStore &store, std::string log_filename_base, int argc, char **argv) :
        PspWorker(id, dmtr::io_queue::SHARED_Q, argc, argv),
        store(store), record_lat(log_filename_base.size() > 0),
        log_filename_base(log_filename_base)
    {
        if (id == 0) {
            // RAISE WARNING
        }
        entry_times.reserve(5000000);
        exit_times.reserve(5000000);
    }
    ~StoreWorker() {
        std::cout << "Worker " << worker_id << " called " << n_accesses << " times" << std::endl;
    }

    int setup() {
        pin_thread(pthread_self(), 4+worker_id);
        networker_qd = get_peer_qd(0);
        if (networker_qd == -1) {
            log_error("Must register networker before starting StoreWorker");
            return -1;
        }
        DMTR_OK(psu.ioqapi.pop(pop_token, networker_qd));
        return 0;
    }

    int dequeue(dmtr_qresult_t &dequeued) {
        int status = psu.wait(&dequeued, pop_token);
        if (status == EAGAIN) {
            return EAGAIN;
        }
        log_debug("StoreWorker Got non-EAGAIN");
        DMTR_OK(status);
        DMTR_OK(psu.ioqapi.pop(pop_token, networker_qd));
        return status;
    }

    int work(int status, dmtr_qresult_t &dequeued) {
        if (status) {
            log_error("StoreWorker work() received non-0 status %d", status);
            return -1;
        }
        if (record_lat) {
            entry_times.push_back(hr_clock::now());
        }
        n_accesses++;
        assert(dequeued.qr_qd == networker_qd);
        assert(dequeued.qr_opcode == DMTR_OPC_POP);
        dmtr_sgarray_t &sga = dequeued.qr_value.sga;
        assert(sga.sga_numsegs == 1);
        auto kvreq = static_cast<KvRequest*>(sga.sga_segs[0].sgaseg_buf);
        assert(kvreq->sga.sga_numsegs == 1);
        std::string req((char*)kvreq->sga.sga_segs[0].sgaseg_buf,
                        kvreq->sga.sga_segs[0].sgaseg_len);
        log_debug("Received request %s", req.c_str());
        KvResponse *kvresp = new KvResponse(kvreq->req_qfd);
        store.process_req(*kvreq, *kvresp);

        dmtr_sgarray_t sga_resp;
        as_sga(*kvresp, sga_resp);
        DMTR_OK(blocking_push_to_peer(0, sga_resp));
        //free(kvreq->sga.sga_buf);
        dmtr_free_mbuf(&kvreq->sga);
        delete kvreq;

        if (record_lat) {
            exit_times.push_back(hr_clock::now());
        }
        return 0;

    }

};

struct ServerOpts {
    std::string ip;
    uint16_t port;
    std::string cmd_file;
    int n_workers;
    std::string choice_fn;
    bool record_latencies;
    bool record_store_latencies;
    CommonOptions common;
};

namespace bpo = boost::program_options;

int parse_server_args(int argc, char **argv, ServerOpts &options) {
    bpo::options_description opts{"KV Server options"};
    opts.add_options()
                    ("ip",
                        bpo::value<std::string>(&options.ip)->default_value("127.0.0.1"),
                        "Server IP")
                    ("port",
                        bpo::value<uint16_t>(&options.port)->default_value(12345),
                        "Server port")
                    ("cmd-file",
                        bpo::value<std::string>(&options.cmd_file)->default_value(""),
                        "Initial commands")
                    ("workers,w",
                        bpo::value<int>(&options.n_workers)->default_value(1),
                        "Number of 'store' workers")
                    ("record-lat,r",
                        bpo::bool_switch(&options.record_latencies),
                        "Turn on latency recording")
                    ("record-store-lat",
                        bpo::bool_switch(&options.record_store_latencies),
                        "Turn on store latency recording")
                    ("strategy,s",
                        bpo::value<std::string>(&options.choice_fn)->default_value("RR"),
                        "Worker delegation strategy (RR or KEY)");
    return parse_args(argc, argv, opts, options.common);
}

int main(int argc, char **argv) {

    ServerOpts opts;
    if (parse_server_args(argc, argv, opts)) {
        return 1;
    }

    NetWorker::worker_choice choice_fn;
    if (opts.choice_fn == "RR") {
        choice_fn = NetWorker::RR;
    } else if (opts.choice_fn == "KEY") {
        choice_fn = NetWorker::KEY;
    } else {
        log_error("Unknown choice function '%s'", opts.choice_fn.c_str());
    }

    log_info("Launching kv store on %s:%u", opts.ip.c_str(), opts.port);

    struct sockaddr_in addr = {};
    addr.sin_family = AF_INET;
    if (inet_pton(AF_INET, opts.ip.c_str(), &addr.sin_addr) != 1) {
        log_error("Could not convert %s to ip", opts.ip.c_str());
        return -1;
    }
    addr.sin_port = htons(opts.port);

    std::string log_file;
    std::string store_log_file;
    if (opts.record_latencies)
        log_file = opts.common.log_dir + "/net_traces";
    if (opts.record_store_latencies)
        store_log_file = opts.common.log_dir + "/store_traces";

    NetWorker n = NetWorker(addr, choice_fn, log_file, argc, argv);

    std::vector<StoreWorker*> store_workers;
    KvStore store(opts.cmd_file);
    for (int i=0; i < opts.n_workers; i++) {
        store_workers.push_back(new StoreWorker(i+1, store, store_log_file, argc, argv));
        PspWorker::register_peers(n, *store_workers[i]);
    }

    auto sig_handler = [](int signal) {
        PspWorker::stop_all();
    };

    std::signal(SIGINT, sig_handler);
    std::signal(SIGTERM, sig_handler);

    bool failed_launch = (n.launch() != 0);
    for (auto w : store_workers) {
        if (w->launch()) {
            failed_launch = true;
            break;
        }
    }
    if (failed_launch) {
        PspWorker::stop_all();
    } else {
        bool stopped = false;
        while (!stopped) {
            if (n.has_exited()) {
                stopped = true;
                PspWorker::stop_all();
                break;
            }
            for (auto w : store_workers) {
                if (w->has_exited()) {
                    stopped = true;
                    PspWorker::stop_all();
                    break;
                }
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(50));
        }
    }
    n.join();
    if (opts.record_latencies)
        n.dump_times();
    for (auto w : store_workers) {
        w->join();
        if (opts.record_store_latencies) {
            w->dump_times();
        }
        delete w;
    }

    log_info("Execution complete");
}

ext: cc

max_stars_repo_name: isaac-ped/demikernel

max_issues_repo_name: isaac-ped/demikernel

max_forks_repo_name: isaac-ped/demikernel