text:
/*****************************************************************************/
/*                                    XDMF                                   */
/*                       eXtensible Data Model and Format                    */
/*                                                                           */
/*  Id : XdmfTemplate.cpp                                                    */
/*                                                                           */
/*  Author:                                                                  */
/*     Andrew Burns                                                          */
/*     andrew.j.burns2@us.army.mil                                           */
/*     US Army Research Laboratory                                           */
/*     Aberdeen Proving Ground, MD                                           */
/*                                                                           */
/*     Copyright @ 2013 US Army Research Laboratory                          */
/*     All Rights Reserved                                                   */
/*     See Copyright.txt for details                                         */
/*                                                                           */
/*     This software is distributed WITHOUT ANY WARRANTY; without            */
/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
/*     for more information.                                                 */
/*                                                                           */
/*****************************************************************************/

#include <sstream>
#include <utility>
#include <climits>
#include <set>
#include "XdmfArray.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfBinaryController.hpp"
#include "XdmfItem.hpp"
#include "XdmfItemFactory.hpp"
#include "XdmfReader.hpp"
#include "XdmfTemplate.hpp"
#include "XdmfError.hpp"
#include "XdmfVisitor.hpp"
#include "XdmfWriter.hpp"

#include "XdmfSystemUtils.hpp"

#include <boost/tokenizer.hpp>

#include <stdio.h>

std::vector<shared_ptr<XdmfHeavyDataController> >
getStepControllers(unsigned int stepId,
                   std::vector<unsigned int> stepDims,
                   std::vector<shared_ptr<XdmfHeavyDataController> > datasetControllers)
{
  std::vector<shared_ptr<XdmfHeavyDataController> > returnVector;
  if (datasetControllers.size() > 0)
  {
    unsigned int sizePerStep = 1;
    for (unsigned int i = 0; i < stepDims.size(); ++i)
    {
      sizePerStep *= stepDims[i];
    }
//    unsigned int offset = (sizePerStep * stepId);
//    unsigned int offsetStepsRemaining = 0;
    unsigned int offset = 0;
    unsigned int offsetStepsRemaining = stepId;
    // grabbing the subset is a little different for each type
    // Right now we assume controllers are of the same type
    unsigned int controllerIndex = 0;
    unsigned int sizeRemaining = sizePerStep;
    unsigned int arrayoffset = 0;
    while (sizeRemaining > 0)
    {
//printf("sizeRemaining = %u\n", sizeRemaining);
      // We don't reset the controller index between runs of the while loop
      // On iterations after the first it should only execute the loop once
      // because offset is set to 0
      while (controllerIndex < datasetControllers.size())
      {
//printf("offset = %u\n", offset);
//printf("%u >= %u\n", offset, datasetControllers[controllerIndex]->getSize());
        //Iterate until we find the controller that the step starts in
        if (offset >= datasetControllers[controllerIndex]->getSize())
        {
          offset -= datasetControllers[controllerIndex]->getSize();
          ++controllerIndex;
        }
        else
        {
          if (offsetStepsRemaining == 0)
          {
            // offset is within the current controller
            break;
          }
          else
          {
            // There are steps left to offset
//printf("accounting for step %d\n", offsetStepsRemaining);
            offset += sizePerStep;
            --offsetStepsRemaining;
          }
        }
      }
//printf("final offset = %u\n", offset);
      std::vector<unsigned int> newDimVector;
      std::vector<unsigned int> newStarts;
//printf("after creating dim vector but before filling it\n");
//printf("%d < %d\n", controllerIndex, datasetControllers.size());
//printf("size left %d\n", sizeRemaining);
      if (offset + sizeRemaining <= datasetControllers[controllerIndex]->getSize())
      {
//printf("step is entirely in the controller\n");
        // step is entirely within this controller
        newStarts.push_back(offset + datasetControllers[controllerIndex]->getStart()[0]); // TODO multidim version
        newDimVector.push_back(sizeRemaining);
        sizeRemaining = 0;
      }
      else
      {
//printf("step is partially in the controller\n");
        if (controllerIndex + 1 >= datasetControllers.size()) {
          // Error, step doesn't fit in the data set provided
          XdmfError::message(XdmfError::FATAL, "Error: Step does not fit in data step provided");
        }
        // step is partially in this controller
        newDimVector.push_back(sizeRemaining -
                               (sizeRemaining - (datasetControllers[controllerIndex]->getSize() - offset)));
        newStarts.push_back(offset+datasetControllers[controllerIndex]->getStart()[0]); // TODO multidim version
        sizeRemaining -= newDimVector[0];
      }
//printf("size for other controllers %d\n", sizeRemaining);
//printf("before creating the new controller\n");
      // Using the remaining space in the controller
      // Slightly differen creation method for each controller
      if (datasetControllers[0]->getName().compare("Binary") == 0) {
        shared_ptr<XdmfBinaryController> createdController =
          XdmfBinaryController::New(datasetControllers[0]->getFilePath(),
                                    datasetControllers[0]->getType(),
                                    shared_dynamic_cast<XdmfBinaryController>(
                                      datasetControllers[0])->getEndian(),
                                    newStarts[0],
                                    newDimVector);
        returnVector.push_back(createdController);
      }
      else if (datasetControllers[0]->getName().compare("HDF") == 0) {
        // TODO
        // The writer should only write to contiguous sets when in this mode.
        // A user would need to do something custom to foul this up.
        std::vector<unsigned int> newStrides;
        newStrides.push_back(1);
        shared_ptr<XdmfHDF5Controller> createdController =
          XdmfHDF5Controller::New(datasetControllers[controllerIndex]->getFilePath(),
                                  shared_dynamic_cast<XdmfHDF5Controller>(
                                    datasetControllers[controllerIndex])->getDataSetPath(),
                                  datasetControllers[0]->getType(),
                                  newStarts,
                                  newStrides,
                                  newDimVector,
                                  shared_dynamic_cast<XdmfHDF5Controller>(
                                    datasetControllers[controllerIndex])->getDataspaceDimensions());
        returnVector.push_back(createdController);
      }
//printf("after creating the new controller\n");
      returnVector[returnVector.size()-1]->setArrayOffset(arrayoffset);
      arrayoffset += returnVector[returnVector.size()-1]->getSize();
      offset = 0;
      ++controllerIndex;
      // Starts at the beggining of the next controller
    }
  }
/*
printf("size of return vector = %d\n", returnVector.size());
for (unsigned int i = 0; i < returnVector.size(); ++i)
{
  shared_ptr<XdmfHDF5Controller> currentController = shared_dynamic_cast<XdmfHDF5Controller>(returnVector[i]);
assert(currentController);
  printf("file = %s\n", currentController->getFilePath().c_str());
  printf("dataset = %s\n", currentController->getDataSetPath().c_str());
  printf("start = %u\n", currentController->getStart()[0]);
  printf("dimension = %u\n", currentController->getDimensions()[0]);
  printf("dataspace = %u\n", currentController->getDataspaceDimensions()[0]);
}
*/
  return returnVector;
}

std::vector<shared_ptr<XdmfHeavyDataController> >
getControllersExcludingStep(unsigned int stepId,
                            std::vector<unsigned int> stepDims,
                            std::vector<shared_ptr<XdmfHeavyDataController> > datasetControllers)
{
  std::vector<shared_ptr<XdmfHeavyDataController> > returnVector;
  if (datasetControllers.size() > 0)
  {
    unsigned int sizePerStep = 1;
    for (unsigned int i = 0; i < stepDims.size(); ++i)
    {
      sizePerStep *= stepDims[i];
    }
    unsigned int offset = sizePerStep * stepId;
    unsigned int sizeRemaining = sizePerStep;
//printf("base offset = %u\nstarting size remaining = %u\ncutting from %u controllers\n", offset, sizeRemaining, datasetControllers.size());
    // grabbing the subset is a little different for each type
    // Right now we assume controllers are of the same type
    for (unsigned int controllerIndex = 0; controllerIndex < datasetControllers.size(); ++controllerIndex)
    {
//printf("offset = %u out of controller size %u\n", offset, datasetControllers[controllerIndex]->getSize());
      if (offset >= datasetControllers[controllerIndex]->getSize())
      {
        // The removed step isn't in the controller provided
        // Simply add it back into the return set
        returnVector.push_back(datasetControllers[controllerIndex]);
        // then subtract the size from the offset
        offset -= datasetControllers[controllerIndex]->getSize();
      }
      else
      {
        // The removed step is inside the controller provided
        if (offset > 0)
        {
//printf("removed step is inside this controller\n");
          // If offset is greater than zero the controller has a section chopped off the front
          std::vector<unsigned int> newDim;
          newDim.push_back(offset);
          // Dataspace is the same
          // stride is the same
          // start is the same
          // TODO dims is reduced to just cover the offset size
          if (datasetControllers[controllerIndex]->getName().compare("Binary") == 0) {
            shared_ptr<XdmfBinaryController> createdController =
              XdmfBinaryController::New(datasetControllers[controllerIndex]->getFilePath(),
                                        datasetControllers[controllerIndex]->getType(),
                                        shared_dynamic_cast<XdmfBinaryController>(
                                          datasetControllers[controllerIndex])->getEndian(),
                                        shared_dynamic_cast<XdmfBinaryController>(
                                          datasetControllers[controllerIndex])->getSeek(),
                                        datasetControllers[controllerIndex]->getStart(),
                                        datasetControllers[controllerIndex]->getStride(),
                                        newDim,
                                        datasetControllers[controllerIndex]->getDataspaceDimensions());
            returnVector.push_back(createdController);
          }
          else if (datasetControllers[controllerIndex]->getName().compare("HDF") == 0) {
            // TODO
            // The writer should only write to contiguous sets when in this mode.
            // A user would need to do something custom to foul this up.
            shared_ptr<XdmfHDF5Controller> createdController =
              XdmfHDF5Controller::New(datasetControllers[controllerIndex]->getFilePath(),
                                      shared_dynamic_cast<XdmfHDF5Controller>(
                                        datasetControllers[controllerIndex])->getDataSetPath(),
                                      datasetControllers[controllerIndex]->getType(),
                                      datasetControllers[controllerIndex]->getStart(),
                                      datasetControllers[controllerIndex]->getStride(),
                                      newDim,
                                      shared_dynamic_cast<XdmfHDF5Controller>(
                                        datasetControllers[controllerIndex])->getDataspaceDimensions());
            returnVector.push_back(createdController);
          }
          // These are the stats for the first half of the dataset
          if (sizeRemaining <= datasetControllers[controllerIndex]->getSize() - offset)
          {
            // The controller is large enough to need to be split into two controllers
            std::vector<unsigned int> newStart; //TODO we're assuming one dim for now
            newStart.push_back(datasetControllers[controllerIndex]->getStart()[0] +sizeRemaining + offset);
            std::vector<unsigned int> newDim;
            newDim.push_back(datasetControllers[controllerIndex]->getSize() - (sizeRemaining + offset));
            // These are the stats of the second controller
            sizeRemaining = 0;
            if (datasetControllers[controllerIndex]->getName().compare("Binary") == 0) {
              shared_ptr<XdmfBinaryController> createdController =
                XdmfBinaryController::New(datasetControllers[controllerIndex]->getFilePath(),
                                          datasetControllers[controllerIndex]->getType(),
                                          shared_dynamic_cast<XdmfBinaryController>(
                                            datasetControllers[controllerIndex])->getEndian(),
                                          shared_dynamic_cast<XdmfBinaryController>(
                                            datasetControllers[controllerIndex])->getSeek(),
                                          newStart,
                                          datasetControllers[controllerIndex]->getStride(),
                                          newDim,
                                          datasetControllers[controllerIndex]->getDataspaceDimensions());
              returnVector.push_back(createdController);
            }
            else if (datasetControllers[controllerIndex]->getName().compare("HDF") == 0) {
              // TODO
              // The writer should only write to contiguous sets when in this mode.
              // A user would need to do something custom to foul this up.
              shared_ptr<XdmfHDF5Controller> createdController =
                XdmfHDF5Controller::New(datasetControllers[controllerIndex]->getFilePath(),
                                        shared_dynamic_cast<XdmfHDF5Controller>(
                                          datasetControllers[controllerIndex])->getDataSetPath(),
                                        datasetControllers[controllerIndex]->getType(),
                                        newStart,
                                        datasetControllers[controllerIndex]->getStride(),
                                        newDim,
                                        shared_dynamic_cast<XdmfHDF5Controller>(
                                          datasetControllers[controllerIndex])->getDataspaceDimensions());
              returnVector.push_back(createdController);
            }
          }
          else {
            // The controller only contains part of the dataset
            sizeRemaining -= (datasetControllers[controllerIndex]->getSize() - offset);
          }
          offset = 0;
        }
        else
        {
          // in the case of 0 offset, we either need to trim from the front or just use the whole controller
          if (sizeRemaining > 0)
          {
            if (sizeRemaining < datasetControllers[controllerIndex]->getSize())
            {
              std::vector<unsigned int> newStart;
              newStart.push_back(sizeRemaining);
              std::vector<unsigned int> newDim;
              newDim.push_back(datasetControllers[controllerIndex]->getSize() - sizeRemaining);
              sizeRemaining = 0;
              if (datasetControllers[controllerIndex]->getName().compare("Binary") == 0) {
                shared_ptr<XdmfBinaryController> createdController =
                  XdmfBinaryController::New(datasetControllers[controllerIndex]->getFilePath(),
                                            datasetControllers[controllerIndex]->getType(),
                                            shared_dynamic_cast<XdmfBinaryController>(
                                            datasetControllers[controllerIndex])->getEndian(),
                                            shared_dynamic_cast<XdmfBinaryController>(
                                              datasetControllers[controllerIndex])->getSeek(),
                                            newStart,
                                            datasetControllers[controllerIndex]->getStride(),
                                            newDim,
                                            datasetControllers[controllerIndex]->getDataspaceDimensions());
                returnVector.push_back(createdController);
              }
              else if (datasetControllers[controllerIndex]->getName().compare("HDF") == 0) {
                // TODO
                // The writer should only write to contiguous sets when in this mode.
                // A user would need to do something custom to foul this up.
                shared_ptr<XdmfHDF5Controller> createdController =
                  XdmfHDF5Controller::New(datasetControllers[controllerIndex]->getFilePath(),
                                          shared_dynamic_cast<XdmfHDF5Controller>(
                                            datasetControllers[controllerIndex])->getDataSetPath(),
                                          datasetControllers[controllerIndex]->getType(),
                                          newStart,
                                          datasetControllers[controllerIndex]->getStride(),
                                          newDim,
                                          shared_dynamic_cast<XdmfHDF5Controller>(
                                            datasetControllers[controllerIndex])->getDataspaceDimensions());
                returnVector.push_back(createdController);
              }
            }
            else {
              sizeRemaining -= datasetControllers[controllerIndex]->getSize();
            }
          }
          else
          {
            // Just use the current controller
            returnVector.push_back(datasetControllers[controllerIndex]);
          }
        }
      }
    }
  }
  return returnVector;
}

class XdmfArrayGatherer : public XdmfVisitor, public Loki::Visitor<XdmfArray>
{
  public:

    static shared_ptr<XdmfArrayGatherer>
    New(std::vector<XdmfArray *> * storageVector)
    {
       shared_ptr<XdmfArrayGatherer> p(new XdmfArrayGatherer(storageVector));
       return p;
    }

    ~XdmfArrayGatherer()
    {
    }

    virtual void
    visit(XdmfArray & array,
          const shared_ptr<XdmfBaseVisitor> visitor)
    {
      ++mDepth;
      if (!array.isInitialized())
      {
//      mStorage->push_back(&array);
        mArrayCollection.insert(&array);
      }
      array.traverse(visitor);
      --mDepth;
      if (mDepth == 0)
      {
        moveToStorage();
      }
    }

    virtual void
    visit(XdmfItem & item,
          const shared_ptr<XdmfBaseVisitor> visitor)
    {
      ++mDepth;
      item.traverse(visitor);
      --mDepth;
      if (mDepth == 0)
      {
        moveToStorage();
      }
    }

    void
    moveToStorage()
    {
      for (std::set<XdmfArray *>::iterator iter = mArrayCollection.begin();
           iter != mArrayCollection.end();
           ++iter)
      {
        mStorage->push_back(*iter);
      }
    }

  private:

    XdmfArrayGatherer(std::vector<XdmfArray *> * storageVector) :
      mDepth(0),
      mStorage(storageVector)
    {
    }

  unsigned int mDepth;
  std::set<XdmfArray *> mArrayCollection;
  std::vector<XdmfArray *> * mStorage;
};

shared_ptr<XdmfTemplate>
XdmfTemplate::New()
{
  shared_ptr<XdmfTemplate> p(new XdmfTemplate());
  return p;
}

XdmfTemplate::XdmfTemplate() :
  mHeavyWriter(shared_ptr<XdmfHeavyDataWriter>()),
  mBase(shared_ptr<XdmfItem>()),
  mCurrentStep(-1),
  mNumSteps(0),
  mItemFactory(shared_ptr<XdmfItemFactory>())
{
}

XdmfTemplate::XdmfTemplate(XdmfTemplate & refTemplate) :
  XdmfItem(refTemplate),
  mBase(refTemplate.mBase),
  mCurrentStep(refTemplate.mCurrentStep),
  mNumSteps(refTemplate.mNumSteps),
  mItemFactory(refTemplate.mItemFactory)
{
}

XdmfTemplate::~XdmfTemplate()
{
}

const std::string XdmfTemplate::ItemTag = "Template";

unsigned int
XdmfTemplate::addStep()
{
  mCurrentStep = this->getNumberSteps();
  std::stringstream datastream;
  if (mTrackedArrays.size() < 1) {
    XdmfError::message(XdmfError::FATAL,
                       "Error: XdmfTemplate attempting to add a step when no arrays are tracked");
  }
  for (unsigned int arrayIndex = 0; arrayIndex < mTrackedArrays.size(); ++arrayIndex) {
    if (mTrackedArrayTypes.size() < mTrackedArrays.size()){
      mTrackedArrayTypes.resize(mTrackedArrays.size());
    }
    if (mTrackedArrayDims.size() < mTrackedArrays.size()){
      mTrackedArrayDims.resize(mTrackedArrays.size());
    }
    if (!mTrackedArrayTypes[arrayIndex]) {
      mTrackedArrayTypes[arrayIndex] = mTrackedArrays[arrayIndex]->getArrayType();
    }
    if (mTrackedArrayDims[arrayIndex].size() == 0) {
      mTrackedArrayDims[arrayIndex] = mTrackedArrays[arrayIndex]->getDimensions();
    }
    // Write the tracked arrays to heavy data if they aren't already
    if (mHeavyWriter) {
      bool revertToAppend = false;
      if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append) {
        // Set to original heavy data controllers for append
        if (mDataControllers.size() > arrayIndex)
        {
          if (mDataControllers[arrayIndex].size() > 0)
          {
            while (mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers() > 0) {
              mTrackedArrays[arrayIndex]->removeHeavyDataController(0);
            }
            for (unsigned int i = 0; i < mDataControllers[arrayIndex].size(); ++i)
            {
              mTrackedArrays[arrayIndex]->insert(mDataControllers[arrayIndex][i]);
            }
          }
        }
        else
        {
          // Creating new Dataset
          // Set to default mode so that it doesn't overlap
          mHeavyWriter->setMode(XdmfHeavyDataWriter::Default);
          revertToAppend = true;
        }
      }
      else if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
        // Use the controller that references the subset that will be overwritten
        if (!(arrayIndex < mDataControllers.size()))
        {
          // When in overwrite mode the dataset must be preallocated
          XdmfError::message(XdmfError::FATAL, "Error: Heavy Data dataset must be preallocated "
                                               "to use Hyperslab mode Templates");
        }
        std::vector<shared_ptr<XdmfHeavyDataController> > overwriteControllers =
          getStepControllers(mCurrentStep, mTrackedArrayDims[arrayIndex], mDataControllers[arrayIndex]);
        mTrackedArrays[arrayIndex]->setHeavyDataController(overwriteControllers);
      }
      mTrackedArrays[arrayIndex]->accept(mHeavyWriter);
      if (revertToAppend)
      {
        mHeavyWriter->setMode(XdmfHeavyDataWriter::Append);
      }
    }
    datastream.str(std::string());
    for (unsigned int controllerIndex = 0; controllerIndex < mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers(); ++controllerIndex) {
      // TODO throw error if controller types don't match
      // For each heavy data controller
      std::string writerPath = XdmfSystemUtils::getRealPath(mHeavyWriter->getFilePath());
      std::string heavyDataPath =
        mTrackedArrays[arrayIndex]->getHeavyDataController(controllerIndex)->getFilePath();
      size_t index = heavyDataPath.find_last_of("/\\");
      if(index != std::string::npos) {
        // If path is not a folder
        // put the directory path into this variable
        const std::string heavyDataDir = heavyDataPath.substr(0, index + 1);
        // If the directory is in the XML File Path
        if(writerPath.find(heavyDataDir) == 0) {
          heavyDataPath =
            heavyDataPath.substr(heavyDataDir.size(),
                                 heavyDataPath.size() - heavyDataDir.size());
          // Pull the file off of the end and place it in the DataPath
        }
        // Otherwise the full path is required
      }
      datastream << heavyDataPath;
      datastream << mTrackedArrays[arrayIndex]->getHeavyDataController(controllerIndex)->getDescriptor();
        datastream << "|";
        for (unsigned int i = 0; i < mTrackedArrays[arrayIndex]->getHeavyDataController(controllerIndex)->getDimensions().size(); ++i) {
          datastream << mTrackedArrays[arrayIndex]->getHeavyDataController(controllerIndex)->getDimensions()[i];
          if (i < mTrackedArrays[arrayIndex]->getHeavyDataController(controllerIndex)->getDimensions().size() - 1) {
            datastream << " ";
          }
        }
        if (controllerIndex + 1 < mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers()) {
          datastream << "|";
        }
    }
    if (mHeavyWriter) {
      if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append) {
        if (mDataControllers.size() > arrayIndex)
        {
          // If controllers already exist
          // Store the overarching controllers again
          mDataControllers[arrayIndex].clear();
          for (unsigned int i = 0; i < mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers(); ++i)
          {
            mDataControllers[arrayIndex].push_back(mTrackedArrays[arrayIndex]->getHeavyDataController(i));
          }
          // Clear controllers from the array
          while (mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers() > 0) {
            mTrackedArrays[arrayIndex]->removeHeavyDataController(0);
          }
          // If append set controller to the correct subsection of the whole
          std::vector<shared_ptr<XdmfHeavyDataController> > readControllers = getStepControllers(mCurrentStep, mTrackedArrayDims[arrayIndex], mDataControllers[arrayIndex]);
          mTrackedArrays[arrayIndex]->setHeavyDataController(readControllers);
          // Replace with updated description
          mDataDescriptions[arrayIndex] = datastream.str();
        }
        else
        {
          // If a new dataset, as normal
          mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
          for (unsigned int i = 0; i < mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers(); ++i) {
            mDataControllers[mDataControllers.size()-1].push_back((mTrackedArrays[arrayIndex]->getHeavyDataController(i)));
          }
          if (mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers() > 0) {
            mDataTypes.push_back(mTrackedArrays[arrayIndex]->getHeavyDataController(0)->getName());
            mDataDescriptions.push_back(datastream.str());
          }
        }
      }
      else if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
        // Hyperslab is already storing the base controller
        // So nothing is done here, the controller should already be pointing to the correct location
        // TODO, to what the file index was before the add, as opposed to 0
        mHeavyWriter->setFileIndex(0);
      }
      else {
        mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
        for (unsigned int i = 0; i < mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers(); ++i) {
          mDataControllers[mDataControllers.size()-1].push_back((mTrackedArrays[arrayIndex]->getHeavyDataController(i)));
        }
        if (mTrackedArrays[arrayIndex]->getNumberHeavyDataControllers() > 0) {
          mDataTypes.push_back(mTrackedArrays[arrayIndex]->getHeavyDataController(0)->getName());
          mDataDescriptions.push_back(datastream.str());
        }
      }
    }
    else {
      mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
      mDataTypes.push_back("XML");
      mDataDescriptions.push_back(mTrackedArrays[arrayIndex]->getValuesString());
    }
  }
  ++mNumSteps;
  this->setIsChanged(true);
  return mCurrentStep;
}

void
XdmfTemplate::clearStep()
{
  for (unsigned int i = 0; i < mTrackedArrays.size(); ++i) {
    mTrackedArrays[i]->release();
    while (mTrackedArrays[i]->getNumberHeavyDataControllers() > 0) {
      mTrackedArrays[i]->removeHeavyDataController(0);
    }
  }
  mCurrentStep = -1;
}

shared_ptr<XdmfItem>
XdmfTemplate::getBase()
{
  return mBase;
}

shared_ptr<XdmfHeavyDataWriter>
XdmfTemplate::getHeavyDataWriter()
{
  return mHeavyWriter;
}

std::map<std::string, std::string>
XdmfTemplate::getItemProperties() const
{
  std::map<std::string, std::string> templateProperties;
/*
  std::stringstream value;
  value << mValue;
  timeProperties.insert(std::make_pair("Value", value.str()));
*/
  return templateProperties;
}

std::string
XdmfTemplate::getItemTag() const
{
  return ItemTag;
}

unsigned int
XdmfTemplate::getNumberSteps() const
{
  return mNumSteps;
}

unsigned int
XdmfTemplate::getNumberTrackedArrays() const
{
  return mTrackedArrays.size();
}

XdmfArray *
XdmfTemplate::getTrackedArray(unsigned int index)
{
  return mTrackedArrays[index];
}

void
XdmfTemplate::populateItem(const std::map<std::string, std::string> & itemProperties,
                           const std::vector<shared_ptr<XdmfItem> > & childItems,
                           const XdmfCoreReader * const reader)
{
  XdmfItem::populateItem(itemProperties, childItems, reader);

  // The first child item is the base
  mBase = childItems[0];
  mCurrentStep = 0;

  std::string referenceHDF5File = "";

  if (childItems.size() > 1) {
    for(std::vector<shared_ptr<XdmfItem> >::const_iterator iter =
          childItems.begin() + 1;
        iter != childItems.end();
        ++iter) {
      if(shared_ptr<XdmfArray> array = shared_dynamic_cast<XdmfArray>(*iter)) {
        // Pull hdf5 reference data from the first provided array
        if (array->getNumberHeavyDataControllers() > 0 & !mHeavyWriter) {
          mHeavyWriter = reader->generateHeavyDataWriter(array->getHeavyDataController(0)->getName(), array->getHeavyDataController(0)->getFilePath());
        }
        if (array->getName().compare("Data Description") == 0) {
          // Split description into substrings based on the " character
          array->read();

          std::string descriptionString;
          if (array->getArrayType() == XdmfArrayType::Int8())
          {
            descriptionString = std::string((char *)array->getValuesInternal());
          }
          else if (array->getArrayType() == XdmfArrayType::String())
          {
            std::stringstream descriptionstream;
            for (unsigned int i = 0; i < array->getSize(); ++i)
            {
              descriptionstream << array->getValue<std::string>(i);
              if (i < array->getSize() - 1)
              {
                descriptionstream << '|';
              }
            }
            descriptionString = descriptionstream.str();
          }

          size_t index = descriptionString.find_first_of("\"");
          size_t previousIndex = 0;

          if (index != std::string::npos) {
            // Removing the prepended "
            previousIndex = index + 1;
            index = descriptionString.find_first_of("\"", previousIndex);
          }

          while (index != std::string::npos) {
            std::string type = descriptionString.substr(previousIndex, index - previousIndex);
            mDataTypes.push_back(type);
            previousIndex = index + 1;
            index = descriptionString.find_first_of("\"", previousIndex);
            if (index - previousIndex > 0) {
              std::string description;
              description = descriptionString.substr(previousIndex, index - previousIndex);
              mDataDescriptions.push_back(description);
              // create controllers here based on the type/description?
              // Is array type already filled?
              // Potentially call "fillControllers" after populating?
              if (index != std::string::npos) {
                previousIndex = index + 1;
                index = descriptionString.find_first_of("\"", previousIndex);
              }
            }
            else {
              XdmfError::message(XdmfError::FATAL, "Error: Type without a description in XdmfTemplate::populateItem");
            }
          }
        }
        else {
          mTrackedArrays.push_back(array.get());
          mTrackedArrayDims.push_back(array->getDimensions());
          mTrackedArrayTypes.push_back(array->getArrayType());
        }
      }
    }
  }
  for (unsigned int i = 0; i < mDataTypes.size(); ++i)
  {
    mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
  }
  mDataControllers.resize(mDataTypes.size());
  if (!mItemFactory) {
    mItemFactory = XdmfItemFactory::New();
  }
  std::map<std::string, std::string> populateProperties;
  if (mHeavyWriter) {
    // The heavy writer provides the XMLDir, which is used to get full paths for the controllers
    // It is assumed that the files that the controllers reference are in the same directory
    // as the file that the writer references
    std::string filepath = XdmfSystemUtils::getRealPath(mHeavyWriter->getFilePath());
    size_t index = filepath.find_last_of("/\\");
    filepath = filepath.substr(0, index + 1);
    populateProperties["XMLDir"] = filepath;
  }
  else
  {
    // Error because a writer is required? TODO
  }
  for (unsigned int i = 0;  i < mDataDescriptions.size(); ++i) {
    populateProperties["Content"] = mDataDescriptions[i];
    std::vector<shared_ptr<XdmfHeavyDataController> > readControllers =
      reader->generateHeavyDataControllers(populateProperties, mTrackedArrayDims[i % mTrackedArrays.size()], mTrackedArrayTypes[i % mTrackedArrays.size()], mDataTypes[i]);
    if (readControllers.size() > 0) {
      // Heavy data controllers reference the data
      for (unsigned int j = 0; j < readControllers.size(); ++j) {
        mDataControllers[i].push_back(readControllers[j]);
      }
    }
  }
  // Compare the first set of controllers to the size of the first array
  unsigned int controllerTotal = 0;

  for (unsigned int i = 0; i < mDataControllers[0].size(); ++i)
  {
    unsigned int previousTotal = controllerTotal;
    controllerTotal += mDataControllers[0][i]->getSize();
    if (previousTotal != controllerTotal - mDataControllers[0][i]->getSize())
    {
      controllerTotal = UINT_MAX;
      break;
    }
  }
  // If the array is smaller, set the writer to append.
  if (controllerTotal > mTrackedArrays[0]->getSize())
  {
    mHeavyWriter->setMode(XdmfHeavyDataWriter::Append);
    mNumSteps = 0;
    unsigned int currentTotal = 0;
    for (unsigned int controllerIndex = 0; controllerIndex < mDataControllers[0].size(); ++controllerIndex)
    {
      currentTotal += mDataControllers[0][controllerIndex]->getSize();
      while (currentTotal >= mTrackedArrays[0]->getSize())
      {
        currentTotal -= mTrackedArrays[0]->getSize();
        ++mNumSteps;
      }
    }
//    mNumSteps = controllerTotal / mTrackedArrays[0]->getSize();
  }
  else {
    mNumSteps = mDataControllers.size() / mTrackedArrays.size();
  }
  this->setStep(0);
}

void
XdmfTemplate::preallocateSteps(unsigned int numSteps)
{
  // Preallocate steps based on the current size of the arrays
  // Use a temporary array to write data to hdf5
  shared_ptr<XdmfArray> tempArray = XdmfArray::New();
  // Set to Default mode so that the new allocations are in new locations
  mHeavyWriter->setMode(XdmfHeavyDataWriter::Default);
  int preallocatedSize = 0;
  int numberSetsPreallocated = 0;
  std::stringstream datastream;
  for (unsigned int i = 0; i < mTrackedArrays.size(); ++i) {
    preallocatedSize = mTrackedArrays[i]->getSize() * numSteps;
///*
    numberSetsPreallocated = 1;

    int adjustment = 1;
    while (preallocatedSize / (numSteps/adjustment) != mTrackedArrays[i]->getSize() || 0 > (int)preallocatedSize) {
//      XdmfError::message(XdmfError::WARNING, "Overflow error");
      ++adjustment;
      while (numSteps % adjustment != 0) {
//printf("%d / %d remainder %d\n", numSteps, adjustment, (numSteps % adjustment));
        ++adjustment;
      }
      numberSetsPreallocated = numberSetsPreallocated * adjustment;
      preallocatedSize = mTrackedArrays[i]->getSize() * (numSteps / adjustment);
//printf("%d / %d = %d ?= %d\n", preallocatedSize , (numSteps/adjustment), preallocatedSize / (numSteps/adjustment), mTrackedArrays[i]->getSize());
    }

    // If adjusted, split one more time, to ensure that the dataset fits.
    if (adjustment > 1) {
      ++adjustment;
      while (numSteps % adjustment != 0) {
        ++adjustment;
      }
      numberSetsPreallocated = numberSetsPreallocated * adjustment;
      preallocatedSize = mTrackedArrays[i]->getSize() * (numSteps / adjustment);
    }

    bool allocateSucceeded = false;
    while (!allocateSucceeded)
    {
      try {
        mHeavyWriter->openFile();
//printf("now size %d allocated %d times\n", preallocatedSize, numberSetsPreallocated);
        for (unsigned int allocateIteration = 0;
             allocateIteration < numberSetsPreallocated;
             ++allocateIteration)
        {
//printf("allocating subsection %u\n", allocateIteration);
//*/
//printf("initializing base array\n");
          tempArray->initialize(mTrackedArrays[i]->getArrayType(), preallocatedSize);
//printf("writing subsection");
          tempArray->accept(mHeavyWriter);
//printf("subsection written\n");
//          mHeavyWriter->clearCache();
///*
          if (mDataControllers.size() <= i) {
            mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
          }
          // Clean Array for the next iteration
          while (tempArray->getNumberHeavyDataControllers() > 0) {
            mDataControllers[i].push_back(tempArray->getHeavyDataController(0));
            if (mDataTypes.size() <= i) {
              mDataTypes.push_back(tempArray->getHeavyDataController(0)->getName());
            }
            tempArray->removeHeavyDataController(0);
          }
          tempArray->release();
//*/
///*
//printf("moving to next allocation\n");
        }
        mHeavyWriter->closeFile();
        allocateSucceeded = true;
//*/
//TODO catch the controllers created by this.
///*
      }
      catch (...)
      {
        while (tempArray->getNumberHeavyDataControllers() > 0) {
          tempArray->removeHeavyDataController(0);
        }
        tempArray->release();
//        XdmfError::message(XdmfError::WARNING, "Array Allocation failed");
        int factor = 2;
        while (preallocatedSize % factor != 0) {
//printf("%d / %d remainder %d\n", preallocatedSize, factor, (preallocatedSize % factor));
          factor = factor + 1;
        }
//printf("adjusted factor %d\n", factor);
        numberSetsPreallocated = numberSetsPreallocated * factor;
        preallocatedSize = (preallocatedSize) / factor;
//printf("now size %d allocated %d times\n", preallocatedSize, numberSetsPreallocated);
      }
    }
//printf("Done writing to hdf5\n");
//*/
/*
    // Transfer controllers to the appropriate slot before clearing them
    if (mDataControllers.size() <= i) {
      mDataControllers.push_back(std::vector<shared_ptr<XdmfHeavyDataController> >());
    }
    // Clean Array for the next iteration
    while (tempArray->getNumberHeavyDataControllers() > 0) {
      mDataControllers[i].push_back(tempArray->getHeavyDataController(0));
      if (mDataTypes.size() <= i) {
        mDataTypes.push_back(tempArray->getHeavyDataController(0)->getName());
      }
      tempArray->removeHeavyDataController(0);
    }
    tempArray->release();
*/
    datastream.str(std::string());
    for (unsigned int controllerIndex = 0; controllerIndex < mDataControllers[i].size(); ++controllerIndex) {
      // TODO throw error if controller types don't match
      // For each heavy data controller
      std::string writerPath = XdmfSystemUtils::getRealPath(mHeavyWriter->getFilePath());
      std::string heavyDataPath =
        mDataControllers[i][controllerIndex]->getFilePath();
      size_t index = heavyDataPath.find_last_of("/\\");
      if(index != std::string::npos) {
        // If path is not a folder
        // put the directory path into this variable
        const std::string heavyDataDir = heavyDataPath.substr(0, index + 1);
        // If the directory is in the XML File Path
        if(writerPath.find(heavyDataDir) == 0) {
          heavyDataPath =
            heavyDataPath.substr(heavyDataDir.size(),
                                 heavyDataPath.size() - heavyDataDir.size());
          // Pull the file off of the end and place it in the DataPath
        }
        // Otherwise the full path is required
      }
      datastream << heavyDataPath;
      datastream << mDataControllers[i][controllerIndex]->getDescriptor();
      datastream << "|";
      for (unsigned int j = 0; j < mDataControllers[i][controllerIndex]->getDimensions().size(); ++j) {
        datastream << mDataControllers[i][controllerIndex]->getDimensions()[j];
        if (j < mDataControllers[i][controllerIndex]->getDimensions().size() - 1) {
          datastream << " ";
        }
      }
      if (controllerIndex + 1 < mDataControllers[i].size()) {
        datastream << "|";
      }
    }
    mDataDescriptions.push_back(datastream.str());
  }
  // To end set the heavy writer to overwrite mode
  mHeavyWriter->setMode(XdmfHeavyDataWriter::Hyperslab);
}

void
XdmfTemplate::removeStep(unsigned int stepId)
{
  if (stepId < this->getNumberSteps()) {
    for (unsigned int i = 0; i < mTrackedArrays.size(); ++i) {
      if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append ||
          mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
        std::vector<shared_ptr<XdmfHeavyDataController> > replacementControllers = getControllersExcludingStep(stepId, mTrackedArrayDims[i], mDataControllers[i]);
        for (unsigned int j = 0; j < replacementControllers.size(); ++j)
        {
          if (mDataControllers[i].size() > j) {
            mDataControllers[i][j] = replacementControllers[j];
          }
          else {
            mDataControllers[i].push_back(replacementControllers[j]);
          }
        }
      }
      else {
        mDataTypes.erase(mDataTypes.begin() + (stepId*mTrackedArrays.size()));
        mDataDescriptions.erase(mDataDescriptions.begin() + (stepId*mTrackedArrays.size()));
        mDataControllers.erase(mDataControllers.begin() + (stepId*mTrackedArrays.size()));
      }
    }
    --mNumSteps;
  }
  mCurrentStep = -1;
  this->setIsChanged(true);
}

void
XdmfTemplate::setBase(shared_ptr<XdmfItem> newBase)
{
  shared_ptr<XdmfArrayGatherer> accumulator = XdmfArrayGatherer::New(&mTrackedArrays);
  newBase->accept(accumulator);
  mBase = newBase;
  this->setIsChanged(true);
}

void
XdmfTemplate::setHeavyDataWriter(shared_ptr<XdmfHeavyDataWriter> writer)
{
  mHeavyWriter = writer;
}

void
XdmfTemplate::setStep(unsigned int stepId)
{
  if (stepId != mCurrentStep) {
    if (!mItemFactory) {
      mItemFactory = XdmfItemFactory::New();
    }
    if (stepId < this->getNumberSteps()) {
      for (unsigned int i = 0; i < mTrackedArrays.size(); ++i) {
        unsigned int arrayIndex = 0;
        if (mHeavyWriter) {
            if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append ||
                mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
              arrayIndex = i;
            }
            else {
              arrayIndex = i+(stepId*mTrackedArrays.size());
            }
          }
          else {
            arrayIndex = i+(stepId*mTrackedArrays.size());
        }
        if (mDataControllers[arrayIndex].size() > 0) {
          if(mHeavyWriter) {
            if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append ||
                mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
              std::vector<shared_ptr<XdmfHeavyDataController> > insertVector =
                getStepControllers(stepId, mTrackedArrayDims[i], mDataControllers[i]);
               mTrackedArrays[i]->setHeavyDataController(insertVector);
            }
            else {
              mTrackedArrays[i]->setHeavyDataController(mDataControllers[i+(stepId*mTrackedArrays.size())]);
            }
          }
          else {
            mTrackedArrays[i]->setHeavyDataController(mDataControllers[i+(stepId*mTrackedArrays.size())]);
          }
        }
        else {
          std::map<std::string, std::string> populateProperties;
          if (mHeavyWriter) {
            // The heavy writer provides the XMLDir, which is used to get full paths for the controllers
            // It is assumed that the files that the controllers reference are in the same directory
            // as the file that the writer references
            std::string filepath = XdmfSystemUtils::getRealPath(mHeavyWriter->getFilePath());
            size_t index = filepath.find_last_of("/\\");
            filepath = filepath.substr(0, index + 1);
            populateProperties["XMLDir"] = filepath;
          }
          populateProperties["Content"] = mDataDescriptions[arrayIndex];
          std::vector<shared_ptr<XdmfHeavyDataController> > readControllers;
          if (mHeavyWriter) {
            if (mHeavyWriter->getMode() == XdmfHeavyDataWriter::Append ||
                mHeavyWriter->getMode() == XdmfHeavyDataWriter::Hyperslab) {
              std::vector<shared_ptr<XdmfHeavyDataController> > totalControllers =
                mItemFactory->generateHeavyDataControllers(populateProperties, mTrackedArrayDims[i], mTrackedArrayTypes[i], mDataTypes[i+(stepId*mTrackedArrays.size())]);
              readControllers = getStepControllers(stepId, mTrackedArrayDims[i], totalControllers);
            }
            else {
              readControllers = mItemFactory->generateHeavyDataControllers(populateProperties, mTrackedArrayDims[i], mTrackedArrayTypes[i], mDataTypes[i+(stepId*mTrackedArrays.size())]);
            }
          }
          else {
            readControllers = mItemFactory->generateHeavyDataControllers(populateProperties, mTrackedArrayDims[i], mTrackedArrayTypes[i], mDataTypes[i+(stepId*mTrackedArrays.size())]);
          }
          if (readControllers.size() > 0) {
            // Heavy data controllers reference the data
            mTrackedArrays[i]->setHeavyDataController(readControllers);
            mDataControllers[arrayIndex] = readControllers;
          }
          else {
            // Data is contained in the content
            std::string content = mDataDescriptions[i+(stepId*mTrackedArrays.size())];

            mTrackedArrays[i]->initialize(mTrackedArrayTypes[i], mTrackedArrayDims[i]);

            unsigned int index = 0;
            boost::char_separator<char> sep(" \t\n");
            boost::tokenizer<boost::char_separator<char> > valtokens(content, sep);
            if(mTrackedArrayTypes[i] == XdmfArrayType::String()) {
              for(boost::tokenizer<boost::char_separator<char> >::const_iterator
                    iter = valtokens.begin();
                  iter != valtokens.end();
                  ++iter, ++index) {
                mTrackedArrays[i]->insert(index, *iter);
              }
            }
            else {
              for(boost::tokenizer<boost::char_separator<char> >::const_iterator
                    iter = valtokens.begin();
                  iter != valtokens.end();
                  ++iter, ++index) {
                mTrackedArrays[i]->insert(index, atof((*iter).c_str()));
              }
            }
          }
        }
      }
    }
    else {
      XdmfError::message(XdmfError::FATAL, "Error: Template attempting to load invalid step");
    }
    mCurrentStep = stepId;
  }
}

void
XdmfTemplate::trackArray(shared_ptr<XdmfArray> newArray)
{
  bool found = false;

  for (unsigned int i = 0; i < mTrackedArrays.size() && !found; ++i) {
    if (mTrackedArrays[i] == newArray.get()) {
      found = true;
    }
  }

  if (!found) {
    mTrackedArrays.push_back(newArray.get());
  }
  this->setIsChanged(true);
}

void
XdmfTemplate::traverse(const shared_ptr<XdmfBaseVisitor> visitor)
{
  // Set to the first step when writing, as the first step is the model for the rest of the template
  // Will fail if there are no steps
  if (this->getNumberSteps() == 0) {
    XdmfError::message(XdmfError::FATAL, "Error: No steps in template in XdmfTemplate::traverse");
  }
  this->clearStep();

  unsigned int arraysize = 1;
  for (unsigned int i = 0; i < mTrackedArrayDims[0].size(); ++i)
  {
    arraysize *= mTrackedArrayDims[0][i];
  }

  unsigned int controllersize = 0;
  for (unsigned int i = 0; i < mDataControllers[0].size(); ++i)
  {
    controllersize += mDataControllers[0][i]->getSize();
  }

  XdmfHeavyDataWriter::Mode originalMode;

  if (mHeavyWriter)
  {
    originalMode = mHeavyWriter->getMode();
    if (controllersize > arraysize) {
      mHeavyWriter->setMode(XdmfHeavyDataWriter::Append);
    }
  }

  this->setStep(0);

  if (mHeavyWriter)
  {
    mHeavyWriter->setMode(originalMode);
  }

  // Sending visitor to the base first so that it appears first when reading.
  mBase->accept(visitor);

  for (unsigned int i = 0; i < mTrackedArrays.size(); ++i) {
    mTrackedArrays[i]->release();
    mTrackedArrays[i]->accept(visitor);
  }

  // Create an array to hold all of the data information strings

  bool originalXPath;

  if (shared_ptr<XdmfWriter> writer =
        shared_dynamic_cast<XdmfWriter>(visitor)) {
    originalXPath = writer->getWriteXPaths();
    writer->setWriteXPaths(false);
  }

  shared_ptr<XdmfArray> dataInfoArray = XdmfArray::New();

  dataInfoArray->setName("Data Description");

  unsigned int i = 0;

  std::stringstream arrayInfo;
  while (i < mDataTypes.size()) {
    arrayInfo << "\"" << mDataTypes[i] << "\"" << mDataDescriptions[i];
    ++i;
  }
  dataInfoArray->insert(0, arrayInfo.str().c_str(), arrayInfo.str().length());
  dataInfoArray->insert(dataInfoArray->getSize(), 0);

  dataInfoArray->accept(visitor);

  if (shared_ptr<XdmfWriter> writer =
        shared_dynamic_cast<XdmfWriter>(visitor)) {
    writer->setWriteXPaths(originalXPath);
  }

  XdmfItem::traverse(visitor);
}

XDMF_ITEM_C_CHILD_WRAPPER(XdmfTemplate, XDMFTEMPLATE)

max_stars_repo_name: txwhhny/vtk

max_issues_repo_name: txwhhny/vtk

max_forks_repo_name: txwhhny/vtk


text:
/*
 * ssl-client.cpp
 *
 *  Created on: Dec 10, 2015
 *      Author: zmij
 */

#include <iostream>
#include <string>
#include <functional>

#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>

#include <boost/program_options.hpp>

enum { max_length = 1024 };

class client {
public:
	typedef boost::asio::ssl::stream< boost::asio::ip::tcp::socket > transport_type;
	typedef boost::asio::io_service io_service;
	typedef boost::asio::ssl::context ssl_context;
public:
	client(io_service& io, ssl_context& context,
			boost::asio::ip::tcp::resolver::iterator endpoint_iterator)
		: transport_(io, context)
	{
		transport_.set_verify_mode(boost::asio::ssl::verify_peer);
		transport_.set_verify_callback(
			std::bind(&client::verify_certificate, this,
					std::placeholders::_1, std::placeholders::_2) );

		boost::asio::async_connect(transport_.lowest_layer(), endpoint_iterator,
			std::bind(&client::handle_connect, this, std::placeholders::_1));
	}
private:
	bool
	verify_certificate(bool preverified, boost::asio::ssl::verify_context& ctx)
	{
		char subject_name[256];
		X509* cert = X509_STORE_CTX_get_current_cert(ctx.native_handle());
		X509_NAME_oneline(X509_get_subject_name(cert), subject_name, 256);
		std::cout << "Verifying " << subject_name << "\n";

		return preverified;
	}

	void
	handle_connect(boost::system::error_code const& ec)
	{
		if (!ec) {
			transport_.async_handshake(boost::asio::ssl::stream_base::client,
				std::bind(&client::handle_handshake, this, std::placeholders::_1)
			);
		} else {
			std::cerr << "Connect failed: " << ec.message() << "\n";
		}
	}

	void
	handle_handshake(boost::system::error_code const& ec)
	{
		if (!ec) {
			std::cout << "Handshake ok\n";
			std::cout << "Enter message: ";
			std::cin.getline(request_, max_length);
			size_t request_length = strlen(request_);

			boost::asio::async_write(transport_,
					boost::asio::buffer(request_, request_length),
					std::bind(&client::handle_write, this,
							std::placeholders::_1,
							std::placeholders::_2));
		} else {
			std::cerr << "Handshake failed: " << ec.message() << "\n";
		}
	}

	void
	handle_write(boost::system::error_code const& ec, size_t bytes_transfered)
	{
		if (!ec) {
			boost::asio::async_read(transport_,
				boost::asio::buffer(reply_, bytes_transfered),
				std::bind(&client::handle_read, this,
					std::placeholders::_1, std::placeholders::_2)
				);
		} else {
			std::cerr << "Write failed: " << ec.message() << "\n";
		}
	}
	void
	handle_read(boost::system::error_code const& ec, size_t bytes_transfered)
	{
		if (!ec) {
			std::cout << "Reply: ";
			std::cout.write(reply_, bytes_transfered);
			std::cout << "\n";
		} else {
			std::cerr << "Read failed: " << ec.message() << "\n";
		}
	}
private:
	transport_type transport_;

	char request_[max_length];
	char reply_[max_length];
};

int
main(int argc, char* argv[])
{
	using boost::asio::ip::tcp;
	try {
		namespace po = boost::program_options;

		std::string host;
		std::string port;
		std::string verify_file;
		std::string cert_file;
		std::string key_file;

		po::options_description options("SSL client options");

		options.add_options()
			("host,h", po::value< std::string >(&host), "Host")
			("port,p", po::value< std::string >(&port), "Port")
			("verify-file,v", po::value< std::string >(&verify_file), "Verify file")
			("cert-file,c", po::value< std::string >(&cert_file), "Certificate file")
			("key-file,k", po::value< std::string >(&key_file), "Private key file")
		;

		po::variables_map vm;
		po::store(po::parse_command_line(argc, argv, options), vm);
		po::notify(vm);

		if (host.empty() || port.empty()) {
			std::cerr << "Usage:\n" << options << "\n";
			return 2;
		}

		if (!cert_file.empty() && key_file.empty()) {
			std::cerr << "Private key required for certificate\n";
			return 2;
		}

		boost::asio::io_service io_service;
		tcp::resolver resolver(io_service);
		tcp::resolver::query query(host, port);
		tcp::resolver::iterator iterator = resolver.resolve(query);

		boost::asio::ssl::context ctx(boost::asio::ssl::context::sslv23);
		if (!verify_file.empty()) {
			ctx.load_verify_file(verify_file);
		}

		if (!cert_file.empty()) {
			ctx.use_certificate_chain_file(cert_file);
			ctx.use_private_key_file(key_file, boost::asio::ssl::context_base::pem);
		}

		client c(io_service, ctx, iterator);

		io_service.run();
	} catch (std::exception const& e) {
		std::cerr << "Exception: " << e.what() << "\n";
		return 1;
	}
	return 0;
}

ext: cpp

max_stars_repo_name: zmij/wire

max_issues_repo_name: zmij/wire

max_forks_repo_name: zmij/wire


text:
/** @file
 *****************************************************************************

 Declaration of interfaces for a secret-key lattice-based additively homomorphic
 vector encryption scheme.

 This includes:
 - class for secret key
 - class for ciphertext
 - key generation algorithm
 - encryption algorithm
 - decryption algorithm
 - operations for homomorphic addition and scalar multiplication of ciphertexts

 The implementation instantiates (a modification of) the LWE-based cryptosystem
 from [LP10] (described in [Pei16, Section 5.2.3]). The implementation encodes
 the message in the low-order bits of the ciphertext.

 References:

  [LP10]: Richard Lindner and Chris Peikert. Better Key Sizes (and Attacks) for
          LWE-Based Encryption. In CT-RSA, 2011.

  [Pei16]: Chris Peikert. A Decade of Lattice Cryptography. Available as
           Report 2015/939 on IACR Cryptology ePrint Archive
           (https://eprint.iacr.org/2015/939.pdf).

 *****************************************************************************
 * @author     Samir Menon, Brennan Shacklett, and David J. Wu
 * @copyright  MIT license (see LICENSE file)
 *****************************************************************************/

#ifndef LWE_HPP_
#define LWE_HPP_

#include <NTL/mat_ZZ_p.h>
#include <random>
#include "lwe_params.hpp"

namespace LWE {

using matrix = NTL::mat_ZZ_p;
using vector = NTL::vec_ZZ_p;
using plaintext = vector;

class secret_key {
public:
    matrix A {NTL::INIT_SIZE, n + pt_dim, n};
    matrix S {NTL::INIT_SIZE, n + pt_dim, pt_dim};
};

class ciphertext {
public:
  // Assignment operator
  ciphertext& operator=(const ciphertext& other);

  // Homomorphic addition
  ciphertext operator+(const ciphertext &other) const;
  ciphertext& operator+=(const ciphertext &other);

  // Homomorphic scalar multiplication
  ciphertext operator*(uint64_t val) const;
  ciphertext operator*(const NTL::ZZ_p &val) const;
  ciphertext& operator*=(uint64_t val);
  ciphertext& operator*=(const NTL::ZZ_p &val);

private:
  vector ctxt;

friend ciphertext encrypt(const secret_key &sk, const plaintext &pt);
friend plaintext  decrypt(const secret_key &sk, const ciphertext &ct);
};

secret_key keygen();
ciphertext encrypt(const secret_key &sk, const plaintext &pt);
plaintext  decrypt(const secret_key &sk, const ciphertext &ct);

ciphertext operator*(uint64_t val, const ciphertext& ct);
ciphertext operator*(const NTL::ZZ_p &val, const ciphertext& ct);

}

#endif // LWE_HPP_

ext: hpp


text:
/**
 * This reads a list of netflows from a file and finds all the triangles.
 * This is used as a check on what is found using the SAM parallel
 * infrastructure.
 */

//#define DEBUG
#define DETAIL_TIMING
#include <fstream>
#include <boost/program_options.hpp>
#include <sam/Util.hpp>
#include <sam/VastNetflow.hpp>

namespace po = boost::program_options;
using namespace sam;

int main(int argc, char** argv) {

  double queryTimeWindow; ///> Amount of time within a triangle can occur.
  std::string infile; ///> The location of the netflows

  po::options_description desc("Reads netflows from a file and counts "
    "how many triangles");
  desc.add_options()
    ("help", "help message")
    ("queryTimeWindow",
      po::value<double>(&queryTimeWindow)->default_value(10),
      "Time window for the query to be satisfied (default: 10).")
    ("infile", po::value<std::string>(&infile),
      "The file with the netflows.")
  ;

  // Parse the command line variables
  po::variables_map vm;
  po::store(po::parse_command_line(argc, argv, desc), vm);
  po::notify(vm);

  // Print out the help and exit if --help was specified.
  if (vm.count("help")) {
    std::cout << desc << std::endl;
    return 1;
  }

  std::vector<VastNetflow> netflows;

  std::ifstream netflowFile;
  netflowFile.open(infile);

  std::string line;
  size_t i = 0;
  while (std::getline(netflowFile, line))
  {
    VastNetflow netflow = makeNetflow(i, line);
    netflows.push_back(netflow);
    i++;
  }

  size_t numTri = sam::numTriangles<VastNetflow, SourceIp, DestIp,
                    TimeSeconds, DurationSeconds>(netflows, queryTimeWindow);

  std::cout << "Number of triangles " << numTri << std::endl;

}

max_stars_repo_name: dirkcgrunwald/SAM

max_issues_repo_name: dirkcgrunwald/SAM

max_forks_repo_name: dirkcgrunwald/SAM


text:
/**
 * BCL to FASTQ file converter
 * Copyright (c) 2007-2017 Illumina, Inc.
 *
 * This software is covered by the accompanying EULA
 * and certain third party copyright/licenses, and any user of this
 * source file is bound by the terms therein.
 *
 * \file FilterFile.cpp
 *
 * \brief Implementation of filter file.
 *
 * \author Marek Balint
 * \author Mauricio Varea
 */

#include <boost/format.hpp>
#include <boost/filesystem/operations.hpp>

#include "io/FileBufWithReopen.hh"
#include "data/FilterFile.hh"

namespace bcl2fastq {

namespace data {

FilterFileBase::FilterFileBase(const common::RawDataBuffer& data,
                               bool                         ignoreErrors,
                               bool                         skipHeader)
    : FileReaderBase(data,
                     ignoreErrors)
    , BinaryAllClustersFileReader(data,
                                  ignoreErrors)
    , skipHeader_(skipHeader)
{
}

bool FilterFileBase::readHeader()
{
    if (skipHeader_)
    {
        return true;
    }

    if (!BinaryAllClustersFileReader::readHeader(clustersCount_))
    {
        return false;
    }

    if (clustersCount_ == 0)
    {
        Header header;
        if (!BinaryAllClustersFileReader::readHeader(header))
        {
            return false;
        }

        if (!validateHeader(header))
        {
            return false;
        }

        clustersCount_ = static_cast<common::ClustersCount>(header.clustersCountIn_);
    }

    return true;
}

bool FilterFileBase::validateHeader(const FilterFile::Header& header)
{
    int errnum = errno;
    if (header.version_ != getHeaderVersion())
    {
        const uint32_t version = header.version_;
        BOOST_THROW_EXCEPTION(common::InputDataError(errnum, (boost::format("Corrupted header of filter file '%s': header_version=%d") % getPath().string() % version).str()));

        return false;
    }

    return true;
}

FilterFile::FilterFile(const common::RawDataBuffer& data,
                       bool                         ignoreErrors,
                       bool                         skipHeader)
    : FileReaderBase(data,
                     ignoreErrors)
    , FilterFileBase(data,
                     ignoreErrors,
                     skipHeader)
{
    if (!skipHeader)
    {
        readHeader();
    }
}

std::size_t FilterFile::read(
    FilterFile::Record *targetBuffer,
    std::size_t targetSize
)
{
    if (!validateCondition(this->isOpen(), "File is not open")) { return 0; }

    return readBytes(reinterpret_cast<char*>(targetBuffer), targetSize);
}

bool FilterFile::doesFileExist(const boost::filesystem::path& inputDir,
                               bool                           aggregateTilesFlag,
                               common::LaneNumber             laneNumber,
                               common::TileNumber             tileNumber,
                               size_t&                        headerSize,
                               boost::filesystem::path&       filePath)
{
    headerSize = 12;
    if( aggregateTilesFlag ) {
        filePath =
            inputDir
            / boost::filesystem::path((boost::format("L%03d") % laneNumber).str())
            / boost::filesystem::path((boost::format("s_%d.filter") % laneNumber).str());
    } else {
        boost::filesystem::path fileName((boost::format("s_%d_%d.filter") % laneNumber % tileNumber).str());
        boost::filesystem::path fileNameZeroPadded((boost::format("s_%d_%04d.filter") % laneNumber % tileNumber).str());
        boost::filesystem::path laneDir((boost::format("L%03d") % laneNumber).str());

        filePath = inputDir / laneDir / fileName;

        if (!boost::filesystem::exists(filePath))
        {
            filePath = inputDir / fileName;
            headerSize = 4;
        }

        if (!boost::filesystem::exists(filePath))
        {
            filePath = inputDir / laneDir / fileNameZeroPadded;
        }

        if (!boost::filesystem::exists(filePath))
        {
            filePath = inputDir / fileNameZeroPadded;
            headerSize = 4;
        }
    }

    return boost::filesystem::exists(filePath);
}

} // namespace data
} // namespace bcl2fastq


text:
/***************************************************************************
 *   Copyright (C) 2008-2021 by Andrzej Rybczak                            *
 *   andrzej@rybczak.net                                                   *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.              *
 ***************************************************************************/

#include <boost/algorithm/string/classification.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
#include <cassert>
#include <cerrno>
#include <cstring>
#include <fstream>
#include <thread>

#include "curses/scrollpad.h"
#include "screens/browser.h"
#include "charset.h"
#include "curl_handle.h"
#include "format_impl.h"
#include "global.h"
#include "helpers.h"
#include "macro_utilities.h"
#include "screens/lyrics.h"
#include "screens/playlist.h"
#include "settings.h"
#include "song.h"
#include "statusbar.h"
#include "title.h"
#include "screens/screen_switcher.h"
#include "utility/string.h"

using Global::MainHeight;
using Global::MainStartY;

Lyrics *myLyrics;

namespace {

std::string removeExtension(std::string filename)
{
	size_t dot = filename.rfind('.');
	if (dot != std::string::npos)
		filename.resize(dot);
	return filename;
}

std::string lyricsFilename(const MPD::Song &s)
{
	std::string filename;
	if (Config.store_lyrics_in_song_dir && !s.isStream())
	{
		if (s.isFromDatabase())
			filename = Config.mpd_music_dir + "/";
		filename += removeExtension(s.getURI());
		removeExtension(filename);
	}
	else
	{
		std::string artist = s.getArtist();
		std::string title  = s.getTitle();
		if (artist.empty() || title.empty())
			filename = removeExtension(s.getName());
		else
			filename = artist + " - " + title;
		removeInvalidCharsFromFilename(filename, Config.generate_win32_compatible_filenames);
		filename = Config.lyrics_directory + "/" + filename;
	}
	filename += ".txt";
	return filename;
}

bool loadLyrics(NC::Scrollpad &w, const std::string &filename)
{
	std::ifstream input(filename);
	if (input.is_open())
	{
		std::string line;
		bool first_line = true;
		while (std::getline(input, line))
		{
			// Remove carriage returns as they mess up the display.
			boost::remove_erase(line, '\r');
			if (!first_line)
				w << '\n';
			w << Charset::utf8ToLocale(line);
			first_line = false;
		}
		return true;
	}
	else
		return false;
}

bool saveLyrics(const std::string &filename, const std::string &lyrics)
{
	std::ofstream output(filename);
	if (output.is_open())
	{
		output << lyrics;
		output.close();
		return true;
	}
	else
		return false;
}

boost::optional<std::string> downloadLyrics(
	const MPD::Song &s,
	std::shared_ptr<Shared<NC::Buffer>> shared_buffer,
	std::shared_ptr<std::atomic<bool>> download_stopper,
	LyricsFetcher *current_fetcher)
{
	std::string s_artist = s.getArtist();
	std::string s_title  = s.getTitle();
	// If artist or title is empty, use filename. This should give reasonable
	// results for google search based lyrics fetchers.
	if (s_artist.empty() || s_title.empty())
	{
		s_artist.clear();
		s_title = s.getName();
		// Get rid of underscores to improve search results.
		std::replace_if(s_title.begin(), s_title.end(), boost::is_any_of("-_"), ' ');
		size_t dot = s_title.rfind('.');
		if (dot != std::string::npos)
			s_title.resize(dot);
	}

	auto fetch_lyrics = [&](auto &fetcher_) {
		{
			if (shared_buffer)
			{
				auto buf = shared_buffer->acquire();
				*buf << "Fetching lyrics from "
				     << NC::Format::Bold
				     << fetcher_->name()
				     << NC::Format::NoBold << "... ";
			}
		}
		auto result_ = fetcher_->fetch(s_artist, s_title);
		if (result_.first == false)
		{
			if (shared_buffer)
			{
				auto buf = shared_buffer->acquire();
				*buf << NC::Color::Red
				     << result_.second
				     << NC::Color::End
				     << '\n';
			}
		}
		return result_;
	};

	LyricsFetcher::Result fetcher_result;
	if (current_fetcher == nullptr)
	{
		for (auto &fetcher : Config.lyrics_fetchers)
		{
			if (download_stopper && download_stopper->load())
				return boost::none;
			fetcher_result = fetch_lyrics(fetcher);
			if (fetcher_result.first)
				break;
		}
	}
	else
		fetcher_result = fetch_lyrics(current_fetcher);

	boost::optional<std::string> result;
	if (fetcher_result.first)
		result = std::move(fetcher_result.second);
	return result;
}

}

Lyrics::Lyrics()
	: Screen(NC::Scrollpad(0, MainStartY, COLS, MainHeight, "", Config.main_color, NC::Border()))
	, m_refresh_window(false)
	, m_scroll_begin(0)
	, m_fetcher(nullptr)
{ }

void Lyrics::resize()
{
	size_t x_offset, width;
	getWindowResizeParams(x_offset, width);
	w.resize(width, MainHeight);
	w.moveTo(x_offset, MainStartY);
	hasToBeResized = 0;
}

void Lyrics::update()
{
	if (m_worker.valid())
	{
		auto buffer = m_shared_buffer->acquire();
		if (!buffer->empty())
		{
			w << *buffer;
			buffer->clear();
			m_refresh_window = true;
		}

		if (m_worker.is_ready())
		{
			auto lyrics = m_worker.get();
			if (lyrics)
			{
				w.clear();
				w << Charset::utf8ToLocale(*lyrics);
				std::string filename = lyricsFilename(m_song);
				if (!saveLyrics(filename, *lyrics))
					Statusbar::printf("Couldn't save lyrics as \"%1%\": %2%",
					                  filename, strerror(errno));
			}
			else
				w << "\nLyrics were not found.\n";
			clearWorker();
			m_refresh_window = true;
		}
	}

	if (m_refresh_window)
	{
		m_refresh_window = false;
		w.flush();
		w.refresh();
	}
}

void Lyrics::switchTo()
{
	using Global::myScreen;
	if (myScreen != this)
	{
		SwitchTo::execute(this);
		m_scroll_begin = 0;
		drawHeader();
	}
	else
		switchToPreviousScreen();
}

std::wstring Lyrics::title()
{
	std::wstring result = L"Lyrics";
	if (!m_song.empty())
	{
		result += L": ";
		result += Scroller(
			Format::stringify<wchar_t>(Format::parse(L"{%a - %t}|{%f}"), &m_song),
			m_scroll_begin,
			COLS - result.length() - (Config.design == Design::Alternative
			                          ? 2
			                          : Global::VolumeState.length()));
	}
	return result;
}

void Lyrics::fetch(const MPD::Song &s)
{
	if (!m_worker.valid() || s != m_song)
	{
		stopDownload();
		w.clear();
		w.reset();
		m_song = s;
		if (loadLyrics(w, lyricsFilename(m_song)))
		{
			clearWorker();
			m_refresh_window = true;
		}
		else
		{
			m_download_stopper = std::make_shared<std::atomic<bool>>(false);
			m_shared_buffer = std::make_shared<Shared<NC::Buffer>>();
			m_worker = boost::async(
				boost::launch::async,
				std::bind(downloadLyrics,
				          m_song, m_shared_buffer, m_download_stopper, m_fetcher));
		}
	}
}

void Lyrics::refetchCurrent()
{
	std::string filename = lyricsFilename(m_song);
	if (std::remove(filename.c_str()) == -1 && errno != ENOENT)
	{
		const char msg[] = "Couldn't remove \"%1%\": %2%";
		Statusbar::printf(msg, wideShorten(filename, COLS - const_strlen(msg) - 25),
		                  strerror(errno));
	}
	else
	{
		clearWorker();
		fetch(m_song);
	}
}

void Lyrics::edit()
{
	if (Config.external_editor.empty())
	{
		Statusbar::print("external_editor variable has to be set in configuration file");
		return;
	}

	Statusbar::print("Opening lyrics in external editor...");

	std::string filename = lyricsFilename(m_song);
	escapeSingleQuotes(filename);
	if (Config.use_console_editor)
	{
		runExternalConsoleCommand(Config.external_editor + " '" + filename + "'");
		fetch(m_song);
	}
	else
		runExternalCommand(Config.external_editor + " '" + filename + "'", false);
}

void Lyrics::toggleFetcher()
{
	if (m_fetcher != nullptr)
	{
		auto fetcher = std::find_if(Config.lyrics_fetchers.begin(),
		                            Config.lyrics_fetchers.end(),
		                            [this](auto &f) { return f.get() == m_fetcher; });
		assert(fetcher != Config.lyrics_fetchers.end());
		++fetcher;
		if (fetcher != Config.lyrics_fetchers.end())
			m_fetcher = fetcher->get();
		else
			m_fetcher = nullptr;
	}
	else
	{
		assert(!Config.lyrics_fetchers.empty());
		m_fetcher = Config.lyrics_fetchers[0].get();
	}

	if (m_fetcher != nullptr)
		Statusbar::printf("Using lyrics fetcher: %s", m_fetcher->name());
	else
		Statusbar::print("Using all lyrics fetchers");
}

void Lyrics::fetchInBackground(const MPD::Song &s, bool notify_)
{
	auto consumer_impl = [this] {
		std::string lyrics_file;
		while (true)
		{
			ConsumerState::Song cs;
			{
				auto consumer = m_consumer_state.acquire();
				assert(consumer->running);
				if (consumer->songs.empty())
				{
					consumer->running = false;
					break;
				}
				lyrics_file = lyricsFilename(consumer->songs.front().song());
				if (!boost::filesystem::exists(lyrics_file))
				{
					cs = consumer->songs.front();
					if (cs.notify())
					{
						consumer->message = "Fetching lyrics for \""
							+ Format::stringify<char>(Config.song_status_format, &cs.song())
							+ "\"...";
					}
				}
				consumer->songs.pop();
			}
			if (!cs.song().empty())
			{
				auto lyrics = downloadLyrics(cs.song(), nullptr, nullptr, m_fetcher);
				if (lyrics)
					saveLyrics(lyrics_file, *lyrics);
			}
		}
	};

	auto consumer = m_consumer_state.acquire();
	consumer->songs.emplace(s, notify_);
	// Start the consumer if it's not running.
	if (!consumer->running)
	{
		std::thread t(consumer_impl);
		t.detach();
		consumer->running = true;
	}
}

boost::optional<std::string> Lyrics::tryTakeConsumerMessage()
{
	boost::optional<std::string> result;
	auto consumer = m_consumer_state.acquire();
	if (consumer->message)
	{
		result = std::move(consumer->message);
		consumer->message = boost::none;
	}
	return result;
}

void Lyrics::clearWorker()
{
	m_shared_buffer.reset();
	m_worker = boost::BOOST_THREAD_FUTURE<boost::optional<std::string>>();
}

void Lyrics::stopDownload()
{
	if (m_download_stopper)
		m_download_stopper->store(true);
}

max_stars_repo_name: doctorfree/mpcplus

max_issues_repo_name: doctorfree/mpcplus

max_forks_repo_name: doctorfree/mpcplus


text:
#include <Backups/BackupCoordinationHelpers.h>
#include <Storages/MergeTree/MergeTreePartInfo.h>
#include <Common/Exception.h>
#include <base/chrono_io.h>
#include <boost/range/adaptor/map.hpp>

namespace DB
{

namespace ErrorCodes
{
    extern const int CANNOT_BACKUP_TABLE;
    extern const int FAILED_TO_SYNC_BACKUP_OR_RESTORE;
    extern const int LOGICAL_ERROR;
}

struct BackupCoordinationReplicatedTablesInfo::HostAndTableName
{
    String host_id;
    DatabaseAndTableName table_name;

    struct Less
    {
        bool operator()(const HostAndTableName & lhs, const HostAndTableName & rhs) const
        {
            return (lhs.host_id < rhs.host_id) || ((lhs.host_id == rhs.host_id) && (lhs.table_name < rhs.table_name));
        }

        bool operator()(const std::shared_ptr<const HostAndTableName> & lhs, const std::shared_ptr<const HostAndTableName> & rhs) const
        {
            return operator()(*lhs, *rhs);
        }
    };
};

class BackupCoordinationReplicatedTablesInfo::CoveredPartsFinder
{
public:
    CoveredPartsFinder() = default;

    void addPart(const String & new_part_name, const std::shared_ptr<const HostAndTableName> & host_and_table_name)
    {
        addPart(MergeTreePartInfo::fromPartName(new_part_name, MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING), host_and_table_name);
    }

    void addPart(MergeTreePartInfo && new_part_info, const std::shared_ptr<const HostAndTableName> & host_and_table_name)
    {
        auto new_min_block = new_part_info.min_block;
        auto new_max_block = new_part_info.max_block;
        auto & parts = partitions[new_part_info.partition_id];

        /// Find the first part with max_block >= `part_info.min_block`.
        auto first_it = parts.lower_bound(new_min_block);
        if (first_it == parts.end())
        {
            /// All max_blocks < part_info.min_block, so we can safely add the `part_info` to the list of parts.
            parts.emplace(new_max_block, PartInfo{std::move(new_part_info), host_and_table_name});
            return;
        }

        {
            /// part_info.min_block <= current_info.max_block
            const auto & part = first_it->second;
            if (new_max_block < part.info.min_block)
            {
                /// (prev_info.max_block < part_info.min_block) AND (part_info.max_block < current_info.min_block),
                /// so we can safely add the `part_info` to the list of parts.
                parts.emplace(new_max_block, PartInfo{std::move(new_part_info), host_and_table_name});
                return;
            }

            /// (part_info.min_block <= current_info.max_block) AND (part_info.max_block >= current_info.min_block), parts intersect.

            if (part.info.contains(new_part_info))
            {
                /// `part_info` is already contained in another part.
                return;
            }
        }

        /// Probably `part_info` is going to replace multiple parts, find the range of parts to replace.
        auto last_it = first_it;
        while (last_it != parts.end())
        {
            const auto & part = last_it->second;
            if (part.info.min_block > new_max_block)
                break;
            if (!new_part_info.contains(part.info))
            {
                throw Exception(
                    ErrorCodes::CANNOT_BACKUP_TABLE,
                    "Intersected parts detected: {} in the table {}.{}{} and {} in the table {}.{}{}. It should be investigated",
                    part.info.getPartName(),
                    part.host_and_table_name->table_name.first,
                    part.host_and_table_name->table_name.second,
                    part.host_and_table_name->host_id.empty() ? "" : (" on the host " + part.host_and_table_name->host_id),
                    new_part_info.getPartName(),
                    host_and_table_name->table_name.first,
                    host_and_table_name->table_name.second,
                    host_and_table_name->host_id.empty() ? "" : (" on the host " + host_and_table_name->host_id));
            }
            ++last_it;
        }

        /// `part_info` will replace multiple parts [first_it..last_it)
        parts.erase(first_it, last_it);
        parts.emplace(new_max_block, PartInfo{std::move(new_part_info), host_and_table_name});
    }

    bool isCoveredByAnotherPart(const String & part_name) const
    {
        return isCoveredByAnotherPart(MergeTreePartInfo::fromPartName(part_name, MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING));
    }

    bool isCoveredByAnotherPart(const MergeTreePartInfo & part_info) const
    {
        auto partition_it = partitions.find(part_info.partition_id);
        if (partition_it == partitions.end())
            return false;

        const auto & parts = partition_it->second;

        /// Find the first part with max_block >= `part_info.min_block`.
        auto it_part = parts.lower_bound(part_info.min_block);
        if (it_part == parts.end())
        {
            /// All max_blocks < part_info.min_block, so there is no parts covering `part_info`.
            return false;
        }

        /// part_info.min_block <= current_info.max_block
        const auto & existing_part = it_part->second;
        if (part_info.max_block < existing_part.info.min_block)
        {
            /// (prev_info.max_block < part_info.min_block) AND (part_info.max_block < current_info.min_block),
            /// so there is no parts covering `part_info`.
            return false;
        }

        /// (part_info.min_block <= current_info.max_block) AND (part_info.max_block >= current_info.min_block), parts intersect.

        if (existing_part.info == part_info)
        {
            /// It's the same part, it's kind of covers itself, but we check in this function whether a part is covered by another part.
            return false;
        }

        /// Check if `part_info` is covered by `current_info`.
        return existing_part.info.contains(part_info);
    }

private:
    struct PartInfo
    {
        MergeTreePartInfo info;
        std::shared_ptr<const HostAndTableName> host_and_table_name;
    };

    using Parts = std::map<Int64 /* max_block */, PartInfo>;
    std::unordered_map<String, Parts> partitions;
};

void BackupCoordinationReplicatedTablesInfo::addDataPath(const String & table_zk_path, const String & table_data_path)
{
    tables[table_zk_path].data_paths.push_back(table_data_path);
}

Strings BackupCoordinationReplicatedTablesInfo::getDataPaths(const String & table_zk_path) const
{
    auto it = tables.find(table_zk_path);
    if (it == tables.end())
        throw Exception(ErrorCodes::LOGICAL_ERROR, "getDataPaths() called for unknown table_zk_path: {}", table_zk_path);
    const auto & replicated_table = it->second;
    return replicated_table.data_paths;
}

void BackupCoordinationReplicatedTablesInfo::addPartNames(
    const String & host_id,
    const DatabaseAndTableName & table_name,
    const String & table_zk_path,
    const std::vector<PartNameAndChecksum> & part_names_and_checksums)
{
    auto & table = tables[table_zk_path];
    auto & part_locations_by_names = table.part_locations_by_names;
    auto host_and_table_name = std::make_shared<HostAndTableName>();
    host_and_table_name->host_id = host_id;
    host_and_table_name->table_name = table_name;

    for (const auto & part_name_and_checksum : part_names_and_checksums)
    {
        const auto & part_name = part_name_and_checksum.part_name;
        const auto & checksum = part_name_and_checksum.checksum;
        auto it = part_locations_by_names.find(part_name);
        if (it == part_locations_by_names.end())
        {
            it = part_locations_by_names.emplace(part_name, PartLocations{}).first;
            it->second.checksum = checksum;
        }
        else
        {
            const auto & existing = it->second;
            if (existing.checksum != checksum)
            {
                const auto & existing_host_and_table_name = **existing.host_and_table_names.begin();
                throw Exception(
                    ErrorCodes::CANNOT_BACKUP_TABLE,
                    "Table {}.{} has part {} which is different from the part of table {}.{}. Must be the same",
                    table_name.first,
                    table_name.second,
                    part_name,
                    existing_host_and_table_name.table_name.first,
                    existing_host_and_table_name.table_name.second);
            }
        }

        auto & host_and_table_names = it->second.host_and_table_names;

        /// `host_and_table_names` should be ordered because we need this vector to be in the same order on every replica.
        host_and_table_names.insert(
            std::upper_bound(host_and_table_names.begin(), host_and_table_names.end(), host_and_table_name, HostAndTableName::Less{}),
            host_and_table_name);
    }
}

Strings BackupCoordinationReplicatedTablesInfo::getPartNames(const String & host_id, const DatabaseAndTableName & table_name, const String & table_zk_path) const
{
    if (!part_names_by_locations_prepared)
        throw Exception(ErrorCodes::LOGICAL_ERROR, "preparePartNamesByLocations() was not called before getPartNames()");

    auto it = tables.find(table_zk_path);
    if (it == tables.end())
        throw Exception(ErrorCodes::LOGICAL_ERROR, "getPartNames() called for unknown table_zk_path: {}", table_zk_path);
    const auto & table = it->second;
    auto it2 = table.part_names_by_locations.find(host_id);
    if (it2 == table.part_names_by_locations.end())
        return {};
    const auto & part_names_by_host_id = it2->second;
    auto it3 = part_names_by_host_id.find(table_name);
    if (it3 == part_names_by_host_id.end())
        return {};
    return it3->second;
}

void BackupCoordinationReplicatedTablesInfo::preparePartNamesByLocations()
{
    if (part_names_by_locations_prepared)
        return;
    part_names_by_locations_prepared = true;

    size_t counter = 0;
    for (auto & table : tables | boost::adaptors::map_values)
    {
        CoveredPartsFinder covered_parts_finder;
        for (const auto & [part_name, part_locations] : table.part_locations_by_names)
            covered_parts_finder.addPart(part_name, *part_locations.host_and_table_names.begin());

        table.part_names_by_locations.clear();
        for (const auto & [part_name, part_locations] : table.part_locations_by_names)
        {
            if (covered_parts_finder.isCoveredByAnotherPart(part_name))
                continue;
            size_t chosen_index = (counter++) % part_locations.host_and_table_names.size();
            const auto & chosen_host_id = part_locations.host_and_table_names[chosen_index]->host_id;
            const auto & chosen_table_name = part_locations.host_and_table_names[chosen_index]->table_name;
            table.part_names_by_locations[chosen_host_id][chosen_table_name].push_back(part_name);
        }
    }
}

BackupCoordinationDistributedBarrier::BackupCoordinationDistributedBarrier(
    const String & zookeeper_path_, zkutil::GetZooKeeper get_zookeeper_, const String & logger_name_, const String & operation_name_)
    : zookeeper_path(zookeeper_path_)
    , get_zookeeper(get_zookeeper_)
    , log(&Poco::Logger::get(logger_name_))
    , operation_name(operation_name_)
{
    createRootNodes();
}

void BackupCoordinationDistributedBarrier::createRootNodes()
{
    auto zookeeper = get_zookeeper();
    zookeeper->createAncestors(zookeeper_path);
    zookeeper->createIfNotExists(zookeeper_path, "");
}

void BackupCoordinationDistributedBarrier::finish(const String & host_id, const String & error_message)
{
    if (error_message.empty())
        LOG_TRACE(log, "Host {} has finished {}", host_id, operation_name);
    else
        LOG_ERROR(log, "Host {} has failed {} with message: {}", host_id, operation_name, error_message);

    auto zookeeper = get_zookeeper();
    if (error_message.empty())
        zookeeper->create(zookeeper_path + "/" + host_id + ":ready", "", zkutil::CreateMode::Persistent);
    else
        zookeeper->create(zookeeper_path + "/" + host_id + ":error", error_message, zkutil::CreateMode::Persistent);
}

void BackupCoordinationDistributedBarrier::waitForAllHostsToFinish(const Strings & host_ids, const std::chrono::seconds timeout) const
{
    auto zookeeper = get_zookeeper();

    bool all_hosts_ready = false;
    String not_ready_host_id;
    String error_host_id;
    String error_message;

    /// Returns true of everything's ready, or false if we need to wait more.
    auto process_nodes = [&](const Strings & nodes)
    {
        std::unordered_set<std::string_view> set{nodes.begin(), nodes.end()};
        for (const String & host_id : host_ids)
        {
            if (set.contains(host_id + ":error"))
            {
                error_host_id = host_id;
                error_message = zookeeper->get(zookeeper_path + "/" + host_id + ":error");
                return;
            }
            if (!set.contains(host_id + ":ready"))
            {
                LOG_TRACE(log, "Waiting for host {} {}", host_id, operation_name);
                not_ready_host_id = host_id;
                return;
            }
        }

        all_hosts_ready = true;
    };

    std::atomic<bool> watch_set = false;
    std::condition_variable watch_triggered_event;

    auto watch_callback = [&](const Coordination::WatchResponse &)
    {
        watch_set = false; /// After it's triggered it's not set until we call getChildrenWatch() again.
        watch_triggered_event.notify_all();
    };

    auto watch_triggered = [&] { return !watch_set; };

    bool use_timeout = (timeout.count() >= 0);
    std::chrono::steady_clock::duration time_left = timeout;
    std::mutex dummy_mutex;

    while (true)
    {
        if (use_timeout && (time_left.count() <= 0))
        {
            Strings children = zookeeper->getChildren(zookeeper_path);
            process_nodes(children);
            break;
        }

        watch_set = true;
        Strings children = zookeeper->getChildrenWatch(zookeeper_path, nullptr, watch_callback);
        process_nodes(children);

        if (!error_message.empty() || all_hosts_ready)
            break;

        {
            std::unique_lock dummy_lock{dummy_mutex};
            if (use_timeout)
            {
                std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now();
                if (!watch_triggered_event.wait_for(dummy_lock, time_left, watch_triggered))
                    break;
                time_left -= (std::chrono::steady_clock::now() - start_time);
            }
            else
                watch_triggered_event.wait(dummy_lock, watch_triggered);
        }
    }

    if (watch_set)
    {
        /// Remove watch by triggering it.
        zookeeper->create(zookeeper_path + "/remove_watch-", "", zkutil::CreateMode::EphemeralSequential);
        std::unique_lock dummy_lock{dummy_mutex};
        watch_triggered_event.wait_for(dummy_lock, timeout, watch_triggered);
    }

    if (!error_message.empty())
    {
        throw Exception(
            ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE,
            "Host {} failed {} with message: {}",
            error_host_id,
            operation_name,
            error_message);
    }

    if (all_hosts_ready)
    {
        LOG_TRACE(log, "All hosts have finished {}", operation_name);
        return;
    }

    throw Exception(
        ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE,
        "Host {} has failed {}: Time ({}) is out",
        not_ready_host_id,
        operation_name,
        to_string(timeout));
}

}

max_stars_repo_name: zzachimed/ClickHouse

max_issues_repo_name: zzachimed/ClickHouse

max_forks_repo_name: zzachimed/ClickHouse