text:
// Copyright (c) Mathieu Malaterre
// SPDX-License-Identifier: BSD-3-Clause
#include "cjpls_options.h"

#include "tuple.h"
#include "version.h"
#include <boost/program_options.hpp>

namespace jlst {

static charls::color_transformation string_to_color_transformation(const char* argument)
{
    if (strcmp(argument, "none") == 0)
        return charls::color_transformation::none;

    if (strcmp(argument, "hp1") == 0)
        return charls::color_transformation::hp1;

    if (strcmp(argument, "hp2") == 0)
        return charls::color_transformation::hp2;

    if (strcmp(argument, "hp3") == 0)
        return charls::color_transformation::hp3;

    throw std::runtime_error("Argument color_transformation needs to be: none, hp1, hp2 or hp3\n");
}

static charls::interleave_mode string_to_interleave_mode(const char* argument)
{
    if (strcmp(argument, "none") == 0)
        return charls::interleave_mode::none;

    if (strcmp(argument, "line") == 0)
        return charls::interleave_mode::line;

    if (strcmp(argument, "sample") == 0)
        return charls::interleave_mode::sample;

    throw std::runtime_error("Argument interleave-mode needs to be: none, line or sample\n");
}

static charls::interleave_mode string_to_planar_configuration(const char* argument)
{
    if (strcmp(argument, "contig") == 0)
        return charls::interleave_mode::sample;

    if (strcmp(argument, "separate") == 0)
        return charls::interleave_mode::none;

    throw std::runtime_error("Argument planar_configuration needs to be: contig or separate\n");
}

bool cjpls_options::process(int argc, char* argv[])
{
    std::vector<std::string> inputs{};
    std::vector<std::string> outputs{};
    typedef tuple<int, 5> pcp_type;  // preset_coding_parameters
    typedef tuple<int, 2> size_type; // frame_info: width + height
    std::string interleave_mode_str{};
    std::string color_transformation_str{};
    std::string planar_configuration_str{};
    pcp_type pcp{};
    size_type size{};
    auto& frame_info = image_info_.frame_info();
    auto& planar_configuration = image_info_.interleave_mode();
    {
        namespace po = boost::program_options;
        po::options_description generic("Generic options (required when no redirects)");
        generic.add_options()                                                      //
            ("input,i", po::value(&inputs) /*->required()*/, "Input filename.")    // input
            ("output,o", po::value(&outputs) /*->required()*/, "Output filename.") // output
            ("type", po::value(&type_), "Input type (pgm, raw...).")               // input type
            ;

        po::options_description jpegls("JPEG-LS output options");
        jpegls.add_options() //
            ("interleave_mode,m", po::value(&interleave_mode_str),
             "Output interleave mode: `none|line|sample`. "
             "Default to input format if not specified.")                                      // interleave mode
            ("near_lossless,n", po::value(&jls_options_.near_lossless), "near lossless value") // near lossless
            ("preset_coding_parameters,k", po::value(&pcp),
             "preset coding parameters: "
             "maximum_sample_value, threshold1, threshold2, threshold3, reset_value") // preset coding parameters
            ("standard_spiff_header", po::value(&jls_options_.standard_spiff_header),
             "Write a standard spiff header: 'yes'/'no'.") // spiff header
            ("color_transformation,t", po::value(&color_transformation_str),
             "Color transformation: `none|hp1|hp2|hp3` (HP extension).") // color transformation
            ;

        po::options_description image("Image input options");
        image.add_options() //
            ("size,s", po::value(&size),
             "Size of image (width, height), unless specified in the format header.") // size width + height
            ("bits_per_sample,b", po::value(&frame_info.bits_per_sample),
             "Bits per sample value, unless specified in the format header.") // bits per sample
            ("component_count,c", po::value(&frame_info.component_count),
             "Component count, unless specified in the format header.") // component count
            ("planar_configuration,p", po::value(&planar_configuration_str),
             "Planar configuration ('contig' or 'separate'), unless specified in the format header.") // planar configuration
            ;

#if CHARLS_VERSION_MAJOR > 2 || (CHARLS_VERSION_MAJOR == 2 && CHARLS_VERSION_MINOR > 2)
        po::options_description encoding("Encoding options");
        encoding.add_options()                                         //
            ("even_destination_size", "even destination size")         //
            ("include_version_number", "include version number")       //
            ("include_pc_parameters_jai", "include pc parameters jai") //
            ;
#endif

        po::options_description desc("Allowed options");
        desc.add_options()("help,h", "print usage message") // help
            ("version", "print version")                    // version
            ;

        desc.add(generic);
        desc.add(jpegls);
#if CHARLS_VERSION_MAJOR > 2 || (CHARLS_VERSION_MAJOR == 2 && CHARLS_VERSION_MINOR > 2)
        desc.add(encoding);
#endif
        desc.add(image);

        po::positional_options_description p;
        // do not allow more than one positional arg for now:
        p.add("input", 1);
        p.add("output", 1);
        po::variables_map vm;
        po::store(po::command_line_parser(argc, argv).options(desc).positional(p).run(), vm);

        if (vm.count("help"))
        {
            std::cout << "usage: cjpls input.ext output.jls\n";
            std::cout << desc << std::endl;
            return false;
        }

        if (vm.count("version"))
        {
            std::cout << "cjpls version: " << JLST_VERSION << "\n";
            std::cout << "charls version: " << charls_get_version_string() << std::endl;
            return false;
        }

        // https://stackoverflow.com/questions/5395503/required-and-optional-arguments-using-boost-library-program-options
        try
        {
            po::notify(vm);

            // let's pretend that input/output are actually required:
            if (vm.count("input"))
            {
                add_inputs(inputs);
                if (!vm.count("type"))
                {
                    type_ = compute_type_from_filenames(inputs);
                }
            }
            else
            {
                add_stdin_input();
            }

            if (vm.count("output"))
            {
                add_outputs(outputs);
            }
            else
            {
                add_stdout_output();
            }
        }
        catch (std::exception&)
        {
            // default value missing. Let's print usage before re-throw:
            std::cout << "usage: cjpls input.ext output.jls\n";
            std::cout << desc << std::endl;
            throw;
        }

        jls_options_.interleave_mode = charls::interleave_mode::none;
        jls_options_.color_transformation = charls::color_transformation::none;
        planar_configuration = charls::interleave_mode::sample;

        if (vm.count("color_transformation"))
        {
            jls_options_.has_color_transformation = true;
            jls_options_.color_transformation = string_to_color_transformation(color_transformation_str.c_str());
        }
        if (vm.count("interleave_mode"))
        {
            jls_options_.has_interleave_mode = true;
            jls_options_.interleave_mode = string_to_interleave_mode(interleave_mode_str.c_str());
        }
        if (vm.count("preset_coding_parameters"))
        {
            const int* val = pcp.values;
            jls_options_.preset_coding_parameters.maximum_sample_value = val[0];
            jls_options_.preset_coding_parameters.threshold1 = val[1];
            jls_options_.preset_coding_parameters.threshold2 = val[2];
            jls_options_.preset_coding_parameters.threshold3 = val[3];
            jls_options_.preset_coding_parameters.reset_value = val[4];
        }
        if (vm.count("size"))
        {
            const int* val = size.values;
            frame_info.width = val[0];
            frame_info.height = val[1];
        }
        if (vm.count("planar_configuration"))
        {
            planar_configuration = string_to_planar_configuration(planar_configuration_str.c_str());
        }
#if CHARLS_VERSION_MAJOR > 2 || (CHARLS_VERSION_MAJOR == 2 && CHARLS_VERSION_MINOR > 2)
        if (vm.count("even_destination_size"))
        {
            jls_options_.encoding_options = jls_options_.encoding_options | charls::encoding_options::even_destination_size;
        }
        if (vm.count("include_version_number"))
        {
            jls_options_.encoding_options = jls_options_.encoding_options | charls::encoding_options::include_version_number;
        }
        if (vm.count("include_pc_parameters_jai"))
        {
            jls_options_.encoding_options =
                jls_options_.encoding_options | charls::encoding_options::include_pc_parameters_jai;
        }
#endif
    }
    return true;
}

} // namespace jlst

max_stars_repo_name: malaterre/charls-tools

max_issues_repo_name: malaterre/charls-tools

max_forks_repo_name: malaterre/charls-tools


text:
/*=============================================================================
    Boost.Wave: A Standard compliant C++ preprocessor library

    Re2C based C++ lexer

    http://www.boost.org/

    Copyright (c) 2001-2005 Hartmut Kaiser. Distributed under the Boost
    Software License, Version 1.0. (See accompanying file
    LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/

#if !defined(CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED)
#define CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED

#include <boost/wave/token_ids.hpp>

///////////////////////////////////////////////////////////////////////////////
namespace boost {
namespace wave {
namespace cpplexer {
namespace re2clex {

///////////////////////////////////////////////////////////////////////////////
//  The scanner function to call whenever a new token is requested
boost::wave::token_id scan(Scanner *s);

///////////////////////////////////////////////////////////////////////////////
}   // namespace re2clex
}   // namespace cpplexer
}   // namespace wave
}   // namespace boost

#endif // !defined(CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED)

ext: hpp

max_stars_repo_name: schinmayee/nimbus

max_issues_repo_name: schinmayee/nimbus

max_forks_repo_name: schinmayee/nimbus


text:
// Copyright 2015-2020 Tier IV, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef TRAFFIC_SIMULATOR__BEHAVIOR__VEHICLE__LANE_CHANGE_ACTION_HPP_
#define TRAFFIC_SIMULATOR__BEHAVIOR__VEHICLE__LANE_CHANGE_ACTION_HPP_

#include <behaviortree_cpp_v3/behavior_tree.h>
#include <behaviortree_cpp_v3/bt_factory.h>

#include <boost/optional.hpp>
#include <memory>
#include <openscenario_msgs/msg/entity_status.hpp>
#include <openscenario_msgs/msg/entity_trajectory.hpp>
#include <string>
#include <traffic_simulator/behavior/vehicle/vehicle_action_node.hpp>
#include <traffic_simulator/entity/vehicle_parameter.hpp>
#include <traffic_simulator/math/hermite_curve.hpp>
#include <vector>

namespace entity_behavior
{
namespace vehicle
{
class LaneChangeAction : public entity_behavior::VehicleActionNode
{
public:
  LaneChangeAction(const std::string & name, const BT::NodeConfiguration & config);
  BT::NodeStatus tick() override;
  static BT::PortsList providedPorts()
  {
    BT::PortsList ports = {BT::InputPort<std::int64_t>("to_lanelet_id")};
    BT::PortsList parent_ports = entity_behavior::VehicleActionNode::providedPorts();
    for (const auto & parent_port : parent_ports) {
      ports.emplace(parent_port.first, parent_port.second);
    }
    return ports;
  }
  const openscenario_msgs::msg::WaypointsArray calculateWaypoints() override;
  const boost::optional<openscenario_msgs::msg::Obstacle> calculateObstacle(
    const openscenario_msgs::msg::WaypointsArray & waypoints) override;
  void getBlackBoardValues();

private:
  boost::optional<traffic_simulator::math::HermiteCurve> curve_;
  double current_s_;
  double target_s_;
  boost::optional<std::int64_t> to_lanelet_id_;
};
}  // namespace vehicle
}  // namespace entity_behavior

#endif  // TRAFFIC_SIMULATOR__BEHAVIOR__VEHICLE__LANE_CHANGE_ACTION_HPP_


text:
/*
 *	DriverManager.cpp
 *	!CHOAS
 *	Created by Bisegni Claudio.
 *
 *    	Copyright 2013 INFN, National Institute of Nuclear Physics
 *
 *    	Licensed under the Apache License, Version 2.0 (the "License");
 *    	you may not use this file except in compliance with the License.
 *    	You may obtain a copy of the License at
 *
 *    	http://www.apache.org/licenses/LICENSE-2.0
 *
 *    	Unless required by applicable law or agreed to in writing, software
 *    	distributed under the License is distributed on an "AS IS" BASIS,
 *    	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    	See the License for the specific language governing permissions and
 *    	limitations under the License.
 */

#include <boost/functional/hash.hpp>

#include <chaos/common/global.h>

#include <chaos/cu_toolkit/driver_manager/DriverManager.h>

namespace common_utility = chaos::common::utility;
namespace cu_driver_manager = chaos::cu::driver_manager;
namespace cu_driver = chaos::cu::driver_manager::driver;
namespace common_plugin = chaos::common::plugin;

#define DMLAPP_ LAPP_ << "[DriverManager] "
#define DMLDBG_ LDBG_ << "[DriverManager] "
#define DMLERR_ LERR_ << "[DriverManager] "

/*
 Constructor
 */
cu_driver_manager::DriverManager::DriverManager() {

}

/*
 Desctructor
 */
cu_driver_manager::DriverManager::~DriverManager() {

}

// Initialize instance
void cu_driver_manager::DriverManager::init(void *initParameter) throw(chaos::CException) {

}

// Start the implementation
void cu_driver_manager::DriverManager::start() throw(chaos::CException) {

}

// Stop the implementation
void cu_driver_manager::DriverManager::stop() throw(chaos::CException) {

}

// Deinit the implementation
void cu_driver_manager::DriverManager::deinit() throw(chaos::CException) {
    boost::unique_lock<boost::shared_mutex> lock(mutextMapAccess);

	//deinitialize all allcoated driver
	mapParameterLiveInstance.clear();
	for(std::map<string, cu_driver::AbstractDriver*>::iterator it = mapDriverUUIDHashLiveInstance.begin();
		it != mapDriverUUIDHashLiveInstance.end();
		it++){

		//deinitialize driver
		try {
			DMLAPP_ << "Deinitializing device driver with uuid = " << it->second->driverUUID;
			chaos::utility::InizializableService::deinitImplementation(it->second, "AbstractDriver", "DriverManager::deinit");
		} catch (...) {
			DMLAPP_ << "Error deinitializing device driver with uuid = " << it->second->driverUUID;

		}
		DMLAPP_ << "Deleting device driver with uuid = " << it->second->driverUUID;
		delete(it->second);
	}
	mapDriverUUIDHashLiveInstance.clear();
	mapParameterLiveInstance.clear();
}

// Register a new driver
void cu_driver_manager::DriverManager::registerDriver(boost::shared_ptr< common_utility::ObjectInstancer<cu_driver::AbstractDriver> > instancer,
													  boost::shared_ptr< common_plugin::PluginInspector > description) throw(chaos::CException) {
    boost::unique_lock<boost::shared_mutex> lock(mutextMapAccess);

	if(!instancer)
		throw chaos::CException(1, "The instancer of the driver is mandatory for his registration", "DriverManager::registerDriver");

	if(!description)
		throw chaos::CException(2, "The description of the driver is mandatory for his registration", "DriverManager::registerDriver");

	std::string composedDriverName = description->getName();
	composedDriverName.append(description->getVersion());
	DMLAPP_ << "Register new driver with alias = \"" << composedDriverName<<"\"";

	if(mapDriverAliasVersionInstancer.count(composedDriverName)) {
		DMLAPP_ << "Driver \""<<composedDriverName<<"\" is already registered";
		return;
	}

	boost::shared_ptr< DriverPluginInfo > driver_plugin_info(new DriverPluginInfo());
	driver_plugin_info->sp_inspector = description;
	driver_plugin_info->sp_instancer = instancer;
	mapDriverAliasVersionInstancer.insert(make_pair(composedDriverName, driver_plugin_info));
}

// Get a new driver accessor for a driver instance
cu_driver::DriverAccessor *cu_driver_manager::DriverManager::getNewAccessorForDriverInstance(cu_driver::DrvRequestInfo& request_info) throw(chaos::CException) {
	boost::unique_lock<boost::shared_mutex> lock(mutextMapAccess);

	std::string composedDriverName;
	std::string stringForMap;
	cu_driver::DriverAccessor *accessor = NULL;
    std::string driverInfo;
	// the hashing of th einstance  is composed by name+version+input_parameter[if given], because
	// different device drive can have same parameter names. If no input is passed the hashing is
	// calculated using only the device name and the version.

	//befor hashing we need to check if the driver with alias and version has been registered
	DMLDBG_ << "Has been requested a driver with the information -> " << request_info.alias << "/" << request_info.version << "/" << request_info.init_parameter;
	composedDriverName = request_info.alias;
	composedDriverName.append(request_info.version);

	stringForMap = composedDriverName;
	stringForMap.append(request_info.init_parameter);
	driverInfo = std::string("\"") + std::string(request_info.alias) + std::string("\" ver=")+std::string(request_info.version);

	DMLDBG_ << "The identification string for the driver is \"" << stringForMap<<"\"";

	if(mapParameterLiveInstance.count(stringForMap)) {
		DMLDBG_ << "Driver \""<<stringForMap<<"\" is already been instantiated";

		if(mapParameterLiveInstance[stringForMap]->getNewAccessor(&accessor)) {
			//new accessor has been allocated
			DMLAPP_ << "Retrive driver accessor with index ="<< accessor->accessor_index << " for driver "<<driverInfo<<" with uuid =" << mapParameterLiveInstance[stringForMap];

			return accessor;
		} else {
			throw chaos::CException(1, "Device driver retriving accessor", "DriverManager::getNewAccessorForDriverInstance");
		}
	}
	DMLDBG_ << "Driver need to be instantiated using alias =\"" << stringForMap << "\" instancer presence = " << mapDriverAliasVersionInstancer.count(composedDriverName);
	//the instance of the driver need to be created
	if(mapDriverAliasVersionInstancer.count(composedDriverName)==0){
        std::map<std::string, boost::shared_ptr< DriverPluginInfo > >::iterator i;
        DMLDBG_ << "Driver "<<driverInfo<<" not found, here a list of possible driver names and versions:";
        for(i=mapDriverAliasVersionInstancer.begin();i!=mapDriverAliasVersionInstancer.end();i++){
            DMLDBG_ << "\""<<(*i).first<<"\"";
        }
        std::string msg ="Driver " + driverInfo +" not found";
		throw chaos::CException(1,msg , "DriverManager::getNewAccessorForDriverInstance");
	}
	// i can create the instance
	cu_driver::AbstractDriver *driverInstance = mapDriverAliasVersionInstancer[composedDriverName]->sp_instancer->getInstance();
	//associate the driver identification string
	driverInstance->identificationString = stringForMap;
	//initialize the newly create instance
	DMLAPP_ << "Initializing device driver " << driverInfo <<", initialization parameters:\""<<request_info.init_parameter<<"\" with uuid = " << driverInstance->driverUUID;
	chaos::utility::InizializableService::initImplementation(driverInstance, (void*)(request_info.init_parameter.c_str()), "AbstractDriver", "DriverManager::getNewAccessorForDriverInstance");

	//here the driver has been initializated and has been associated with the hash of the parameter
	DMLAPP_ << "Add device driver with hash = " << driverInstance->identificationString;
	mapParameterLiveInstance.insert(make_pair(stringForMap, driverInstance));
	mapDriverUUIDHashLiveInstance.insert(make_pair(driverInstance->driverUUID, driverInstance));
	//now can get new accessor
	if(driverInstance->getNewAccessor(&accessor)) {
		DMLAPP_ << "Got new driver accessor with index =\""<< accessor->accessor_index << "\" for driver:"<<driverInfo<<" driver with uuid =" << driverInstance->driverUUID;
	}
	return accessor;
}

//! release the accessor instance
void cu_driver_manager::DriverManager::releaseAccessor(cu_driver::DriverAccessor *accessor) {
	boost::unique_lock<boost::shared_mutex> lock(mutextMapAccess);

	if(!mapDriverUUIDHashLiveInstance.count(accessor->driver_uuid)) return;

	cu_driver::AbstractDriver *dInstance = mapDriverUUIDHashLiveInstance[accessor->driver_uuid];
	//we can relase the accessor
	if(!dInstance->releaseAccessor(accessor)) {
		throw chaos::CException(1, "This errore never have to appen", "DriverManager::releaseAccessor");
	}
	if(!dInstance->accessors.size()) {
		DMLAPP_ << "The driver with uuid =" << dInstance->driverUUID << " has no more accessor allocated so we can deinitialize it";
		//deinitialize driver
		try {
			DMLAPP_ << "Deinitializing device driver with uuid = " << dInstance->driverUUID <<" (\""<<dInstance->identificationString + "\")";
			chaos::utility::InizializableService::deinitImplementation(dInstance, "AbstractDriver", "DriverManager::deinit");
		} catch (...) {
			DMLAPP_ << "Error deinitializing device driver with uuid = " << dInstance->driverUUID<<" (\""<<dInstance->identificationString + "\")";

		}
		DMLAPP_ << "Deleting device driver with uuid = " << dInstance->driverUUID;
		mapParameterLiveInstance.erase(dInstance->identificationString);
		mapDriverUUIDHashLiveInstance.erase(dInstance->driverUUID);
		delete(dInstance);
	}
}

ext: cpp


text:
#ifndef FM_BACKEND_CLIENTBACKEND_HPP
#define FM_BACKEND_CLIENTBACKEND_HPP

#include "IClient.hpp"

#include "traffic/ChannelImpl.hpp"

#include "core/CoreClient.hpp"

#include "backend/HeartbeatHandler.hpp"
#include "backend/ChannelsHandler.hpp"

#include "facade/control/facade_service.grpc.pb.h"
#include <grpc++/grpc++.h>

#include <boost/asio.hpp>
#include <boost/log/trivial.hpp>

#include <memory>
#include <thread>

namespace fm
{

namespace backend
{

/**
 * Created by: Bartosz Nawrot
 * Date: 2018-12-07
 * Description:
 */
class ClientBackend
{
public:
    ClientBackend(IClient&, IClient::Listener&, boost::asio::io_service&, boost::property_tree::ptree&, core::CoreClient*);

    core::CoreClient& getCore();

    HeartbeatHandler& getHeartbeatHandler();

    ChannelsHandler& getChannelsHandler();

    boost::asio::io_service& getIoService();

    boost::property_tree::ptree& getConfiguration();

    std::unique_ptr<com::fleetmgr::interfaces::Location> getLocation();

    void openFacadeConnection(const com::fleetmgr::interfaces::AttachResponse&);

    void openFacadeConnection(const com::fleetmgr::interfaces::OperateResponse&);

    void closeFacadeConnection();

    void send(const com::fleetmgr::interfaces::facade::control::ClientMessage& message);

    // TODO Bartek argument should be changed to recursive template for optimization
    void log(const boost::log::trivial::severity_level&, const std::string& message);

private:
    enum GrpcTag
    {
        OPEN,
        CLOSE,
        WRITE,
        READ,
    };

    IClient& client;
    IClient::Listener& listener;

    boost::asio::io_service& ioService;

    boost::property_tree::ptree configuration;

    std::unique_ptr<core::CoreClient> core;

    HeartbeatHandler heartbeatHandler;
    ChannelsHandler channelsHandler;

    std::shared_ptr<grpc::Channel> channel;

    std::unique_ptr<
    com::fleetmgr::interfaces::facade::control::FacadeService::Stub> stub;

    std::mutex sendingLock;
    std::atomic<bool> sending;
    std::deque<com::fleetmgr::interfaces::facade::control::ClientMessage> sendingQueue;

    grpc::ClientContext context;
    grpc::CompletionQueue completionQueue;

    std::shared_ptr<com::fleetmgr::interfaces::facade::control::ControlMessage> toRead;

    std::unique_ptr<grpc::ClientAsyncReaderWriter<
    com::fleetmgr::interfaces::facade::control::ClientMessage,
    com::fleetmgr::interfaces::facade::control::ControlMessage>> stream;

    std::atomic<bool> keepReader;

    void openFacadeConnection(const std::string&, const int, const int, const bool);

    void proceedGrpcQueue();

    void readCert(const std::string&, std::string&);
};

} // backend

} // fm

#endif // FM_BACKEND_CLIENTBACKEND_HPP

max_stars_repo_name: skydiveuas/fm-client-sdk-cpp

max_issues_repo_name: skydiveuas/fm-client-sdk-cpp

max_forks_repo_name: skydiveuas/fm-client-sdk-cpp


text:
/*
 * vi:ts=4:shiftwidth=4:expandtab
 * vim600:fdm=marker
 *
 * ngramstat.cpp  - Implements Nagao 1994 N-gram extraction algorithm
 *
 * Copyright (C) 2002 by Zhang Le <ejoy@users.sourceforge.net>
 * Begin       : 30-Oct-2002
 * Last Change : 30-May-2004.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include <cstdlib>
#include <cstdio>
#include <cassert>
#include <cwchar>
#include <cwctype>
#include <algorithm>
#include <stdexcept>
#include <boost/progress.hpp>
#include <boost/ref.hpp>

#include "iconvert.hpp"
#include "mmapfile.hpp"

using namespace std;
using boost::progress_display;

template <typename CharT, typename Traits >
const CharT NGramStat<CharT, Traits>::s_terminal = CharT();

/**
 * Constructor
 *
 * Construct a NGramStat object.
 * @param memory indicate how many memory to use (in MB)
 *        The memory allocated will be freed either by calling
 *        \ref clear() or when the destructor is called.
 * @param file_name_base base name for ptable and ltable file
 *        if empty ltable will be built in memory.
 * @param use_mmap use mmap() call for disk operation.
 */
template <typename CharT, typename Traits>
NGramStat<CharT, Traits>::NGramStat(unsigned memory,
        const string& file_name_base, bool use_mmap)
:
m_is_use_mmap(use_mmap),
m_filename_base(file_name_base),
m_mem_size(memory),
m_buffersize(0),
m_buffer(0),
m_ptable(0),
m_ltable(0)
{
    alloc_mem();
}

template <typename CharT,typename Traits>
NGramStat<CharT, Traits>::~NGramStat(){
    clear();
}

/**
 * Free all memory allocated to save memory for other processing
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::clear() {
    delete[] m_buffer;
    m_buffer = 0;
    delete m_ptable;
    m_ptable = 0;
    delete m_ltable;
    m_ltable = 0;
}

template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::alloc_mem() {
    assert(!m_buffer);
    assert(!m_ptable);
    assert(!m_ltable);

    cerr << "Try to allocate " << m_mem_size/1024 << " MB for processing" <<
        endl;

    unsigned char_count;

    if (m_filename_base.empty())
        //LTable is in memory
        char_count = m_mem_size * 1024 / (sizeof(CharT) + sizeof(unsigned) +
                sizeof(unsigned char));
    else
        char_count = m_mem_size * 1024 / (sizeof(CharT) + sizeof(unsigned));

    //reserve memory in advance to prevent unexcepted mem allocation
    //when vector grows, which may exceed the system's limit
    m_ptable = new vector<unsigned>;
    m_ptable->reserve(char_count);
    if (m_filename_base.empty()) {
        m_ltable = new vector<unsigned char>;
        m_ltable->reserve(char_count);
    }

    m_buffersize = char_count;
    m_buffer = new CharT[m_buffersize + 1 + m_extra_buffersize];
    m_buffer[m_buffersize + m_extra_buffersize] = s_terminal;

    cerr << "Use "
        << sizeof(CharT) * (m_buffersize + 1 + m_extra_buffersize) / (1024 *
                1024)
        << " MB memory for text buffer" << endl;
    cerr << "Use " << char_count * sizeof(unsigned) / (1024 * 1024)
        << " MB memory for ptable entry" << endl;
    if (m_filename_base.empty())
        cerr << "Use " << char_count * sizeof(char) / (1024 * 1024)
            << " MB memory for ltable entry" << endl;
    if (m_is_use_mmap)
        cerr << "Use mmap() for faster operation if necessary" << endl;
}

/**
 * Indicating the beginning of parsing text stream.
 *
 * This function resets all counter to initial states.
 * It also allocates memory if text buffer is deleted
 * in \ref extract_ngram()
 *
 * This function must be called before \ref parse_buf()
 * The correct steps are:
 * 1. call \ref parse_begin() to initiaze inner text buffers and states
 * 2. call \ref parse_buf() one or more times to to parse text stream(s)
 * 3. call \ref parse_end() to finish parsing text.
 * Now you can extract ngram with \ref extract_ngram()
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::parse_begin(){
    if (!m_buffer)
        alloc_mem();

    m_ptable->clear();
    if (m_ltable)
        m_ltable->clear();
    m_start_offset  = 0;
    m_buffer_offset = 0;
    m_last_word_end = 0;
    m_buf_remain.clear();
    m_tempfiles.clear();

    if (!m_filename_base.empty()) {
        m_ngramfile.open((m_filename_base + ".ngram").c_str(),ios::binary);
        if (!m_ngramfile){
            cerr << "unable to open ngramfile:" << m_filename_base
                << ".ngram to write!" << endl;
            throw runtime_error("unable to open ngramfile to write");
        }
    }

}

/**
 * Signal the ending of parsing text stream.
 *
 * Also write/merge PTable/LTable if necessary
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::parse_end() {
    cmp_ptable cmp_obj(m_buffer);
    if (m_filename_base.empty()) { //in memory operation
        cerr << "Sorting ptable..." << endl;
        sort(m_ptable->begin(),m_ptable->end(),cmp_obj);
        cerr << "N-gram buffer size(in CharT):" <<(m_buffer_offset + 1)<<endl;
        cerr << "ptable size:" << m_ptable->size() <<endl;
        //LTable is in memory
        calc_ltable();
    } else {
        if (m_buf_remain.size() > 0)
            save_temp_buffer();

        //no disk merme needed {{{
        //save ptable directly
        if (m_tempfiles.empty()) {
            if (m_buffer_offset > m_buffersize) {
                //add ptable nodes in extra buffer
                add_ptable_node(m_last_word_end,m_buffer_offset);
            }
            cerr << "Sorting ptable..." << endl;
            sort(m_ptable->begin(),m_ptable->end(),cmp_obj);
            cerr << "N-gram buffer size(in CharT):" <<(m_buffer_offset + 1)<<endl;
            cerr << "ptable size:" << m_ptable->size() <<endl;

            write_ptable(m_filename_base + ".ptable",0u);
            write_ltable();
            m_ngramfile.write((char*)m_buffer,
                    (m_buffer_offset + 1) * sizeof(CharT)); //includinm the last L'\0'
            m_ngramfile.close();
            clear();
            //}}}
        } else {
            //need disk merge {{{
            //save current buffer and ptable(including extra buffer) to disk
            if (m_buffer_offset > m_buffersize) {
                //add ptable nodes in extra buffer
                add_ptable_node(m_last_word_end,m_buffer_offset);
            }
            sort(m_ptable->begin(),m_ptable->end(),cmp_obj);

            string filename = next_temp_ptable_filename();
            m_tempfiles.push_back(filename);
            write_ptable(filename,m_start_offset);

            m_ngramfile.write((char*)m_buffer,
                    (m_buffer_offset + 1) * sizeof(CharT)); //includinm the last L'\0'
            m_ngramfile.close();

            //delete allocated memory to save memory for
            //disk merge
            clear();

            merge_ptables(); //also calculate ltable here
        }//}}}
    }
}

/**
 * Parse text in buf (CharT) and store it into inner text buffer.
 *
 * update PTable entry for each token and
 * write temporary ptable to disk if necessary
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::parse_buf(const string_type& buf) {
    assert(!buf.empty());

    if (buf.size() > m_buffersize) {
        //damn! the string is too big to be fit in memory
        //ignore it
        cerr << "string too long, try to allocate more memory to solve this problem"  << endl;
        return;
    }

    if (m_buffer_offset + 30 >= m_buffersize + m_extra_buffersize) {
        //text buffer is full
        //save current buffer and ptable to disk
        save_temp_buffer();
        cerr << "Continue parsing..." << endl;
    }

    //copy buf to the end of buffer
    //start from m_buffer_offset
    //may extend to extra buffer
    unsigned start = m_buffer_offset;
    unsigned n = min(buf.size(),m_buffersize + m_extra_buffersize - start - 20);

    Traits::copy(m_buffer + m_buffer_offset,buf.data(),n);
    m_buffer_offset += n;
    m_buffer[m_buffer_offset] = s_terminal;
    if (n < buf.size())
        m_buf_remain.assign(buf,n,buf.size() - n);

    add_ptable_node(start,min(start + buf.size(),m_buffersize));
}

/**
 * save current buffer to disk for disk merging later
 * then move un-processed text to the beginning of text buffer
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::save_temp_buffer() {
    assert(!m_ptable->empty());

    if (m_filename_base.empty())
        throw runtime_error("Text Buffer full with no external ngram file name given!");

    cerr << "Sorting temporary ptable for disk merging later..." << endl;
    cmp_ptable cmp_obj(m_buffer);
    sort(m_ptable->begin(),m_ptable->end(),cmp_obj);

    string filename = next_temp_ptable_filename();
    m_tempfiles.push_back(filename);
    //write temp ptable
    write_ptable(filename,m_start_offset);
    m_ptable->clear();

    //not include the last L'\0' when writing
    //temp ngram buffer
    m_ngramfile.write((char*)m_buffer,m_last_word_end * sizeof(CharT));

    //copy the rest N-Gram to the beginning of the buffer
    //wcscpy(m_buffer,m_buffer + m_last_word_end);
    size_t n = Traits::length(m_buffer + m_last_word_end);
    Traits::copy(m_buffer,m_buffer + m_last_word_end,n);
    m_buffer[n] = s_terminal;
    if (m_buf_remain.size() > 0) {
        Traits::copy(m_buffer + n,m_buf_remain.data(),m_buf_remain.size());
        m_buffer[n + m_buf_remain.size()] = s_terminal;
        m_buf_remain.clear();
    }

    m_start_offset += m_last_word_end;
    m_last_word_end = 0;
    m_buffer_offset = Traits::length(m_buffer);

    add_ptable_node(0,m_buffer_offset);
}

/**
 * calculate in memory LTable
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::calc_ltable(){
    assert(m_buffer);
    assert(m_ptable);
    assert(m_ltable && m_ltable->empty());
    assert(m_filename_base.empty());

    //set first entry to zero
    m_ltable->push_back(0);

    unsigned char count;
    vector<unsigned>& ptable = *m_ptable;
    for (unsigned i = 1;i < ptable.size(); ++i) {
        assert(ptable[i - 1] < m_buffer_offset && ptable[i] < m_buffer_offset);

        count = calc_common_words(&m_buffer[ptable[i- 1]],&m_buffer[ptable[i]]);
        //m_ltable->push_back(calc_common_words(&m_buffer[ptable->at(i - 1)],&m_buffer[ptable->at(i]]));
        m_ltable->push_back(count);
    }
}

/**
 * calculate and write a ltable into ltable file(*.ltable)
 * this ltable is calculated from in memory ptable and ngram table
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::write_ltable() const {
    assert(m_buffer);
    assert(m_ptable);
    assert(!m_filename_base.empty());

    string name = m_filename_base + ".ltable";

    cerr << "Writing ltable: " << name << endl;

    ofstream file(name.c_str(),ios::binary);
    if (!file) {
        cerr << "unable to open ltable file:" << name << "to write!" << endl;
        throw runtime_error("unable to open ltable for writing");
    }

    unsigned char count = 0;
    //set first entry to zero
    file.write((char*)&count,sizeof(unsigned char));

    vector<unsigned>& ptable = *m_ptable;
    for (unsigned i = 1;i < ptable.size(); ++i) {
        assert(ptable[i - 1] < m_buffer_offset && ptable[i] < m_buffer_offset);

        count = calc_common_words(&m_buffer[ptable[i - 1]],
                &m_buffer[ptable[i]]);
        file.write((char*)&count,sizeof(unsigned char));
    }
    file.close();
}

/**
 * calculate and write ltable into ltable file(*.ltable)
 * this ltable is calculated from the given ptable and ngram table
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::write_ltable(const CharT* ngram_table,const string ptable_filename) const{
    assert(!m_filename_base.empty());

    string ltable_filename = m_filename_base + ".ltable";
    cerr << "Writing ltable:" << ltable_filename << endl;

    ofstream file(ltable_filename.c_str(),ios::binary);
    if (!file) {
        cerr << "unable to open ltable file:" << ltable_filename << "to write!" << endl;
        throw runtime_error("unable to open ltable for writing");
    }

    unsigned char count = 0;

    //set first entry to zero
    file.write((char*)&count,sizeof(unsigned char));

    if (m_is_use_mmap) {
        unsigned* ptable     = 0;
        unsigned  ptable_size = 0;

        //mmap ptable file for calculating ltable
        MmapFile fm(ptable_filename.c_str());
        if (!fm.open())
            throw runtime_error("mmap call failed");

        ptable = (unsigned*)fm.addr();
        ptable_size = fm.size();

        unsigned* ptable_end = ptable + ptable_size / sizeof(unsigned) - 1;

        for (const unsigned* i = ptable;i < ptable_end;++i) {
            count = calc_common_words(&ngram_table[*i],
                    &ngram_table[*(i+1)]);
            file.write((char*)&count,sizeof(unsigned char));
        }
        fm.close();
    } else {
        /*
           int fd;
           fd = open(ptable_filename.c_str(),O_RDONLY);
           if (fd == -1) {
           perror("unable to open ptable file");
           exit(EXIT_FAILURE);
           }
           */

        FILE* fp;
        fp = fopen(ptable_filename.c_str(),"rb");
        if (fp == NULL) {
            perror("unable to open ptable file");
            exit(EXIT_FAILURE);
        }

        /*
           ifstream fs;
           fs.open(ptable_filename.c_str(),ios::binary);
           if (!fs) {
           perror("unable to open ptable file");
           exit(EXIT_FAILURE);
           }
           */

        struct stat st;
        if (stat(ptable_filename.c_str(),&st) == -1) {
            perror("unable to stat ptable file size");
            exit(EXIT_FAILURE);
        }

        struct {
            unsigned offset1;
            unsigned offset2;
        }offset;

        unsigned end = st.st_size - sizeof(unsigned);

        for (unsigned pos = 0;pos < end; pos += sizeof(unsigned) ) {
            /*
               if (lseek(fd,pos,SEEK_SET) == -1 ||
               read(fd,&offset,sizeof(offset)) != sizeof(offset)) {
               perror("error reading ptable file");
               exit(EXIT_FAILURE);
               }
               */
            if (fseek(fp,pos,SEEK_SET) == -1 ||
                    fread(&offset,sizeof(offset),1,fp) != 1) {
                perror("error reading ptable file");
                exit(EXIT_FAILURE);
            }
            /*
               cerr << fs.seekg(pos,ios::beg);
               cerr << "read:"<< fs.read(&offset,sizeof(offset));
               */

            count = calc_common_words(&ngram_table[offset.offset1],
                    &ngram_table[offset.offset2]);
            file.write((char*)&count,sizeof(unsigned char));
        }

        file.close();
    }
}

/**
 * write in memory ptable into ptable file(*.ptable)
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::write_ptable(const string& name,unsigned start_offset) const{
    assert(m_ptable);

    cerr << "Writing ptable: " << name << endl;

    ofstream file(name.c_str(),ios::binary);
    if (!file) {
        cerr << "unable to open ptable file:" << name << "to write!" << endl;
        return;
    }

    unsigned offset;
    vector<unsigned>& ptable = *m_ptable;
    for (unsigned i = 0;i < ptable.size(); ++i) {
        offset = ptable[i] + start_offset;
        file.write((char*)&offset,sizeof(unsigned));
        /* for debug
        ustring ws(&g_buffer[offset],255);
        string s;
        g_iConvert->convert(ws,s);
        cerr << offset << ":" << s << endl;
        */
    }
    file.close();
}

/**
 *fill ptable entry in m_buffer[start,end)
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::add_ptable_node(unsigned start,unsigned end){
    for (unsigned j = start;j < end;++j) {
        m_ptable->push_back(j);
        m_last_word_end = j + 1;
    }
}

struct PTable_attr {
    FILE* m_fp;
    int          m_fd;

    //mmap attr
    unsigned*    mp_addr;    //mmap addr
    unsigned*    mp_end;     //end of mmap addr
    unsigned*    mp_pos;     //current pos

    //file attr
    off_t        m_pos;      //current pos in file
    unsigned     m_offset;   //current ngram  offset in ngram file
    off_t        m_end;      //end offset
    unsigned     m_size;
};

/**
 * merge several temp ptable file into one
 * and save it to m_filename_base + ".ptable"
 * Merging temporary ptable file using mmap() is faster than normal
 * file operation. But some system (such as Win32) has a memory limitation
 * of 2G. So if you want to process large file(1-2 Gb), do not use mmap().
 * However, we still need to mmap ngram file for fast random access.
 * That is to say, you can not process file larger than 2G on Win32 platform.
 * Use Linux instead.
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::merge_ptables() {
    string ptable_filename = m_filename_base + ".ptable";
    vector<PTable_attr > ptables(m_tempfiles.size());

    for (unsigned i = 0;i < ptables.size(); ++i) {
        ptables[i].m_fd     = -1;

        ptables[i].mp_addr  = 0;
        ptables[i].mp_end   = 0;
        ptables[i].mp_pos   = 0;

        ptables[i].m_pos    = 0;
        ptables[i].m_offset = 0;
        ptables[i].m_end    = 0;
        ptables[i].m_size   = 0;
    }

    unsigned files_to_merge = 0;

    vector<MmapFile*> fm_objs;

    for (unsigned i = 0;i < m_tempfiles.size(); ++i, ++files_to_merge) {
        if (m_is_use_mmap) {
            //open m_tempfiles using mmap()
            MmapFile* fm = new MmapFile(m_tempfiles[i].c_str());
            if (fm_objs[i]->open()) {
                ptables[i].mp_addr = (unsigned*)fm_objs[i]->addr();
                ptables[i].m_size = fm_objs[i]->size();
                ptables[i].mp_pos = ptables[i].mp_addr;
                ptables[i].mp_end = ptables[i].mp_addr + ptables[i].m_size /
                    sizeof(unsigned);
                fm_objs.push_back(fm);
            } else {
                cerr << "unable to mmap() file:" << m_tempfiles[i] << endl;
                delete fm;
                throw runtime_error("unable to mmap() temp ptable file");
            }
        } else {
            //ptables[i].m_fd = open(m_tempfiles[i].c_str(),O_RDONLY);
            ptables[i].m_fp = fopen(m_tempfiles[i].c_str(),"rb");
            if (ptables[i].m_fp == NULL) {
                perror("unable to open temporary ptable file");
                throw runtime_error("unable to open temporary ptable file");
            }
            ptables[i].m_fd = fileno(ptables[i].m_fp);

            if (ptables[i].m_fd == -1) {
                perror("unable to open temporary ptable file");
                throw runtime_error("unable to open temporary ptable file");
            }

            struct stat st;
            if (fstat(ptables[i].m_fd,&st) == -1) {
                perror("unable to stat temporary ptable file size");
                throw runtime_error("unable to stat temporary ptable file size");
            }

            ptables[i].m_size = st.st_size;
            ptables[i].m_end  = st.st_size;
        }
    }

    //mmap ngram table
#if !defined (HAVE_SYSTEM_MMAP)
#error the ptable merging code needs mmap(2) support, which is missing on this system
#endif
    CharT* ngram_table = 0;
    unsigned ngram_table_size = 0;
    MmapFile fm_ngram(string(m_filename_base + ".ngram").c_str());
    if (!fm_ngram.open()) {
        cerr << "unable to mmap:" << m_filename_base + ".ngram" << endl;
        throw runtime_error("unable to mmap() ngram file");
    }
    ngram_table = (CharT*)fm_ngram.addr();
    ngram_table_size = fm_ngram.size();

    ngram_table_size /= sizeof(CharT);

    ofstream ptable_file(ptable_filename.c_str(),ios::binary);
    if (!ptable_file) {
        cerr << "unable to write to ptable file:" << ptable_filename << endl;
        throw runtime_error("unable to open ptable file to write");
    }

    //now merging
    cerr << "Merging " << files_to_merge << " temporary ptables..." << endl;

    assert (files_to_merge == ptables.size());
    int rc;

    if (m_is_use_mmap) { //{{{
        PTable_attr* min;
        while (files_to_merge > 0) {
            min = 0;
            //set min to first ptable needed to merge
            for (unsigned i = 0;i < ptables.size(); ++i ) {
                if (ptables[i].mp_pos < ptables[i].mp_end) {
                    min = &ptables[i];
                    break;
                }
            }

            assert(min);

            for (unsigned i = 0;i < ptables.size(); ++i ) {
                PTable_attr& attr = ptables[i];
                if (attr.mp_pos < attr.mp_end && min != &ptables[i]) {
//                    rc = wcsncmp((const wchar_t*)&ngram_table[*attr.mp_pos],
//                            (const wchar_t*)&ngram_table[*(min->mp_pos)],255);
                    rc = Traits::compare((const CharT*)&ngram_table[*attr.mp_pos],
                            (const CharT*)&ngram_table[*(min->mp_pos)],255);
                    if (rc < 0 || (rc == 0 && *attr.mp_pos < *(min->mp_pos)))
                        min = &ptables[i];
                }
            }

            assert(min);

            ptable_file.write((char*)min->mp_pos,sizeof(unsigned));

            if (++(min->mp_pos) >= min->mp_end) {
//                mmap files will be closed on clean up
//                if (min->m_size > 0)
//                    munmap_file(min->mp_addr,min->m_size);
                --files_to_merge;
            }
        }//}}}
    } else { //do not use mmap() {{{
        PTable_attr* min;
        while (files_to_merge > 0) {
            min = 0;
            //set min to first ptable needed to merge
            for (unsigned i = 0;i < ptables.size(); ++i ) {
                if (ptables[i].m_pos < ptables[i].m_end) {
                    min = &ptables[i];
                    break;
                }
            }

            assert(min);

            if (fseek(min->m_fp,min->m_pos,SEEK_SET) == -1 ||
                    fread(&min->m_offset,sizeof(unsigned),1,min->m_fp) != 1 ) {
                perror("error reading temporary ptable file");
                exit(EXIT_FAILURE);
            }
            /*
            if (lseek(min->m_fd,min->m_pos,SEEK_SET) == -1 ) {
                perror("error reading temporary ptable file");
                exit(EXIT_FAILURE);
            }
            */
            //int size;
            //size = read(min->m_fd,&min->m_offset,sizeof(unsigned));
            /*
            size = fread(&min->m_offset,sizeof(unsigned),1,min->m_fp);
            if (size != 1)
                cerr << "read return:"<<size<< endl;
                */
            /*
            if (lseek(min->m_fd,min->m_pos,SEEK_SET) == -1 ||
                    read(min->m_fd,&min->m_offset,sizeof(unsigned)) != sizeof(unsigned)) {
                perror("error reading temporary ptable file");
                exit(EXIT_FAILURE);
            }
            */

            for (unsigned i = 0;i < ptables.size(); ++i ) {
                PTable_attr& attr = ptables[i];
                if (attr.m_pos < attr.m_end && min != &ptables[i]) {
                    if (fseek(attr.m_fp,attr.m_pos,SEEK_SET) == -1 ||
                            fread(&attr.m_offset,sizeof(unsigned),1,attr.m_fp) != 1) {
                        perror("error reading temporary ptable file");
                        exit(EXIT_FAILURE);
                    }
                    //size = read(attr.m_fd,&attr.m_offset,sizeof(unsigned));
                    /*
                    size = fread(&attr.m_offset,sizeof(unsigned),1,attr.m_fp);

                    if (size != 1)
                        cerr << "read2 return:"<<size<< endl;
                        */
                    /*
                       if (lseek(attr.m_fd,attr.m_pos,SEEK_SET) == -1 ||
                       read(attr.m_fd,&attr.m_offset,sizeof(unsigned)) != sizeof(unsigned)) {
                       perror("error reading temporary ptable file");
                       exit(EXIT_FAILURE);
                       }
                       */

//                    rc = wcsncmp((const wchar_t*)&ngram_table[attr.m_offset],
//                            (const wchar_t*)&ngram_table[min->m_offset],255);
                    rc = Traits::compare((const CharT*)&ngram_table[attr.m_offset],
                            (const CharT*)&ngram_table[min->m_offset],255);
                    if (rc < 0 || (rc == 0 && attr.m_offset < min->m_offset)) {
                        min = &ptables[i];
                    }
                }
            }

            assert(min);

            ptable_file.write((char*)&min->m_offset,sizeof(unsigned));

            min->m_pos += sizeof(unsigned);

            if (min->m_pos >= min->m_end) {
                if (min->m_size > 0)
                    if (fclose(min->m_fp) == EOF)
                    //if (close(min->m_fd) == -1)
                        perror("unable to close temporary ptable file");
                --files_to_merge;
            }
        }
    } //}}}

    ptable_file.close();

    write_ltable(ngram_table,ptable_filename);

    //clean up
    // munmap_file(ngram_table,ngram_table_size);

    for (size_t i = 0;i < fm_objs.size(); ++i)
        delete fm_objs[i];

    for (unsigned i = 0;i < m_tempfiles.size();++i)
        if (remove(m_tempfiles[i].c_str())) {
            string s = "unable to remove file:";
            s += m_tempfiles[i];
            perror(s.c_str());
        }
}

//return next temp ptable filename
template <typename CharT,typename Traits>
string NGramStat<CharT, Traits>::next_temp_ptable_filename() const {
    return next_temp_filename("NGramStat");
}
/**
 * calculate common words(max 255) from the beginning of s1 and s2
 */
template <typename CharT,typename Traits>
unsigned char NGramStat<CharT, Traits>::calc_common_words(const CharT* s1,const CharT* s2) const {
    assert(*s1 && *s2);

    unsigned char count = 0;
        //simply counting every char including blanks and punctuations
        while (count < 255 && *s1 != s_terminal && *s2 != s_terminal &&
                *s1++ == *s2++)
            ++count;
    return count;
}

/**
 * parse a string in given encoding
 */
//template <typename CharT,typename Traits>
//void NGramStat<CharT, Traits>::parse(const string& str, const string& encoding/*"UTF-8"*/){
//    IConvert iconv(encoding);
//    ustring ws;
//    if (iconv.convert(str,ws) == false)
//        throw runtime_error("fail to convert input string into UNICODE");
//
//    parse_begin();
//    parse_buf(ws);
//    parse_end();
//}

/**
 * parse several files in given encoding
 */
//template <typename CharT,typename Traits>
//void NGramStat<CharT, Traits>::parse(const vector<string>& files, const string& encoding/*"UTF-8"*/){
//    IConvert iconv(encoding);
//    string s;
//    ustring ws;
//
//    parse_begin();
//
//    for (unsigned i = 0;i < files.size(); ++i) {
//        ifstream f(files[i].c_str());
//        if (!f)
//            throw runtime_error("unable to open file for parsing");
//
//        cerr << "parsing file: " << files[i] << endl;
//        while (getline(f,s)) {
//            if (s.empty())
//                continue;
//
//            if (iconv.convert(s,ws) == false) {
//                cerr << '\"' << s << "\" can not be converted into UNICODE" << endl;
//                continue;
//            }
//
//            parse_buf(ws);
//        }
//    }
//
//    parse_end();
//}

/**
 * parse a string(array) of CharT
 */
//template <typename CharT,typename Traits>
//void NGramStat<CharT, Traits>::parse(const string_type& str) {
//    parse_begin();
//    parse_buf(str);
//    parse_end();
//}
//
/**
 * fetch a N gram from the beginning of s
 * if no N-gram found at s,set ngram to L""
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::fetch_ngram(unsigned N,CharT* s,string_type& ngram) const{
    ngram.clear();

    //Simply counting every char including punctuations and blanks
    while (N > 0 && *s) {
        ngram += *s++;
        --N;
    }

    if (N != 0)
        ngram.clear();
}

/**
 * this function behaves essentially like the above function
 * except it fetches N to M-gram(N<=M) from the beginning of s
 * and store them in a vector of ustring
 * if N-gram is not found in s,store a L""
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::fetch_ngrams(unsigned N,unsigned M,CharT* s,vector<NGram>& ngrams) const{
    assert(N <= M);
    assert(ngrams.size() >= M + 1);

    string_type  ngram;
    unsigned n;

    for (n = N;n <= M; ++n)
        ngrams[n].m_text.clear();

    //Simply counting every char including punctuations and blanks
    for (n = 1; n <= M && *s;++n) {
        ngram += *s++;
        if (n >= N) {
            ngrams[n].m_text = ngram;
            ngrams[n].m_count = 1;
        }
    }
}

/**
 * fetch a entry of ptable from m_ptable, mmaped ptable or ptable file
 * depending on access method used
 */
template <typename CharT,typename Traits>
unsigned NGramStat<CharT, Traits>::ptable_entry(unsigned* ptable,FILE* fp,unsigned pos) const {
    if (ptable) {
        assert(m_is_use_mmap);
        return ptable[pos];
    } else if (fp) {
        unsigned entry;
        if (fseek(fp,pos * sizeof(unsigned),SEEK_SET) == -1 ||
                fread(&entry,sizeof(unsigned),1,fp) != 1) {
            perror("error reading ptable file");
            throw runtime_error("error when reading ptable file");
        }
        /*
           if (lseek(g_ptable_fd,pos * sizeof(unsigned),SEEK_SET) == -1 ||
           read(g_ptable_fd,&entry,sizeof(unsigned)) != sizeof(unsigned)) {
           perror("error reading ptable file");
           exit(EXIT_FAILURE);
           }
           */
        return entry;
    } else {
        assert(m_ptable);
        return (*m_ptable)[pos];
    }
}

/**
 * fetch a entry of ltable from m_ltable, mmaped ltable or ltable file
 * depending on access method used
 */
template <typename CharT,typename Traits>
unsigned char NGramStat<CharT, Traits>::ltable_entry(unsigned char* ltable,FILE* fp,unsigned pos) const {
    if (ltable) {
        assert(m_is_use_mmap);
        return ltable[pos];
    } else if (fp) {
        unsigned char entry;
        /*
           if (lseek(g_ltable_fd,pos * sizeof(unsigned char),SEEK_SET) == -1 ||
           read(g_ltable_fd,&entry,sizeof(unsigned char)) != sizeof(unsigned char)) {
           perror("error reading ltable file");
           exit(EXIT_FAILURE);
           }
           */
        if (fseek(fp,pos * sizeof(unsigned char),SEEK_SET) == -1 ||
                fread(&entry,sizeof(unsigned char),1,fp) != 1) {
            perror("error reading ltable file");
            throw runtime_error("error when reading ltable file");
        }

        return entry;
    } else {
        assert(m_ltable);
        return (*m_ltable)[pos];
    }
}

/**
 * Extract N-M ngram with frequency >= freq from NGramStat object.
 *
 * Extract N-gram to M-gram from the inner text buffer.
 *
 * @param N smallest N-gram to extract
 * @param M largest M-gram to extract (M >= N)
 * @param freq only N-grams whose frequency >= freq are extracted
 * @param output a helper function object for extracting N-grams
 *        the first argument is the ngram the second argument is the count of
 *        the ngram
 * TODO: more document on OutputFunc
 */
template <typename CharT,typename Traits>
void NGramStat<CharT, Traits>::extract_ngram(unsigned N, unsigned M, unsigned freq,
        OutputFunc& output) {
    CharT*         ngramtable      = m_buffer;
    unsigned       ngramtable_size = m_buffer_offset;
    unsigned*      ptable          = 0;
    unsigned       ptable_size     = m_ptable->size();
    unsigned char* ltable          = 0;
    unsigned       ltable_size     = m_ltable ? m_ltable->size():0;
    FILE*          ptable_fp       = 0;
    FILE*          ltable_fp       = 0;
    //int            ptable_fd;
    //int            ltable_fd;

    string ngram_filename  = m_filename_base + ".ngram";
    string ptable_filename = m_filename_base + ".ptable";
    string ltable_filename = m_filename_base + ".ltable";

    vector<MmapFile*> fm_objs;

    //init: mmap ngram table,ptable and ltable if necessary {{{
    if (!m_filename_base.empty()) {
        //free memory(if any)
        clear();

        MmapFile* fm = new MmapFile(ngram_filename.c_str());
        if (!fm->open()) {
            cerr << "could not mmap file:" << ngram_filename << endl;
            throw runtime_error("unable to mmap ngram table file when extracting NGram");
        }
        fm_objs.push_back(fm);
        ngramtable = (CharT*)fm->addr();
        ngramtable_size = fm->size();
        assert(ngramtable);
        ngramtable_size /= sizeof(CharT);

        if (m_is_use_mmap) {
            MmapFile* fm = new MmapFile(ptable_filename.c_str());
            if (!fm->open()) {
                cerr << "could not mmap file:" << ptable_filename << endl;
                throw runtime_error("unable to mmap ptable file when extracting NGram");
            }
            fm_objs.push_back(fm);
            ptable = (unsigned*)fm->addr();
            ptable_size = fm->size();
            assert(ptable);
            ptable_size /= sizeof(unsigned);

            MmapFile* fm2 = new MmapFile(ltable_filename.c_str());
            if (!fm2->open()) {
                cerr << "could not mmap file:" << ltable_filename << endl;
                throw runtime_error("unable to mmap ltable file when extracting NGram");
            }
            fm_objs.push_back(fm2);
            ltable = (unsigned char*)fm2->addr();
            ltable_size = fm2->size();
            assert(ltable);
            ltable_size /= sizeof(unsigned char);
        } else {
            ptable_fp = fopen(ptable_filename.c_str(),"rb");
            if (ptable_fp == NULL) {
                perror("unable to open ptable file");
                throw runtime_error("unable to open ptable file when extracting NGram");
            }
            struct stat st;
            if (stat(ptable_filename.c_str(),&st) == -1) {
                perror("unable to stat ptable file size");
                throw runtime_error("unable to stat ptable file size when extracting NGram");
            }
            ptable_size = st.st_size / sizeof(unsigned);

            ltable_fp = fopen(ltable_filename.c_str(),"rb");
            if (ltable_fp == NULL) {
                perror("unable to open ltable file");
                throw runtime_error("unable to open ltable file when extracting NGram");
            }

            if (stat(ltable_filename.c_str(),&st) == -1) {
                perror("unable to stat ltable file size");
                throw runtime_error("unable to stat ltable file size when extracting NGram");
            }

            ltable_size = st.st_size / sizeof(unsigned char);
            /*  old implementation{{{
                ptable_fd = open(ptable_filename.c_str(),O_RDONLY);
                if (ptable_fd == -1) {
                perror("unable to open ptable file");
                exit(EXIT_FAILURE);
                }

                struct stat st;
                if (fstat(ptable_fd,&st) == -1) {
                perror("unable to stat ptable file size");
                exit(EXIT_FAILURE);
                }

                ptable_size = st.st_size;

                ltable_fd = open(g_ltable_filename.c_str(),O_RDONLY);
                if (ltable_fd == -1) {
                perror("unable to open ltable file");
                exit(EXIT_FAILURE);
                }

                if (fstat(ltable_fd,&st) == -1) {
                perror("unable to stat ltable file size");
                exit(EXIT_FAILURE);
                }

                ltable_size = st.st_size;
                }}}*/
        }

        assert(ptable_size == ltable_size);
    }//}}}

    if (false) { //dump ngram table and ltable {{{
        /*
        assert(m_is_use_mmap);

        unsigned char* l = g_ltable;
        unsigned* p = g_ptable;
        unsigned* pend = g_ptable + g_ptable_size/sizeof(unsigned);
        wchar_t ws[26];
        string s;
        ws[25] = L'\0';

        cerr << "Ngramtable_size:" << ngramtable_size/sizeof(wchar_t) << endl;
        cerr << "PTable_size:" << g_ptable_size/sizeof(unsigned) << endl;
        cerr << "LTable_size:" << g_ltable_size/sizeof(unsigned char) << endl;

        while (p < pend) {
            assert(*p < ngramtable_size/sizeof(wchar_t));
            wcsncpy(ws,&ngramtable[*p],25);
            if(g_iConvert->convert(ws,s)) {
                cout << int(*l) << "\t==>" << s << endl;
            }
            else {
                cerr << "error occurs when calling iconv()" << endl;
            }
            ++p;
            ++l;
        }
        */
        //}}}

    } else if (N == M) { //extract N-gram with a fixed N extraction algorithm {{{
        //which is a little faster than N-M extract algorithm below
        string_type  ngram;
        unsigned i;
        unsigned count = 1;
        unsigned size  = ltable_size;
        progress_display* progress = 0;
        if (true)
            progress = new progress_display(size,cerr);

        //set first ngram
        fetch_ngram(N,&ngramtable[ptable_entry(ptable,ptable_fp,0)],ngram);
        for (i = 1;i < size; ++i) {
            if (progress)
                ++(*progress);

            if (ltable_entry(ltable,ltable_fp,i) >= N) {
                ++count;
            } else {
                //this node is a n-gram which n < N
                //so we print out current ngram (n >=N )and its count
                if (count >= freq && !ngram.empty())
                    output(ngram,count);

                //fetch new ngram
                fetch_ngram(N,&ngramtable[ptable_entry(ptable,ptable_fp,i)],ngram);
                count = 1;
            }
        }//for

        //output last entry
        if (count >= freq && !ngram.empty())
            output(ngram,count);

        if (progress) {
            delete progress;
            cerr << endl;
        }
        //}}}

    } else { //extract N-gram in range[N,M] {{{
        vector<NGram> ngrams(M + 1); //so we can access the Nth-gram with ngrams[N]
        unsigned i;
        unsigned j;
        unsigned l;  //store ltable[l]:the co-occurence count of the two adjancent ngrams
        unsigned size = ltable_size;
        progress_display* progress = 0;
        if (true)
            progress = new progress_display(size,cerr);

        //get first N-Mngrams
        fetch_ngrams(N,M,&ngramtable[ptable_entry(ptable,ptable_fp,0)],ngrams);
        for (i = 1;i < size; ++i) {
            if (progress)
                ++(*progress);

            l = ltable_entry(ltable,ltable_fp,i);
            if (l < N) {
                //this ltable node is a n-gram which n < N
                //so we print out current ngrams (n in [N,M]) and their counts
                for (j = N;j <= M;++j)
                    if (ngrams[j].m_count >= freq && !ngrams[j].m_text.empty())
                        output(ngrams[j].m_text,ngrams[j].m_count);

                //and fetch new ngrams
                fetch_ngrams(N,M,&ngramtable[ptable_entry(ptable,ptable_fp,i)],ngrams);
            } else if (l >= M){
                //increasing N-gram count in [N,M]
                for (j = N;j <= M;++j)
                    ++ngrams[j].m_count;
            } else {
                // N <= l < M
                //increasing N-gram count in [N,l]
                for (j = N;j <= l;++j)
                    ++ngrams[j].m_count;

                //need to output ngrams[l + 1,M]
                for (j = l + 1;j <= M;++j)
                    if (ngrams[j].m_count >= freq && !ngrams[j].m_text.empty())
                        output(ngrams[j].m_text,ngrams[j].m_count);
                fetch_ngrams(l + 1,M,&ngramtable[ptable_entry(ptable,ptable_fp,i)],ngrams);
            }
        }//for

        //output last entry
        for (j = N;j <= M;++j)
            if (ngrams[j].m_count >= freq && !ngrams[j].m_text.empty())
                output(ngrams[j].m_text,ngrams[j].m_count);

        if (progress) {
            delete progress;
            cerr << endl;
        }
    } //}}}

    // clean up {{{
    for (size_t i = 0; i < fm_objs.size(); ++i)
        delete fm_objs[i];

    if (!m_filename_base.empty()) {
        if (!m_is_use_mmap) {
            assert(ptable_fp && ltable_fp);
            fclose(ptable_fp);
            fclose(ltable_fp);
        }
    }
    //}}}
}

//helper output function object
//struct DefaultOutputHelper{
//    DefaultOutputHelper(ostream& os,const string& encoding)
//    :os(&os),iconv(encoding){}
//
//    void operator()(const ustring& ws,unsigned count) const {
//        static string s;
//        if (iconv.convert(ws,s))
//            (*os) << s << '\t' << count << endl;
//    }
//    private:
//    ostream* os;
//    mutable IConvert iconv;
//};
//
///**
// * this function behaves essentially like above function(just a wrapper)
// * except it output the result N-Grams to a ostream using given
// * encoding
// */
//template <typename CharT,typename Traits>
//void NGramStat<CharT, Traits>::extract_ngram(unsigned N, unsigned M, unsigned freq,
//        std::ostream& os/*std::cout*/,const string& encoding/*"UTF-8"*/) {
//    DefaultOutputHelper out(os,encoding);
//    OutputFunc f = boost::ref(out);
//    extract_ngram(N,M,freq,f);
//}

ext: tcc

max_stars_repo_name: jimregan/ngramtool

max_issues_repo_name: jimregan/ngramtool

max_forks_repo_name: jimregan/ngramtool


text:
#include <CGAL/Simple_cartesian.h>
#include <CGAL/Monge_via_jet_fitting.h>

#include <fstream>
#include <cassert>

#include <CGAL/property_map.h>

#include <boost/program_options.hpp>
namespace po = boost::program_options;

using namespace std;

#include "PolyhedralSurf.h"
#include "PolyhedralSurf_operations.h"
#include "PolyhedralSurf_rings.h"

//Kernel of the PolyhedralSurf
typedef double                DFT;
typedef CGAL::Simple_cartesian<DFT>  Data_Kernel;
typedef Data_Kernel::Point_3  DPoint;
typedef Data_Kernel::Vector_3 DVector;

//HDS
typedef PolyhedralSurf::Vertex_handle Vertex_handle;
typedef PolyhedralSurf::Vertex Vertex;
typedef PolyhedralSurf::Halfedge_handle Halfedge_handle;
typedef PolyhedralSurf::Halfedge Halfedge;
typedef PolyhedralSurf::Vertex_iterator Vertex_iterator;
typedef PolyhedralSurf::Facet_handle Facet_handle;
typedef PolyhedralSurf::Facet Facet;

struct Hedge_cmp{
  bool operator()(Halfedge_handle a,  Halfedge_handle b) const{
    return &*a < &*b;
  }
};

struct Facet_cmp{
  bool operator()(Facet_handle a, Facet_handle b) const{
    return &*a < &*b;
  }
};

//Vertex property map, with std::map
typedef std::map<Vertex*, int> Vertex2int_map_type;
typedef boost::associative_property_map< Vertex2int_map_type > Vertex_PM_type;
typedef T_PolyhedralSurf_rings<PolyhedralSurf, Vertex_PM_type > Poly_rings;

//Hedge property map, with enriched Halfedge with its length
// typedef HEdge_PM<PolyhedralSurf> Hedge_PM_type;
// typedef T_PolyhedralSurf_hedge_ops<PolyhedralSurf, Hedge_PM_type> Poly_hedge_ops;
//Hedge property map, with std::map
typedef std::map<Halfedge_handle, double, Hedge_cmp> Hedge2double_map_type;
typedef boost::associative_property_map<Hedge2double_map_type> Hedge_PM_type;
typedef T_PolyhedralSurf_hedge_ops<PolyhedralSurf, Hedge_PM_type> Poly_hedge_ops;

// //Facet property map with enriched Facet with its normal
// typedef Facet_PM<PolyhedralSurf> Facet_PM_type;
// typedef T_PolyhedralSurf_facet_ops<PolyhedralSurf, Facet_PM_type> Poly_facet_ops;
//Facet property map, with std::map
typedef std::map<Facet_handle, Vector_3, Facet_cmp> Facet2normal_map_type;
typedef boost::associative_property_map<Facet2normal_map_type> Facet_PM_type;
typedef T_PolyhedralSurf_facet_ops<PolyhedralSurf, Facet_PM_type> Poly_facet_ops;

typedef double                   LFT;
typedef CGAL::Simple_cartesian<LFT>     Local_Kernel;
typedef CGAL::Monge_via_jet_fitting<Data_Kernel> My_Monge_via_jet_fitting;
typedef My_Monge_via_jet_fitting::Monge_form My_Monge_form;

// default parameter values and global variables
unsigned int d_fitting = 2;
unsigned int d_monge = 2;
unsigned int nb_rings = 0;//seek min # of rings to get the required #pts
unsigned int nb_points_to_use = 0;//
bool verbose = false;
unsigned int min_nb_points = (d_fitting + 1) * (d_fitting + 2) / 2;

//gather points around the vertex v using rings on the
//polyhedralsurf. the collection of points resorts to 3 alternatives:
// 1. the exact number of points to be used
// 2. the exact number of rings to be used
// 3. nothing is specified
void gather_fitting_points(Vertex* v,
                           std::vector<DPoint> &in_points,
                           Vertex_PM_type& vpm)
{
  //container to collect vertices of v on the PolyhedralSurf
  std::vector<Vertex*> gathered;
  //initialize
  in_points.clear();

  //OPTION -p nb_points_to_use, with nb_points_to_use != 0. Collect
  //enough rings and discard some points of the last collected ring to
  //get the exact "nb_points_to_use"
  if ( nb_points_to_use != 0 ) {
    Poly_rings::collect_enough_rings(v, nb_points_to_use, gathered, vpm);
    if ( gathered.size() > nb_points_to_use ) gathered.resize(nb_points_to_use);
  }
  else { // nb_points_to_use=0, this is the default and the option -p is not considered;
    // then option -a nb_rings is checked. If nb_rings=0, collect
    // enough rings to get the min_nb_points required for the fitting
    // else collect the nb_rings required
    if ( nb_rings == 0 )
      Poly_rings::collect_enough_rings(v, min_nb_points, gathered, vpm);
    else Poly_rings::collect_i_rings(v, nb_rings, gathered, vpm);
  }

  //store the gathered points
  std::vector<Vertex*>::iterator
    itb = gathered.begin(), ite = gathered.end();
  CGAL_For_all(itb,ite) in_points.push_back((*itb)->point());
}

///////////////MAIN///////////////////////////////////////////////////////
#if defined(CGAL_USE_BOOST_PROGRAM_OPTIONS) && ! defined(DONT_USE_BOOST_PROGRAM_OPTIONS)
int main(int argc, char *argv[])
#else
int main()
#endif
{
  string if_name_string;
  string if_name; //input file name
  string w_if_name;  //as above, but / replaced by _
  string res4openGL_fname;
  string verbose_fname;
  std::ofstream out_4ogl, out_verbose;

  try {
#if defined(CGAL_USE_BOOST_PROGRAM_OPTIONS) && ! defined(DONT_USE_BOOST_PROGRAM_OPTIONS)
    po::options_description desc("Allowed options");
    desc.add_options()
      ("help,h", "produce help message.")
      ("input-file,f", po::value<string>(&if_name_string)->default_value(CGAL::data_file_path("meshes/ellipe0.003.off")),
       "name of the input off file")
      ("degree-jet,d", po::value<unsigned int>(&d_fitting)->default_value(2),
       "degree of the jet, 1 <= degre-jet <= 4")
      ("degree-monge,m", po::value<unsigned int>(&d_monge)->default_value(2),
       "degree of the Monge rep, 1 <= degree-monge <= degree-jet")
      ("nb-rings,a", po::value<unsigned int>(&nb_rings)->default_value(0),
       "number of rings to collect neighbors. 0 means collect enough rings to make appro possible a>=1 fixes the nb of rings to be collected")
      ("nb-points,p", po::value<unsigned int>(&nb_points_to_use)->default_value(0),
       "number of neighbors to use.  0 means this option is not considered, this is the default p>=1 fixes the nb of points to be used")
      ("verbose,v", po::value<bool>(&verbose)->default_value(false),
       "verbose output on text file")
      ;

    po::variables_map vm;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);

    if (vm.count("help")) {
      cout << desc << "\n";
      return 1;
    }
#else
    std::cerr << "Command-line options require Boost.ProgramOptions" << std::endl;
    if_name_string = CGAL::data_file_path("meshes/ellipe0.003.off");
    d_fitting = 2;
    d_monge = 2;
    nb_rings = 0;
    nb_points_to_use = 0;
    verbose = false;
#endif
  }
  catch(exception& e) {
    cerr << "error: " << e.what() << "\n";
    return 1;
  }
  catch(...) {
    cerr << "Exception of unknown type!\n";
  }

  //modify global variables which are fct of options:
  min_nb_points = (d_fitting + 1) * (d_fitting + 2) / 2;
  if (nb_points_to_use < min_nb_points && nb_points_to_use != 0)
    {std::cerr << "the nb of points asked is not enough to perform the fitting" << std::endl; exit(0);}

  //prepare output file names
  //--------------------------
  std::cerr << "if_name_string" << if_name_string  << std::endl;
  if_name = if_name_string;

  w_if_name = if_name;
  for(unsigned int i=0; i<w_if_name.size(); i++)
    if (w_if_name[i] == '/') w_if_name[i]='_';
  cerr << if_name << '\n';
  cerr << w_if_name << '\n';

  res4openGL_fname = w_if_name + ".4ogl.txt";
std::cerr << "res4openGL_fname" << res4openGL_fname  << std::endl;
  out_4ogl.open(res4openGL_fname.c_str(), std::ios::out);
  assert(out_4ogl.good());
  //if verbose only...
  if(verbose){
    verbose_fname  = w_if_name + ".verb.txt";
    out_verbose.open(verbose_fname.c_str(), std::ios::out);
    assert(out_verbose.good());
    CGAL::IO::set_pretty_mode(out_verbose);
  }
  unsigned int nb_vertices_considered = 0;//count vertices for verbose

  //load the model from <mesh.off>
  //------------------------------
  PolyhedralSurf P;
  std::ifstream stream(if_name.c_str());
  stream >> P;
  std::cout << "loadMesh...  "<< "Polysurf with " << P.size_of_vertices()
            << " vertices and " << P.size_of_facets()
            << " facets. " << std::endl;

  if(verbose)
    out_verbose << "Polysurf with " << P.size_of_vertices()
                << " vertices and " << P.size_of_facets()
                << " facets. " << std::endl;
  //exit if not enough points in the model
  if (min_nb_points > P.size_of_vertices())    exit(0);

  //create property maps
  //-----------------------------
  //Vertex, using a std::map
  Vertex2int_map_type vertex2props;
  Vertex_PM_type vpm(vertex2props);

  //Hedge, with enriched hedge
  //HEdgePM_type hepm = get_hepm(boost::edge_weight_t(), P);
  //Hedge, using a std::map
  Hedge2double_map_type hedge2props;
  Hedge_PM_type hepm(hedge2props);

  //Facet PM, with enriched Facet
  //FacetPM_type fpm = get_fpm(boost::vertex_attribute_t(), P);
  //Facet PM, with std::map
  Facet2normal_map_type facet2props;
  Facet_PM_type fpm(facet2props);

  //initialize Polyhedral data : length of edges, normal of facets
  Poly_hedge_ops::compute_edges_length(P, hepm);
  Poly_facet_ops::compute_facets_normals(P, fpm);

  //MAIN LOOP: perform calculation for each vertex
  //----------------------------------------------
  std::vector<DPoint> in_points;  //container for data points
  Vertex_iterator vitb, vite;

  //initialize the tag of all vertices to -1
  vitb = P.vertices_begin(); vite = P.vertices_end();
  CGAL_For_all(vitb,vite) put(vpm, &(*vitb), -1);

  vitb = P.vertices_begin(); vite = P.vertices_end();
  for (; vitb != vite; vitb++) {
    //initialize
    Vertex* v = &(*vitb);
    in_points.clear();
    My_Monge_form monge_form;

    //gather points around the vertex using rings
    gather_fitting_points(v, in_points, vpm);

    //skip if the nb of points is to small
    if ( in_points.size() < min_nb_points )
      {std::cerr << "not enough pts for fitting this vertex" << in_points.size() << std::endl;
        continue;}

    // perform the fitting
    My_Monge_via_jet_fitting monge_fit;
    monge_form = monge_fit(in_points.begin(), in_points.end(),
                           d_fitting, d_monge);
    //switch min-max ppal curv/dir wrt the mesh orientation
    const DVector normal_mesh = Poly_facet_ops::compute_vertex_average_unit_normal(v, fpm);
    monge_form.comply_wrt_given_normal(normal_mesh);

    //OpenGL output. Scaling for ppal dir, may be optimized with a
    //global mean edges length computed only once on all edges of P
    DFT scale_ppal_dir = Poly_hedge_ops::compute_mean_edges_length_around_vertex(v, hepm)/2;

    out_4ogl << v->point()  << " ";
    monge_form.dump_4ogl(out_4ogl, scale_ppal_dir);

    //verbose txt output
    if (verbose) {
      std::vector<DPoint>::iterator itbp = in_points.begin(), itep = in_points.end();
      out_verbose << "in_points list : " << std::endl ;
      for (;itbp!=itep;itbp++) out_verbose << *itbp << std::endl ;

      out_verbose << "--- vertex " <<  ++nb_vertices_considered
                  <<        " : " << v->point() << std::endl
                  << "number of points used : " << in_points.size() << std::endl
        ;// << monge_form;
    }
  } //all vertices processed

  //cleanup filenames
  //------------------
  out_4ogl.close();
  if(verbose) {
    out_verbose.close();
  }
  return 0;
}

max_stars_repo_name: ffteja/cgal

max_issues_repo_name: ffteja/cgal

max_forks_repo_name: ffteja/cgal


text:
/*
 * Copyright (c) 2013 Turi Inc.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://turicreate.com
 *
 */

#define BOOST_TEST_MODULE
#include <boost/test/unit_test.hpp>
#include <util/test_macros.hpp>
#include <iostream>
#include <typeinfo>       // operator typeid
#include <cstddef>
#include <functional>
#include <iterator>
#include <type_traits>
#include <iterator>

#include <generics/gl_string.hpp>
#include <util/testing_utils.hpp>

using namespace turi;

int random_int() {
  return random::fast_uniform<int>(0, std::numeric_limits<int>::max());
}

void verify_serialization(const gl_string& v) {

  // into gl_string
  gl_string v1;
  save_and_load_object(v1, v);
  DASSERT_TRUE(v == v1);

  // Into std
  std::string v2;
  save_and_load_object(v2, v);
  DASSERT_EQ(v2.size(), v.size());
  DASSERT_TRUE(std::equal(v2.begin(), v2.end(), v.begin()));

  // from std into empty
  std::string v3;
  v3 = v;
  gl_string v4;
  save_and_load_object(v4, v3);
  DASSERT_TRUE(v4 == v);

  // from std into non-empty
  save_and_load_object(v1, v3);
  DASSERT_TRUE(v1 == v);
}

void verify_consistency(const gl_string& v) {

  // Constructor 1
  {
    gl_string v2(v);
    DASSERT_TRUE(v == v2);
  }

  // Constructor 2
  {
    gl_string v2(v.begin(), v.end());
    DASSERT_TRUE(v == v2);
  }

  // Assignment
  {
    gl_string v2;
    v2 = v;
    DASSERT_TRUE(v == v2);
  }

  // Assignment
  {
    gl_string v2;
    v2.assign(v.begin(), v.end());
    DASSERT_TRUE(v == v2);
  }

  // Assignment by insert
  {
    gl_string v2;
    v2.insert(v2.end(), v.begin(), v.end());
    DASSERT_TRUE(v == v2);
  }

  // Assignment by insert into cleared vector
  {
    gl_string v2;
    v2.resize(1);
    v2.clear();
    v2.insert(v2.end(), v.begin(), v.end());
    DASSERT_TRUE(v == v2);
  }

  // construction by empty, then resize, then index fill
  {
    gl_string v2;
    v2.resize(v.size());
    for(size_t i = 0; i < v.size(); ++i) {
      v2[i] = v[i];
    }
    DASSERT_TRUE(v == v2);
  }

  // construction by empty, then resize, then
  {
    gl_string v2;
    v2.reserve(v.size());
    for(auto e : v) {
      v2.push_back(e);
    }
    DASSERT_TRUE(v == v2);
  }

  // construction by empty, then resize, then iteration
  {
    gl_string v2;
    v2.resize(v.size());
    auto it = v2.begin();
    for(size_t i = 0; i < v.size(); ++i, ++it) {
      *it = v[i];
    }
    DASSERT_TRUE(v == v2);
  }

  // construction by empty, then resize, then reverse iteration
  {
    gl_string v2;
    v2.resize(v.size());
    auto it = v2.rbegin();
    for(size_t i = v.size(); (i--) != 0; ++it) {
      *it = v[i];
    }
    DASSERT_TRUE(v == v2);
  }

  // Assignment by insert into cleared vector.
  {
    gl_string v2;
    v2.resize(1);
    v2.insert(v2.begin(), v.begin(), v.end());
    v2.resize(v.size());
    DASSERT_TRUE(v == v2);
  }

  // Assignment by insert into cleared vector, then erase
  {
    gl_string v2;
    v2.resize(1);
    v2.insert(v2.end(), v.begin(), v.end());
    v2.erase(v2.begin());
    DASSERT_TRUE(v == v2);
  }

  ////////////////////////////////////////////////////////////////////////////////
  // test casting.
  {
    std::string v_stl;

    v_stl = v;

    TS_ASSERT_EQUALS(v_stl.size(), v.size());
    TS_ASSERT(std::equal(v.begin(), v.end(), v_stl.begin()));

    // Assignment
    gl_string v2(20);

    v2 = v_stl;

    TS_ASSERT_EQUALS(v2, v);

    // Construction
    gl_string v3(v_stl);

    TS_ASSERT_EQUALS(v3, v);
  }
}

////////////////////////////////////////////////////////////////////////////////

void stress_test(size_t n_tests) {

  auto gen_element = [](){ return char(random::fast_uniform<int>(32, 127)); };

  gl_string v;
  std::string v_ref;

  std::vector<std::function<void()> > operations;

  // Push back
  operations.push_back([&]() {
      auto e = gen_element();
      v.push_back(e);
      v_ref.push_back(e);
    });

  // Insert, 1 element.
  operations.push_back([&]() {
      auto e = gen_element();
      v.insert(v.begin(), e);
      v_ref.insert(v_ref.begin(), e);
    });

  operations.push_back([&]() {
      auto e = gen_element();
      size_t idx = random::fast_uniform<size_t>(0, v.size());
      v.insert(v.begin() + idx, e);
      v_ref.insert(v_ref.begin() + idx, e);
    });

  operations.push_back([&]() {
      auto e = gen_element();
      v.insert(v.end(), e);
      v_ref.insert(v_ref.end(), e);
    });

  // Insert, multiple elements.
  operations.push_back([&]() {
      auto e = gen_element();
      v.insert(v.begin(), 3, e);
      v_ref.insert(v_ref.begin(), 3, e);
    });

  operations.push_back([&]() {
      auto e = gen_element();
      size_t idx = random::fast_uniform<size_t>(0, v.size());
      v.insert(v.begin() + idx, 3, e);
      v_ref.insert(v_ref.begin() + idx, 3, e);
    });

  operations.push_back([&]() {
      auto e = gen_element();
      v.insert(v.end(), 3, e);
      v_ref.insert(v_ref.end(), 3, e);
    });

  // Insert, move element
  operations.push_back([&]() {
      auto e = gen_element();
      auto e2 = e;
      v.insert(v.begin(), std::move(e));
      v_ref.insert(v_ref.begin(), std::move(e2));
    });

  operations.push_back([&]() {
      auto e = gen_element();
      auto e2 = e;
      size_t idx = random::fast_uniform<size_t>(0, v.size());
      v.insert(v.begin() + idx, std::move(e));
      v_ref.insert(v_ref.begin() + idx, std::move(e2));
    });

  operations.push_back([&]() {
      auto e = gen_element();
      auto e2 = e;
      v.insert(v.end(), std::move(e));
      v_ref.insert(v_ref.end(), std::move(e2));
    });

  // Insert, 3 elements.
  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      v.insert(v.begin(), ev.begin(), ev.end());
      v_ref.insert(v_ref.begin(), ev.begin(), ev.end());
    });

  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      size_t idx = random::fast_uniform<size_t>(0, v.size());
      v.insert(v.begin() + idx, ev.begin(), ev.end());
      v_ref.insert(v_ref.begin() + idx, ev.begin(), ev.end());
    });

  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      v.insert(v.end(), ev.begin(), ev.end());
      v_ref.insert(v_ref.end(), ev.begin(), ev.end());
    });

  // Erase, single element.
  operations.push_back([&]() {
      if(v.empty()) return;
      size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
      v.erase(v.begin() + idx);
      v_ref.erase(v_ref.begin() + idx);
    });

  // Erase, block
  operations.push_back([&]() {
      if(v.empty()) return;
      size_t idx_1 = random::fast_uniform<size_t>(0, v.size() - 1);
      size_t idx_2 = random::fast_uniform<size_t>(0, v.size() - 1);
      v.erase(v.begin() + std::min(idx_1, idx_2), v.begin() + std::max(idx_1, idx_2));
      v_ref.erase(v_ref.begin() + std::min(idx_1, idx_2), v_ref.begin() + std::max(idx_1, idx_2));
    });

  // Erase, to end
  operations.push_back([&]() {
      if(v.empty()) return;
      size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
      v.erase(v.begin() + idx, v.end());
      v_ref.erase(v_ref.begin() + idx, v_ref.end());
    });

  // Erase, from beginning
  operations.push_back([&]() {
      if(v.empty()) return;
      size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
      v.erase(v.begin(), v.begin() + idx);
      v_ref.erase(v_ref.begin(), v_ref.begin() + idx);
    });

  // Clear everything.
  operations.push_back([&]() {
      v.clear();
      v_ref.clear();
    });

  // Total clear.
  operations.push_back([&]() {
      gl_string v_empty;
      std::string vr_empty;
      v.swap(v_empty);
      v_ref.swap(vr_empty);
    });

  // Assignment to init list
  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      v = {ev[0], ev[1], ev[2]};
      v_ref = {ev[0], ev[1], ev[2]};
    });

  // Assignment by iterator
  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      v.assign(ev.begin(), ev.end());
      v_ref.assign(ev.begin(), ev.end());
    });

  // Assignment by move equality
  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      v = std::move(gl_string(ev.begin(), ev.end()));
      v_ref = std::move(std::string(ev.begin(), ev.end()));
    });

  // pop_back
  operations.push_back([&]() {
      if(v.empty()) return;
      v.pop_back();
      v_ref.pop_back();
    });

  // swap front and back
  operations.push_back([&]() {
      if(v.empty()) return;
      std::swap(v.back(), v.front());
      std::swap(v_ref.back(), v_ref.front());
    });

  // shuffle by index
  operations.push_back([&]() {
      for(size_t j = 0; j < v.size(); ++j) {
        size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
        std::swap(v[j], v[idx]);
        std::swap(v_ref[j], v_ref[idx]);
      }
    });

  // shuffle by iterator
  operations.push_back([&]() {
      for(size_t j = 0; j < v.size(); ++j) {
        size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
        std::swap(*(v.begin() + j), *(v.begin() + idx));
        std::swap(*(v_ref.begin() + j), *(v_ref.begin() + idx));
      }
    });

  // shuffle by reverse iterator
  operations.push_back([&]() {
      for(size_t j = 0; j < v.size(); ++j) {
        size_t idx = random::fast_uniform<size_t>(0, v.size() - 1);
        std::swap(*(v.rbegin() + j), *(v.rbegin() + idx));
        std::swap(*(v_ref.rbegin() + j), *(v_ref.rbegin() + idx));
      }
    });

  // swap and insert.
  operations.push_back([&]() {
      std::string ev = {gen_element(), gen_element(), gen_element()};
      gl_string v2 = {ev[0], ev[1], ev[2]};
      std::string v2_ref = {ev[0], ev[1], ev[2]};
      v.swap(v2);
      v_ref.swap(v2_ref);

      size_t idx = random::fast_uniform<size_t>(0, v.size());
      v.insert(v.begin() + idx, v2.begin(), v2.end());
      v_ref.insert(v_ref.begin() + idx, v2_ref.begin(), v2_ref.end());
    });

  // setting through an std::string.
  operations.push_back([&]() {

      std::string v2 = v;
      v.assign(v2.begin(), v2.end());

      gl_string v2_ref(v_ref.begin(), v_ref.end());
      v_ref.assign(v2_ref.begin(), v2_ref.end());
    });

  // string deserialization
  operations.push_back([&]() {
      std::string s = serialize_to_string(v);
      v.clear();
      deserialize_from_string(s, v);
    });

  // string deserialization via vector
  operations.push_back([&]() {
      std::string v2 = v;
      std::string s = serialize_to_string(v2);
      deserialize_from_string(s, v);
    });

  // setting through an std::string.
  operations.push_back([&]() {

      std::string v2 = v;
      v.assign(v2.begin(), v2.end());

      gl_string v2_ref(v_ref.begin(), v_ref.end());
      v_ref.assign(v2_ref.begin(), v2_ref.end());
    });

  // substring
  operations.push_back([&]() {

      size_t n1 = random::fast_uniform<size_t>(0, v.size());
      size_t n2 = random::fast_uniform<size_t>(0, v.size());

      if(n1 > n2) std::swap(n1, n2);

      v = v.substr(n1, n2 - n1);
      v_ref = v_ref.substr(n1, n2 - n1);
    });

  // self-append
  operations.push_back([&]() {
      v+= v;
      v_ref += v_ref;
    });

  for(size_t i = 0; i < n_tests; ++i) {
    size_t idx = random::fast_uniform<size_t>(0, operations.size() - 1);
    operations[idx]();

    ASSERT_EQ(v.size(), v_ref.size());

    for(size_t j = 0; j < v.size(); ++j) {
      ASSERT_TRUE(v[j] == v_ref[j]);
    }

    if((i + 1) % 1000 == 0) {
      verify_serialization(v);
      verify_consistency(v);
    }
  }
}

struct gl_string_stress_test   {
 public:

  void test_stress() {
    random::seed(0);
    stress_test(5000000);
  }
};

BOOST_FIXTURE_TEST_SUITE(_gl_string_stress_test, gl_string_stress_test)
BOOST_AUTO_TEST_CASE(test_stress) {
  gl_string_stress_test::test_stress();
}
BOOST_AUTO_TEST_SUITE_END()

ext: cxx

max_stars_repo_name: TimothyRHuertas/turicreate

max_issues_repo_name: tashby/turicreate

max_forks_repo_name: tashby/turicreate