text:
/*
 * Copyright 2018 University of Leeds
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 * This is being developed for the TANGO Project: http://tango-project.eu
 */

#include <string>
#include <iostream>
#include <fstream>
#include <chrono>
#include <sys/stat.h>
#include <jsoncpp/json/json.h>
#include <NTL/ZZ.h>
#include "inner.h"
#include "HE1CiphertextCPU.h"

inline bool file_exists (const char* name) {
  struct stat buffer;
  return (stat(name, &buffer) == 0);
}

void usage() {
    std::cerr << "[ERROR] Bad number of parameters" << std::endl;
    std::cout << "    Usage: inner <num mappers> <num ints per line> <path to parameters file> <prefix for map input files> <prefix for map output files> <prefix for map results path> <path to output file> <path to timings file>" << std::endl;
}

void invoke_map_task(int tasknum, int numPerLine, const char* parametersPath, std::string& inputPath, std::string& resultsPath){
	std::cout << "Map task " << tasknum << ":" << std::endl;
	file parametersFile = strdup(parametersPath);
	std::cout << "\tparameters file: " << parametersPath << std::endl;
	std::string inFile = inputPath + '_' + std::to_string(tasknum);
	file splitFile = strdup(inFile.c_str());
	std::cout << "\tinput file: " << inFile << std::endl;
	std::string resultsFile = resultsPath + '_' + std::to_string(tasknum);
	file resFile = strdup(resultsFile.c_str());
	std::cout << "\tresults file: " << resultsFile << std::endl;
	run_map_task(numPerLine, parametersFile, splitFile, resFile);
}

int main(int argc, char **argv) {
	int numPerLine;
	std::size_t numMappers;
	std::string inputPath, outputPath, resultsPath;
	const char* parametersPath;
	const char* timingsPath;
	if (argc < 8){
		usage();
		return -1;
	}
	else{
		numMappers = atoi(argv[1]);
		numPerLine = atoi(argv[2]);
		parametersPath = argv[3];
		inputPath = argv[4];
		resultsPath = argv[5];
		outputPath = argv[6];
		timingsPath = argv[7];
	}

	auto startJob = std::chrono::high_resolution_clock::now();

	std::cout << "Starting inner with the following configuration:" << std::endl;
	std::cout << "\tNumber of workers to use: " << numMappers << std::endl;
	std::cout << "\tDegree of polynomial to compute: " << numPerLine << std::endl;
	std::cout << "\tParameters file: " << parametersPath << std::endl;
	std::cout << "\tInput file prefix: " << inputPath << std::endl;
	std::cout << "\tMap results file prefix: " << resultsPath << std::endl;
	std::cout << "\tOutput file: " << outputPath << std::endl;
	std::cout << "\tTimings file: " << timingsPath << std::endl;

	compss_on();
	//invoke map tasks
	for (std::size_t i = 0; i < numMappers; i++){
		invoke_map_task(i, numPerLine, parametersPath, inputPath, resultsPath);
	}
	compss_barrier();

	//Read in parameters from file
	std::ifstream ifs(parametersPath);
	if (!ifs.is_open()) throw std::ios_base::failure("Could not open cipher parameters file.");
	std::string parameters;
	getline(ifs,parameters);
	ifs.close();

	//Do prefix sum reduce and collate statistics
	HE1CiphertextCPU::setModulus(parameters);
	NTL::ZZ zero;
	HE1CiphertextCPU result(zero);
	unsigned long totalExecTime=0;
	unsigned long totalSumTime=0;
	unsigned long totalAdditions=0;
	unsigned long totalProductTime=0;
	unsigned long totalMultiplications=0;
	for (std::size_t i = 0; i < numMappers; i++){
		std::string resultsFile = resultsPath + '_' + std::to_string(i);
		std::ifstream results_ifs(resultsFile);
		if (results_ifs.is_open()){
			std::string json, line;
			while(getline(results_ifs,line)) json += line;
			Json::Value root;   // will contains the root value after parsing.
			Json::Reader reader;
			bool parsingSuccessful = reader.parse(json,root);
			if (parsingSuccessful){
				totalExecTime += root["task_execution_time"].asUInt64();
				totalAdditions += root["number_additions"].asUInt64();
				totalSumTime += root["total_sum_time"].asUInt64();
				totalMultiplications += root["number_multiplications"].asUInt64();
				totalProductTime += root["total_product_time"].asUInt64();
				std::string resultStr = root["result"].asString();
				HE1CiphertextCPU mapOutput(resultStr);
				auto start = std::chrono::high_resolution_clock::now();
				result += mapOutput;
				auto finish = std::chrono::high_resolution_clock::now();
				totalAdditions++;
				totalSumTime+= std::chrono::duration_cast<std::chrono::nanoseconds>(finish-start).count();
			}
		}
	}

	compss_off();

	//write computational result to file
	std::ofstream output(outputPath);
	if (output.is_open()){
		output << result << std::endl;
	}
	output.close();

	auto finishJob = std::chrono::high_resolution_clock::now();
	auto jobExecutionTime = std::chrono::duration_cast<std::chrono::nanoseconds>(finishJob-startJob).count();

	//write out statistics to file
	bool fileExists = file_exists(timingsPath);
	std::ofstream timings(timingsPath,std::ios::out | std::ios::app);
	if (!fileExists) timings << "jobExecTime,avgTaskTime,avgSumTime,avgProductTime" << std::endl;
	double avgTaskTime = (double)totalExecTime/numMappers;
	double avgSumTime = (double)totalSumTime/totalAdditions;
	double avgProductTime = (double)totalProductTime/totalMultiplications;
	timings << jobExecutionTime << "," << std::fixed << avgTaskTime << "," << avgSumTime << "," << avgProductTime << std::endl;
	return 0;
}

ext: cc

max_stars_repo_name: TANGO-Project/cryptango

max_issues_repo_name: TANGO-Project/cryptango

max_forks_repo_name: TANGO-Project/cryptango


text:
// Copyright (C) 2021 T. Zachary Laine
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_STL_INTERFACES_USE_CONCEPTS

void compile_seq_cont_constrained_pop_back() {}

#else

#include <initializer_list>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include <boost/stl_interfaces/sequence_container_interface.hpp>

template<class T>
struct container : boost::stl_interfaces::sequence_container_interface<
                       container<T>,
                       boost::stl_interfaces::element_layout::contiguous>
{
public:
    using super = boost::stl_interfaces::sequence_container_interface<
        container<T>,
        boost::stl_interfaces::element_layout::contiguous>;
    using container_type = std::vector<T>;

    using value_type = typename container_type::value_type;
    using reference = typename container_type::reference;
    using const_reference = typename container_type::const_reference;
    using iterator = typename container_type::iterator;
    using const_iterator = typename container_type::const_iterator;
    using difference_type = typename container_type::difference_type;
    using size_type = typename container_type::size_type;

    container() = default;
    container(const container &) = default;
    container(container &&) = default;
    ~container() = default;
    container & operator=(const container &) = default;
    container & operator=(container &&) = default;

    container(size_type count, const_reference value) : c(count, value) {}
    container(std::initializer_list<value_type> init) : c(init) {}

    template<class InputIt>
    container(InputIt first, InputIt last) : c(first, last)
    {}

    iterator begin() noexcept { return c.begin(); }
    iterator end() noexcept { return c.end(); }

    size_type max_size() const noexcept { return c.max_size(); }

    template<class InputIt>
    iterator insert(const_iterator pos, InputIt first, InputIt last)
    {
        return c.insert(pos, first, last);
    }

    template<class... Args>
    iterator emplace(const_iterator pos, Args &&... args)
    {
        return c.emplace(pos, std::forward<Args>(args)...);
    }

    iterator erase(const_iterator first, const_iterator last)
    {
        return c.erase(first, last);
    }

    template<class... Args>
    requires(std::constructible_from<value_type, Args &&...>) reference
        emplace_back(Args &&... args)
    {
        return c.emplace_back(std::forward<Args>(args)...);
    }

    void swap(container & other) { c.swap(other.c); }

    using super::begin;
    using super::end;
    using super::insert;
    using super::erase;

private:
    container_type c;
};

void compile_seq_cont_constrained_pop_back()
{
    {
        std::vector<int> v;
        v.emplace_back(0);
        v.pop_back();

        container<int> c;
        c.emplace_back(0);
        c.pop_back();
    }

    {
        std::vector<std::unique_ptr<int>> v;
        v.emplace_back(new int);
        v.pop_back();

        container<std::unique_ptr<int>> c;
        c.emplace_back(new int);
        c.pop_back();
    }
}

#endif

ext: cpp


text:
#include "../common/mycache2.h"
#include "../common/mylogger.h"
#include <boost/lexical_cast.hpp>

static LoggerPtr logger(Logger::getLogger("mycache"));

using mycache::KVMap;

#include "mycache2_data.h"

void dump(const UserT& u)
{
	LOG4CXX_INFO(logger, "charid=" << u.charid << ",name=" << u.name << ",sex=" << u.sex << ", age=" << u.age);
}

void test_async(const std::string& arg, int num)
{
	mycache::async->connect(mycache::LV_LOCAL, "redis://127.0.0.1:6379");
	mycache::async->connect(mycache::LV_GLOBAL, "redis://127.0.0.1:6380");

	boost::thread_group thrds;

	for (int i = 0; i < num; ++i)
	{
		std::ostringstream oss;
		oss << "test:" << i;
		std::string key = oss.str();

		if ("set" == arg)
		{
			struct SetUser : public mycache::Set<UserT>
			{
				void result(bool result)
				{
					LOG4CXX_INFO(logger, "setuser: " << key_ << " " << result);
				}
			};

			UserT u;
			u.charid = i;
			u.name = key;
			u.sex = 1;
			u.age = i+10;
			mycache::async->set_to(new SetUser, key, u, mycache::LV_GLOBAL);
		}
		else if ("get" == arg)
		{
			struct GetUser : public mycache::Get<UserT>
			{
				void result(const UserT* user)
				{
					if (user)
					{
						dump(*user);
					}
					else
					{
						LOG4CXX_WARN(logger, "missing..");
					}
				}
			};

			mycache::async->get_from(new GetUser, key, mycache::LV_LOCAL);
			mycache::async->get_from(new GetUser, key, mycache::LV_GLOBAL);
		}
		else if ("del" == arg)
		{
			struct DelUser : public mycache::Del
			{
				void result(bool result)
				{
					LOG4CXX_INFO(logger, "deluser: " << key_ << " " << result);
				}
			};

			mycache::async->del_from(new DelUser, key, mycache::LV_GLOBAL);
		}
		else if ("get2" == arg)
		{
			struct GetUser : public mycache::Get<UserT>
			{
				void result(const UserT* user)
				{
					if (user)
					{
						dump(*user);
					}
					else
					{
						LOG4CXX_WARN(logger, "missing..");
					}
				}
			};

			mycache::async->get(new GetUser, key);
		}
		else if ("get3" == arg)
		{
			struct GetUser : public mycache::Get<UserT>
			{
				void result(const UserT* user)
				{
					if (user)
					{
						dump(*user);
					}
					else
					{
						LOG4CXX_WARN(logger, "missing..");
					}
				}
			};

			mycache::async->get_and_cache(new GetUser, key, mycache::LV_LOCAL);
		}
		else if ("del2" == arg)
		{
			mycache::async->del(key);
		}
	}

	EventLoop::instance()->loop();

	thrds.join_all();
}

int main(int argc, const char* argv[])
{
	BasicConfigurator::configure();

	if (argc > 1)
	{
		std::string arg1 = argv[1];
		std::string arg2 = (argc > 2 ? argv[2] : "");
		int num = (argc > 3 ? atol(argv[3]) : 1);

		if ("sync" == arg1)
		{
			mycache::sync->connect(mycache::LV_LOCAL, "redis://127.0.0.1:6379/0");
			mycache::sync->connect(mycache::LV_GLOBAL, "redis://127.0.0.1:6379/1");

			{
				LOG4CXX_INFO(logger, "set------------");

				UserT u;
				u.charid = 1;
				u.name = "david";
				u.sex = 1;
				u.age = 29;

				mycache::sync->set("test:1", u, mycache::LV_INPROC);
				mycache::sync->set("test:1", u, mycache::LV_LOCAL);

				u.charid = 2;
				mycache::sync->set("test:2", u, mycache::LV_GLOBAL);
				mycache::sync->dump_inproc();
			}

			{
				LOG4CXX_INFO(logger, "get------------");

				UserT u;
				mycache::sync->get("test:1", u);
				dump(u);

				mycache::sync->get_and_cache("test:2", u, mycache::LV_LOCAL);
				dump(u);

				mycache::sync->dump_inproc();
			}

			{
				LOG4CXX_INFO(logger, "del------------");

				mycache::sync->del("test:2");
				mycache::sync->dump_inproc();
			}

			mycache::sync->close();
		}
		else if ("async" == arg1)
		{
			test_async(arg2, num);
		}
	}
}

max_stars_repo_name: david-pp/tinyworld

max_issues_repo_name: david-pp/tinyworld-servers

max_forks_repo_name: david-pp/tinyworld-servers


text:
/*
 * @Description: pre-integrator interface
 * @Author: Ge Yao
 * @Date: 2020-11-29 15:47:49
 */

#ifndef LIDAR_LOCALIZATION_MODELS_PRE_INTEGRATOR_PRE_INTEGRATOR_HPP_
#define LIDAR_LOCALIZATION_MODELS_PRE_INTEGRATOR_PRE_INTEGRATOR_HPP_

#include <yaml-cpp/yaml.h>

#include <Eigen/Core>
#include <Eigen/Dense>
#include <Eigen/Geometry>

namespace lidar_localization {

class PreIntegrator {
public:
    /**
     * @brief  whether the pre-integrator is inited:
     * @return true if inited false otherwise
     */
    double IsInited(void) const { return is_inited_; }

    /**
     * @brief  get pre-integrator time
     * @return pre-integrator time as double
     */
    double GetTime(void) const { return time_; }

protected:
    PreIntegrator() {}

    // init:
    bool is_inited_ = false;

    // time:
    double time_;
};

} // namespace lidar_localization

#endif // LIDAR_LOCALIZATION_MODELS_PRE_INTEGRATOR_PRE_INTEGRATOR_HPP_

ext: hpp


text:
#include <iostream>
#include <cmath>
#include <algorithm>
#include <vector>
#include <boost/multiprecision/cpp_int.hpp>
using namespace boost::multiprecision;
using namespace std;
cpp_int Combination(int n, int r) {
    cpp_int u = 1, d = 1;
    for (int i = 2; i <= n; i++) {
        u *= i;
        if (i <= n - r) d *= i;
    }
    for (int i = 2; i <= r; i++) d *= i;
    return u / d;
}
int main() {
    int l; cin >> l;
    cpp_int ans = Combination(l - 1, 11);
    cout << ans << endl;
}


text:
/**********************************************************************
*  Copyright (c) 2008-2014, Alliance for Sustainable Energy.
*  All rights reserved.
*
*  This library is free software; you can redistribute it and/or
*  modify it under the terms of the GNU Lesser General Public
*  License as published by the Free Software Foundation; either
*  version 2.1 of the License, or (at your option) any later version.
*
*  This library is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
*  Lesser General Public License for more details.
*
*  You should have received a copy of the GNU Lesser General Public
*  License along with this library; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
**********************************************************************/

#ifndef UTILITIES_IDD_IDDFILE_HPP
#define UTILITIES_IDD_IDDFILE_HPP

#include <utilities/UtilitiesAPI.hpp>

#include <utilities/idd/IddObject.hpp>

#include <utilities/core/Logger.hpp>
#include <utilities/core/Path.hpp>

#include <string>
#include <ostream>
#include <vector>

#include <boost/shared_ptr.hpp>

namespace openstudio{

// forward declarations
class IddFactorySingleton;
namespace detail{
  class IddFile_Impl;
} // detail

/** IddFile provides parsing and printing of text files in Input Data Definition (IDD) format.
 *  IDD is a schema definition format defined by the EnergyPlus team, adopted with slight
 *  modifications for the OpenStudio project. IddFile is a shared object. */
class UTILITIES_API IddFile {
 public:
  /** @name Constructors */
  //@{

  /** Default constructor returns an IddFile with no version, no header, and no objects. */
  IddFile();

  /** Copy constructor returns an IddFile that shares its data with other. */
  IddFile(const IddFile& other);

  /** Returns an IddFile containing a single, default constructed IddObject of Catchall type. */
  static IddFile catchallIddFile();

  //@}
  /** @name Getters */
  //@{

  /** Returns the file version (as specified in the header). */
  std::string version() const;

  /** Returns the file header. */
  std::string header() const;

  /** Returns all the objects in the file. */
  std::vector<IddObject> objects() const;

  /** Returns all groups in the file (e.g. "" and "Simulation Parameters"). */
  std::vector<std::string> groups() const;

  /** Returns all objects in group (e.g. "Simulation Parameters"). */
  std::vector<IddObject> getObjectsInGroup(const std::string& group) const;

  /** Returns the objects whose names match objectRegex. */
  std::vector<IddObject> getObjects(const boost::regex& objectRegex) const;

  /** Returns the version IddObject for this file, if possible. */
  boost::optional<IddObject> versionObject() const;

  /** Returns the object named objectName (case-insensitive). Return value will evaluate to false
   *  if there is no such object in the IddFile. */
  boost::optional<IddObject> getObject(const std::string& objectName) const;

  /** Returns the object of type objectType, if possible. Return value will evaluate to false if
   *  there is no such object in the IddFile, or if IddObjectType == IddObjectType::UserCustom. */
  boost::optional<IddObject> getObject(IddObjectType objectType) const;

  /** Returns all objects in the file that are required. An IdfFile or Workspace conforming to
   *  this IddFile must contain at least one object of each of these types to be valid at
   *  StrictnessLevel::Final. */
  std::vector<IddObject> requiredObjects() const;

  /** Returns all objects in the file that are marked as unique. An IdfFile or Workspace
   *  conforming to this IddFile must contain at most one object of each of these types to be
   *  valid at StrictnessLevel::Final. */
  std::vector<IddObject> uniqueObjects() const;

  //@}
  /** @name Serialization */
  //@{

  /** Load an IddFile from std::istream, if possible. */
  static boost::optional<IddFile> load(std::istream& is);

  /** Load an IddFile from path p, if possible. */
  static boost::optional<IddFile> load(const openstudio::path& p);

  /** Prints this file to std::ostream os. */
  std::ostream& print(std::ostream& os) const;

  /** Saves file to path p. Will construct the parent folder if necessary and if its parent
   *  folder already exists. Will only overwrite an existing file if overwrite==true. If no
   *  extension is provided will use 'idd'. */
  bool save(const openstudio::path& p, bool overwrite=false);

  //@}
 protected:
  friend class IddFactorySingleton;

  /// set version
  void setVersion(const std::string& version);

  /// set header
  void setHeader(const std::string& header);

  /// add an object
  void addObject(const IddObject& object);

 private:
  // impl
  boost::shared_ptr<detail::IddFile_Impl> m_impl;

  IddFile(const boost::shared_ptr<detail::IddFile_Impl>& impl);

  // configure logging
  REGISTER_LOGGER("utilities.idd.IddFile");
};

/** \relates IddFile */
typedef boost::optional<IddFile> OptionalIddFile;

/** \relates IddFile */
typedef std::vector<IddFile> IddFileVector;

/** \relates IddFile */
UTILITIES_API std::ostream& operator<<(std::ostream& os, const IddFile& iddFile);

} // openstudio

#endif // UTILITIES_IDD_IDDFILE_HPP

max_stars_repo_name: ORNL-BTRIC/OpenStudio

max_issues_repo_name: ORNL-BTRIC/OpenStudio

max_forks_repo_name: ORNL-BTRIC/OpenStudio


text:
/*
Copyright 2015-2016 Joanna Hulboj <j@hulboj.org>
Copyright 2016 Milosz Hulboj <m@hulboj.org>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include <cornelich/util/test_helpers.h>
#include <cornelich/util/files.h>

#include <cstdint>

#include <boost/filesystem.hpp>

#include <catch.hpp>

using namespace cornelich::util;
namespace fs = boost::filesystem;

TEST_CASE_METHOD(clean_up_fixture, "File utils - create_mapping", "[util/files]")
{
    GIVEN("A temporary directory")
    {
        fs::create_directory(path());
        REQUIRE(exists());
        auto p = (path() / "foo");

        SECTION("We can create a mapping")
        {
            auto mapping = create_mapping(p.string(), 16384);
            REQUIRE(fs::exists(p));
            REQUIRE(fs::file_size(p) == 16384);
        }
    }
}

TEST_CASE_METHOD(clean_up_fixture, "File utils - touch & size", "[util/files]")
{
    GIVEN("A temporary directory")
    {
        fs::create_directory(path());
        REQUIRE(exists());
        auto p = (path() / "foo");

        SECTION("Touch and create a file")
        {
            touch(p.string());
            REQUIRE(fs::exists(p));
        }
    }
}

max_issues_repo_name: jfhk/cornelich

max_forks_repo_name: jfhk/cornelich


text:
/**
 * Copyright Soramitsu Co., Ltd. 2018 All Rights Reserved.
 * http://soramitsu.co.jp
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "ametsuchi/impl/postgres_block_query.hpp"
#include <boost/range/adaptor/transformed.hpp>
#include <boost/range/algorithm/for_each.hpp>
#include "backend/protobuf/from_old_model.hpp"

namespace iroha {
  namespace ametsuchi {

    PostgresBlockQuery::PostgresBlockQuery(pqxx::nontransaction &transaction,
                                           FlatFile &file_store)
        : block_store_(file_store),
          transaction_(transaction),
          log_(logger::log("PostgresBlockIndex")),
          execute_{makeExecuteOptional(transaction_, log_)} {}

    rxcpp::observable<BlockQuery::wBlock> PostgresBlockQuery::getBlocks(
        shared_model::interface::types::HeightType height, uint32_t count) {
      shared_model::interface::types::HeightType last_id =
          block_store_.last_id();
      auto to = std::min(last_id, height + count - 1);
      if (height > to or count == 0) {
        return rxcpp::observable<>::empty<wBlock>();
      }
      return rxcpp::observable<>::range(height, to).flat_map([this](auto i) {
        // TODO IR-975 victordrobny 12.02.2018 convert directly to
        // shared_model::proto::Block after FlatFile will be reworked to new
        // model
        auto block = block_store_.get(i) | [](const auto &bytes) {
          return model::converters::stringToJson(bytesToString(bytes));
        } | [this](const auto &d) {
          return serializer_.deserialize(d);
        } | [](const auto &block_old) {
          return std::make_shared<shared_model::proto::Block>(
              shared_model::proto::from_old(block_old));
        };
        return rxcpp::observable<>::create<PostgresBlockQuery::wBlock>(
            [block{std::move(block)}](auto s) {
              if (block) {
                s.on_next(block);
              }
              s.on_completed();
            });
      });
    }

    rxcpp::observable<BlockQuery::wBlock> PostgresBlockQuery::getBlocksFrom(
        shared_model::interface::types::HeightType height) {
      return getBlocks(height, block_store_.last_id());
    }

    rxcpp::observable<BlockQuery::wBlock> PostgresBlockQuery::getTopBlocks(
        uint32_t count) {
      auto last_id = block_store_.last_id();
      count = std::min(count, last_id);
      return getBlocks(last_id - count + 1, count);
    }

    std::vector<shared_model::interface::types::HeightType>
    PostgresBlockQuery::getBlockIds(
        const shared_model::interface::types::AccountIdType &account_id) {
      return execute_(
                 "SELECT DISTINCT height FROM height_by_account_set WHERE "
                 "account_id = "
                 + transaction_.quote(account_id) + ";")
                 | [&](const auto &result)
                 -> std::vector<shared_model::interface::types::HeightType> {
        return transform<shared_model::interface::types::HeightType>(
            result, [&](const auto &row) {
              return row.at("height")
                  .template as<shared_model::interface::types::HeightType>();
            });
      };
    }

    boost::optional<shared_model::interface::types::HeightType>
    PostgresBlockQuery::getBlockId(const shared_model::crypto::Hash &hash) {
      boost::optional<uint64_t> blockId;
      return execute_("SELECT height FROM height_by_hash WHERE hash = "
                      + transaction_.quote(pqxx::binarystring(
                            hash.blob().data(), hash.blob().size()))
                      + ";")
                 | [&](const auto &result)
                 -> boost::optional<
                     shared_model::interface::types::HeightType> {
        if (result.size() == 0) {
          return boost::none;
        }
        return result[0]
            .at("height")
            .template as<shared_model::interface::types::HeightType>();
      };
    }

    std::function<void(pqxx::result &result)> PostgresBlockQuery::callback(
        const rxcpp::subscriber<wTransaction> &subscriber, uint64_t block_id) {
      return [this, &subscriber, block_id](pqxx::result &result) {
        auto block = block_store_.get(block_id) | [this](auto bytes) {
          // TODO IR-975 victordrobny 12.02.2018 convert directly to
          // shared_model::proto::Block after FlatFile will be reworked to new
          // model
          return boost::optional<shared_model::proto::Block>(
              shared_model::proto::from_old(*serializer_.deserialize(
                  *model::converters::stringToJson(bytesToString(bytes)))));
        };
        boost::for_each(
            result | boost::adaptors::transformed([](const auto &x) {
              return x.at("index").template as<size_t>();
            }),
            [&](auto x) {
              subscriber.on_next(PostgresBlockQuery::wTransaction(
                  clone(*block->transactions().at(x))));
            });
      };
    }

    rxcpp::observable<BlockQuery::wTransaction>
    PostgresBlockQuery::getAccountTransactions(
        const shared_model::interface::types::AccountIdType &account_id) {
      return rxcpp::observable<>::create<wTransaction>(
          [this, account_id](auto subscriber) {
            auto block_ids = this->getBlockIds(account_id);
            if (block_ids.empty()) {
              subscriber.on_completed();
              return;
            }

            for (const auto &block_id : block_ids) {
              execute_(
                  "SELECT DISTINCT index FROM index_by_creator_height WHERE "
                  "creator_id = "
                  + transaction_.quote(account_id)
                  + " AND height = " + transaction_.quote(block_id) + ";")
                  | this->callback(subscriber, block_id);
            }
            subscriber.on_completed();
          });
    }

    rxcpp::observable<BlockQuery::wTransaction>
    PostgresBlockQuery::getAccountAssetTransactions(
        const shared_model::interface::types::AccountIdType &account_id,
        const shared_model::interface::types::AssetIdType &asset_id) {
      return rxcpp::observable<>::create<wTransaction>([this,
                                                        account_id,
                                                        asset_id](
                                                           auto subscriber) {
        auto block_ids = this->getBlockIds(account_id);
        if (block_ids.empty()) {
          subscriber.on_completed();
          return;
        }

        for (const auto &block_id : block_ids) {
          execute_(
              "SELECT DISTINCT index FROM index_by_id_height_asset WHERE id = "
              + transaction_.quote(account_id)
              + " AND height = " + transaction_.quote(block_id)
              + " AND asset_id = " + transaction_.quote(asset_id) + ";")
              | this->callback(subscriber, block_id);
        }
        subscriber.on_completed();
      });
    }

    rxcpp::observable<boost::optional<BlockQuery::wTransaction>>
    PostgresBlockQuery::getTransactions(
        const std::vector<shared_model::crypto::Hash> &tx_hashes) {
      return rxcpp::observable<>::create<boost::optional<wTransaction>>(
          [this, tx_hashes](auto subscriber) {
            std::for_each(tx_hashes.begin(),
                          tx_hashes.end(),
                          [that = this, &subscriber](auto tx_hash) {
                            subscriber.on_next(that->getTxByHashSync(tx_hash));
                          });
            subscriber.on_completed();
          });
    }

    boost::optional<BlockQuery::wTransaction>
    PostgresBlockQuery::getTxByHashSync(
        const shared_model::crypto::Hash &hash) {
      return getBlockId(hash) |
          [this](auto blockId) { return block_store_.get(blockId); } |
          [](auto bytes) {
            // TODO IR-975 victordrobny 12.02.2018 convert directly to
            // shared_model::proto::Block after FlatFile will be reworked to new
            // model
            return model::converters::stringToJson(bytesToString(bytes));
          }
      | [&](const auto &json) { return serializer_.deserialize(json); } |
          [](const auto &block) {
            return boost::optional<shared_model::proto::Block>(
                shared_model::proto::from_old(block));
          }
      | [&](const auto &block) {
          boost::optional<PostgresBlockQuery::wTransaction> result;
          auto it =
              std::find_if(block.transactions().begin(),
                           block.transactions().end(),
                           [&hash](auto tx) { return tx->hash() == hash; });
          if (it != block.transactions().end()) {
            result = boost::optional<PostgresBlockQuery::wTransaction>(clone(**it));
          }
          return result;
        };
    }

  }  // namespace ametsuchi
}  // namespace iroha

max_stars_repo_name: steephengeorge/iroha

max_issues_repo_name: steephengeorge/iroha

max_forks_repo_name: steephengeorge/iroha


text:
// ***************************************************************** -*- C++ -*-
/*
 * Copyright (C) 2009 Brad Schick <schickb@gmail.com>
 *
 * This file is part of the organize tool.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, 5th Floor, Boston, MA 02110-1301 USA.
 */
/*
 File:      organize.cpp
 Version:   $Rev: 2932 $
 Author(s): Brad Schick (brad) <schickb@gmail.com>
 History:   19-Jan-09, brad: created
*/
// *****************************************************************************

#include <boost/program_options.hpp>
#include <boost/regex.hpp>
#include <boost/array.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <exiv2/image.hpp>
#include <exiv2/error.hpp>
#include <exiv2/basicio.hpp>
#include <iostream>
#include <iomanip>
#include <cassert>
#include <limits>
#include "MD5.h"
#include "helpers.hpp"

typedef Exiv2::byte md5digest[16];

namespace po = boost::program_options;

bool g_verbose = false;
bool g_neednewline = false;

// Array size should match number of SLOTs
boost::array<int,4> g_run_order = {{-1, -1, -1, -1}};
const int EXIF_SLOT = 0;
const int IPTC_SLOT = 1;
const int XMP_SLOT  = 2;
const int FILE_SLOT = 3;

const unsigned DOT_EVERY = 55;

struct Pattern {
    std::string pat;
    std::string desc;
    pfunc funcs[4];  // order should always be exif, iptc, xmp, file
};

struct PathPart {
    std::string pre;
    const Pattern *pat;
    std::string post;
    PathPart(std::string pre_, const Pattern *pat_, std::string post_)
         : pre(pre_), pat(pat_), post(post_) {}
};

std::vector<PathPart> g_path_parts;

// Instead of making these all global
struct ProcessParams {
    const fs::path &dest_dir;
    const bool dry_run;
    const bool ignore_dups;
    const bool ignore_unsorted;
    const bool force;
    const bool rename;
    const bool symlink;
    const bool verify;
    const bool move;
    const long limit_depth;
    const fs::path &dups_dir;
    const fs::path &unsorted_dir;
    const std::vector<std::string> &excludes;
    unsigned dups_count;
    unsigned unsorted_count;
    unsigned dir_err_count;
    unsigned file_err_count;
    unsigned ok_count;
    unsigned dups_ignored_count;
    unsigned unsorted_ignored_count;
    unsigned dir_ex_count;
    unsigned file_ex_count;
};

void process_directory(const fs::path &directory, const long depth,
    ProcessParams &params);

const Pattern g_patterns[] = {
    {"@date", "date captured (2009-01-19)",
        {exif_date, iptc_date, NULL, file_date} },
    {"@year", "year captured (2009)",
        {exif_year, iptc_year, NULL, file_year} },
    {"@month", "month captured (01)",
        {exif_month, iptc_month, NULL, file_month} },
    {"@day", "day captured (19)",
        {exif_day, iptc_day, NULL, file_day} },
    {"@time", "time captured (14-35-27)",
        {exif_time, iptc_time, NULL, file_time} },
    {"@hour", "hour captured (14)",
        {exif_hour, iptc_hour, NULL, file_hour} },
    {"@min", "minute captured (35)",
        {exif_minute, iptc_minute, NULL, file_minute} },
    {"@sec", "second captured (27)",
        {exif_second, iptc_second, NULL, file_second} },
    {"@dim", "pixel dimension (2272-1704)",
        {exif_dimension, NULL, NULL, file_dimension} },
    {"@x", "pixel width (2272)",
        {exif_width, NULL, NULL, file_width} },
    {"@y", "pixel height (1704)",
        {exif_height, NULL, NULL, file_height} },
    {"@make", "device make (Canon)",
        {exif_make, NULL, NULL, NULL} },
    {"@model", "device model (Canon PowerShot S40)",
        {exif_model, NULL, NULL, NULL} },
    {"@speed", "shutter speed (1-60)",
        {exif_speed, NULL, NULL, NULL} },
    {"@aper", "aperture (F3.2)",
        {exif_aperture, NULL, NULL, NULL} },
    {"@iso", "iso speed (400)",
        {exif_iso, NULL, NULL, NULL} },
    {"@focal", "focal length (8.6 mm)",
        {exif_focal, NULL, NULL, NULL} },
    {"@dist", "subject distance (1.03 m)",
        {exif_distance, NULL, NULL, NULL} },
    {"@meter", "meter mode (multi-segment)",
        {exif_meter, NULL, NULL, NULL} },
    {"@macro", "macro mode (Off)",
        {exif_macro, NULL, NULL, NULL} },
    {"@orient", "orientation (top_left)",
        {exif_orientation, NULL, NULL, NULL} },
    {"@lens", "lens name (Tamron 90mm f-2.8)",
        {exif_lens, NULL, NULL, NULL} },
    {"@key", "first keyword (Family)",
        {exif_keyword, iptc_keyword, NULL, NULL} },

    {"", "", {NULL, NULL, NULL, NULL} }
};

// Check that 'opt1' and 'opt2' are not specified at the same time.
void conflicting(const po::variables_map& vm,
    const char* opt1, const char* opt2)
{
    if (vm.count(opt1) && !vm[opt1].defaulted()
        && vm.count(opt2) && !vm[opt2].defaulted()) {
        throw std::logic_error(std::string("conflicting options '")
            + opt1 + "' and '" + opt2 + "'");
    }
}

// Check that 'required' is present
void required(const po::variables_map& vm, const char* required)
{
    if (!vm.count(required) || vm[required].defaulted()) {
        throw std::logic_error(std::string("required parameter '") + required
            + "' is missing");
    }
}

void info(const std::string &msg)
{
    if(g_verbose) {
        std::cout << msg << "\n";
        g_neednewline = false;
    }
}

void error(const std::exception &e, const std::string &msg)
{
    if(g_neednewline) {
        std::cout << "\n";
        g_neednewline = false;
    }
    std::cerr << e.what() << "\n";
    std::cerr << msg << std::endl;
}

void usage_header(const char* exname)
{
    std::cout << "Usage: " << exname << " [options] source-dir dest-dir pattern\n";
}

void usage_full(const po::options_description &options, const char* exname)
{
    usage_header(exname);
    std::cout << "\n  Creates groups of files in new directories defined by a metadata 'pattern'.\n" <<
        "  Files are copied, moved, or linked from 'source-dir' to 'dest-dir'.\n" <<
        "  The destination directory should not be within the source directory.\n\n";
    std::cout << options;

    std::cout << "\nPattern values:\n";
    for( const Pattern *pattern = g_patterns; pattern->pat.length(); ++pattern) {
        std::cout << "  " << std::setw(8) << std::left << pattern->pat;
        std::cout << pattern->desc << "\n";
    }

    std::cout << "\nExamples:\n";
    std::cout << "  `" <<  exname << " -m mess clean @year-@month'\n";
    std::cout << "     Moves files from 'mess' into directories of 'clean' according to\n" <<
                 "     year-month the file was captured (clean/2006-11/...)\n\n";
    std::cout << "  `" <<  exname << " -o ie source find width-@x/height-@y'\n";
    std::cout << "     Copies files into directories according first to pixel width then pixel\n" <<
                 "     height. Check iptc then exif metadata (find/width-2272/height-1704/...)\n\n";
    std::cout << "  `" <<  exname << " -lf source find @aper/@hour'\n";
    std::cout << "     Force create symlinks in directories according first to aperture then\n" <<
                 "     hour captured (find/F3.2/15/...)\n";

    std::cout << std::endl;
}

void version()
{
    std::cout << "organized 0.1\n" <<
        "Copyright (C) 2009 Brad Schick. <schickb@gmail.com>\n\n" <<
        "This program is free software; you can redistribute it and/or\n"
        "modify it under the terms of the GNU General Public License\n"
        "as published by the Free Software Foundation; either version 2\n"
        "of the License, or (at your option) any later version.\n"
        "\n"
        "This program is distributed in the hope that it will be useful,\n"
        "but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
        "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n"
        "GNU General Public License for more details.\n"
        "\n"
        "You should have received a copy of the GNU General Public\n"
        "License along with this program; if not, write to the Free\n"
        "Software Foundation, Inc., 51 Franklin Street, Fifth Floor,\n"
        "Boston, MA 02110-1301 USA" << std::endl;
}

// Returns empty string if the destination subdirectory could not be determined
// for the supplied source file.
std::string build_dest(const fs::path &source_file)
{
    std::string dest;

    Exiv2::Image::AutoPtr image;
    try {
        image = Exiv2::ImageFactory::open(source_file.string());
        image->readMetadata();
    }
    catch(const Exiv2::AnyError&) {
        // No metadata, let things continue to try file info
    }

    std::vector<PathPart>::iterator iter = g_path_parts.begin();
    std::vector<PathPart>::iterator end = g_path_parts.end();
    for( ; iter != end; ++iter) {
        dest += iter->pre;
        std::string result;

        const Pattern *pat = iter->pat;
        for(unsigned fx = 0; fx < g_run_order.size(); ++fx) {
            if(g_run_order[fx] != -1 && pat->funcs[g_run_order[fx]]) {
                if(g_run_order[fx] == FILE_SLOT) {
                    // Always run file operations
                    result = pat->funcs[g_run_order[fx]](image.get(), source_file);
                }
                else if(image.get()) {
                    // No point in running metadata operations without an image
                    result = pat->funcs[g_run_order[fx]](image.get(), source_file);
                }
                if(result.length())
                    break;
            }
        }
        // If we found no data, even for part of pattern, give up and
        // return no destination
        if(!result.length())
            return result;

        dest += (result + iter->post);
    }
    return dest;
}

bool md5sum(const fs::path &path, md5digest &digest)
{
    try {
        Exiv2::FileIo io(path.string());
        if (io.open() != 0)
            return false;
        Exiv2::IoCloser closer(io);

        Exiv2::byte buff[4096];
        MD5_CTX context;
        MD5Init(&context);

        long read_count = io.read(buff, 4096);
        while(read_count) {
            MD5Update(&context, buff, read_count);
            read_count = io.read(buff, 4096);
        }
        MD5Final(digest, &context);
        return true;
    }
    catch (std::exception& ) {
        return false;
    }
}

int main(int argc, char* argv[])
{
    po::options_description options("Options");
    // Don't use default values because the help print it ugly and too wide
    options.add_options()
        ("move,m", "move files rather than copy")
        ("symlink,s", "symlink files rather than copy (posix only)")
        ("order,o", po::value<std::string>(),
            "order and types of metadata to read\ne=exif, i=iptc, f=file (default: eif)")
        ("unsorted,u", po::value<std::string>(),
            "special directory to store unsorted files (default: unsorted)")
        ("dups,d", po::value<std::string>(),
            "special directory to store files with duplicate names (default: duplicates)")
        ("force,f", "overwrite duplicate files instead of using special directory")
        ("rename,r", "rename duplicate files instead of using special directory")
        ("ignore,i", "ignore both unsorted and duplicate files instead of using special directories")
        ("ignore-unsorted", "ignore unsorted files instead of using special directory")
        ("ignore-dups", "ignore duplicate files instead of using special directory")
        ("verify", "verify copied or moved files and exit if incorrect")
        ("exclude,x", po::value< std::vector<std::string> >(),
            "exclude directories and files that contain arg (case sensitive on all platforms)")
        ("limit-depth,l", po::value<long>(),
            "limit recursion to specified depth (0 disables recursion)")
        ("verbose,v", "prints operations as they happen")
        ("dry-run,n", "do not make actual changes (implies verbose)")
        ("help,h", "show this help message then exit")
        ("version,V", "show program version then exit")
        ;

    po::options_description hidden("Hidden Options");
    hidden.add_options()
        ("source-dir", po::value< std::string >(), "directory of files to organize, may end in file wildcard")
        ("dest-dir", po::value< std::string >(), "desination directory for files, may not be within source-dir")
        ("pattern", po::value< std::string >(), "subdirectory pattern for grouping files within dest-dir")
        ;

    po::options_description cmdline;
    cmdline.add(options).add(hidden);

    po::positional_options_description positional;
    positional.add("source-dir", 1);
    positional.add("dest-dir", 1);
    positional.add("pattern", 1);

    try {
        po::variables_map vm;
        po::store(po::command_line_parser(argc, argv).
        options(cmdline).positional(positional).run(), vm);
        po::notify(vm);

        if (vm.count("help")) {
            usage_full(options, argv[0]);
            return 0;
        }

        if (vm.count("version")) {
            version();
            return 0;
        }

        conflicting(vm, "verify", "symlink");
        conflicting(vm, "move", "symlink");
        conflicting(vm, "unsorted", "ignore");
        conflicting(vm, "unsorted", "ignore-unsorted");
        conflicting(vm, "dups", "ignore");
        conflicting(vm, "dups", "ignore-dups");
        conflicting(vm, "force", "ignore");
        conflicting(vm, "force", "ignore-dups");
        conflicting(vm, "force", "rename");
        conflicting(vm, "rename", "ignore");
        conflicting(vm, "rename", "ignore-dups");
        required(vm, "source-dir");
        required(vm, "dest-dir");
        required(vm, "pattern");

        const bool dry_run = vm.count("dry-run") != 0;
        g_verbose = (vm.count("verbose") != 0 || dry_run);

        std::string order = "eif";
        if(vm.count("order")) {
            order = vm["order"].as<std::string>();

            boost::to_lower(order);
            if(order.length() > 3) {
                throw std::logic_error(std::string("order is longer than 4 characters"));
            }
        }

        unsigned i = 0;
        std::string::iterator end = order.end();
        for(std::string::iterator iter = order.begin(); iter != end && i < 4; ++iter, ++i) {
            switch(*iter) {
                case 'e':
                    g_run_order[i] = EXIF_SLOT;
                    break;
                case 'i':
                    g_run_order[i] = IPTC_SLOT;
                    break;
                case 'x':
                    throw std::logic_error(std::string("xmp not implemented yet '") +
                        *iter + "'");
                    break;
                case 'f':
                    g_run_order[i] = FILE_SLOT;
                    break;
                default:
                    throw std::logic_error(std::string("unknown order character '") +
                        *iter + "'");
            }
        }

        const fs::path source_dir( vm["source-dir"].as<std::string>() );
        if( !exists(source_dir) || !is_directory(source_dir) ) {
            throw std::logic_error(std::string("source '") +
                source_dir.string() + "' must exist and be a directory");
        }

        const fs::path dest_dir( vm["dest-dir"].as<std::string>() );
        if( exists(dest_dir) && !is_directory(dest_dir) ) {
            throw std::logic_error(std::string("destination '") +
                dest_dir.string() + "' must be a directory");
        }

        // Boost doesn't seem to have a way to get a canonical path, so this
        // simple test is easy to confuse with some ../../'s in the paths. Oh
        // well, this is good enough for now.
        fs::path test_dest(dest_dir);
        for(; !test_dest.empty(); test_dest = test_dest.parent_path()) {
            if(fs::equivalent(source_dir, test_dest)) {
                throw std::logic_error(std::string("dest-dir must not be within source-dir"));
            }
        }

        // Disect the pattern
        std::string pattern = vm["pattern"].as<std::string>();
        boost::regex regex( "([^@]*)(@[[:alpha:]]+)([^@]*)");
        boost::sregex_iterator m_iter = make_regex_iterator(pattern, regex);
        boost::sregex_iterator m_end;
        for( ; m_iter != m_end; ++m_iter) {
            const boost::smatch &match = *m_iter;
            const std::string &pre = match[1];
            const std::string &pat = match[2];
            const std::string &post = match[3];

            // Should put this in a map, but there aren't that many options now
            bool found = false;
            for( const Pattern *pattern = g_patterns; pattern->pat.length(); ++pattern) {
                if(pattern->pat == pat) {
                    PathPart part(pre, pattern, post);
                    g_path_parts.push_back(part);
                    found = true;
                    break;
                }
            }

            if(!found) {
                throw std::logic_error(std::string("unknown pattern '") + pat + "'");
            }
        }

        // Assign defaults to params that need them
        const bool ignore = vm.count("ignore") != 0;
        std::vector<std::string> excludes;
        if(vm.count("exclude"))
            excludes = vm["exclude"].as< std::vector<std::string> >();
        long limit_depth = LONG_MAX;
        if(vm.count("limit-depth")) {
            limit_depth = vm["limit-depth"].as<long>();
            // Boost program_options doesn't work with unsigned, so do it manually
            if( limit_depth < 0 )
                throw std::logic_error(std::string("recursion depth limit must be positive"));
        }
        std::string dups = "duplicates";
        if(vm.count("dups"))
            dups = vm["dups"].as<std::string>();
        const fs::path dups_dir = dest_dir / dups;

        std::string unsorted = "unsorted";
        if(vm.count("unsorted"))
            unsorted = vm["unsorted"].as<std::string>();
        const fs::path unsorted_dir = dest_dir / unsorted;

        ProcessParams params = {
            dest_dir,
            dry_run,
            (vm.count("ignore-dups") != 0 || ignore),
            (vm.count("ignore-unsorted") != 0 || ignore),
            vm.count("force") != 0,
            vm.count("rename") != 0,
            vm.count("symlink") != 0,
            vm.count("verify") != 0,
            vm.count("move") != 0,
            limit_depth,
            dups_dir,
            unsorted_dir,
            excludes,
            0, 0, 0, 0, 0, 0, 0, 0, 0
        };

        process_directory(source_dir, 0, params);

        std::string op = "copied";
        if(params.symlink)
            op = "linked";
        else if(params.move)
            op = "moved";

        if(dry_run)
            op = std::string("would be ") + op;

        if(g_neednewline)
            std::cout << "\n";

        std::cout << "\n" << params.ok_count << " files " << op << "\n";
        std::cout << "   " << params.dups_count << " duplicates\n";
        std::cout << "   " << params.unsorted_count << " unsorted\n";
        if(params.dups_ignored_count)
            std::cout << params.dups_ignored_count << " duplicates ignored\n";
        if(params.unsorted_ignored_count)
            std::cout << params.unsorted_ignored_count << " unsorted ignored\n";
        if(params.dir_ex_count)
            std::cout << params.dir_ex_count << " directories excluded\n";
        if(params.file_ex_count)
            std::cout << params.file_ex_count << " files excluded\n";
        if(params.dir_err_count)
            std::cout << params.dir_err_count << " directory errors\n";
        if(params.file_err_count)
            std::cout << params.file_err_count << " file errors\n";

        return 0;
    }
    catch (Exiv2::AnyError& e) {
        error(e, std::string("Aborting"));
        return -1;
    }
    catch(std::logic_error& e) {
        error(e, "");
        usage_header(argv[0]);
        std::cout << argv[0] << " -h    for more help" << std::endl;
        return -2;
    }
    catch(std::exception& e) {
        error(e, "Aborting");
        return -3;
    }
}

boost::regex uregex("(.*?)\\(([[:digit:]]{1,2})\\)$");

fs::path uniquify(const fs::path &dest)
{
    std::string ext = dest.extension().string();
    std::string fname = dest.stem().string();
    fs::path parent = dest.parent_path();

    unsigned number = 1;
    std::string newfname;
    fs::path newdest;

    boost::smatch match;
    if(boost::regex_search(fname, match, uregex)) {
        // Matches are indexes into fname, so don't change it while reading values
        newfname = match[1];
        number = boost::lexical_cast<short>(match[2]);
        fname = newfname;
    }

    do {
        newfname = fname + "(" + boost::lexical_cast<std::string>(++number) + ")" + ext;
        newdest = parent / newfname;
    } while(fs::exists(newdest));

    return newdest;
}

void process_directory(const fs::path &directory, const long depth,
    ProcessParams &params)
{
    // Exclude entire directories
    bool exclude = false;
    std::vector<std::string>::const_iterator x_iter = params.excludes.begin();
    std::vector<std::string>::const_iterator x_end = params.excludes.end();
    for( ; x_iter != x_end; ++x_iter ) {
        if(boost::contains(directory.string(), *x_iter)) {
            exclude = true;
            break;
        }
    }
    if(exclude) {
        info(std::string("excluding directory: ") + directory.string() +
            " matched: " + *x_iter);
        ++params.dir_ex_count;
        return;
    }

    try {
        fs::directory_iterator p_iter(directory), p_end;
        for( ; p_iter != p_end; ++p_iter) {
            if( is_directory(*p_iter) ) {
                // recurse if we haven't hit the limit
                if(depth < params.limit_depth)
                    process_directory(p_iter->path(), depth + 1, params);
                else {
                    info(std::string("depth reached, skipping: ") +
                        p_iter->path().string());
                }
            }
            else if( is_regular_file(*p_iter) ) {

                // Check again for excluding file names
                exclude = false;
                x_iter = params.excludes.begin();
                for( ; x_iter != x_end; ++x_iter ) {
                    if(boost::contains(p_iter->path().string(), *x_iter)) {
                        exclude = true;
                        break;
                    }
                }
                if(exclude) {
                    info(std::string("excluding file: ") + p_iter->path().string() +
                        " matched: " + *x_iter);
                    ++params.file_ex_count;
                    continue;
                }

                try {
                    const fs::path dest_subdir = build_dest(*p_iter);
                    fs::path dest_file;
                    if(!dest_subdir.empty())
                        dest_file = params.dest_dir / dest_subdir;
                    else if(params.ignore_unsorted) {
                        info(std::string("ignoring unsorted: ") + p_iter->path().string());
                        ++params.unsorted_ignored_count;
                        continue;
                    }
                    else {
                        info(std::string("unsorted file (missing metadata): ") + p_iter->path().string());
                        dest_file = params.unsorted_dir;
                        ++params.unsorted_count;
                    }

                    dest_file /= p_iter->path().filename();

                    if(fs::exists(dest_file)) {
                        if(params.ignore_dups) {
                            info(std::string("ignoring: ") + p_iter->path().string() +
                                " duplicates: " +  dest_file.string());
                            ++params.dups_ignored_count;
                            continue;
                        }
                        else {
                            if(params.force) {
                                info(std::string("force removing: ") + dest_file.string() + " for: "
                                    + p_iter->path().string());
                                if(!params.dry_run)
                                    fs::remove(dest_file);
                            }
                            else if(params.rename) {
                                info(std::string("renaming: ") + p_iter->path().string() +
                                    " duplicates: " +  dest_file.string());
                                dest_file = uniquify(dest_file);
                            }
                            else {
                                info(std::string("duplicate file: ") + p_iter->path().string() +
                                    " of: " +  dest_file.string());
                                dest_file = params.dups_dir / dest_subdir / p_iter->path().filename();
                                // Ugh, more dup possibilities
                                if(fs::exists(dest_file)) {
                                    info(std::string("renaming: ") + p_iter->path().string() +
                                        " duplicates: " +  dest_file.string());
                                    dest_file = uniquify(dest_file);
                                }
                            }
                            ++params.dups_count;
                        }
                    }

                    if(!params.dry_run)
                        fs::create_directories(dest_file.parent_path());

                    if(params.symlink) {
                        info(std::string("linking from: ") + p_iter->path().string() +
                            " to: " + dest_file.string());
                        if(!params.dry_run) {
                            // The target of a symlink must be either absolute (aka complete) or
                            // relative to the location of the link. Easiest solution is to make
                            // a complete path.
                            fs::path target;
                            if(p_iter->path().is_complete())
                                target = p_iter->path();
                            else
                                target = fs::initial_path() / p_iter->path();
                            fs::create_symlink(target, dest_file);
                        }
                    }
                    else {
                        info(std::string("copying from: ") + p_iter->path().string() +
                            " to: " + dest_file.string());
                        if(!params.dry_run) {
                            // Copy the file and restore its write time (needed for posix)
                            std::time_t time = fs::last_write_time(*p_iter);
                            fs::copy_file(*p_iter, dest_file);
                            fs::last_write_time(dest_file, time);
                            if(params.verify) {
                                md5digest src_digest, dst_digest;
                                bool ok = md5sum(p_iter->path(), src_digest);
                                if(ok)
                                    ok = md5sum(dest_file, dst_digest);
                                if(ok)
                                    ok = (memcmp(src_digest,dst_digest, sizeof(md5digest))==0);
                                if(!ok) {
                                    // Should probably find a more appropriate exception for this
                                    throw std::runtime_error(std::string("File verification failed: '")
                                        + p_iter->path().string() + "' differs from '" +
                                        dest_file.string() + "'");
                                }
                                else {
                                    info(std::string("verification passed"));
                                }
                            }
                        }
                    }
                    if(params.move) {
                        info(std::string("removing: ") + p_iter->path().string());
                        if(!params.dry_run)
                            fs::remove(*p_iter);
                    }

                    if(!g_verbose && (params.ok_count % DOT_EVERY)==0) {
                        std::cout << "." << std::flush;
                        g_neednewline = true;
                    }
                    ++params.ok_count;
                }
                catch(fs::filesystem_error& e) {
                    error(e, std::string("skipping file: " + p_iter->path().string()));
                    ++params.file_err_count;
                }
            }
        }
    }
    catch(fs::filesystem_error& e) {
        error(e, std::string("skipping directory: " + directory.string()));
        ++params.dir_err_count;
    }
}