text:
/*************************************************************************
 * Copyright (C) 2017-2019 Barcelona Supercomputing Center               *
 *                         Centro Nacional de Supercomputacion           *
 * All rights reserved.                                                  *
 *                                                                       *
 * This file is part of NORNS, a service that allows other programs to   *
 * start, track and manage asynchronous transfers of data resources      *
 * between different storage backends.                                   *
 *                                                                       *
 * See AUTHORS file in the top level directory for information regarding *
 * developers and contributors.                                          *
 *                                                                       *
 * This software was developed as part of the EC H2020 funded project    *
 * NEXTGenIO (Project ID: 671951).                                       *
 *     www.nextgenio.eu                                                  *
 *                                                                       *
 * Permission is hereby granted, free of charge, to any person obtaining *
 * a copy of this software and associated documentation files (the       *
 * "Software"), to deal in the Software without restriction, including   *
 * without limitation the rights to use, copy, modify, merge, publish,   *
 * distribute, sublicense, and/or sell copies of the Software, and to    *
 * permit persons to whom the Software is furnished to do so, subject to *
 * the following conditions:                                             *
 *                                                                       *
 * The above copyright notice and this permission notice shall be        *
 * included in all copies or substantial portions of the Software.       *
 *                                                                       *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       *
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    *
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND                 *
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS   *
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN    *
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN     *
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE      *
 * SOFTWARE.                                                             *
 *************************************************************************/

#include <boost/random.hpp>
#include <boost/generator_iterator.hpp>
#include "utils.hpp"
#include "compare-files.hpp"
#include "test-env.hpp"
#include "catch.hpp"

namespace {

constexpr static const std::size_t tar_header_size =
    norns::utils::tar::TAR_BLOCK_SIZE;
constexpr static const std::size_t tar_record_size =
    norns::utils::tar::TAR_BLOCK_SIZE;
constexpr static const std::size_t tar_eof_size =
    2*norns::utils::tar::TAR_BLOCK_SIZE;

constexpr std::size_t
tar_aligned_size(std::size_t sz) {
    return sz % tar_record_size != 0 ?
            ((sz & ~(tar_record_size - 1)) + tar_record_size) :
            sz;
}

using rng = boost::mt19937;
using distribution = boost::uniform_int<>;
using size_generator =
    boost::variate_generator<rng, distribution>;

bfs::path
create_hierarchy(test_env& env,
                 size_generator& size_gen,
                 const bfs::path& subdir,
                 const bfs::path& parent,
                 std::size_t subdirs_per_level,
                 std::size_t files_per_subdir,
                 std::size_t levels) {

    if(levels == 0) {
        return {};
    }

    bfs::path self = env.create_directory(subdir, parent);

    for(std::size_t i = 0; i < files_per_subdir; ++i) {
        std::string fname = "/regular_file" + std::to_string(i);
        auto p = env.create_file(fname, self, size_gen());
    }

    if(levels == 1) {
        return self;
    }

    for(std::size_t i = 0; i < subdirs_per_level; ++i) {
        std::string dname = "/subdir" + std::to_string(i);
        auto child = env.create_directory(dname, self);

        (void) create_hierarchy(env, size_gen, dname, self, subdirs_per_level,
                                files_per_subdir, levels - 1);
    }

    return self;
}

std::string
exec(const std::string& cmd) {
    std::array<char, 128> buffer;
    std::string result;
    int status;

    std::cout << "exec(\"" << cmd << "\")";

    auto deleter = [&status](FILE* stream) {
        status = ::pclose(stream);
    };

    std::unique_ptr<FILE, decltype(deleter)> pipe(
            ::popen(cmd.c_str(), "r"), deleter);

    if(!pipe) {
        throw std::runtime_error("popen() failed!");
    }

    while(::fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
        result += buffer.data();
    }

    return result;
}

std::error_code
extract(const bfs::path archive_path,
        const bfs::path parent_dir) {

    std::error_code ec;

    if(!bfs::exists(parent_dir)) {
        boost::system::error_code bec;
        bfs::create_directories(parent_dir, bec);

        if(bec) {
            ec.assign(bec.value(), std::generic_category());
            return ec;
        }
    }

    std::string cmd("tar xf " + archive_path.string() + " -C " +
                    parent_dir.string());
    auto res = ::exec(cmd);

    std::cout << res << "\n";

    return ec;
}

}

SCENARIO("tar creation", "[utils::tar::tar(filename, tar::create)]") {

    GIVEN("an invalid output path") {
        test_env env;
        using norns::utils::tar;

        bfs::path p = env.basedir();

        WHEN("creating a tar archive") {
            std::error_code ec;
            tar t(p, tar::create, ec);

            THEN("constructor fails") {
                REQUIRE(ec);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid output path") {
        test_env env;
        using norns::utils::tar;

        bfs::path p = env.basedir() / "archive.tar";

        WHEN("creating a tar archive") {
            std::error_code ec;
            tar t(p, tar::create, ec);

            THEN("constructor succeeds") {
                REQUIRE(!ec);
                REQUIRE(bfs::exists(p));
            }
        }

        env.notify_success();
    }
}

SCENARIO("appending files to a TAR archive",
         "[utils::tar::add_file(path, alias, error_code)]") {

    GIVEN("an invalid tar instance") {
        test_env env;
        using norns::utils::tar;

        std::error_code ec;
        tar t("", tar::create, ec);
        REQUIRE(ec);

        WHEN("trying to add a file") {

            bfs::path file = env.create_file("/regular_file", env.basedir(), 0);

            THEN("add_file() returns EINVAL") {
                t.add_file(file, bfs::relative(file, env.basedir()), ec);
                REQUIRE(ec);
                REQUIRE(ec.value() == EINVAL);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance") {
        test_env env;
        using norns::utils::tar;

        std::array<std::string, 6> filenames = {
            "/regular_file0", "/regular_file1", "/regular_file2",
            "/regular_file3", "/regular_file4", "/regular_file5",
        };

        std::array<std::size_t, 6> sizes = {
            0, 10000, 20000, 2134513, 42, 50124887
        };

        std::vector<bfs::path> file_paths;

        for(std::size_t i = 0; i < filenames.size(); ++i) {
            file_paths.emplace_back(
                    env.create_file(filenames[i], env.basedir(), sizes[i]));
        }

        bfs::path p = env.basedir() / "archive.tar";

        std::error_code ec;
        tar t(p, tar::create, ec);
        REQUIRE(!ec);

        WHEN("adding an empty file") {

            t.add_file(file_paths[0],
                    bfs::relative(file_paths[0], env.basedir()), ec);

            THEN("add_file() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added file") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / filenames[0]));
                    REQUIRE(compare_files(
                                file_paths[0], parent_dir / filenames[0]));
                }
            }
        }

        WHEN("adding a non-empty file") {
            t.add_file(file_paths[1],
                        bfs::relative(file_paths[1], env.basedir()), ec);

            THEN("add_file() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added file") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / filenames[1]));
                    REQUIRE(compare_files(
                                file_paths[1], parent_dir / filenames[1]));
                }
            }
        }

        WHEN("adding several files") {

            std::vector<std::error_code> ecs;

            for(std::size_t i = 0; i < file_paths.size(); ++i) {
                t.add_file(file_paths[i],
                           bfs::relative(file_paths[i], env.basedir()), ec);

                ecs.emplace_back(ec);

            }

            THEN("add_file() succeeds") {

                for(const auto err : ecs) {
                    REQUIRE(!err);
                }

                AND_THEN("the archive contains all added files") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);
                    REQUIRE(!ec);

                    for(std::size_t i = 0; i < file_paths.size(); ++i) {
                        REQUIRE(bfs::exists(parent_dir / filenames[i]));
                        REQUIRE(compare_files(
                                    file_paths[i], parent_dir / filenames[i]));
                    }
                }
            }
        }

        env.notify_success();
    }

}

SCENARIO("appending directories to a TAR archive",
         "[utils::tar::add_directory(path, alias, error_code)]") {

    GIVEN("an invalid tar instance") {
        test_env env;
        using norns::utils::tar;

        std::error_code ec;
        tar t("", tar::create, ec);
        REQUIRE(ec);

        WHEN("trying to add a directory") {

            bfs::path subdir = env.create_directory("/subdir", env.basedir());

            THEN("add_directory() returns EINVAL") {
                t.add_directory(subdir,
                        bfs::relative(subdir, env.basedir()), ec);
                REQUIRE(ec);
                REQUIRE(ec.value() == EINVAL);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance") {
        test_env env;
        using norns::utils::tar;

        std::array<std::string, 6> filenames = {
            "/regular_file0", "/regular_file1", "/regular_file2",
            "/regular_file3", "/regular_file4", "/regular_file5",
        };

        std::array<std::size_t, 6> sizes = {
            0, 10000, 20000, 2134513, 42, 5012488//7
        };

        std::vector<bfs::path> file_paths;

        bfs::path subdir =
            env.create_directory("/subdir", env.basedir());

        for(std::size_t i = 0; i < filenames.size(); ++i) {
            file_paths.emplace_back(
                    env.create_file(filenames[i], subdir, sizes[i]));
        }

        bfs::path p = env.basedir() / "archive.tar";

        std::error_code ec;
        tar t(p, tar::create, ec);
        REQUIRE(!ec);

        WHEN("adding an empty directory") {

            bfs::path empty_subdir =
                env.create_directory("/empty_subdir", env.basedir());

            t.add_directory(empty_subdir,
                    bfs::relative(empty_subdir, env.basedir()), ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / "/empty_subdir"));
                    REQUIRE(compare_directories(
                                empty_subdir, parent_dir / "/empty_subdir"));
                }
            }
        }

        WHEN("adding a non-empty directory containing only files and "
             "defining a 1-level alias (e.g. subdir0/ -> subdir1/ ) ") {

            t.add_directory(subdir,
                        bfs::relative(subdir, env.basedir()), ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / "/subdir"));

					AND_THEN("both directories are identical") {
						REQUIRE(compare_directories(
									subdir, parent_dir / "/subdir"));
					}
                }
            }
        }

        WHEN("adding a non-empty directory containing only files and defining "
             "a multi-level alias (e.g. subdir0/ -> subdir1/a/b/c/ )") {

            t.add_directory(subdir,
                        bfs::relative(subdir, env.basedir()) / "a/b/c/", ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / "/subdir"));
                    REQUIRE(bfs::exists(parent_dir / "/subdir/a/"));
                    REQUIRE(bfs::exists(parent_dir / "/subdir/a/b/"));
                    REQUIRE(bfs::exists(parent_dir / "/subdir/a/b/c/"));

					AND_THEN("contents are identical") {
						REQUIRE(compare_directories(
									subdir, parent_dir / "/subdir/a/b/c"));
					}
                }
            }
        }

        WHEN("adding a non-empty directory containing only files and "
             "defining an empty alias") {

            t.add_directory(subdir, "", ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive does not contain the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(!bfs::exists(parent_dir / "/subdir"));

					AND_THEN("contents are identical") {
						REQUIRE(compare_directories(subdir, parent_dir));
					}
                }
            }
        }

        WHEN("adding a non-empty directory containing only files and "
             "defining '/' as an alias") {

            t.add_directory(subdir, "/", ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive does not contain the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(!bfs::exists(parent_dir / "/subdir"));

					AND_THEN("contents are identical") {
						REQUIRE(compare_directories(subdir, parent_dir));
					}
                }
            }
        }

        WHEN("adding a non-empty directory containing files and subdirs, and "
             "defining a 1-level alias (e.g. subdir0/ -> subdir1/ ) ") {

			size_generator gen(rng{42}, distribution{0, 42});

            bfs::path complex_subdir =
                create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                                 2, 5, 5);

            t.add_directory(complex_subdir,
                        bfs::relative(complex_subdir, env.basedir()), ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / "/complex_subdir"));

					AND_THEN("both directories are identical") {
						REQUIRE(compare_directories(
									complex_subdir,
                                    parent_dir / "/complex_subdir"));
					}
                }
            }
        }

        WHEN("adding a non-empty directory containing files and subdirs, and "
             "defining a multi-level alias (e.g. subdir0/ -> "
             "subdir1/a/b/c/ )") {

			size_generator gen(rng{42}, distribution{0, 42});

            bfs::path complex_subdir =
                create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                                 2, 5, 5);

            t.add_directory(
                complex_subdir,
                bfs::relative(complex_subdir, env.basedir()) / "a/b/c/", ec);

            THEN("add_directory() succeeds") {
                REQUIRE(!ec);

                AND_THEN("the archive contains the added directory") {

                    t.release(); // make sure data has been written to archive

                    const bfs::path parent_dir(env.basedir() / "tmp");
                    ec = ::extract(t.path(), parent_dir);

                    REQUIRE(!ec);
                    REQUIRE(bfs::exists(parent_dir / "/complex_subdir"));
                    REQUIRE(bfs::exists(parent_dir / "/complex_subdir/a/"));
                    REQUIRE(bfs::exists(parent_dir / "/complex_subdir/a/b/"));
                    REQUIRE(bfs::exists(parent_dir / "/complex_subdir/a/b/c/"));

					AND_THEN("both directories are identical") {
						REQUIRE(compare_directories(
									complex_subdir,
                                    parent_dir / "/complex_subdir/a/b/c"));
					}
                }
            }
        }

        env.notify_success();
    }
}

SCENARIO("extracting from a tar archive",
         "[utils::tar::extract(destination_path, error_code)]") {

    GIVEN("an invalid tar instance") {
        test_env env;
        using norns::utils::tar;

        std::error_code ec;
        tar t("", tar::open, ec);
        REQUIRE(ec);

        WHEN("trying to extract the archive") {

			std::error_code ec;
			bfs::path tmpdir = env.create_directory("/tmp", env.basedir());
			t.extract(tmpdir, ec);

            THEN("extract() returns EINVAL") {
                REQUIRE(ec);
                REQUIRE(ec.value() == EINVAL);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
		size_generator gen(rng(42), distribution(0, 42));
		bfs::path complex_subdir =
			create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                             2, 5, 5);
        bfs::path p = env.basedir() / "archive.tar";

		{
			std::error_code ec;
			tar t(p, tar::create, ec);
			REQUIRE(!ec);
			t.add_directory(complex_subdir, "", ec);
			REQUIRE(!ec);
		}

        WHEN("trying to extract the archive to \"\"") {

            std::error_code ec;
            tar t(p, tar::open, ec);
            REQUIRE(!ec);

			t.extract("", ec);

            THEN("extract() returns EINVAL") {
                REQUIRE(ec);
                REQUIRE(ec.value() == EINVAL);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
		size_generator gen(rng(42), distribution(0, 42));
		bfs::path complex_subdir =
			create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                             2, 5, 5);
        bfs::path p = env.basedir() / "archive.tar";

		{
			std::error_code ec;
			tar t(p, tar::create, ec);
			REQUIRE(!ec);
			t.add_directory(complex_subdir, "", ec);
			REQUIRE(!ec);
		}

        WHEN("trying to extract the archive to a path leading to a file") {

            std::error_code ec;
            tar t(p, tar::open, ec);
            REQUIRE(!ec);

			t.extract(p, ec);

            THEN("extract() returns EINVAL") {
                REQUIRE(ec);
                REQUIRE(ec.value() == EINVAL);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
        bfs::path file = env.create_file("/fake.tar", env.basedir(), 0);

        WHEN("trying to extract from something that looks like an archive ") {

            std::error_code ec;
            tar t(file, tar::open, ec);

            THEN("constructor() fails") {
                REQUIRE(ec);
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance and an archive with alias \"\"") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
		size_generator gen(rng(42), distribution(0, 42));
		bfs::path complex_subdir =
			create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                             2, 5, 5);
        bfs::path p = env.basedir() / "archive.tar";

        std::error_code ec;
		{
			tar t(p, tar::create, ec);
			REQUIRE(!ec);

			t.add_directory(complex_subdir, "", ec);
			REQUIRE(!ec);
		}

        WHEN("trying to extract the archive to a path leading to a directory") {

			tar t(p, tar::open, ec);
			REQUIRE(!ec);

            THEN("extract() succeeds") {
                t.extract(tmp_dir, ec);
                REQUIRE(!ec);

                AND_THEN("the specified directory contains the archive "
                         "contents and they are identical to the original") {
                    REQUIRE(compare_directories(complex_subdir, tmp_dir));
                }
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance and an archive with a 1-level alias") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
		size_generator gen(rng(42), distribution(0, 42));
		bfs::path complex_subdir =
			create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                             2, 5, 5);
        bfs::path p = env.basedir() / "archive.tar";

        std::error_code ec;
		{
			tar t(p, tar::create, ec);
			REQUIRE(!ec);

			t.add_directory(complex_subdir, "/alias_subdir", ec);
			REQUIRE(!ec);
		}

        WHEN("trying to extract the archive to a path leading to a directory") {

			tar t(p, tar::open, ec);
			REQUIRE(!ec);

            THEN("extract() succeeds") {
                t.extract(tmp_dir, ec);
                REQUIRE(!ec);

                AND_THEN("the specified directory contains the archive "
                         "contents and they are identical to the original") {
                    REQUIRE(bfs::exists(tmp_dir / "/alias_subdir"));
                    REQUIRE(compare_directories(
                                complex_subdir, tmp_dir / "/alias_subdir"));
                }
            }
        }

        env.notify_success();
    }

    GIVEN("a valid tar instance and an archive with a multi-level alias") {
        test_env env;
        using norns::utils::tar;

		bfs::path tmp_dir = env.create_directory("/tmp", env.basedir());
		size_generator gen(rng(42), distribution(0, 42));
		bfs::path complex_subdir =
			create_hierarchy(env, gen, "/complex_subdir", env.basedir(),
                             2, 5, 5);
        bfs::path p = env.basedir() / "archive.tar";

        std::error_code ec;
		{
			tar t(p, tar::create, ec);
			REQUIRE(!ec);

			t.add_directory(complex_subdir, "/alias_subdir/a/b/c", ec);
			REQUIRE(!ec);
		}

        WHEN("trying to extract the archive to a path leading to a directory") {

			tar t(p, tar::open, ec);
			REQUIRE(!ec);

            THEN("extract() succeeds") {
                t.extract(tmp_dir, ec);
                REQUIRE(!ec);

                AND_THEN("the specified directory contains the archive "
                         "contents and they are identical to the original") {
                    REQUIRE(bfs::exists(tmp_dir / "/alias_subdir/a/b/c"));
                    REQUIRE(compare_directories(
                                complex_subdir,
                                tmp_dir / "/alias_subdir/a/b/c"));
                }
            }
        }

        env.notify_success();
    }

}

SCENARIO("tar size estimation", "[utils::tar::estimate_packed_size()]") {

    GIVEN("a path to a regular file of 0 bytes") {

        test_env env;
        using norns::utils::tar;

        bfs::path file = env.create_file("/regular_file", env.basedir(), 0);

        THEN("estimated size follows formula") {
            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(file, ec);

            REQUIRE(psize == tar_header_size +
                             tar_eof_size);

            AND_THEN("size of generated TAR archive matches estimated one") {

                bfs::path file_tar = file;
                file_tar.replace_extension(".tar");

                {
                    tar t(file_tar, tar::create, ec);
                    REQUIRE(!ec);

                    t.add_file(file, bfs::relative(file, env.basedir()), ec);
                    REQUIRE(!ec);
                }

                // this check needs to be done after tar::~tar() has been
                // called, since it appends the EOF blocks
                REQUIRE(psize == bfs::file_size(file_tar));
            }
        }

        env.notify_success();
    }

    GIVEN("a path to a regular file of tar_record_size bytes") {

        test_env env;
        using norns::utils::tar;

        bfs::path file =
            env.create_file("/regular_file", env.basedir(), tar_header_size);

        THEN("estimated size follows formula") {
            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(file, ec);

            REQUIRE(psize == tar_header_size +
                             tar_header_size +
                             tar_eof_size);

            AND_THEN("size of generated TAR archive matches estimated one") {

                bfs::path file_tar = file;
                file_tar.replace_extension(".tar");

                {
                    tar t(file_tar, tar::create, ec);
                    REQUIRE(!ec);

                    t.add_file(file, bfs::relative(file, env.basedir()), ec);
                    REQUIRE(!ec);
                }

                // this check needs to be done after tar::~tar() has been
                // called, since it appends the EOF blocks
                REQUIRE(psize == bfs::file_size(file_tar));
            }
        }

        env.notify_success();
    }

    GIVEN("a path to a regular file of 2*tar_record_size bytes") {

        test_env env;
        using norns::utils::tar;

        bfs::path file =
            env.create_file("/regular_file", env.basedir(), 2*tar_header_size);

        THEN("estimated size follows formula") {
            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(file, ec);

            REQUIRE(psize == tar_header_size +
                             2*tar_header_size +
                             tar_eof_size);

            AND_THEN("size of generated TAR archive matches estimated one") {

                bfs::path file_tar = file;
                file_tar.replace_extension(".tar");

                {
                    tar t(file_tar, tar::create, ec);
                    REQUIRE(!ec);

                    t.add_file(file, bfs::relative(file, env.basedir()), ec);
                    REQUIRE(!ec);
                }

                // this check needs to be done after tar::~tar() has been
                // called, since it appends the EOF blocks
                REQUIRE(psize == bfs::file_size(file_tar));
            }
        }

        env.notify_success();
    }

    GIVEN("a path to a regular file of 10000 bytes") {

        test_env env;
        using norns::utils::tar;

        bfs::path file =
            env.create_file("/regular_file", env.basedir(), 10000);

        THEN("estimated size follows formula") {
            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(file, ec);

            REQUIRE(psize == tar_header_size +
                             tar_aligned_size(10000) +
                             tar_eof_size);

            AND_THEN("size of generated TAR archive matches estimated one") {

                bfs::path file_tar = file;
                file_tar.replace_extension(".tar");

                {
                    tar t(file_tar, tar::create, ec);
                    REQUIRE(!ec);

                    t.add_file(file, bfs::relative(file, env.basedir()), ec);
                    REQUIRE(!ec);
                }

                // this check needs to be done after tar::~tar() has been
                // called, since it appends the EOF blocks
                REQUIRE(psize == bfs::file_size(file_tar));
            }
        }

        env.notify_success();
    }

    GIVEN("a path to an empty directory") {

        test_env env;
        using norns::utils::tar;

        bfs::path subdir = env.create_directory("/subdir", env.basedir());

        THEN("estimated size follows formula") {
            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(subdir, ec);

            REQUIRE(psize == tar_header_size +
                             tar_eof_size);

            AND_THEN("size of generated TAR archive matches estimated one") {

                bfs::path subdir_tar = subdir;
                subdir_tar.replace_extension(".tar");

                {
                    tar t(subdir_tar, tar::create, ec);
                    REQUIRE(!ec);

                    t.add_directory(subdir,
                                    bfs::relative(subdir, env.basedir()), ec);
                    REQUIRE(!ec);
                }

                // this check needs to be done after tar::~tar() has been
                // called, since it appends the EOF blocks
                REQUIRE(psize == bfs::file_size(subdir_tar));
            }
        }

        env.notify_success();
    }

    GIVEN("a path to a directory hierarchy containing empty files") {

        test_env env;
        using norns::utils::tar;

        size_generator gen(rng{42}, distribution{0, 0});

        bfs::path subdir =
            create_hierarchy(env, gen, "/subdir", env.basedir(), 5, 5, 5);

        THEN("size of generated TAR archive matches estimated one") {

            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(subdir, ec);

            bfs::path subdir_tar = subdir;
            subdir_tar.replace_extension(".tar");

            {
                tar t(subdir_tar, tar::create, ec);
                REQUIRE(!ec);

                t.add_directory(subdir,
                                bfs::relative(subdir, env.basedir()), ec);
                REQUIRE(!ec);
            }

            // this check needs to be done after tar::~tar() has been
            // called, since it appends the EOF blocks
            REQUIRE(psize == bfs::file_size(subdir_tar));
        }

        env.notify_success();
    }

    GIVEN("a path to a directory hierarchy containing empty files and an "
          "empty alias") {

        test_env env;
        using norns::utils::tar;

        size_generator gen(rng{42}, distribution{0, 0});

        bfs::path subdir =
            create_hierarchy(env, gen, "/subdir", env.basedir(), 5, 5, 5);

        THEN("size of generated TAR archive matches estimated one") {

            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(subdir, ec);

            bfs::path subdir_tar = subdir;
            subdir_tar.replace_extension(".tar");

            {
                tar t(subdir_tar, tar::create, ec);
                REQUIRE(!ec);

                t.add_directory(subdir, "", ec);
                REQUIRE(!ec);
            }

            // this check needs to be done after tar::~tar() has been
            // called, since it appends the EOF blocks
            REQUIRE(psize == bfs::file_size(subdir_tar));
        }

        env.notify_success();
    }

    GIVEN("a path to a directory hierarchy containing arbitrary files") {

        test_env env;
        using norns::utils::tar;

        size_generator gen(rng{42}, distribution{100, 42*1024});

        bfs::path subdir =
            create_hierarchy(env, gen, "/subdir", env.basedir(), 5, 5, 5);

        THEN("size of generated TAR archive matches estimated one") {

            std::error_code ec;
            std::size_t psize = tar::estimate_size_once_packed(subdir, ec);

            bfs::path subdir_tar = subdir;
            subdir_tar.replace_extension(".tar");

            {
                tar t(subdir_tar, tar::create, ec);
                REQUIRE(!ec);

                t.add_directory(subdir,
                                bfs::relative(subdir, env.basedir()), ec);
                REQUIRE(!ec);
            }

            // this check needs to be done after tar::~tar() has been
            // called, since it appends the EOF blocks
            REQUIRE(psize == bfs::file_size(subdir_tar));
        }

        env.notify_success();
    }

}

max_stars_repo_name: bsc-ssrg/NORNS

max_issues_repo_name: bsc-ssrg/NORNS

max_forks_repo_name: bsc-ssrg/NORNS


text:
//==============================================================================
//         Copyright 2003 - 2011   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#include <boost/config.hpp>

#ifdef BOOST_MSVC
#pragma warning(disable: 4996) // std::transform on pointers may be unsafe
#endif

#include <nt2/sdk/memory/buffer.hpp>
#include <nt2/sdk/memory/composite_buffer.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/simd/memory/aligned_array.hpp>

#include <boost/fusion/include/io.hpp>
#include <boost/fusion/include/make_vector.hpp>
#include <nt2/sdk/memory/buffer.hpp>
#include <nt2/sdk/unit/module.hpp>
#include <nt2/sdk/unit/tests/basic.hpp>
#include <nt2/sdk/unit/tests/relation.hpp>
#include <nt2/sdk/unit/tests/type_expr.hpp>

struct foo
{
  double d; float f; short c;
};

BOOST_FUSION_ADAPT_STRUCT(foo,(double,d)(float,f)(short,c))

struct data_type_
{
  template<class T> struct apply { typedef typename T::data_t type; };
};

//==============================================================================
// Checks proper composite internal typedefs based on buffer
//==============================================================================
NT2_TEST_CASE( composite_buffer_typing )
{
  using boost::mpl::_;
  using nt2::meta::value_type_;
  using nt2::meta::reference_;
  using nt2::meta::const_reference_;
  using nt2::meta::pointer_;
  using nt2::meta::const_pointer_;
  using nt2::memory::composite_buffer;
  using nt2::container::composite_reference;
  using nt2::memory::buffer;

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , data_type_
                    , (boost::fusion::vector3<buffer<double>,buffer<float>,buffer<short> >)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , value_type_<_>
                    , (foo)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , reference_<_>
                    , (composite_reference<foo>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , const_reference_<_>
                    , (composite_reference<foo const>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , pointer_<_>
                    , (boost::fusion::vector3<double*,float*,short*>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< buffer<foo> >())
                    , const_pointer_<_>
                    , (boost::fusion::vector3<double const*,float const*,short const*>)
                    );
}

//==============================================================================
// Checks proper composite internal typedefs based on aligned_array
//==============================================================================
NT2_TEST_CASE( composite_aligned_array_typing )
{
  using boost::mpl::_;
  using nt2::meta::value_type_;
  using nt2::meta::reference_;
  using nt2::meta::const_reference_;
  using nt2::meta::pointer_;
  using nt2::meta::const_pointer_;
  using nt2::memory::composite_buffer;
  using nt2::container::composite_reference;
  using boost::simd::aligned_array;

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3,16> >())
                    , data_type_
                    , ( boost::fusion::
                        vector3 < aligned_array<double,3,16>
                                , aligned_array<float,3,16>
                                , aligned_array<short,3,16>
                                >
                      )
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3> >())
                    , value_type_<_>
                    , (foo)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3> >())
                    , reference_<_>
                    , (composite_reference<foo>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3> >())
                    , const_reference_<_>
                    , (composite_reference<foo const>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3> >())
                    , pointer_<_>
                    , (boost::fusion::vector3<double*,float*,short*>)
                    );

  NT2_TEST_EXPR_TYPE( (composite_buffer< aligned_array<foo,3> >())
                    , const_pointer_<_>
                    , (boost::fusion::vector3<double const*,float const*,short const*>)
                    );
}

//==============================================================================
// composite_buffer default constructor
//==============================================================================
NT2_TEST_CASE( composite_buffer_default_ctor )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > b;

  NT2_TEST(b.empty());
  NT2_TEST_EQUAL(b.size()     , 0u      );
  NT2_TEST_EQUAL(b.capacity() , 0u      );
  NT2_TEST_EQUAL(b.begin()    , b.end() );
}

//==============================================================================
// composite_buffer allocator constructor
//==============================================================================
NT2_TEST_CASE( composite_buffer_allocator_ctor )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> >::allocator_type allocs;
  composite_buffer< buffer<foo> > b(allocs);

  NT2_TEST(b.empty());
  NT2_TEST_EQUAL(b.size()     , 0u      );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 0u      );
  NT2_TEST_EQUAL(b.begin()    , b.end() );
}

//==============================================================================
// Test for size composite_buffer ctor
//==============================================================================
NT2_TEST_CASE( composite_buffer_size_ctor )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > b(5);

  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 5u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// Test for size+allocator composite_buffer ctor
//==============================================================================
NT2_TEST_CASE( composite_buffer_size_allocator_ctor )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> >::allocator_type allocs;
  composite_buffer< buffer<foo> > b(5,allocs);

  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 5u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// Test for dynamic composite_buffer copy ctor
//==============================================================================
NT2_TEST_CASE( buffer_data_ctor)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > b(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  composite_buffer< buffer<foo> > x(b);

  NT2_TEST(!x.empty());
  NT2_TEST_EQUAL(x.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(x.capacity() , 5u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// Test for buffer resize
//==============================================================================
NT2_TEST_CASE(buffer_resize)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > b(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  b.resize(3);
  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 3u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 5u    );

  for ( std::size_t i = 0; i < 3; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }

  b.resize(5);
  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 5u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }

  b.resize(9);
  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 9u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 9u    );

  for ( std::size_t i = 0; i < 9; ++i )
  {
    foo f = {-1.*i,i/2.f,short(i+1)};
    b[i] = f;
  }

  for ( std::size_t i = 0; i < 9; ++i )
  {
    foo f = {-1.*i,i/2.f,short(i+1)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }

  b.resize(1);
  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 1u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 9u    );

  for ( std::size_t i = 0; i < 1; ++i )
  {
    foo f = {-1.*i,i/2.f,short(i+1)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// Test for buffer assignment
//==============================================================================
NT2_TEST_CASE(buffer_assignment)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x, b(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  x = b;

  NT2_TEST(!x.empty());
  NT2_TEST_EQUAL(x.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(x.capacity() , 5u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// Test for buffer swap
//==============================================================================
NT2_TEST_CASE(buffer_swap)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x(7), b(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  for ( std::size_t i = 0; i < 7; ++i )
  {
    foo f = {2.+i,3.f-i,short(1+i)};
    x[i] = f;
  }

  swap(x,b);

  NT2_TEST(!x.empty());
  NT2_TEST_EQUAL(x.size()     , 5u    );
  NT2_TEST_GREATER_EQUAL(x.capacity() , 5u    );

  NT2_TEST(!b.empty());
  NT2_TEST_EQUAL(b.size()     , 7u    );
  NT2_TEST_GREATER_EQUAL(b.capacity() , 7u    );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }

  for ( std::size_t i = 0; i < 7; ++i )
  {
    foo f = {2.+i,3.f-i,short(1+i)};
    NT2_TEST_EQUAL( b[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// buffer Range interface
//==============================================================================
struct f_
{
  template<class T> T operator()(T const& e) const
  {
    T that(e);
    boost::fusion::at_c<0>(that) *= 2;
    boost::fusion::at_c<1>(that) *= 10;
    boost::fusion::at_c<2>(that) *= 3;
    return e;
  }
};

NT2_TEST_CASE(buffer_iterator)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x(5);
  composite_buffer< buffer<foo> > const xc(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    x[i] = f;
  }

  f_ f;

  composite_buffer< buffer<foo> >::iterator b = x.begin();
  composite_buffer< buffer<foo> >::iterator e = x.end();

  std::transform( b, e, b, f );

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo fx = {4.*i,30.f*i,short(3*i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(fx) );
  }
}

//==============================================================================
// buffer push_back single value
//==============================================================================
NT2_TEST_CASE_TPL(buffer_push_back, NT2_TYPES )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x(5);

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    x[i] = f;
  }

  for ( std::size_t i = 0; i < 7; ++i )
  {
    foo f = {-1.*i,3.f/i,short(i/2)};
    x.push_back(f);
  }

  NT2_TEST_EQUAL( x.size(), 12u );
  std::ptrdiff_t i = 0;
  for ( ; i < 5;  ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }

  for ( ; i < 12;  ++i )
  {
    std::ptrdiff_t j = i-5;
    foo f = {-1.*j,3.f/j,short(j/2)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }

  std::cout << "capacity = " << x.capacity() << std::endl;
}

//==============================================================================
// Test for buffer append range
//==============================================================================
NT2_TEST_CASE(buffer_append_range)
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x(7), b(5);

  for ( std::size_t i = 0; i < 7; ++i )
  {
    foo f = {2.+i,3.f-i,short(1+i)};
    x[i] = f;
  }

  for ( std::size_t i = 0; i < 5; ++i )
  {
    foo f = {2.*i,3.f*i,short(i)};
    b[i] = f;
  }

  x.append(b.begin(),b.end());

  NT2_TEST_EQUAL(x.size() , 12u );
  NT2_TEST_GREATER_EQUAL(x.capacity(), 12u );

  std::size_t i = 0;

  for( ; i < 7; ++i )
  {
    foo f = {2.+i,3.f-i,short(1+i)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }

  for( ; i < 5; ++i )
  {
    std::size_t j = i - 7;
    foo f = {2.*j,3.f*j,short(j)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }
}

//==============================================================================
// buffer push_back single value in empty buffer
//==============================================================================
NT2_TEST_CASE_TPL(buffer_push_back_def, NT2_TYPES )
{
  using nt2::memory::buffer;
  using nt2::memory::composite_buffer;

  composite_buffer< buffer<foo> > x;

  for ( std::size_t i = 0; i < 7; ++i )
  {
    foo f = {-1.*i,3.f/i,short(i/2)};
    x.push_back(f);
  }

  NT2_TEST_EQUAL( x.size(), 7u );

  std::ptrdiff_t i = 0;
  for ( ; i < 7;  ++i )
  {
    foo f = {-1.*i,3.f/i,short(i/2)};
    NT2_TEST_EQUAL( x[i], boost::fusion::as_vector(f) );
  }

  std::cout << "capacity = " << x.capacity() << std::endl;
}

ext: cpp


text:
#include <cctbx/boost_python/flex_fwd.h>

#include <scitbx/array_family/tiny.h>
#include <scitbx/array_family/small.h>
#include <scitbx/vec3.h>
#include <scitbx/mat3.h>
#include <scitbx/sym_mat3.h>
#include <scitbx/array_family/boost_python/c_grid_flex_conversions.h>
#include <scitbx/boost_python/container_conversions.h>
#include <scitbx/boost_python/utils.h>
#include <cctbx/coordinates.h>
#include <cctbx/miller.h>
#include <cctbx/hendrickson_lattman.h>
#include <cctbx/maptbx/accessors/c_grid_p1.h>
#include <cctbx/maptbx/accessors/c_grid_padded_p1.h>
#include <boost/python/module.hpp>
#include <boost/python/def.hpp>
#include <boost/python/class.hpp>

namespace scitbx { namespace af { namespace boost_python {

  void wrap_flex_hendrickson_lattman();
  void wrap_flex_miller_index(boost::python::object const& flex_root_scope);
  void wrap_flex_xray_scatterer();

namespace {

  void
  register_cctbx_to_tuple()
  {
    // used by flex_hendrickson_lattman.cpp
    using namespace scitbx::boost_python::container_conversions;
    to_tuple_mapping<af::tiny<af::shared<double>, 4> >();
  }

  void register_cctbx_tuple_mappings()
  {
    using namespace scitbx::boost_python::container_conversions;

    tuple_mapping_fixed_size<cctbx::cartesian<> >();
    tuple_mapping_fixed_size<cctbx::fractional<> >();
    tuple_mapping_fixed_size<cctbx::grid_point<> >();
    tuple_mapping_fixed_size<cctbx::hendrickson_lattman<> >();
    tuple_mapping_fixed_size<cctbx::miller::index<> >();
  }

  void register_cctbx_c_grid_conversions()
  {
    using scitbx::af::boost_python::c_grid_flex_conversions;

    c_grid_flex_conversions<long, cctbx::maptbx::c_grid_p1<3> >();
    c_grid_flex_conversions<long, cctbx::maptbx::c_grid_padded_p1<3> >();
    c_grid_flex_conversions<float, cctbx::maptbx::c_grid_p1<3> >();
    c_grid_flex_conversions<float, cctbx::maptbx::c_grid_padded_p1<3> >();
    c_grid_flex_conversions<double, cctbx::maptbx::c_grid_p1<3> >();
    c_grid_flex_conversions<double, cctbx::maptbx::c_grid_padded_p1<3> >();
    c_grid_flex_conversions<std::complex<double>,
                            cctbx::maptbx::c_grid_p1<3> >();
    c_grid_flex_conversions<std::complex<double>,
                            cctbx::maptbx::c_grid_padded_p1<3> >();
  }

  void init_module()
  {
    using namespace boost::python;

    object flex_root_scope(scitbx::boost_python::import_module(
      "scitbx_array_family_flex_ext"));

    wrap_flex_hendrickson_lattman();
    wrap_flex_miller_index(flex_root_scope);
    wrap_flex_xray_scatterer();

    // The flex module will be used from all cctbx extension modules.
    // Therefore it is convenient to register all tuple mappings from here.
    register_cctbx_tuple_mappings();
    register_cctbx_c_grid_conversions();
    register_cctbx_to_tuple();
  }

} // namespace <anonymous>
}}} // namespace scitbx::af::boost_python

BOOST_PYTHON_MODULE(cctbx_array_family_flex_ext)
{
  scitbx::af::boost_python::init_module();
}


text:
/*
// Licensed to DynamoBI Corporation (DynamoBI) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  DynamoBI licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at

//   http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
*/

#include "fennel/common/CommonPreamble.h"
#include "fennel/test/SegmentTestBase.h"
#include "fennel/segment/SegPageLock.h"
#include "fennel/cache/Cache.h"
#include "fennel/cache/CacheStats.h"

#include <boost/test/test_tools.hpp>

using namespace fennel;

class RandomAllocationSegmentTest : virtual public SegmentTestBase
{
    // TODO jvs 13-Jan-2008:  Need many more tests here...

    void testAllocateAndDeallocate();

public:
    explicit RandomAllocationSegmentTest()
    {
        FENNEL_UNIT_TEST_CASE(
            RandomAllocationSegmentTest, testAllocateAndDeallocate);
    }
};

void RandomAllocationSegmentTest::testAllocateAndDeallocate()
{
    openStorage(DeviceMode::createNew);
    openRandomSegment();

    // Allocate 100 pages
    std::vector<PageId> pageList;
    uint n = 100;
    SegmentAccessor segAccessor(pRandomSegment, pCache);
    SegPageLock segPageLock(segAccessor);
    for (uint i = 0; i < n; ++i) {
        PageId pageId = segPageLock.allocatePage();
        BOOST_CHECK(pRandomSegment->isPageIdAllocated(pageId));
        pageList.push_back(pageId);

        // Perform a dummy write
        segPageLock.getPage().getWritableData();
        segPageLock.unlock();
    }

    // Verify that all are still allocated
    for (uint i = 0; i < n; ++i) {
        BOOST_CHECK(pRandomSegment->isPageIdAllocated(pageList[i]));
    }

    // Verify that the page count matches what's been allocated
    uint nAllocated = pRandomSegment->getAllocatedSizeInPages();
    BOOST_REQUIRE(nAllocated == n);

    // Verify that the high water page occupied count exceeds the allocated
    // count
    uint highWaterMarkBefore = pRandomSegment->getNumPagesOccupiedHighWater();
    BOOST_REQUIRE(highWaterMarkBefore > n);

    // Save cache stats before deallocation
    CacheStats statsBefore;
    pCache->collectStats(statsBefore);

    // Deallocate all
    for (uint i = 0; i < n; ++i) {
        PageId pageId = pageList[i];
        pRandomSegment->deallocatePageRange(pageId, pageId);
        BOOST_CHECK(!pRandomSegment->isPageIdAllocated(pageId));
    }

    // Make sure the high water mark stays the same even after deallocation
    uint highWaterMarkAfter = pRandomSegment->getNumPagesOccupiedHighWater();
    BOOST_REQUIRE(highWaterMarkAfter == highWaterMarkBefore);

    // Get cache stats after deallocation and compare
    CacheStats statsAfter;
    pCache->collectStats(statsAfter);

    // Count of unused pages should go up since deallocation results
    // in discard.
    BOOST_CHECK(statsAfter.nMemPagesUnused > statsBefore.nMemPagesUnused);

    // Verify that all are still deallocated
    for (uint i = 0; i < n; ++i) {
        BOOST_CHECK(!pRandomSegment->isPageIdAllocated(pageList[i]));
    }

    // Allocate the pages again, and recheck the highwater mark to make
    // sure it's still the same
    for (uint i = 0; i < n; ++i) {
        PageId pageId = segPageLock.allocatePage();
    }
    highWaterMarkAfter = pRandomSegment->getNumPagesOccupiedHighWater();
    BOOST_REQUIRE(highWaterMarkAfter == highWaterMarkBefore);
}

FENNEL_UNIT_TEST_SUITE(RandomAllocationSegmentTest);

// End RandomAllocationSegmentTest.cpp


text:
//
// udp_multicast_receiver.cpp - udp multicast receiver implementation
//
// lecnet network library, part of the liblec library
// Copyright (c) 2018 Alec Musasa (alecmus at live dot com)
//
// Released under the MIT license. For full details see the
// file LICENSE.txt
//

#if not defined(_WINSOCKAPI_)
	#define _WINSOCKAPI_	// prevent winsock 1 from being defined
#endif

#include "../../udp.h"
#include "../../auto_mutex/auto_mutex.h"

#include <future>

#define ASIO_STANDALONE

#if not defined(_WIN32_WINNT)
	#define _WIN32_WINNT 0x0601
#endif

#if defined(_WINSOCKAPI_)
	#undef _WINSOCKAPI_
	#include <boost/asio.hpp>
	#include <boost/bind.hpp>
	#include <boost/thread.hpp>
	#define _WINSOCKAPI_
#else
	#include <boost/asio.hpp>
	#include <boost/bind.hpp>
	#include <boost/thread.hpp>
#endif

//----------------------------------------------------------------------

//
// This class manages socket timeouts by applying the concept of a deadline.
// Each asynchronous operation is given a deadline by which it must complete.
// Deadlines are enforced by an "actor" that persists for the lifetime of the
// client object:
//
//  +----------------+
//  |                |
//  | check_deadline |<---+
//  |                |    |
//  +----------------+    | async_wait()
//              |         |
//              +---------+
//
// If the actor determines that the deadline has expired, any outstanding
// socket operations are cancelled. The socket operations themselves are
// implemented as transient actors:
//
//   +---------------+
//   |               |
//   |    receive    |
//   |               |
//   +---------------+
//           |
//  async_-  |    +----------------+
// receive() |    |                |
//           +--->| handle_receive |
//                |                |
//                +----------------+
//
// The client object runs the io_service to block thread execution until the
// actor completes.
//

class _client {
public:
	_client(boost::asio::io_service& io_service,
		const boost::asio::ip::address& listen_address,
		const boost::asio::ip::address& multicast_address,
		unsigned short multicast_port,
		long long timeout_milliseconds) :

		_listen_address(listen_address),
		_multicast_address(multicast_address),
		_multicast_port(multicast_port),
		_timeout_milliseconds(timeout_milliseconds),
		_io_service(&io_service),
		_socket(io_service),
		_deadline(io_service) {}

	bool receive(std::string& message, std::string& error) {
		// Create the socket so that multiple may be bound to the same address.
		boost::asio::ip::udp::endpoint listen_endpoint(_listen_address,
			_multicast_port);
		_socket.open(listen_endpoint.protocol());
		_socket.set_option(boost::asio::ip::udp::socket::reuse_address(true));
		_socket.bind(listen_endpoint);

		// Join the multicast group.
		_socket.set_option(boost::asio::ip::multicast::join_group(_multicast_address));

		// No deadline is required until the first socket operation is started. We
		// set the deadline to positive infinity so that the actor takes no action
		// until a specific deadline is set.
		_deadline.expires_at(boost::posix_time::pos_infin);

		// Start the persistent actor that checks for deadline expiry.
		check_deadline();

		boost::system::error_code ec;
		std::size_t n = do_receive(boost::asio::buffer(_data),
			boost::posix_time::milliseconds(_timeout_milliseconds),
			ec);

		bool result;

		if (ec) {
			error = ec.message();
			result = false;
		}
		else {
			std::stringstream ss;
			ss.write(_data, n);
			message = ss.str();
			result = true;
		}

		try {
			// close the socket if it's still open
			if (_socket.is_open()) {
				_socket.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);

				_socket.close();
			}
		}
		catch (const std::exception&) {
			// ignore this if it occurs ... the underlying descriptor is closed anyway
		}

		// stop the service, if it's still running
		if (!_io_service->stopped())
			_io_service->stop();

		return result;
	}

	void stop() {
		if (_socket.is_open())
			_deadline.expires_from_now(boost::posix_time::milliseconds(0));
	}

private:
	std::size_t do_receive(const boost::asio::mutable_buffer& buffer,
		boost::posix_time::time_duration timeout,
		boost::system::error_code& ec) {
		// Set a deadline for the asynchronous operation.
		_deadline.expires_from_now(timeout);

		// Set up the variables that receive the result of the asynchronous
		// operation. The error code is set to would_block to signal that the
		// operation is incomplete. Asio guarantees that its asynchronous
		// operations will never fail with would_block, so any other value in
		// ec indicates completion.
		ec = boost::asio::error::would_block;
		std::size_t length = 0;

		// Start the asynchronous operation itself. The handle_receive function
		// used as a callback will update the ec and length variables.
		_socket.async_receive(boost::asio::buffer(buffer),
			boost::bind(&_client::handle_receive, _1, _2, &ec, &length));

		// Block until the asynchronous operation has completed.
		do _io_service->run_one(); while (ec == boost::asio::error::would_block);

		return length;
	}

	static void handle_receive( const boost::system::error_code& ec, std::size_t length,
		boost::system::error_code* out_ec, std::size_t* out_length) {
		*out_ec = ec;
		*out_length = length;
	}

	void check_deadline() {
		// Check whether the deadline has passed. We compare the deadline against
		// the current time since a new asynchronous operation may have moved the
		// deadline before this actor had a chance to run.
		if (_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) {
			// The deadline has passed. The outstanding asynchronous operation needs
			// to be cancelled so that the blocked receive() function will return.
			//
			// Please note that cancel() has portability issues on some versions of
			// Microsoft Windows, and it may be necessary to use close() instead.
			// Consult the documentation for cancel() for further information.
			_socket.cancel();

			// There is no longer an active deadline. The expiry is set to positive
			// infinity so that the actor takes no action until a new deadline is set.
			_deadline.expires_at(boost::posix_time::pos_infin);
		}

		// Put the actor back to sleep.
		_deadline.async_wait(boost::bind(&_client::check_deadline, this));
	}

private:
	boost::asio::deadline_timer _deadline;
	boost::asio::io_service* _io_service;
	boost::asio::ip::address _listen_address;
	boost::asio::ip::address _multicast_address;
	unsigned short _multicast_port;
	long long _timeout_milliseconds;

private:
	boost::asio::ip::udp::socket _socket;
	boost::asio::ip::udp::endpoint _sender_endpoint;
	enum { max_length = 1024 };
	char _data[max_length];
};

///////////////////////////////////////////////////////////////////////////////////////////////////

/// <summary>
/// Structure for receiver results.
/// </summary>
struct receive_result {
	/// <summary>
	/// The receive result. True if successful, else false.
	/// </summary>
	bool result = false;

	/// <summary>
	/// Error information, in the case that <see cref="result"/> is false.
	/// </summary>
	std::string error;

	/// <summary>
	/// The received message, in the case that <see cref="result"/> is true.
	/// </summary>
	std::string message;
};

class liblec::lecnet::udp::multicast::receiver::receiver_impl {
public:
	receiver_impl(unsigned short port,
		std::string multicast_address,
		std::string listen_address) :

		_port(port),
		_multicast_address(multicast_address),
		_listen_address(listen_address),
		_p_current_client(nullptr) {}

	~receiver_impl() {}

	static void receiver_func(receiver* p_current);

private:
	std::future<void> _fut;
	_client* _p_current_client;
	unsigned short _port;
	std::string _multicast_address;
	std::string _listen_address;
	long long _timeout_milliseconds;

	// for capturing the result of the the receiver thread
	receive_result _result;
	liblec::mutex _result_lock;

	friend receiver;
};

liblec::lecnet::udp::multicast::receiver::receiver(unsigned short multicast_port,
	std::string multicast_address,
	std::string listen_address) {
	_d = new receiver_impl(multicast_port,
		multicast_address,
		listen_address);
}

liblec::lecnet::udp::multicast::receiver::~receiver() {
	stop();

	// ensure the async operation is completed before deleting
	if (_d->_fut.valid())
		_d->_fut.get();

	delete _d;
	_d = nullptr;
}

void liblec::lecnet::udp::multicast::receiver::receiver_impl::receiver_func(receiver* p_current) {
	try {
		boost::asio::io_service io_service;
		_client client(io_service,
			boost::asio::ip::address::from_string(p_current->_d->_listen_address),
			boost::asio::ip::address::from_string(p_current->_d->_multicast_address),
			p_current->_d->_port, p_current->_d->_timeout_milliseconds);

		p_current->_d->_p_current_client = &client;

		// run the client
		std::string message, error;
		bool result = client.receive(message, error);

		p_current->_d->_p_current_client = nullptr;

		// only here can _result be changed to true. Only here. This is absolutely important.
		liblec::auto_mutex lock(p_current->_d->_result_lock);
		p_current->_d->_result.result = result;
		p_current->_d->_result.error = error;
		p_current->_d->_result.message = message;
	}
	catch (std::exception& e) {
		liblec::auto_mutex lock(p_current->_d->_result_lock);
		p_current->_d->_result.result = false;
		p_current->_d->_result.error = e.what();
		p_current->_d->_result.message.clear();
	}
}

bool liblec::lecnet::udp::multicast::receiver::run(long long timeout_milliseconds,
	std::string& error) {
	if (running()) {
		// allow only one thread
		return true;
	}

	_d->_timeout_milliseconds = timeout_milliseconds;

	try {
		// run receiver task asynchronously
		_d->_fut = std::async(std::launch::async,
			_d->receiver_func, this);
	}
	catch (std::exception& e) {
		error = e.what();
		return false;
	}

	return true;
}

bool liblec::lecnet::udp::multicast::receiver::running() {
	if (_d->_fut.valid())
		return _d->_fut.wait_for(std::chrono::seconds{ 0 }) != std::future_status::ready;
	else
		return false;
}

bool liblec::lecnet::udp::multicast::receiver::get(std::string& message,
	std::string& error) {
	liblec::auto_mutex lock(_d->_result_lock);

	// capture the result
	bool result = _d->_result.result;
	error = _d->_result.error;
	message = _d->_result.message;

	// reset the result
	_d->_result.result = false;
	_d->_result.error.clear();
	_d->_result.message.clear();

	return result;
}

void liblec::lecnet::udp::multicast::receiver::stop() {
	try {
		if (running()) {
			if (_d->_p_current_client)
				_d->_p_current_client->stop();	// to-do: make this safer

			// wait for receiver to stop running
			while (running())
				boost::this_thread::sleep(boost::posix_time::milliseconds(1));
		}
	}
	catch (std::exception&) {}
}

max_stars_repo_name: alecmus/lecnet

max_issues_repo_name: alecmus/lecnet

max_forks_repo_name: alecmus/lecnet


text:
#include <fastad_bits/reverse/core/var.hpp>
#include <fastad_bits/reverse/core/eq.hpp>
#include <fastad_bits/reverse/core/unary.hpp>
#include <fastad_bits/reverse/core/binary.hpp>
#include <fastad_bits/reverse/core/eval.hpp>
#include <fastad_bits/reverse/core/pow.hpp>
#include <fastad_bits/reverse/core/prod.hpp>
#include <benchmark/benchmark.h>
#ifdef USE_ADEPT
#include <adept_arrays.h>
#endif

// Finite-difference

static inline double f_test(const std::vector<double>& x)
{
    double prod = 1.;
    for (const auto& xi : x) {
        prod *= xi * xi;
    }
    return prod * prod + std::cos(prod);
}

static void BM_prod_fd(benchmark::State& state)
{
    constexpr double h = 1e-10;
    std::vector<double> x(10);
    for (size_t i = 0; i < x.size(); ++i) {
        x[i] = i;
    }
    for (auto _ : state) {
        double f = f_test(x);
        for (size_t i = 0; i < x.size(); ++i) {
            x[i] += h;
            double f_h = f_test(x);
            double dfdx_i = (f_h - f)/h;
            benchmark::DoNotOptimize(dfdx_i);
            x[i] -= h;
        }
    }
}

BENCHMARK(BM_prod_fd);

// FastAD
static void BM_prod_fastad(benchmark::State& state)
{
    using namespace ad;
    std::vector<Var<double>> vec;
    for (size_t i = 0; i < 1e1; ++i) {
        vec.emplace_back(i);
    }
    Var<double> w4, w5;
    auto prod_expr = ad::prod(vec.begin(), vec.end()
        , [](const auto& x) {
            return x * x;
        });
    auto expr = ad::bind((w4 = prod_expr, w5 = w4 * w4 + ad::cos(w4)));

    for (auto _ : state) {
        autodiff(expr);
        benchmark::ClobberMemory();
    }
}

BENCHMARK(BM_prod_fastad);

#ifdef USE_ADEPT

// Adept
static void BM_prod_adept(benchmark::State& state)
{
    using namespace adept;
    for (auto _ : state) {
        Stack stack;
        aVector x = {0.,1.,2.,3.,4.,5.,6.,7.,8.,9.};
        stack.new_recording();
        aReal y = product(x * x);
        aReal J = pow(y, 2) + cos(y);
        J.set_gradient(1.);
        stack.reverse();
        benchmark::ClobberMemory();
    }
}

BENCHMARK(BM_prod_adept);

#endif

max_stars_repo_name: kilasuelika/FastAD

max_issues_repo_name: kilasuelika/FastAD

max_forks_repo_name: kilasuelika/FastAD


text:
#include <toynet/math.h>
#include <algorithm>
#include <cmath>
#include <functional>
#include <boost/numeric/ublas/matrix_proxy.hpp>

namespace toynet {

ublas::vector<double> softmax(const ublas::vector<double>& v)
{
    return stable_softmax(v);
}

ublas::vector<double> naive_softmax(const ublas::vector<double>& v)
{
    double sum = 0;
    for (auto x : v)
        sum += exp(x);
    ublas::vector<double> ret(v.size());
    for (int i = 0;  i < v.size();  ++i)
        ret[i] = exp(v[i]) / sum;
    return ret;
}

ublas::vector<double> stable_softmax(const ublas::vector<double> &v)
{
    double max = v.empty() ? 0.0 : *std::max_element(v.begin(), v.end());
    double sum = 0;
    for (auto x : v)
        sum += exp(x - max);
    ublas::vector<double> ret(v.size());
    for (int i = 0;  i < v.size();  ++i)
        ret[i] = exp(v[i] - max) / sum;
    return ret;
}

double magnitude(const ublas::vector<double>& v)
{
    return ublas_magnitude(v);
}

double naive_magnitude(const ublas::vector<double>& v)
{
    double ret = 0.0;
    for (auto x : v)
        ret += x * x;
    return sqrt(ret);
}

double ublas_magnitude(const ublas::vector<double>& v)
{
    return ublas::norm_2(v);
}

double dot_product(const ublas::vector<double>& v1, const ublas::vector<double>& v2)
{
    return ublas_dot_product(v1, v2);
}

double naive_dot_product(const ublas::vector<double>& v1, const ublas::vector<double>& v2)
{
    double ret = 0.0;
    for (int i = 0;  i < v1.size();  ++i)
        ret += v1[i] * v2[i];
    return ret;
}

double ublas_dot_product(const ublas::vector<double>& v1, const ublas::vector<double>& v2)
{
    return ublas::inner_prod(v1, v2);
}

double cosine_distance(const ublas::vector<double>& v1, const ublas::vector<double>& v2)
{
    return dot_product(v1, v2) / (magnitude(v1) * magnitude(v2));
}

std::vector<int> nearest_neighbors(const ublas::vector<double>& v, const std::vector<ublas::vector<double>>& points)
{
    std::vector<std::pair<double, int>> distances(points.size());
    for (int i = 0;  i < points.size();  ++i)
        distances[i] = std::make_pair(cosine_distance(v, points[i]), i);
    std::sort(distances.begin(), distances.end(), std::greater<>());
    std::vector<int> ret(points.size());
    for (int i = 0;  i < points.size();  ++i)
        ret[i] = distances[i].second;
    return ret;
}

void add(std::vector<ublas::vector<double>>& to, const std::vector<ublas::vector<double>>& other)
{
    for (int i = 0;  i < to.size();  ++i)
        to[i] += other[i];
}

void add(std::vector<ublas::matrix<double>>& to, const std::vector<ublas::matrix<double>>& other)
{
    for (int i = 0;  i < to.size();  ++i)
        to[i] += other[i];
}

void normalize(std::vector<ublas::vector<double>>& to, double denom)
{
    for (auto& d : to)
        d /= denom;
}

void normalize(std::vector<ublas::matrix<double>>& to, double denom)
{
    for (auto& d : to)
        d /= denom;
}

} // namespace toynet

max_forks_repo_name: pbrunelle/toynet


text:
#include "ZeroMQIPC.h"

#include <iostream>

#include <boost/lambda/bind.hpp>
#include <mongo/bson/bson.h>

#include "defs.h"

#define INTERNAL_SEND_CHANNEL "inproc://sender"
#define INTERNAL_PUBLISH_CHANNEL "inproc://channeler"

#ifndef DEBUG
#define DEBUG false
#endif

static void messageString(zmq::message_t &msg, const string &str) {
    size_t len = str.size();
    msg.rebuild(len);
    memcpy(msg.data(), str.data(), len);
}

static void noteZmqError(const zmq::error_t &err) {
    std::cerr << "ZMQ error: " << err.what() << std::endl << std::flush;
}

template <class T>
static void noteZmqError(T prefix, const zmq::error_t &err) {
    std::cerr << '[' << prefix << "] ";
    noteZmqError(err);
}

template <class T1, class T2>
static void noteZmqError(T1 p1, T2 p2, const zmq::error_t &err) {
    std::cerr << '[' << p1 << ' ' << p2 << "] ";
    noteZmqError(err);
}

/** This worker manages the external socket. It proxies messages to send out
the external socket, and external messages are proxied into the internal pub-sub
system. We need to do this extra layer because each socket in ZMQ must be
created and managed by exactly one thread. Since the usage model of IPC is that
the main thread sends messages, but listening threads handle incoming messages,
the actual external socket must be managed by yet its own thread. */
void ZeroMQIPCMessageService::mainWorker(
        const string &serverAddress, bool bind) {
    zmq::socket_t external(*ctx, ZMQ_ROUTER);
    external.setsockopt(ZMQ_IDENTITY, id.data(), id.length());

    if (bind) {
        try {
            external.bind(serverAddress.c_str());
        } catch (const zmq::error_t &err) {
            noteZmqError("mainWorker external.bind", serverAddress, err);
            exit(1);
        }
    } else {
        try {
            const int one = 1;
            external.connect(serverAddress.c_str());
            external.setsockopt(ZMQ_ROUTER_MANDATORY, &one, sizeof(one));
        } catch (const zmq::error_t &err) {
            noteZmqError("mainWorker external.connect", serverAddress, err);
            exit(1);
        }
    }

    /* The mailbox will receive messages to send to the external socket. */
    zmq::socket_t mailbox(*ctx, ZMQ_PAIR);
    try {
        mailbox.connect(INTERNAL_SEND_CHANNEL);
    } catch (const zmq::error_t &err) {
        noteZmqError("mainWorker mailbox.connect", INTERNAL_SEND_CHANNEL, err);
    }

    /* The publisher will publish messages received from the external socket */
    zmq::socket_t publisher(*ctx, ZMQ_PUB);
    try {
        publisher.bind(INTERNAL_PUBLISH_CHANNEL);
    } catch (const zmq::error_t &err) {
        noteZmqError("mainWorker publisher.bind", INTERNAL_PUBLISH_CHANNEL, err);
    }

    ready.unlock();

    /* Now, poll both external and sender, and forward messages */
    zmq::pollitem_t items[] = {
        { external, 0, ZMQ_POLLIN, 0 },
        { mailbox, 0, ZMQ_POLLIN, 0 }
    };

    while (true) {
        try {
            zmq::poll(items, sizeof(items)/sizeof(items[0]), -1);

            if (items[0].revents & ZMQ_POLLIN) {
                // copy message from external to publisher... however:
                // message frames are:              <addr>,<channel>,<type>,<message>
                // these need to be re-shuffled to: <channel>,<addr>,<type>,<message>
                // This is so that the PUB?SUB mechanism can select on the channel.
                zmq::message_t addrFrame;
                zmq::message_t channelFrame;

                external.recv(&addrFrame);
                if (!addrFrame.more()) continue;

                external.recv(&channelFrame);
                bool more = channelFrame.more();

                publisher.send(channelFrame, ZMQ_SNDMORE);
                publisher.send(addrFrame, more ? ZMQ_SNDMORE : 0);

                while (more) {
                    zmq::message_t dataFrame;
                    external.recv(&dataFrame);
                    more = dataFrame.more();
                    publisher.send(dataFrame, more ? ZMQ_SNDMORE : 0);
                }
            }

            if (items[1].revents & ZMQ_POLLIN) {
                // copy message from mailbox to external.
                zmq::message_t address;
                zmq::message_t channel;
                zmq::message_t type;
                zmq::message_t payload;

                do {
                    mailbox.recv(&address);     if (!address.more()) break;
                    mailbox.recv(&channel);     if (!channel.more()) break;
                    mailbox.recv(&type);        if (!type.more()) break;
                    mailbox.recv(&payload);
                    if (payload.more()) {
                        for (bool more = true; more;) {
                            zmq::message_t extra;
                            mailbox.recv(&extra);
                            more = extra.more();
                        }
                        break;
                    }

                    bool retry = false;
                    for (int attempt = bind ? 1 : 30; attempt > 0; --attempt) {
                        try {
                            external.send(address, ZMQ_SNDMORE);
                            external.send(channel, ZMQ_SNDMORE);
                            external.send(type, ZMQ_SNDMORE);
                            external.send(payload);
                            if (retry) {
                                std::cerr << "[ZMQIPC mainWorker external.forward] succeeded" << std::endl;
                            }
                            break;
                        } catch (const zmq::error_t &err) {
                            noteZmqError("mainWorker external.forward", err);
                            if (attempt > 1 && err.num() == EHOSTUNREACH) {
                                std::cerr << "[ZMQIPC mainWorker external.forward], sleeping" << std::endl;
                                usleep(500*1000);
                                retry = true;
                            } else {
                                std::cerr << "[ZMQIPC mainWorker external.forward], gave up" << std::endl;
                                break;
                            }
                        }
                    }
                } while (false);
            }
        } catch (const zmq::error_t &err) {
            noteZmqError("mainWorker", err);
            throw;
        }
    }
}

/** This worker subscribes to a given channel, and builds and processes messages
that it receives. */
void ZeroMQIPCMessageService::subWorker(const string &channelId,
        IPCMessageFactory *factory, IPCMessageProcessor *processor) {

    {
        // ensure the mainWorker is ready before proceeding
        boost::lock_guard<boost::mutex> lock(ready);
    }

    zmq::socket_t subscriber(*ctx, ZMQ_SUB);
    try {

        subscriber.setsockopt(ZMQ_SUBSCRIBE, channelId.data(), channelId.length());
        subscriber.connect(INTERNAL_PUBLISH_CHANNEL);
    } catch (const zmq::error_t &err) {
        noteZmqError("subWorker subscriber.connect", INTERNAL_PUBLISH_CHANNEL, err);
        exit(2);
    }

    while (true) {
        try {
            zmq::message_t channel;
            zmq::message_t addr;
            zmq::message_t type;
            zmq::message_t payload;

            subscriber.recv(&channel);
            if (!channel.more()) continue;

            subscriber.recv(&addr);
            if (!addr.more()) continue;

            subscriber.recv(&type);
            if (!type.more()) continue;

            subscriber.recv(&payload);
            bool more = payload.more();

            while (more) {
                zmq::message_t extra;
                subscriber.recv(&extra);
                more = extra.more();
            }

            if (type.size() == 1) {
                IPCMessage *msg = factory->buildForType(static_cast<const char*>(type.data())[0]);
                msg->from_BSON(static_cast<const char*>(payload.data()));

                if (DEBUG) {
                    std::string ch(static_cast<const char*>(channel.data()), channel.size());
                    std::string from(static_cast<const char*>(addr.data()), addr.size());

                    std::cerr << "[ZMQIPC subWorker subscribe.recv] " << id
                        << " ch " << ch << " from " << from
                        << std::endl << msg->str();
                }

                processor->process(static_cast<const char*>(addr.data()), channelId, static_cast<const char*>(channel.data()), *msg);
                delete msg;
            }
        } catch (const zmq::error_t &err) {
            noteZmqError("subWorker", err);
        }
    }
}

ZeroMQIPCMessageService::ZeroMQIPCMessageService(
        const string& serverAddress, const string _id, bool bind) : id(_id) {
    string address = serverAddress.size() > 0 ? serverAddress : ZEROMQ_ADDRESS;

    ready.lock();

    ctx = new zmq::context_t(1);

    try {
        // the first socket of a pair must be bound first, and since the other end
        // is in a thread, we don't know when that will actually get created.
        this->sender = new zmq::socket_t(*this->ctx, ZMQ_PAIR);
        this->sender->bind(INTERNAL_SEND_CHANNEL);
    } catch (const zmq::error_t &err) {
        noteZmqError("ZeroMQIPCMessageService::ctor sender.bind", INTERNAL_SEND_CHANNEL, err);
        exit(3);
    }

    boost::thread t(boost::bind(&ZeroMQIPCMessageService::mainWorker, this,
                                address, bind));
    t.detach();
}

void ZeroMQIPCMessageService::listen(const string &channelId, IPCMessageFactory *factory, IPCMessageProcessor *processor, bool block) {
    boost::thread t(boost::bind(&ZeroMQIPCMessageService::subWorker, this,
                                channelId, factory, processor));
    if (block)
        t.join();
    else
        t.detach();
}

bool ZeroMQIPCMessageService::send(const string &channelId, const string &to, IPCMessage& msg) {
    try {
        if (DEBUG) {
            std::cerr << "[ZMQIPC send] " << id
                << "ch " << channelId << " to " << to
                << std::endl << msg.str();
        }

        zmq::message_t destination;
        messageString(destination, to);

        zmq::message_t channel;
        messageString(channel, channelId);

        zmq::message_t type(1);
        static_cast<char *>(type.data())[0] = (char)msg.get_type();

        const char* msgBSONBytes = msg.to_BSON();
        size_t msgBSONSize = mongo::BSONObj(msgBSONBytes).objsize();
        zmq::message_t payload(msgBSONSize);
        memcpy(payload.data(), msgBSONBytes, msgBSONSize);
        delete msgBSONBytes;

        this->sender->send(destination, ZMQ_SNDMORE);
        this->sender->send(channel, ZMQ_SNDMORE);
        this->sender->send(type, ZMQ_SNDMORE);
        this->sender->send(payload);
    } catch (const zmq::error_t &err) {
        noteZmqError("send", err);
        return false;
    }

    return true;
}

IPCMessageService* IPCMessageServiceFactory::forServer(const string& server, const string& id) {
    return new ZeroMQIPCMessageService(server, id, true);
}
IPCMessageService* IPCMessageServiceFactory::forClient(const string& server, const string& id) {
    return new ZeroMQIPCMessageService(server, id, false);
}
IPCMessageService* IPCMessageServiceFactory::forProxy(const string& server, const string& id) {
    return new ZeroMQIPCMessageService(server, id, false);
}

ext: cc


text:
#define BOOST_TEST_DYN_LINK
#define BOOST_TEST_MODULE ForwardBackward
#include <boost/test/unit_test.hpp>
#include <HSMM.hpp>
#include <ForwardBackward.hpp>
#include <memory>
#include <exception>
#include <set>

#define EPSILON 1e-6

using namespace arma;
using namespace hsmm;
using namespace std;

mat transition = {{0.0, 0.1, 0.4, 0.5},
                  {0.3, 0.0, 0.6, 0.1},
                  {0.2, 0.2, 0.0, 0.6},
                  {0.4, 0.4, 0.2, 0.0}};

mat duration = {{0.0, 0.1, 0.4, 0.5},
                {0.3, 0.0, 0.6, 0.1},
                {0.2, 0.2, 0.0, 0.6},
                {0.4, 0.4, 0.2, 0.0}};

int min_duration = 4;

vec pi = {0.25, 0.25, 0.25, 0.25};

int ndurations = duration.n_cols;

int nstates = duration.n_rows;

// Emission parameters.
vec means = {0, 5, 10, 15};
vec std_devs =  {0.5, 1.0, 0.1, 2.0};

BOOST_AUTO_TEST_CASE( ForwardBackwardWithLabels ) {
    shared_ptr<AbstractEmission> ptr_emission(new DummyGaussianEmission(
            means, std_devs));
    HSMM dhsmm(ptr_emission, transition, pi, duration, min_duration);
    ivec hiddenStates, hiddenDurations;
    int nSampledSegments = 50;
    field<mat> samples = dhsmm.sampleSegments(nSampledSegments, hiddenStates,
            hiddenDurations);
    int nobs = samples.n_elem;

    // Output parameters of the Forward-Backward algorithm.
    mat alpha(nstates, nobs, fill::zeros);
    mat beta(nstates, nobs, fill::zeros);
    mat alpha_s(nstates, nobs, fill::zeros);
    mat beta_s(nstates, nobs, fill::zeros);
    vec beta_s_0(nstates, fill::zeros);
    cube eta(nstates, ndurations, nobs, fill::zeros);
    cube zeta(nstates, nstates, nobs - 1, fill::zeros);

    cube logpdf = dhsmm.computeEmissionsLogLikelihood(samples);
    Labels full_labels;
    Labels sparse_labels;
    set<int> sparse_segment_ids = {10, 30, 40, 45};

    // Required for generating the runs of unobserved segments.
    vector<pair<int, int>> seq_unobserved_segments;
    int starting_unobserved_segment_idx = 0;

    int current_idx = 0;
    double expected_llikelihood_fulllabels = 0;
    for(int i = 0; i < nSampledSegments; i++) {
        int hs = hiddenStates(i);
        int d = hiddenDurations(i);
        expected_llikelihood_fulllabels += logpdf(hs, current_idx,
                d - min_duration) + log(duration(hs, d - min_duration));
        if (i == 0)
            expected_llikelihood_fulllabels += log(pi(hs));
        else
            expected_llikelihood_fulllabels += log(transition(hiddenStates(
                            i - 1), hs));
        int starting_idx = current_idx;
        current_idx += d;
        int ending_idx = current_idx - 1;
        full_labels.setLabel(ending_idx, d, hs);
        if (sparse_segment_ids.find(i) != sparse_segment_ids.end()) {
            sparse_labels.setLabel(ending_idx, d, hs);
            seq_unobserved_segments.push_back(make_pair(
                    starting_unobserved_segment_idx, starting_idx - 1));
            starting_unobserved_segment_idx = ending_idx + 1;
        }
    }
    seq_unobserved_segments.push_back(make_pair(starting_unobserved_segment_idx,
                current_idx - 1));

    // Testing with full labels.
    logsFB(log(transition), log(pi), log(duration), logpdf, full_labels,
            alpha, beta, alpha_s, beta_s, beta_s_0, eta, zeta, min_duration,
            nobs);
    eta = exp(eta);
    zeta = exp(zeta);
    double llikelihood = logsumexp(alpha.col(nobs - 1));
	BOOST_CHECK(fabs(llikelihood - expected_llikelihood_fulllabels) < EPSILON);
    current_idx = 0;
    for(int i = 0; i < nSampledSegments - 1; i++) {
        int hs = hiddenStates(i);
        int next_hs = hiddenStates(i + 1);
        int d = hiddenDurations(i);
        current_idx += d;

        // Checking that the observed transitions have probability one and the
        // rest have zero probability.
        for(int j = 0; j < nstates; j++)
            for(int k = 0; k < nstates; k++)
                if (j == hs && k == next_hs)
                    BOOST_CHECK(fabs(zeta(j, k, current_idx - 1) - 1.0) < EPSILON);
                else
                    BOOST_CHECK(fabs(zeta(j, k, current_idx - 1) - 0.0) < EPSILON);
    }

    // Testing with sparse labels.
    logsFB(log(transition), log(pi), log(duration), logpdf, sparse_labels,
            alpha, beta, alpha_s, beta_s, beta_s_0, eta, zeta, min_duration,
            nobs);
    eta = exp(eta);
    zeta = exp(zeta);
    for(auto p : seq_unobserved_segments) {
        double sum_starting = 0;
        double sum_ending = 0;
        int start_seg = p.first;
        int end_seg = p.second;
        for(int i = 0; i < nstates; i++)
            for(int d = 0; d < ndurations; d++) {
                sum_starting += eta(i, d, start_seg + min_duration + d - 1);
                sum_ending += eta(i, d, end_seg);
            }

        // There must be a segment which explains the starting part of an
        // unobserved run.
        BOOST_CHECK(fabs(sum_starting - 1.0) < EPSILON);

        // The same applies for the ending part.
        BOOST_CHECK(fabs(sum_ending - 1.0) < EPSILON);
    }

    current_idx = 0;
    for(int i = 0; i < nSampledSegments; i++) {
        int hs = hiddenStates(i);
        int d = hiddenDurations(i);
        int starting_idx = current_idx;
        current_idx += d;
        int ending_idx = current_idx - 1;
        if (sparse_segment_ids.find(i) != sparse_segment_ids.end()) {

            // Checking that the provided labels apper as ones in eta.
            BOOST_CHECK(fabs(eta(hs, d - min_duration, ending_idx) - 1) <
                    EPSILON);
            for(int j = 0; j < nstates; j++) {

                double sum_eta_last = 0;
                double sum_eta_first = 0;
                for(int k = 0; k < ndurations; k++) {
                    sum_eta_last += eta(j, k, starting_idx - 1);
                    sum_eta_first += eta(j, k, ending_idx + 1);
                }

                // Checking that the expected transition from an unobserved
                // state to an observed one (or viceversa) is equal to the
                // expected value of being in the unobserved state.
                BOOST_CHECK(fabs(sum_eta_last - zeta(j, hs, starting_idx - 1))
                        < EPSILON);
                BOOST_CHECK(fabs(sum_eta_first - zeta(hs, j, ending_idx + 1))
                        < EPSILON);

                // Checking that unfeasible transitions have 0 posterior mass.
                for(int k = 0; k < nstates; k++)
                    if (k != hs)
                        BOOST_CHECK(zeta(j, k, starting_idx - 1) < EPSILON &&
                                zeta(k, j, ending_idx + 1) < EPSILON);

            }
        }
    }

    // Checking that the expected number of segments in the observation
    // sequence is equal to the actual value. This is expected because
    // the actual parameters are used for inference.
    BOOST_CHECK(fabs(accu(eta) - nSampledSegments) < EPSILON);
}

BOOST_AUTO_TEST_CASE( LogSumExp ) {
    int n = 10;
    vec ones_v = ones<vec>(n);
    double sum = exp(logsumexp(log(ones_v)));
    BOOST_CHECK(fabs(sum - n) < EPSILON);
    int nzeros = 3;
    ones_v.subvec(0, nzeros - 1).fill(0.0);
    sum = exp(logsumexp(log(ones_v)));
    BOOST_CHECK(fabs(sum - (n-nzeros)) < EPSILON);
}

max_stars_repo_name: DiegoAE/BOSD

max_issues_repo_name: DiegoAE/BOSD

max_forks_repo_name: DiegoAE/BOSD


text:
/*
 * p_Copyright (c) 2018 Nicholas Corgan (n.corgan@gmail.com)
 *
 * p_Distributed under the MIT License (MIT) (See accompanying file LICENSE.txt
 * p_or copy at http://opensource.org/licenses/MIT)
 */

#include "cpp_to_c.hpp"
#include "error_internal.hpp"

#include <boost/assert.hpp>
#include <boost/thread/mutex.hpp>

#include <pkmn-c/daycare.h>

enum pkmn_error pkmn_daycare_init(
    enum pkmn_game game,
    struct pkmn_daycare* p_daycare_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare_out);

    PKMN_CPP_TO_C(
        pkmn::daycare::sptr cpp = pkmn::daycare::make(static_cast<pkmn::e_game>(game));

        pkmn::c::init_daycare(
            cpp,
            p_daycare_out
        );
    )
}

enum pkmn_error pkmn_daycare_free(
    struct pkmn_daycare* p_daycare
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);

    p_daycare->game = PKMN_GAME_NONE;
    p_daycare->can_breed_pokemon = false;
    p_daycare->levelup_pokemon_capacity = 0;
    p_daycare->breeding_pokemon_capacity = 0;

    PKMN_CPP_TO_C(
        pkmn::c::delete_pointer_and_set_to_null(
            reinterpret_cast<pkmn_daycare_internal_t**>(&p_daycare->p_internal)
        );
    )
}

const char* pkmn_daycare_strerror(
    struct pkmn_daycare* p_daycare
)
{
    if(!p_daycare)
    {
        return nullptr;
    }

    try
    {
        pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
        if(!p_internal)
        {
            return nullptr;
        }

        boost::mutex::scoped_lock lock(p_internal->error_mutex);
        return p_internal->last_error.c_str();
    }
    catch(...)
    {
        return nullptr;
    }
}

enum pkmn_error pkmn_daycare_get_levelup_pokemon(
    const struct pkmn_daycare* p_daycare,
    size_t index,
    struct pkmn_pokemon* p_levelup_pokemon_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_PARAM_WITH_HANDLE(p_levelup_pokemon_out, p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        pkmn::c::init_pokemon(
            p_internal->cpp->get_levelup_pokemon(
                static_cast<int>(index)
            ),
            p_levelup_pokemon_out
        );
    )
}

enum pkmn_error pkmn_daycare_set_levelup_pokemon(
    const struct pkmn_daycare* p_daycare,
    size_t index,
    const struct pkmn_pokemon* p_pokemon
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_WRAPPER_PARAM_WITH_HANDLE(p_pokemon, p_internal);

    pkmn_pokemon_internal_t* p_pokemon_internal = POKEMON_INTERNAL_RCAST(p_pokemon->p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        p_internal->cpp->set_levelup_pokemon(
            static_cast<int>(index),
            p_pokemon_internal->cpp
        );
    )
}

enum pkmn_error pkmn_daycare_get_levelup_pokemon_as_list(
    const struct pkmn_daycare* p_daycare,
    struct pkmn_pokemon_list* p_levelup_pokemon_list_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_PARAM_WITH_HANDLE(p_levelup_pokemon_list_out, p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        pkmn::c::pokemon_list_cpp_to_c(
            p_internal->cpp->get_levelup_pokemon_as_vector(),
            p_levelup_pokemon_list_out
        );
    )
}

enum pkmn_error pkmn_daycare_get_breeding_pokemon(
    const struct pkmn_daycare* p_daycare,
    size_t index,
    struct pkmn_pokemon* p_breeding_pokemon_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_PARAM_WITH_HANDLE(p_breeding_pokemon_out, p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        pkmn::c::init_pokemon(
            p_internal->cpp->get_breeding_pokemon(
                static_cast<int>(index)
            ),
            p_breeding_pokemon_out
        );
    )
}

enum pkmn_error pkmn_daycare_set_breeding_pokemon(
    const struct pkmn_daycare* p_daycare,
    size_t index,
    const struct pkmn_pokemon* p_pokemon
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_WRAPPER_PARAM_WITH_HANDLE(p_pokemon, p_internal);

    pkmn_pokemon_internal_t* p_pokemon_internal = POKEMON_INTERNAL_RCAST(p_pokemon->p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        p_internal->cpp->set_breeding_pokemon(
            static_cast<int>(index),
            p_pokemon_internal->cpp
        );
    )
}

enum pkmn_error pkmn_daycare_get_breeding_pokemon_as_list(
    const struct pkmn_daycare* p_daycare,
    struct pkmn_pokemon_list* p_breeding_pokemon_list_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_PARAM_WITH_HANDLE(p_breeding_pokemon_list_out, p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        pkmn::c::pokemon_list_cpp_to_c(
            p_internal->cpp->get_breeding_pokemon_as_vector(),
            p_breeding_pokemon_list_out
        );
    )
}

enum pkmn_error pkmn_daycare_get_egg(
    const struct pkmn_daycare* p_daycare,
    struct pkmn_pokemon* p_egg_out
)
{
    PKMN_CHECK_NULL_PARAM(p_daycare);
    pkmn_daycare_internal_t* p_internal = DAYCARE_INTERNAL_RCAST(p_daycare->p_internal);
    PKMN_CHECK_NULL_PARAM_WITH_HANDLE(p_egg_out, p_internal);

    PKMN_CPP_TO_C_WITH_HANDLE(p_internal,
        pkmn::c::init_pokemon(
            p_internal->cpp->get_egg(),
            p_egg_out
        );
    )
}

max_stars_repo_name: ncorgan/libpkmn

max_issues_repo_name: PMArkive/libpkmn

max_forks_repo_name: PMArkive/libpkmn