text:
// This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

/* behavior_manager.cc
   Jeremy Barnes, 7 August 2012
   Copyright (c) 2012 mldb.ai inc.  All rights reserved.

*/

#define __STDC_FORMAT_MACROS 1
#include "inttypes.h"
#include <stdio.h>
#include <math.h>
#include <unistd.h>
#include <utime.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <ftw.h>

#include <boost/algorithm/string/split.hpp>
#include "mldb/compiler/filesystem.h"

#include "mldb/arch/fslock.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/vm.h"
#include "mldb/utils/file_functions.h"
#include "mldb/utils/string_functions.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/base/scope.h"
#include "mldb/base/parallel.h"
#include "mldb/types/date.h"
#include "mldb/vfs/fs_utils.h"
#include "mldb/base/scope.h"

#include "mapped_behavior_domain.h"
#include "merged_behavior_domain.h"
#include "mutable_behavior_domain.h"
#include "behavior_svd.h"

#include "behavior_manager.h"

using namespace std;
namespace fs = std::filesystem;
using namespace MLDB;

namespace {

string
getErrorString(int errNum)
{
    char buffer[128]; /* error messages are < 80 chars most of the time */

    char * msg = strerror_r(errNum, buffer, sizeof(buffer));
    if (msg == nullptr) {
        throw MLDB::Exception(errno, "strerror_r");
    }

    return string(msg);
}

/* This function returns a cache filename for the corresponding remote entry. It
 * takes its etag, strips the double-quotes from it and appends the ".beh"
 * suffix. */
string
getRemoteCacheName(const FsObjectInfo & info)
{
    string etag = info.etag;

    // Remove the quotes which makes scripting harder.
    if (!etag.empty() && etag[0] == '"' && etag[etag.size()-1] == '"')
        etag = etag.substr(1, etag.size() - 2);

    return etag + ".beh";
}

/* This function updates the "last access time" field of the given file
 * metadata. It is used to ensure that this field is properly updated even
 * when the "noatime" flag is set as parameter for mounting the underlying
 * file system. */
void
touchATime(const string & filename)
{
    /* compute the new atime */
    struct timeval newATime;
    int res = ::gettimeofday(&newATime, nullptr);
    if (res == -1) {
        throw MLDB::Exception("gettimeofday");
    }

    /* fetch the mtime */
    struct timeval oldMTime;
    struct stat filestat;
    res = ::stat(filename.c_str(), &filestat);
    if (res == -1) {
        throw MLDB::Exception("stat");
    }
    oldMTime.tv_sec = filestat.st_mtim.tv_sec;
    oldMTime.tv_usec = filestat.st_mtim.tv_nsec / 1000;

    /* apply the new values */
    struct timeval times[] = {newATime, oldMTime};
    res = ::utimes(filename.c_str(), times);
    if (res == -1) {
        throw MLDB::Exception("utimes");
    }
}

} // file scope

namespace MLDB {

/*****************************************************************************/
/* BEHAVIOR MANAGER                                                         */
/*****************************************************************************/

BehaviorManager::
BehaviorManager()
    : cacheLoadedFiles(false), touchPages(true)
{
}

BehaviorManager::
~BehaviorManager()
{
}

std::shared_ptr<BehaviorDomain>
BehaviorManager::
get(const std::vector<std::string> & inputFiles,
    bool preIndex,
    CacheAction cacheAction,
    bool skipBadFiles,
    std::function<bool (Json::Value)> onProgress)
{
    std::shared_ptr<BehaviorDomain> behs;
    if (inputFiles.empty())
        return std::make_shared<MutableBehaviorDomain>();
    else if (inputFiles.size() == 1) {
        behs = get(inputFiles[0], cacheAction, onProgress);
    }
    else {
        vector<std::shared_ptr<BehaviorDomain> > toMerge(inputFiles.size());

        auto loadFile = [&] (int i)
            {
                string filename = inputFiles[i];
                try {
                    toMerge[i] = get(filename, cacheAction);
                } catch (const std::exception & exc) {
                    if (!skipBadFiles)
                        throw;
                    else cerr << "beh mgr: skipping bad file " << filename << endl;
                }
            };

        parallelMap(0, inputFiles.size(), loadFile);

        int numBadFiles = 0, numGoodFiles = 0;

        vector<std::shared_ptr<BehaviorDomain> > filtered;
        for (auto f: toMerge) {
            if (f) {
                filtered.push_back(f);
                ++numGoodFiles;
            }
            else {
                ++numBadFiles;
            }
        }

        if (numBadFiles > 0)
            cerr << ("beh mgr: skipping " + to_string(numBadFiles)
                     + " of " + to_string(toMerge.size()) + " bad files\n");

        behs = std::make_shared<MergedBehaviorDomain>(filtered, preIndex);
    }

    return behs;
}

std::shared_ptr<BehaviorDomain>
BehaviorManager::
get(const std::string & filename, CacheAction cacheAction,
    std::function<bool (Json::Value)> onProgress)
{
    {
        std::unique_lock<std::mutex> guard(lock);
        if (cache.count(filename))
            return cache[filename];
    }

    std::shared_ptr<BehaviorDomain> result;

    try {
        string scheme = getUriScheme(filename);

        if (scheme == "file") {
            result = getFile(getUriPath(filename), onProgress);
        }
        else {
            result = getRemote(filename, onProgress);
        }
    } catch (const std::exception & exc) {
        throw MLDB::Exception("exception loading beh file " + filename + ": " + exc.what());
    } catch (...) {
        throw MLDB::Exception("exception loading beh file " + filename);
    }

    if (cacheAction == CACHE_ALWAYS
        || (cacheAction == CACHE_CONFIG && cacheLoadedFiles)) {
        std::unique_lock<std::mutex> guard(lock);
        if (cache.count(filename))
            return cache[filename];
        cache[filename] = result;
    }

    return result;
}

void touchByte(const char * c)
{
    __asm__
        (" # [in]"
         :
         : [in] "r" (*c)
         :
         );
}

/** Touch the given range of memory, to ensure that its contents will be
    available to the application without page faults.

    The progress function that can be passed in allows a calling process to
    know how it is progressing and cancel it (by returning false from the
    lambda).  A null function means don't report progress.
*/
bool touch(const char * start, size_t size,
           const std::function<bool (size_t) > & progress)
{
    int res;

    if (progress)
        if (!progress(0))
            return false;

    /* This madvise call is harmless and should be there.  It tells the kernel that
       we're about to use this memory sequentially and so it can enable read-ahead
       on the disk file underneath.  This may allow it to be loaded more efficiently
       from disk.
    */
    res = madvise((void *)start, size,
                  MADV_WILLNEED | MADV_SEQUENTIAL);
    if (res == -1 && errno != EBADF)
        throw MLDB::Exception(errno, "madvise");

    /* Block size at which we report progress.  */
    enum {
        BLOCKSIZE = 4096 * 4096
    };

    if (progress)
        if (!progress(1))
            return false;

    /* Iterate through our file a "block" at at time, and for each of those
       blocks touch each page exactly once.  This forces the entire set of
       page tables to be populated and the memory to be paged in from disk
       sequentially, which means that further access will be low latency.
    */
    for (size_t i = 0;  i < size;  i += BLOCKSIZE) {
        for (size_t j = i;  j < size && j < i + BLOCKSIZE;  j += page_size) {
            touchByte(start + j);
        }
        if (progress)
            if (!progress(i))
                return false;
    }

    if (progress)
        return progress(size);

    else return true;
}

bool touchWithProgress(const std::string & filename,
                       const MLDB::File_Read_Buffer & file,
                       const std::function<bool (Json::Value)> & onProgress)
{
    Date start = Date::now();

    auto onProgress2 = [&] (size_t bytesDone)
        {
            if (!onProgress)
                return true;

            double seconds = Date::now().secondsSince(start);

            Json::Value val;
            val["process"] = "memory mapping file";
            val["file"] = filename;
            val["mappedMB"] = bytesDone / 1000000.0;
            val["totalBytes"] = file.size() / 1000000.0;
            val["percentDone"] = 100.0 * bytesDone / file.size();
            val["elapsedSeconds"] = seconds;
            val["rateMbPerSecond"] = bytesDone / 1000000.0 / seconds;

            return onProgress(val);
        };

    return touch(file.start(), file.size(), onProgress2);
}

void
BehaviorManager::
preloadRemoteCache(const set<string> & filenames)
    const
{
    cerr << "WARNING: BehaviorManager::preloadRemoteCache is a no-op for now\n";
#if 0
    map<string, pair<string, string>> files;

    // Date start = Date::now();

    if (s3CacheDir.empty()) {
        throw MLDB::Exception("no s3 cache dir set");
    }

    for (const string & filename: filenames) {
        auto s3UrlParts = S3Api::parseUri(filename);
        const string & s3ObjectName = s3UrlParts.second;
        auto s3 = getS3ApiForUri(filename);
        auto objectInfo = s3->getObjectInfo(filename);

        if (canCacheS3File(filename, objectInfo.size)) {
            string fileCacheDir = s3CacheDir + "/" + s3ObjectName;
            fs::create_directories(fileCacheDir);

            string basename = getS3CacheName(objectInfo);
            files[filename] = make_pair(fileCacheDir, basename);
        }
    }

    while (!files.empty()) {
        bool doSleep(true);
        vector<string> toRemove;

        for (const auto & entry: files) {
            const string & fileCacheDir = entry.second.first;
            const string & basename = entry.second.second;
            string fullname = fileCacheDir + "/" + basename;

            MLDB::GuardedFsLock fsLock(fullname);
            if (fsLock.tryLock()) {
                if (fs::exists(fullname)) {
                    toRemove.push_back(entry.first);
                }
                else {
                    /* we need to download and save it */
                    cerr << ("beh mgr: [" + to_string(getpid()) + "]"
                             + " preloading '" + entry.first + "'\n");
                    cacheS3File(entry.first, objectInfo.size,
                                nullptr, fullname);
                }
                doSleep = false;
                fsLock.unlock();
            }
        }

        for (const string & filename: toRemove) {
            files.erase(filename);
        }

        if (doSleep) {
            MLDB::sleep(0.2);
        }
    }

    // Date end = Date::now();
    // double delta = end.secondsSinceEpoch() - start.secondsSinceEpoch();
    // ::fprintf(stderr, "%d: preloaded %ld files in %f secs\n", ::getpid(),
    // filenames.size(), delta);
#endif
}

/* load entire file contents in a std::string, using an filter_istream */
string
BehaviorManager::
readFile(const string & filename,
         std::function<bool (uint64_t)> onProgress)
    const
{
    string contents;
    char readBuf[1024*1024];

    uint64_t totalRead = 0;

    if (onProgress)
        if (!onProgress(totalRead))
            return "";

    filter_istream stream(filename);
    while (stream) {
        stream.read(readBuf, sizeof(readBuf));
        ssize_t read = stream.gcount();
        contents.append(readBuf, read);
        totalRead += read;
        if (onProgress)
            if (!onProgress(totalRead))
                return "";
    }

    if (onProgress)
        if (!onProgress(totalRead))
            return "";

    return contents;
}

bool
BehaviorManager::
prepareRemoteCache(uint64_t objectSize)
    const
{
    // Check whether there is enough empty space there...
    struct statfs fsstats;
    int res = ::statfs(remoteCacheDir.c_str(), &fsstats);
    if (res == -1) {
        cerr << ("beh mgr: statfs on remoteCacheDir: "
                 + getErrorString(errno) + "\n");
        return false;
    }

    uint64_t totalSize = fsstats.f_blocks * fsstats.f_bsize;
    if (totalSize < objectSize) {
        cerr << ("beh mgr: disk space in '" + remoteCacheDir
                 + "' too small to hold a " + to_string(objectSize)
                 + " bytes files\n");
        return false;
    }

    uint64_t freeSpace = fsstats.f_bavail * fsstats.f_bsize;

    /* An alternative method here is to deduce the freeSpace from the
       objectSize, so that only the required disk space would be recovered.
       However, in that case, a race condition between two processes could
       lead to the freed space being reused right away by a concurrent
       instance. It is thus better to free a bit more space than required. */
    if (freeSpace < objectSize && !recoverRemoteCacheDiskSpace(objectSize)) {
        cerr << ("beh mgr: not enough disk space could be recovered"
                 " to store a " + to_string(objectSize) + " bytes files\n");
        return false;
    }

    return true;
}

namespace {

enum FileAction {
    FA_CONTINUE = FTW_CONTINUE,
    FA_SKIP_SIBLINGS = FTW_SKIP_SIBLINGS,
    FA_SKIP_SUBTREE = FTW_SKIP_SUBTREE,
    FA_STOP = FTW_STOP
};

enum FileType {
    FT_FILE = FTW_F,
    FT_DIR  = FTW_D,
    FT_DIR_INACCESSIBLE = FTW_DNR,
    FT_FILE_INACCESSIBLE = FTW_NS
};

typedef std::function<FileAction (std::string dir,
                                    std::string basename,
                                    const struct stat & stats,
                                    FileType type,
                                    int depth)>
    OnFileFound;

struct ScanFilesData;

static __thread ScanFilesData * scanFilesThreadData = 0;

struct ScanFilesData {
    ScanFilesData(const OnFileFound & onFileFound,
                  int maxDepth)
        : onFileFound(onFileFound),
          maxDepth(maxDepth),
          isThrown(false)
    {
    }

    OnFileFound onFileFound;
    int maxDepth;
    std::exception_ptr thrown;
    bool isThrown;

    static int onFile (const char *fpath, const struct stat *sb,
                       int typeflag, struct FTW *ftwbuf)
    {
        ScanFilesData * d = scanFilesThreadData;
        ExcAssert(d);
        try {
            if (d->maxDepth != -1 && ftwbuf->level > d->maxDepth)
                return FTW_SKIP_SIBLINGS;
            string dir(fpath, fpath + ftwbuf->base);
            string basename(fpath + ftwbuf->base);

            FileAction action = d->onFileFound(dir, basename, *sb,
                                               (FileType)typeflag,
                                               ftwbuf->level);
            return action;
        } MLDB_CATCH_ALL {
            d->thrown = std::current_exception();
            d->isThrown = true;
            return FTW_STOP;
        }
    }
};

static void scanFiles(const std::string & path,
                      OnFileFound onFileFound,
                      int maxDepth = -1)
{
    ScanFilesData data(onFileFound, maxDepth);

    auto * oldData = scanFilesThreadData;
    Scope_Exit(scanFilesThreadData = oldData);
    scanFilesThreadData = &data;

    int res = nftw(path.c_str(),
                   &ScanFilesData::onFile,
                   maxDepth == -1 ? 100 : maxDepth + 1,
                   FTW_ACTIONRETVAL);

    if (data.isThrown) {
        auto exc = data.thrown;
        rethrow_exception(exc);
    }

    if (res == -1) {
        if (errno == ENOENT)
            return;
        throw MLDB::Exception(errno, "ftw");
    }
}

} // file scope

vector<BehaviorManager::RemoteCacheEntry>
BehaviorManager::
getRemoteCacheEntries()
    const
{
    ExcAssert(!remoteCacheDir.empty());
    vector<RemoteCacheEntry> cacheFiles;

    auto onFileFound = [&] (std::string dir,
                            std::string basename,
                            const struct stat & stats,
                            FileType type,
                            int depth) {
        if (S_ISREG(stats.st_mode)) {
            RemoteCacheEntry newFile{dir + basename,
                                Date::fromTimespec(stats.st_atim),
                                stats.st_size};
            cacheFiles.emplace_back(move(newFile));
        }
        return FA_CONTINUE;
    };

    scanFiles(remoteCacheDir, onFileFound, -1);

    return cacheFiles;
}

vector<BehaviorManager::RemoteCacheEntry>
BehaviorManager::
getRemoteRecoverableEntries(uint64_t objectSize)
    const
{
    vector<RemoteCacheEntry> recoverableEntries;

    /* retrieve and sort the entries by access time in descending order */
    vector<RemoteCacheEntry> entries = getRemoteCacheEntries();
    auto sortByDate = [&] (const RemoteCacheEntry & left,
                           const RemoteCacheEntry & right) -> int {
        return ((right.accessTime > left.accessTime)
                ? 1 : 0);
    };
    sort(entries.begin(), entries.end(), sortByDate);

    /* build a list of files that needs to be deleted in order to recover
       "objectSize" bytes */
    uint64_t recoverable(0);
    for (const RemoteCacheEntry & file: entries) {
        if (recoverable >= objectSize) {
            break;
        }
        recoverableEntries.push_back(file);
        recoverable += file.size;
    }

    if (recoverable < objectSize) {
        cerr << ("beh mgr: not enough recoverable entries for a "
                 + to_string(objectSize) + " bytes file\n");
        recoverableEntries.clear();
    }

    return recoverableEntries;
}

bool
BehaviorManager::
recoverRemoteCacheDiskSpace(uint64_t objectSize)
    const
{
    vector<RemoteCacheEntry> entries = getRemoteRecoverableEntries(objectSize);
    if (entries.size() == 0) {
        return false;
    }

    uint64_t recovered(0);
    for (const RemoteCacheEntry & file: entries) {
        if (tryEraseUriObject(file.name)) {
            recovered += file.size;
        }
        else {
            cerr << ("beh mgr: file '" + file.name
                     + "' could not be deleted\n");
        }
    }

    if (recovered < objectSize) {
        cerr << ("beh mgr: requested: " + to_string(objectSize) + " bytes;"
                 " recovered: " + to_string(recovered) + "\n");
    }

    return (recovered >= objectSize);
}

/* Concurrency scenarios:
   1) process 1 and process 2 request the cleanup of a distinct list of files,
      in which case enough disk space can be recovered for both files
   2) process 1 has enough disk space to store a file and has not begun to
      store it yet, process 2 initially has enough disk space too. Both
      processes start downloading and storing their file, leading to ENOSPC.
      This case is handled by calling the disk space recovery functions as
      many times as pwrite fails with ENOSPC, with a maximum of 10 attempts
      for the same call to pwrite
   3) process 1 and process 2 request disk space but clean up a list of files
      which have one or more entries in common: the total disk space actually
      recovered will be less than the requested disk space. Again, this is
      handled by calling the disk space recovery functions as many times are
      pwrite fails with ENOSPC
 */
bool
BehaviorManager::
cacheRemoteFile(const string & filename, uint64_t fileSize,
            std::function<bool (uint64_t)> onProgress,
            const string & cacheFile)
    const
{
    char buffer[1024*1024];
    uint64_t totalRead = 0;

    ExcAssert(remoteCacheDir.size() > 0);
    ExcAssert(remoteCacheDir.size() < (1024*1024 - 14));

    ::sprintf(buffer, "%s/remote-download-XXXXXX", remoteCacheDir.c_str());
    int fd = mkstemp(buffer);
    /* we put the filename in "tmp" so that it can be remembered while the
       buffer is used for reading file data */
    string tmp(buffer, ::strlen(buffer));

    if (fd == -1) {
        throw MLDB::Exception(errno, "creating temporary file: " + tmp);
    }

    bool ok = false;

    Scope_Exit(
        if(!ok) {
            ::unlink(tmp.c_str());
        }

        ::close(fd);
               );

    //Date start = Date::now();

    if (onProgress)
        if (!onProgress(totalRead))
            return false;

    uint64_t ofs = 0;
    filter_istream stream(filename);
    while (stream) {
        stream.read(buffer, sizeof(buffer));
        ssize_t read = stream.gcount();

        /* If the write operation fails due to lack of disk space, we attempt
           to recover the disk space equivalent to the size of the original remote
           file. In spite of the compression, using that quantity as reference
           enables to reduce the number of recovery operations, which is
           presumably quite costly. Ideally, we would approximate the
           remaining size by comparing the number of read bytes from both the
           compressed and the uncompressed streams, but there is unfortunately
           no standard way of doing that. */
        int failedAttempts(0);
        while (true) {
            ssize_t res = ::pwrite(fd, buffer, read, ofs);
            if (res > -1) {
                break;
            }

            if (errno == EINTR) {
                continue;
            }

            failedAttempts++;
            if (errno == ENOSPC
                && failedAttempts < 11
                && recoverRemoteCacheDiskSpace(fileSize)) {
                continue;
            }

            cerr << ("beh mgr: error writing cache file: "
                     + getErrorString(errno) + "\n");
            return false;
        }

        ofs += read;

        totalRead += read;
        if (onProgress)
            if (!onProgress(totalRead))
                return false;
    }

    if (onProgress)
        if (!onProgress(totalRead))
            return false;

    int res = ::rename(tmp.c_str(), cacheFile.c_str());
    if (res == -1) {
        throw MLDB::Exception(errno, "error renaming cache file " + tmp);
    }

    ok = true;

    return true;
}

std::shared_ptr<BehaviorDomain>
BehaviorManager::
getRemote(const std::string & filename,
      std::function<bool (Json::Value)> onProgress)
{
    ExcAssert(!filename.empty());

    auto objectInfo = getUriObjectInfo(filename);
    string remoteObjectName = getUriPath(filename);
    string cacheFile;

    bool useCache = (remoteCacheDir != "" && fs::exists(remoteCacheDir));
    bool cacheFileExists = false;

    if (useCache && fs::exists(remoteCacheDir)) {
        string fileCacheDir = remoteCacheDir + "/" + remoteObjectName;

        string basename = getRemoteCacheName(objectInfo);
        cacheFile = remoteCacheDir + "/" + basename; //fileCacheDir + "/" + basename;

        cacheFileExists = fs::exists(cacheFile);

        /* The objectInfo.size value is a bad reference value when storing
           compressed files, since it will be always smaller than the real
           decompressed size. However, the cache cleanup will occur later
           again from "cacheRemoteFile" in the case where write operation fails
           due to lack of disk space. */
        if (!prepareRemoteCache(objectInfo.size)) {
            cacheStats_.cacheBytesUnsaved += objectInfo.size;
            cacheStats_.cacheFilesUnsaved++;
            useCache = false;
            cerr << ("beh mgr: skipped cache due to unability to cache"
                     " requested file\n");
        }
    }

    if (useCache && cacheFileExists) {
        try {
            MLDB::File_Read_Buffer file(cacheFile);
            touchATime(cacheFile);

            if(touchPages) {
                if (!touchWithProgress(cacheFile, file, onProgress))
                    return nullptr;
            }

            cacheStats_.cacheBytesRead += file.size();
            cacheStats_.cacheFilesRead++;

            return std::make_shared<MappedBehaviorDomain>(file);
        }
        catch (const std::exception & exc) {
            cerr << ("beh mgr: error loading local cache file "
                     + cacheFile + ": " + exc.what() + "\n");
        }
    }

    Date start = Date::now();

    bool cancelled = false;
    auto onFileProgress = [&] (uint64_t bytesRead)
        {
            if (onProgress) {
                double seconds = Date::now().secondsSince(start);

                Json::Value val;
                val["process"] = "downloading";
                val["file"] = filename;
                val["readMB"] = bytesRead / 1000000.0;
                val["elapsedSeconds"] = seconds;
                val["rateMbPerSecond"] = bytesRead / 1000000.0 / seconds;

                cancelled = !onProgress(val);
                return !cancelled;
            }

            return true;  // continue
        };

    if (useCache) {
        if (cacheRemoteFile(filename, objectInfo.size,
                        onFileProgress, cacheFile)) {
            if (cancelled)
                return nullptr;

            MLDB::File_Read_Buffer file(cacheFile);
            touchATime(cacheFile);

            if(touchPages) {
                if (!touchWithProgress(cacheFile, file, onProgress))
                    return nullptr;
            }

            return std::make_shared<MappedBehaviorDomain>(file);
        }
    }

    auto contents = std::make_shared<string>();
    *contents = readFile(filename, onFileProgress);
    if (cancelled)
        return nullptr;

    auto onDone = [contents] () {
        shared_ptr<string> localPtr = contents;
        localPtr.reset();
    };

    MLDB::File_Read_Buffer file(contents->c_str(), contents->size(),
                              filename, onDone);
    auto result = std::make_shared<MappedBehaviorDomain>(file);
    return result;
}

std::shared_ptr<BehaviorDomain>
BehaviorManager::
getFile(const std::string & filename,
        std::function<bool (Json::Value)> onProgress)
{
    ExcAssert(!filename.empty());

    if (MLDB::endsWith(filename, ".gz") || MLDB::endsWith(filename, ".lz4")) {
        auto contents = std::make_shared<string>();

        bool cancelled = false;
        auto onFileProgress = [&] (uint64_t bytesRead)
            {
                if (onProgress) {
                    Json::Value val;
                    val["file"] = filename;
                    val["readMB"] = bytesRead / 1000000.0;
                    cancelled = !onProgress(val);
                    return !cancelled;
                }

                return true;  // continue
            };

        *contents = readFile(filename, onFileProgress);
        if (cancelled)
            return nullptr;

        auto onDone = [contents] () {
            shared_ptr<string> localPtr = contents;
            localPtr.reset();
        };
        MLDB::File_Read_Buffer file(contents->c_str(), contents->size(),
                                  filename, onDone);
        return std::make_shared<MappedBehaviorDomain>(file);
    }

    File_Read_Buffer file(filename);

    if(touchPages) {
        if (!touchWithProgress(filename, file, onProgress))
            return nullptr;
    }

    return std::make_shared<MappedBehaviorDomain>(file);
}

void
BehaviorManager::
save(BehaviorDomain & behs,
     const std::string & filename)
    const
{
    if (filename == "") return;

    filter_ostream stream(filename);
    behs.saveToStream(stream);
}

void
BehaviorManager::
saveSvd(const BehaviorSvd & svd,
        const std::string & filename)
{
    filter_ostream stream(filename);
    DB::Store_Writer store(stream);
    svd.serialize(store);
}

std::shared_ptr<BehaviorSvd>
BehaviorManager::
getSvd(const std::string & filename)
{
    filter_istream stream(filename);
    DB::Store_Reader store(stream);
    auto result = std::make_shared<BehaviorSvd>();
    result->reconstitute(store);
    return result;
}

void
BehaviorManager::
setRemoteCacheDir(const std::string & dir)
{
    remoteCacheDir = dir;
}

std::map<std::string, std::string>
BehaviorManager::
parseSpecs(const std::string & specs) const
{
    vector<string> splitStr;
    boost::split(splitStr, specs, boost::is_any_of(" "));
    map<std::string, std::string> result;
    for (auto k_vStr : splitStr) {
        vector<string> key_value;
        boost::split(key_value, k_vStr, boost::is_any_of(":"));
        if (key_value.size() != 2)
            throw MLDB::Exception("invalid key " + k_vStr + " in specs");
        result[key_value[0]] = key_value[1];
    }
    return result;
}

} // namespace MLDB

max_stars_repo_name: tomzhang/mldb

max_issues_repo_name: tomzhang/mldb

max_forks_repo_name: tomzhang/mldb


text:
/******************************************************************************
*       SOFA, Simulation Open-Framework Architecture, development version     *
*                (c) 2006-2017 INRIA, USTL, UJF, CNRS, MGH                    *
*                                                                             *
* This program is free software; you can redistribute it and/or modify it     *
* under the terms of the GNU General Public License as published by the Free  *
* Software Foundation; either version 2 of the License, or (at your option)   *
* any later version.                                                          *
*                                                                             *
* This program is distributed in the hope that it will be useful, but WITHOUT *
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       *
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    *
* more details.                                                               *
*                                                                             *
* You should have received a copy of the GNU General Public License along     *
* with this program. If not, see <http://www.gnu.org/licenses/>.              *
*******************************************************************************
* Authors: The SOFA Team and external contributors (see Authors.txt)          *
*                                                                             *
* Contact information: contact@sofa-framework.org                             *
******************************************************************************/

#ifndef SOFA_STANDARDTEST_TETRAHEDRONDIFFUSIONFEMFORCEFIELD_TEST_H
#define SOFA_STANDARDTEST_TETRAHEDRONDIFFUSIONFEMFORCEFIELD_TEST_H

#include <SofaSimulationGraph/DAGSimulation.h>
#include <SceneCreator/SceneCreator.h>

#include <SofaTest/Sofa_test.h>
#include <SofaBaseMechanics/MechanicalObject.h>
#include <SofaBaseTopology/RegularGridTopology.h>
#include <SofaSimpleFem/TetrahedronDiffusionFEMForceField.h>
#include <SofaBaseMechanics/DiagonalMass.h>

#include <sofa/defaulttype/Vec.h>

#include <iostream>
#include <fstream>

#if defined(WIN32) && _MSC_VER<=1700  // before or equal to visual studio 2012
   #include <boost/math/special_functions/erf.hpp>
   #define ERFC(x) boost::math::erfc(x)
#else
   #define ERFC(x) std::erfc(x)
#endif

namespace sofa {

/** @brief Comparison of the result of simulation with theoretical values in Diffusion case
 *
 * @author Talbot Hugo, 2016
 *
 */
template <typename _ForceFieldType>
struct TetrahedronDiffusionFEMForceField_test : public Sofa_test<typename _ForceFieldType::DataTypes::Real>
{
    typedef _ForceFieldType ForceField;
    typedef typename ForceField::DataTypes DataTypes;
    typedef typename DataTypes::VecCoord VecCoord;
    typedef typename DataTypes::VecDeriv VecDeriv;
    typedef typename DataTypes::Coord Coord;
    typedef typename DataTypes::Deriv Deriv;
    typedef typename Coord::value_type Real;

    typedef component::container::MechanicalObject<DataTypes> DOF;
    typedef typename component::topology::RegularGridTopology RegularGridTopology;
    typedef typename component::forcefield::TetrahedronDiffusionFEMForceField<DataTypes> TetrahedronDiffusionFEMForceField;
    typedef typename component::mass::DiagonalMass<DataTypes, Real> DiagonalMass;

    /// @name Scene elements
    /// {
    typename DOF::SPtr dof;
    simulation::Node::SPtr root, tetraNode, temperatureNode;
    std::string sceneFilename;
    Coord theorX;
    /// }

    /// @name Precision and control parameters
    /// {
    Real diffusionCoefficient;
    SReal massDensity;
    defaulttype::Vec<3,SReal> beamDimension;
    defaulttype::Vec3i beamResolution;
    SReal errorMax;
    SReal timeEvaluation;
    SReal timeStep;
    unsigned long idMiddlePoint;
    bool debug;
    /// }

    /// @name Tested API
    /// {
    static const unsigned char TEST_ALL = UCHAR_MAX; ///< testing everything
    unsigned char flags; ///< testing options. (all by default). To be used with precaution.
    /// }

    TetrahedronDiffusionFEMForceField_test()
        : sceneFilename(std::string(SOFASIMPLEFEM_TEST_SCENES_DIR) + "/" + "TetrahedronDiffusionFEMForceField.scn")
        , diffusionCoefficient( 1.0 )
        , massDensity( 1.0 )
        , beamDimension(1.0, 0.5, 0.5)
        , beamResolution( defaulttype::Vec3i(21,11,11) )
        , errorMax( 1e-3 )
        , timeEvaluation( 0.05 )
    {
        timeEvaluation = 0.05;
        timeStep = 0.0001;
        idMiddlePoint = 1270;

        simulation::Simulation* simu;
        sofa::simulation::setSimulation(simu = new sofa::simulation::graph::DAGSimulation());

        /// Load the scene
        root = simu->createNewGraph("root");
        root = sofa::simulation::getSimulation()->load(sceneFilename.c_str());

    }

    void init_scene()
    {
        tetraNode = root->getChild("Tetra");
        temperatureNode = tetraNode->getChild("Temperature");

        if(!tetraNode || !temperatureNode)
        {
          std::cerr << "Node not found in TetrahedronDiffusionFEMForceField_test.scn, test will break" << std::endl;
          return;
        }

        // Beam setup
        typename RegularGridTopology::SPtr grid = root->get<RegularGridTopology>(root->SearchDown);
        if(grid)
        {
            grid->setNumVertices(beamResolution);
            grid->setPos(0.0, beamDimension[0], 0.0, beamDimension[1], 0.0, beamDimension[2]);
        }

        // Init simulation
        sofa::simulation::getSimulation()->init(this->root.get());

        // Mass parameters
        typename DiagonalMass::SPtr mass = temperatureNode->get<DiagonalMass>(temperatureNode->SearchDown);
        if(mass)
        {
            mass->setMassDensity(massDensity);
        }

        ///  Get mechanical object of tracked points
        dof = temperatureNode->get<DOF>(temperatureNode->SearchDown);

        // Diffusion parameters
        typename TetrahedronDiffusionFEMForceField::SPtr diffusionFF = temperatureNode->get<TetrahedronDiffusionFEMForceField>(temperatureNode->SearchDown);
        if(diffusionFF)
        {
            diffusionFF->setDiffusionCoefficient(diffusionCoefficient);
        }
    }

    void animate_scene()
    {
        //Animate simulation
        unsigned int nbSteps = timeEvaluation/timeStep;
        unsigned int stepId;
        for (stepId = 0; stepId < nbSteps; ++stepId)
            sofa::simulation::getSimulation()->animate(root.get(),timeStep);
    }

    void compute_theory()
    {
        // For a Dirac heat of T=1 and a fixed BC T=0, the temperature at time = TTTT in the middle of the beam is:
        SReal temp = 1.0 / (4.0 * sqrt(timeEvaluation));
        theorX[0] = 1.0 * ERFC( temp );
    }

    void run_test_theoretical_diffusion()
    {
        typename DOF::ReadVecCoord sofaX= this->dof->readPositions();
        SReal value_SOFA = 0.0;
        if(sofaX.size() > idMiddlePoint)
            value_SOFA = sofaX[idMiddlePoint][0];

        if( fabs(value_SOFA - theorX[0]) > errorMax)
            ADD_FAILURE()<<"Wrong computation for diffusion effect" << std::endl;
    }
};

// ========= Define the list of types to instanciate.
//using testing::Types;
typedef testing::Types<component::forcefield::TetrahedronDiffusionFEMForceField<defaulttype::Vec1Types> > TestTypes; // the types to instanciate.

// ========= Tests to run for each instanciated type
TYPED_TEST_CASE(TetrahedronDiffusionFEMForceField_test, TestTypes);

// test case
TYPED_TEST( TetrahedronDiffusionFEMForceField_test , extension )
{
    this->debug = false;

    // run test
    this->init_scene();
    this->animate_scene();
    this->compute_theory();

    this->run_test_theoretical_diffusion();
}

} // namespace sofa

#undef ERFC

#endif /* SOFA_STANDARDTEST_TETRAHEDRONDIFFUSIONFEMFORCEFIELD_TEST_H */

ext: cpp

max_stars_repo_name: sofa-framework/issofa

max_issues_repo_name: sofa-framework/issofa

max_forks_repo_name: sofa-framework/issofa


text:
/*
Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/)
Copyright 2017-2019 University of Huddersfield.
Licensed under the BSD-3 License.
See license.md file in the project root for full license information.
This project has received funding from the European Research Council (ERC)
under the European Union’s Horizon 2020 research and innovation programme
(grant agreement No 725899).
*/

#pragma once

#include "FluidEigenMappings.hpp"
#include "../../data/FluidIndex.hpp"
#include "../../data/TensorTypes.hpp"
#include <Eigen/Core>
#include <cassert>
#include <cmath>

namespace fluid {
namespace algorithm {

class WeightedStats
{
public:
  Eigen::ArrayXd process(Eigen::Ref<Eigen::ArrayXd> input,
                         Eigen::Ref<Eigen::ArrayXd> weights, double low,
                         double mid, double high)
  {
    using namespace Eigen;
    using namespace std;
    index   length = input.size();
    ArrayXd out = ArrayXd::Zero(7);
    double  mean = (weights * input).sum();
    double  stdev = sqrt((weights * (input - mean).square()).sum());
    double  skewness =
        (weights * ((input - mean) / (stdev == 0 ? 1 : stdev)).cube()).sum();
    double kurtosis =
        (weights * ((input - mean) / (stdev == 0 ? 1 : stdev)).pow(4)).sum();
    ArrayXd sorted = input;
    ArrayXidx perm = ArrayXidx::LinSpaced(length, 0, length - 1);
    std::sort(perm.data(), perm.data() + length,
              [&](index i, index j) { return input(i) < input(j); });
    index  level = 0;
    double lowVal{input(perm(0))};
    double midVal{input(perm(lrint(length - 1) / 2))};
    double hiVal{input(perm(lrint(length - 1)))};
    double acc = weights(perm(0)), prevAcc = 0;
    for (index i = 1; i < length; i++)
    {
      acc += weights(perm(i));
      if (level == 0 && acc >= low)
      {
        lowVal = abs(prevAcc - low) <= abs(acc - low) ? input(perm(i - 1))
                                                      : input(perm(i));
        level = 1;
      }
      if (level == 1 && acc >= mid)
      {
        midVal = abs(prevAcc - mid) < abs(acc - mid) ? input(perm(i - 1))
                                                     : input(perm(i));
        level = 2;
      }
      if (level == 2 && acc >= high)
      {
        hiVal = abs(prevAcc - high) < abs(acc - high) ? input(perm(i - 1))
                                                      : input(perm(i));
        break;
      }
      prevAcc = acc;
    }
    out << mean, stdev, skewness, kurtosis, lowVal, midVal, hiVal;
    return out;
  }
};
} // namespace algorithm
} // namespace fluid

ext: hpp


text:
//==============================================================================
//         Copyright 2003 - 2011 LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011 LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef NT2_ELLIPTIC_FUNCTIONS_SCALAR_ELLIK_HPP_INCLUDED
#define NT2_ELLIPTIC_FUNCTIONS_SCALAR_ELLIK_HPP_INCLUDED
#include <nt2/elliptic/functions/ellik.hpp>
#include <boost/math/special_functions/ellint_1.hpp>
#include <nt2/include/constants/eps.hpp>
#include <nt2/include/constants/nan.hpp>
#include <nt2/include/constants/one.hpp>
#include <nt2/include/constants/pi.hpp>
#include <nt2/include/constants/pio_2.hpp>
#include <nt2/include/functions/scalar/abs.hpp>
#include <nt2/include/functions/scalar/atan.hpp>
#include <nt2/include/functions/scalar/average.hpp>
#include <nt2/include/functions/scalar/ellint_1.hpp>
#include <nt2/include/functions/scalar/is_eqz.hpp>
#include <nt2/include/functions/scalar/is_ltz.hpp>
#include <nt2/include/functions/scalar/log.hpp>
#include <nt2/include/functions/scalar/oneminus.hpp>
#include <nt2/include/functions/scalar/oneplus.hpp>
#include <nt2/include/functions/scalar/sqrt.hpp>
#include <nt2/include/functions/scalar/tan.hpp>
#include <nt2/include/functions/scalar/toint.hpp>
#include <nt2/sdk/error/policies.hpp>

namespace nt2 { namespace ext
{
  BOOST_DISPATCH_IMPLEMENT  ( ellik_, tag::cpu_
                            , (A0)
                            , (scalar_< double_<A0> >)
                              (scalar_< double_<A0> >)
                            )
  {
    typedef A0 result_type;
    NT2_FUNCTOR_CALL_REPEAT(2)
    {
      if (a1>nt2::One<A0>()||(nt2::is_ltz(a1))) return nt2::Nan<A0>();
      if (nt2::is_eqz(a1))  return A0(a0);
      return boost::math::ellint_1(nt2::sqrt(a1), a0, nt2_policy());
    }
  };

  BOOST_DISPATCH_IMPLEMENT  ( ellik_, tag::cpu_
                            , (A0)
                            , (scalar_< single_<A0> >)(scalar_< single_<A0> >)
                            )
  {
    typedef A0 result_type;
    NT2_FUNCTOR_CALL_REPEAT(2)
    {
      if (a1>nt2::One<A0>()||(nt2::is_ltz(a1))) return nt2::Nan<A0>();
      if (nt2::is_eqz(a1))  return a0;
      A0 phi = nt2::abs(a0);
      A0 m = a1;
      A0 a = nt2::One<A0>();
      A0 b = nt2::oneminus(m);
      if( nt2::is_eqz(b) )   return nt2::log(nt2::tan(nt2::average(Pio_2<A0>(),phi)));
      b = nt2::sqrt(b);
      A0 c = nt2::sqrt(m);
      A0 d = nt2::One<A0>();
      A0 t = nt2::tan(phi);
      int mod = nt2::toint((phi+nt2::Pio_2<A0>())/nt2::Pi<A0>());
      while( nt2::abs(c) > nt2::abs(a)*nt2::Eps<A0>() )
      {
        A0 temp = b/a;
        phi += nt2::atan(t*temp) + mod*nt2::Pi<A0>();
        mod = nt2::toint((phi+nt2::Pio_2<A0>())/nt2::Pi<A0>());
        t = t*nt2::oneplus(temp)/( nt2::oneminus(temp*t*t));
        c = nt2::average(a,-b);
        temp = nt2::sqrt(a*b);
        a = nt2::average(a,b);
        b = temp;
        d += d;
      }
      A0 temp = (nt2::atan(t) + mod * nt2::Pi<A0>())/(d * a);
      if( nt2::is_ltz(a0) )  temp = -temp;
      return temp;
    }
  };
} }

#endif


text:
/**********************************************************************
*  Copyright (c) 2008-2013, Alliance for Sustainable Energy.
*  All rights reserved.
*
*  This library is free software; you can redistribute it and/or
*  modify it under the terms of the GNU Lesser General Public
*  License as published by the Free Software Foundation; either
*  version 2.1 of the License, or (at your option) any later version.
*
*  This library is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
*  Lesser General Public License for more details.
*
*  You should have received a copy of the GNU Lesser General Public
*  License along with this library; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
**********************************************************************/

#include <generateiddfactory/GenerateIddFactory.hpp>

#include <utilities/core/Path.hpp>
#include <utilities/core/GeneratorApplicationPathHelpers.hpp>

#include <boost/program_options.hpp>
#include <boost/foreach.hpp>
#include <boost/filesystem/fstream.hpp>

#include <iostream>
#include <exception>

int main(int argc, char *argv[])
{
  try {
    std::stringstream ss;

    // set up program options
    std::string outdir;
    boost::program_options::options_description opts("Options",100);
    opts.add_options()
      ("help,h","prints help message")
      ("outdir,o",
       boost::program_options::value<std::string>(&outdir)->default_value(std::string(".")),
       "specify the output directory")
      ("idd,i",
       boost::program_options::value< std::vector<std::string> >(),
       "name,path of an IDD file; ex. EnergyPlus,C:\\EnergyPlus\\Energy+.idd; if no name provided, will use file name")
    ;
    boost::program_options::positional_options_description posOpts;
    posOpts.add("idd",-1);

    // parse command line
    boost::program_options::variables_map vm;
    boost::program_options::store(boost::program_options::command_line_parser(argc,argv).
        options(opts).positional(posOpts).run(),vm);
    boost::program_options::notify(vm);

    // provide help message
    if (vm.count("help")) {
      std::cout << opts;
      std::cout << "In addition to an outdir, at least one name,path IDD pair is required." << std::endl;
      return 0;
    }

    // get valid output directory, if possible (otherwise throws)
    openstudio::path outPath = openstudio::getApplicationOutputDirectory(outdir);

    // get the idd file information, if possible (otherwise throws)
    openstudio::IddFileFactoryDataVector iddFiles =
        openstudio::constructIddFileObjects(vm["idd"].as< std::vector<std::string> >());

    // data common to the start of each file
    std::stringstream fileHeader;
    fileHeader
      << "/**********************************************************************" << std::endl
      << "*  Copyright (c) 2008-2013, Alliance for Sustainable Energy." << std::endl
      << "*  All rights reserved." << std::endl
      << "*" << std::endl
      << "*  This library is free software; you can redistribute it and/or" << std::endl
      << "*  modify it under the terms of the GNU Lesser General Public" << std::endl
      << "*  License as published by the Free Software Foundation; either" << std::endl
      << "*  version 2.1 of the License, or (at your option) any later version." << std::endl
      << "*" << std::endl
      << "*  This library is distributed in the hope that it will be useful," << std::endl
      << "*  but WITHOUT ANY WARRANTY; without even the implied warranty of" << std::endl
      << "*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU" << std::endl
      << "*  Lesser General Public License for more details." << std::endl
      << "*" << std::endl
      << "*  You should have received a copy of the GNU Lesser General Public" << std::endl
      << "*  License along with this library; if not, write to the Free Software" << std::endl
      << "*  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA" << std::endl
      << "**********************************************************************/" << std::endl;

    // create and initialize output files
    openstudio::GenerateIddFactoryOutFiles outFiles(outPath,fileHeader.str(),iddFiles);
    openstudio::initializeOutFiles(outFiles,iddFiles);

    // parse input files and write pieces of output files that require object text
    int i(0);
    BOOST_FOREACH(openstudio::IddFileFactoryData& idd,iddFiles) {
      idd.parseFile(outPath,fileHeader.str(),outFiles,i);
      ++i;
    }

    // complete writing and close output files
    openstudio::completeOutFiles(iddFiles,outFiles);

  }
  catch (std::runtime_error& e) {
    std::cout << e.what() << std::endl;
    return 1;
  }
  return 0;
}

max_stars_repo_name: bobzabcik/OpenStudio

max_issues_repo_name: bobzabcik/OpenStudio

max_forks_repo_name: bobzabcik/OpenStudio


text:
#include "optim.h"
#include <limits>
// #include <cstddef>
#include <boost/range/combine.hpp>
#include <boost/range/adaptor/reversed.hpp>

//////////////////////////////////////////////////////////////////
//////////////////////// COO Matrix Struct ///////////////////////
//////////////////////////////////////////////////////////////////

matCOO::matCOO(Eigen::SparseMatrix<double> const & m){
    nr = m.rows();
    nc = m.cols();
    for (int k=0; k < m.outerSize(); ++k){
        for (Eigen::SparseMatrix<double>::InnerIterator it(m,k); it; ++it)
        {
            row.push_back(it.row());
            col.push_back(it.col());
            val.push_back(it.value());
        }
    }
};

Eigen::SparseMatrix<double> matCOO::toSparseEigen(){
  std::vector<T> tripletList;
  tripletList.reserve(row.size());
  int i,j;
  double v;
  for(auto tup : boost::combine(row,col,val)){
    boost::tie(i,j,v) = tup;
    tripletList.push_back(T(i,j,v));
  }
  Eigen::SparseMatrix<double> mat(nr,nc);
  mat.setFromTriplets(tripletList.begin(), tripletList.end());
  return mat;
};

Eigen::SparseMatrix<double> matCOO::Extract(std::vector<bool> Mask ,std::vector<int> const & NewIndice, int size){
    std::vector<T> tripletList;
    tripletList.reserve(row.size());
    int i,j;
    double v;
    for(auto tup : boost::combine(row,col,val)){
        boost::tie(i,j,v) = tup;
        if (Mask[i] && Mask[j]){
            tripletList.push_back(T(NewIndice[i],NewIndice[j],v));
        }

    }
    Eigen::SparseMatrix<double> mat(size,size);
    mat.setFromTriplets(tripletList.begin(), tripletList.end());
    return mat;
};

//////////////////////////////////////////////////////////////////
/////////////////////////// BFGS CLASS ///////////////////////////
//////////////////////////////////////////////////////////////////

BFGS::BFGS() : memorySize(15), problemSize(1000){
}
BFGS::BFGS(uint memSize,uint pbSize) : memorySize(memSize), problemSize(pbSize){
    this->memoryX    = std::list<Eigen::VectorXd>();
    this->memoryGrad = std::list<Eigen::VectorXd>();
}
void BFGS::addDirection(double step, Eigen::VectorXd direction, Eigen::VectorXd gradNew, Eigen::VectorXd gradOld){
    if(this->memoryX.size()==memorySize)    this->memoryX.pop_back();
    if(this->memoryGrad.size()==memorySize) this->memoryGrad.pop_back();
    this->memoryX.push_front(step*direction);
    this->memoryGrad.push_front(gradOld - gradNew);
}

Eigen::VectorXd BFGS::findDirection(Eigen::VectorXd gradf){
    if(this->memoryX.size() == 0){
        return gradf;
    }
    else{
        Eigen::VectorXd r = -1.*gradf;
        std::vector<double> rhoVect = std::vector<double>(this->memoryX.size());
        Eigen::VectorXd x,grad;
        int compt = 0;
        double dotValue;
        for(auto tup : boost::combine(this->memoryX,this->memoryGrad)){
            boost::tie(x, grad) = tup;
            dotValue = grad.dot(x);
            rhoVect[compt] = std::abs(dotValue) < 1e-42 ? 0 : 1./dotValue;
            compt++;
        }
        double gamma = this->memoryX.front().dot(this->memoryGrad.front())/(this->memoryGrad.front().dot(this->memoryGrad.front()));
        std::vector<double> alpha = std::vector<double>(this->memoryGrad.size());
        compt = 0;
        for(auto tup : boost::combine(this->memoryX,this->memoryGrad)){
            boost::tie(x, grad) = tup;
            alpha[compt] = rhoVect[compt]*x.dot(r);
            r -= alpha[compt]*grad;
            compt++;
        }
        r *= gamma;
        double beta;
        compt = this->memoryX.size()-1;
        for(auto tup : boost::adaptors::reverse(boost::combine(this->memoryX,this->memoryGrad))){
            boost::tie(x, grad) = tup;
            beta = rhoVect[compt]*grad.dot(r);
            r += (alpha[compt] - beta)*x;
            compt--;
        }
        return -r;

    }
}

//////////////////////////////////////////////////////////////////
////////////////// gradientDescent CLASS /////////////////////////
//////////////////////////////////////////////////////////////////
gradientDescent::gradientDescent() : stepSize(1e-2), gradNormThres(1e-5), iterMax(2000) {
}
gradientDescent::gradientDescent(double step, double gradN, int nbIter) : stepSize(step),
                                gradNormThres(gradN), iterMax((uint) nbIter){
}

//////////////////////////////////////////////////////////////////
///////////////// gradientStrongWolf CLASS ///////////////////////
//////////////////////////////////////////////////////////////////
gradientStrongWolf::gradientStrongWolf() : stepSize(1e-2), gradNormThres(1e-5),
                                           alpha1(0.001), alpha2(0.9), maxStepReduction(12), iterMax(2000) {
}
gradientStrongWolf::gradientStrongWolf(double step, double gradN, double wOne, double wTwo,int innerLoop, int nbIter) :
                                stepSize(step), gradNormThres(gradN), alpha1(wOne), alpha2(wTwo),
                                maxStepReduction((uint) innerLoop), iterMax((uint) nbIter){
}

//////////////////////////////////////////////////////////////////
////////////////// barzilaiBorwein CLASS /////////////////////////
//////////////////////////////////////////////////////////////////
barzilaiBorwein::barzilaiBorwein() : gradNormThres(1e-5), iterMax(2000){
}
barzilaiBorwein::barzilaiBorwein(double gradN,int nbIter) : gradNormThres(gradN), iterMax((uint) nbIter) {
}

//////////////////////////////////////////////////////////////////
/////////////////// barzilaiBorweinLS CLASS //////////////////////
//////////////////////////////////////////////////////////////////
barzilaiBorweinLS::barzilaiBorweinLS() : gradNormThres(1e-5), lineSearchFactor(.70),gammaWolfe(1e-4), iterMax(2000),memorySize(10){
}
barzilaiBorweinLS::barzilaiBorweinLS(double gradN,double lineSearchFactor,double gammaW,int nbIter,int memSize) :
                 gradNormThres(gradN), lineSearchFactor(.70),gammaWolfe(gammaW),iterMax((uint) nbIter),memorySize((uint) memSize) {
}
void barzilaiBorweinLS::addCost(double cost){
    if(costFunctionList.size() > memorySize){
        costFunctionList.pop_back();
    }
    costFunctionList.push_front(cost);
}
double barzilaiBorweinLS::minCost(){
    if(costFunctionList.size() < 2){
        return std::numeric_limits<double>::min();
    }
    else{
        double minValue = -2.0*costFunctionList.front()-1.;
        for( const auto & elm : costFunctionList){
            if( elm < minValue ){
                minValue = elm;
            }
        }
        return minValue;
    }
}

//////////////////////////////////////////////////////////////////
/////////////////////// heavyBall CLASS //////////////////////////
//////////////////////////////////////////////////////////////////
heavyBall::heavyBall() : Lips(1000.), gradNormThres(1e-5), iterMax(2000){
}
heavyBall::heavyBall(double LispchitzConstant,double gradN,int nbIter) : Lips(LispchitzConstant),gradNormThres(gradN), iterMax((uint) nbIter) {
}

ext: cxx


text:
//
// Copyright (c) 2019 Vinnie Falco (vinnie.falco@gmail.com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Official repository: https://github.com/cppalliance/json
//

#include <boost/json/value.hpp>

#include <boost/json/monotonic_resource.hpp>
#include <boost/json/static_resource.hpp>

#include "test_suite.hpp"

BOOST_JSON_NS_BEGIN

class natvis_test
{
public:
    static
    value
    make_key(string_view s)
    {
        value jv;
        char* dest = nullptr;
        detail::value_access::construct_value(
            &jv, &dest, s.size(), storage_ptr());
        std::memcpy(dest, s.data(), s.size());
        return jv;
    }

    void
    run()
    {
        // These variables may be inspected in the
        // Visual Studio IDE to verify that the .natvis
        // file is working correctly

        value jv0 = { 1, 2, 3 };
        value jv1 = { {"a",1}, {"b",2}, {"c",3} };
        value jv2 = "dynamically-allocated-string";
        value jv3 = make_key("key");
        value jv4 = "string";
        value jv5 = 3.141;
        value jv6 = -1;
        value jv7 = 42u;
        value jv8 = true;
        value jv9 = nullptr;

        array&  x0 = jv0.get_array();  (void)x0;
        object& x1 = jv1.get_object(); (void)x1;
        string& x2 = jv2.get_string(); (void)x2;
        string& x3 = jv3.get_string(); (void)x3;
        string& x4 = jv4.get_string(); (void)x4;

        array y0;
        y0.emplace_back(jv0);
        y0.emplace_back(jv1);
        y0.emplace_back(jv2);
        y0.emplace_back(jv4);
        y0.emplace_back(jv5);
        y0.emplace_back(jv6);
        y0.emplace_back(jv7);
        y0.emplace_back(jv8);
        y0.emplace_back(jv9);

        object y1;
        y1.emplace("a", jv0);
        y1.emplace("b", jv1);
        y1.emplace("c", jv2);
        y1.emplace("d", jv4);
        y1.emplace("f", jv5);
        y1.emplace("g", jv6);
        y1.emplace("h", jv7);
        y1.emplace("i", jv8);
        y1.emplace("j", jv9);

        key_value_pair& z0 = y1.begin()[0]; (void)z0;
        key_value_pair& z1 = y1.begin()[1]; (void)z1;
        key_value_pair& z2 = y1.begin()[2]; (void)z2;
        key_value_pair& z3 = y1.begin()[3]; (void)z3;
        key_value_pair& z4 = y1.begin()[4]; (void)z4;
        key_value_pair& z5 = y1.begin()[5]; (void)z5;
        key_value_pair& z6 = y1.begin()[6]; (void)z6;
        key_value_pair& z7 = y1.begin()[7]; (void)z7;
        key_value_pair& z8 = y1.begin()[8]; (void)z8;

        {
            auto sp = y1.storage();
        }
        {
            unsigned char buf[1024];
            storage_ptr sp;
            sp = make_counted_resource<
                monotonic_resource>();
            sp = make_counted_resource<
                static_resource>(buf);
            sp = {};
        }
        {
            unsigned char buf[1024];
            monotonic_resource mr1;
            static_resource mr2(buf);
            storage_ptr sp;

            sp = &mr1;
            sp = &mr2;
            sp = {};
        }
    }
};

TEST_SUITE(natvis_test, "json.natvis");

BOOST_JSON_NS_END

max_stars_repo_name: djarek/json

max_issues_repo_name: djarek/json

max_forks_repo_name: djarek/json


text:
#pragma once

#include <ctime>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>
#include <limits>

namespace SparCraft
{
	class RandomInt;
}

class SparCraft::RandomInt
{
	boost::random::mt19937 gen;
	boost::random::uniform_int_distribution<> dist;

public:

	RandomInt(int min, int max, int seed)
		: dist(min, max)
	{
		gen.seed(seed);
	}

	int nextInt()
	{
		return dist(gen);
	}

	void seed(int seed)
	{
		gen.seed(seed);
	}

	int reset(int min, int max, int seed)
	{
		dist = boost::random::uniform_int_distribution<>(min, max);
		gen.seed(seed);
	}
};

max_stars_repo_name: abeshry/StarcraftAI

max_issues_repo_name: abeshry/StarcraftAI

max_forks_repo_name: abeshry/StarcraftAI