text:
// Copyright (c) 2017 The Denarius developers
// Copyright (c) 2009-2012 The Darkcoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "fortuna.h"
#include "main.h"
#include "init.h"
#include "util.h"
#include "fortunastake.h"
#include "ui_interface.h"

#include <openssl/rand.h>

#include <boost/algorithm/string/replace.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/lexical_cast.hpp>

#include <algorithm>
#include <boost/assign/list_of.hpp>

using namespace std;
using namespace boost;

CCriticalSection cs_fortuna;

/** The main object for accessing fortuna */
CForTunaPool forTunaPool;
/** A helper object for signing messages from fortunastakes */
CForTunaSigner forTunaSigner;
/** The current fortunas in progress on the network */
std::vector<CFortunaQueue> vecFortunaQueue;
/** Keep track of the used fortunastakes */
std::vector<CTxIn> vecFortunastakesUsed;
// keep track of the scanning errors I've seen
map<uint256, CFortunaBroadcastTx> mapFortunaBroadcastTxes;
//
CActiveFortunastake activeFortunastake;
// count peers we've requested the list from
int RequestedFortunaStakeList = 0;

//MIN_MN_PROTO_VERSION
int MIN_MN_PROTO_VERSION = 33900; // D v3.3.9.2 - Proto - 33900

/* *** BEGIN FORTUNA MAGIC  **********
    Copyright 2014, Darkcoin Developers
        eduffield - evan@darkcoin.io
    Copyright 2018-2019, Denarius Developers
        carsenk - admin@denarius.io
        enkayz - enkayz@denarius.io
*/

int randomizeList (int i) { return std::rand()%i;}

// Recursively determine the rounds of a given input (How deep is the fortuna chain for a given input)
int GetInputFortunaRounds(CTxIn in, int rounds)
{
    if(rounds >= 17) return rounds;

    std::string padding = "";
    padding.insert(0, ((rounds+1)*5)+3, ' ');

    CWalletTx tx;
    if(pwalletMain->GetTransaction(in.prevout.hash,tx))
    {
        // bounds check
        if(in.prevout.n >= tx.vout.size()) return -4;

        if(tx.vout[in.prevout.n].nValue == FORTUNA_FEE) return -3;

        //make sure the final output is non-denominate
        if(rounds == 0 && !pwalletMain->IsDenominatedAmount(tx.vout[in.prevout.n].nValue)) return -2; //NOT DENOM

        bool found = false;
        BOOST_FOREACH(CTxOut out, tx.vout)
        {
            found = pwalletMain->IsDenominatedAmount(out.nValue);
            if(found) break; // no need to loop more
        }
        if(!found) return rounds - 1; //NOT FOUND, "-1" because of the pre-mixing creation of denominated amounts

        // find my vin and look that up
        BOOST_FOREACH(CTxIn in2, tx.vin)
        {
            if(pwalletMain->IsMine(in2))
            {
                //printf("rounds :: %s %s %d NEXT\n", padding.c_str(), in.ToString().c_str(), rounds);
                int n = GetInputFortunaRounds(in2, rounds+1);
                if(n != -3) return n;
            }
        }
    }

    return rounds-1;
}

void CForTunaPool::Reset(){
    cachedLastSuccess = 0;
    vecFortunastakesUsed.clear();
    UnlockCoins();
    SetNull();
}

void CForTunaPool::SetNull(bool clearEverything){
    finalTransaction.vin.clear();
    finalTransaction.vout.clear();

    entries.clear();

    state = POOL_STATUS_ACCEPTING_ENTRIES;

    lastTimeChanged = GetTimeMillis();

    entriesCount = 0;
    lastEntryAccepted = 0;
    countEntriesAccepted = 0;
    lastNewBlock = 0;

    sessionUsers = 0;
    sessionDenom = 0;
    sessionFoundFortunastake = false;
    vecSessionCollateral.clear();
    txCollateral = CTransaction();

    if(clearEverything){
        myEntries.clear();

        if(fFortunaStake){
            sessionID = 1 + (rand() % 999999);
        } else {
            sessionID = 0;
        }
    }

    // -- seed random number generator (used for ordering output lists)
    unsigned int seed = 0;
    GetRandBytes((unsigned char*)&seed, sizeof(seed));
    std::srand(seed);
}

bool CForTunaPool::SetCollateralAddress(std::string strAddress){
    CBitcoinAddress address;
    if (!address.SetString(strAddress))
    {
        printf("CForTunaPool::SetCollateralAddress - Invalid ForTuna collateral address\n");
        return false;
    }
    collateralPubKey= GetScriptForDestination(address.Get());
    return true;
}

//
// Unlock coins after Fortuna fails or succeeds
//
void CForTunaPool::UnlockCoins(){
    BOOST_FOREACH(CTxIn v, lockedCoins)
        pwalletMain->UnlockCoin(v.prevout);

    lockedCoins.clear();
}

//
// Check for various timeouts (queue objects, fortuna, etc)
//
void CForTunaPool::CheckTimeout(){
    if(!fFortunaStake) return;

    // catching hanging sessions
    if(!fFortunaStake) {
        if(state == POOL_STATUS_TRANSMISSION) {
            if(fDebug) printf("CForTunaPool::CheckTimeout() -- Session complete -- Running Check()\n");
            // Check();
        }
    }

    // check fortuna queue objects for timeouts
    int c = 0;
    vector<CFortunaQueue>::iterator it;
    for(it=vecFortunaQueue.begin();it<vecFortunaQueue.end();it++){
        if((*it).IsExpired()){
            if(fDebug) printf("CForTunaPool::CheckTimeout() : Removing expired queue entry - %d\n", c);
            vecFortunaQueue.erase(it);
            break;
        }
        c++;
    }

    /* Check to see if we're ready for submissions from clients */
    if(state == POOL_STATUS_QUEUE && sessionUsers == GetMaxPoolTransactions()) {
        CFortunaQueue dsq;
        dsq.nDenom = sessionDenom;
        dsq.vin = activeFortunastake.vin;
        dsq.time = GetTime();
        dsq.ready = true;
        dsq.Sign();
        dsq.Relay();

        UpdateState(POOL_STATUS_ACCEPTING_ENTRIES);
    }

    int addLagTime = 0;
    if(!fFortunaStake) addLagTime = 10000; //if we're the client, give the server a few extra seconds before resetting.

    if(state == POOL_STATUS_ACCEPTING_ENTRIES || state == POOL_STATUS_QUEUE){
        c = 0;

        // if it's a fortunastake, the entries are stored in "entries", otherwise they're stored in myEntries
        std::vector<CForTunaEntry> *vec = &myEntries;
        if(fFortunaStake) vec = &entries;

        // check for a timeout and reset if needed
        vector<CForTunaEntry>::iterator it2;
        for(it2=vec->begin();it2<vec->end();it2++){
            if((*it2).IsExpired()){
                if(fDebug) printf("CForTunaPool::CheckTimeout() : Removing expired entry - %d\n", c);
                vec->erase(it2);
                if(entries.size() == 0 && myEntries.size() == 0){
                    SetNull(true);
                    UnlockCoins();
                }
                if(fFortunaStake){
                    RelayForTunaStatus(forTunaPool.sessionID, forTunaPool.GetState(), forTunaPool.GetEntriesCount(), FORTUNASTAKE_RESET);
                }
                break;
            }
            c++;
        }

        if(GetTimeMillis()-lastTimeChanged >= (FORTUNA_QUEUE_TIMEOUT*1000)+addLagTime){
            lastTimeChanged = GetTimeMillis();

            // reset session information for the queue query stage (before entering a fortunastake, clients will send a queue request to make sure they're compatible denomination wise)
            sessionUsers = 0;
            sessionDenom = 0;
            sessionFoundFortunastake = false;
            vecSessionCollateral.clear();

            UpdateState(POOL_STATUS_ACCEPTING_ENTRIES);
        }
    } else if(GetTimeMillis()-lastTimeChanged >= (FORTUNA_QUEUE_TIMEOUT*1000)+addLagTime){
        if(fDebug) printf("CForTunaPool::CheckTimeout() -- Session timed out (30s) -- resetting\n");
        SetNull();
        UnlockCoins();

        UpdateState(POOL_STATUS_ERROR);
        lastMessage = _("Session timed out (30 seconds), please resubmit.");
    }

    if(state == POOL_STATUS_SIGNING && GetTimeMillis()-lastTimeChanged >= (FORTUNA_SIGNING_TIMEOUT*1000)+addLagTime ) {
        if(fDebug) printf("CForTunaPool::CheckTimeout() -- Session timed out -- restting\n");
        SetNull();
        UnlockCoins();
        //add my transactions to the new session

        UpdateState(POOL_STATUS_ERROR);
        lastMessage = _("Signing timed out, please resubmit.");
    }
}

// check to see if the signature is valid
bool CForTunaPool::SignatureValid(const CScript& newSig, const CTxIn& newVin){
    CTransaction txNew;
    txNew.vin.clear();
    txNew.vout.clear();

    int found = -1;
    CScript sigPubKey = CScript();
    unsigned int i = 0;

    BOOST_FOREACH(CForTunaEntry e, entries) {
        BOOST_FOREACH(const CTxOut out, e.vout)
            txNew.vout.push_back(out);

        BOOST_FOREACH(const CForTunaEntryVin s, e.sev){
            txNew.vin.push_back(s.vin);

            if(s.vin == newVin){
                found = i;
                sigPubKey = s.vin.prevPubKey;
            }
            i++;
        }
    }

    if(found >= 0){ //might have to do this one input at a time?
        int n = found;
        txNew.vin[n].scriptSig = newSig;
        if(fDebug) printf("CForTunaPool::SignatureValid() - Sign with sig %s\n", newSig.ToString().substr(0,24).c_str());
        if (!VerifyScript(txNew.vin[n].scriptSig, sigPubKey, txNew, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, 0)){
            if(fDebug) printf("CForTunaPool::SignatureValid() - Signing - Error signing input %u\n", n);
            return false;
        }
    }

    if(fDebug) printf("CForTunaPool::SignatureValid() - Signing - Succesfully signed input\n");
    return true;
}

// check to make sure the collateral provided by the client is valid
bool CForTunaPool::IsCollateralValid(const CTransaction& txCollateral){
    if(txCollateral.vout.size() < 1) return false;
    if(txCollateral.nLockTime != 0) return false;

    int64_t nValueIn = 0;
    int64_t nValueOut = 0;
    bool missingTx = false;

    BOOST_FOREACH(const CTxOut o, txCollateral.vout){
        nValueOut += o.nValue;

        if(!o.scriptPubKey.IsNormalPaymentScript()){
            printf("CForTunaPool::IsCollateralValid - Invalid Script %s\n", txCollateral.ToString().c_str());
            return false;
        }
    }

    BOOST_FOREACH(const CTxIn i, txCollateral.vin){
        CTransaction tx2;
        uint256 hash;
        //if(GetTransaction(i.prevout.hash, tx2, hash, true)){
    if(GetTransaction(i.prevout.hash, tx2, hash)){
            if(tx2.vout.size() > i.prevout.n) {
                nValueIn += tx2.vout[i.prevout.n].nValue;
            }
        } else{
            missingTx = true;
        }
    }

    if(missingTx){
        if(fDebug) printf("CForTunaPool::IsCollateralValid - Unknown inputs in collateral transaction - %s\n", txCollateral.ToString().c_str());
        return false;
    }

    //collateral transactions are required to pay out FORTUNA_COLLATERAL as a fee to the miners
    if(nValueIn-nValueOut < FORTUNA_COLLATERAL) {
        if(fDebug) printf("CForTunaPool::IsCollateralValid - did not include enough fees in transaction %lu\n%s\n", nValueOut-nValueIn, txCollateral.ToString().c_str());
        return false;
    }

    if(fDebug) printf("CForTunaPool::IsCollateralValid %s\n", txCollateral.ToString().c_str());

    CValidationState state;
    //if(!AcceptableInputs(mempool, state, txCollateral)){
    bool* pfMissingInputs;
    if(!AcceptableInputs(mempool, txCollateral, false, pfMissingInputs)){
        if(fDebug) printf("CForTunaPool::IsCollateralValid - didn't pass IsAcceptable\n");
        return false;
    }

    return true;
}

//
// Add a clients transaction to the pool
//
bool CForTunaPool::AddEntry(const std::vector<CTxIn>& newInput, const int64_t& nAmount, const CTransaction& txCollateral, const std::vector<CTxOut>& newOutput, std::string& error){
    if (!fFortunaStake) return false;

    BOOST_FOREACH(CTxIn in, newInput) {
        if (in.prevout.IsNull() || nAmount < 0) {
            if(fDebug) printf("CForTunaPool::AddEntry - input not valid!\n");
            error = _("Input is not valid.");
            sessionUsers--;
            return false;
        }
    }

    if (!IsCollateralValid(txCollateral)){
        if(fDebug) printf("CForTunaPool::AddEntry - collateral not valid!\n");
        error = _("Collateral is not valid.");
        sessionUsers--;
        return false;
    }

    if((int)entries.size() >= GetMaxPoolTransactions()){
        if(fDebug) printf("CForTunaPool::AddEntry - entries is full!\n");
        error = _("Entries are full.");
        sessionUsers--;
        return false;
    }

    BOOST_FOREACH(CTxIn in, newInput) {
        if(fDebug) printf("looking for vin -- %s\n", in.ToString().c_str());
        BOOST_FOREACH(const CForTunaEntry v, entries) {
            BOOST_FOREACH(const CForTunaEntryVin s, v.sev){
                if(s.vin == in) {
                    if(fDebug) printf("CForTunaPool::AddEntry - found in vin\n");
                    error = _("Already have that input.");
                    sessionUsers--;
                    return false;
                }
            }
        }
    }

    if(state == POOL_STATUS_ACCEPTING_ENTRIES) {
        CForTunaEntry v;
        v.Add(newInput, nAmount, txCollateral, newOutput);
        entries.push_back(v);

        if(fDebug) printf("CForTunaPool::AddEntry -- adding %s\n", newInput[0].ToString().c_str());
        error = "";

        return true;
    }

    if(fDebug) printf("CForTunaPool::AddEntry - can't accept new entry, wrong state!\n");
    error = _("Wrong state.");
    sessionUsers--;
    return false;
}

bool CForTunaPool::AddScriptSig(const CTxIn newVin){
    if(fDebug) printf("CForTunaPool::AddScriptSig -- new sig  %s\n", newVin.scriptSig.ToString().substr(0,24).c_str());

    BOOST_FOREACH(const CForTunaEntry v, entries) {
        BOOST_FOREACH(const CForTunaEntryVin s, v.sev){
            if(s.vin.scriptSig == newVin.scriptSig) {
                printf("CForTunaPool::AddScriptSig - already exists \n");
                return false;
            }
        }
    }

    if(!SignatureValid(newVin.scriptSig, newVin)){
        if(fDebug) printf("CForTunaPool::AddScriptSig - Invalid Sig\n");
        return false;
    }

    if(fDebug) printf("CForTunaPool::AddScriptSig -- sig %s\n", newVin.ToString().c_str());

    if(state == POOL_STATUS_SIGNING) {
        BOOST_FOREACH(CTxIn& vin, finalTransaction.vin){
            if(newVin.prevout == vin.prevout && vin.nSequence == newVin.nSequence){
                vin.scriptSig = newVin.scriptSig;
                vin.prevPubKey = newVin.prevPubKey;
                if(fDebug) printf("CForTunaPool::AddScriptSig -- adding to finalTransaction  %s\n", newVin.scriptSig.ToString().substr(0,24).c_str());
            }
        }
        for(unsigned int i = 0; i < entries.size(); i++){
            if(entries[i].AddSig(newVin)){
                if(fDebug) printf("CForTunaPool::AddScriptSig -- adding  %s\n", newVin.scriptSig.ToString().substr(0,24).c_str());
                return true;
            }
        }
    }

    printf("CForTunaPool::AddScriptSig -- Couldn't set sig!\n" );
    return false;
}

// check to make sure everything is signed
bool CForTunaPool::SignaturesComplete(){
    BOOST_FOREACH(const CForTunaEntry v, entries) {
        BOOST_FOREACH(const CForTunaEntryVin s, v.sev){
            if(!s.isSigSet) return false;
        }
    }
    return true;
}

// Incoming message from fortunastake updating the progress of fortuna
//    newAccepted:  -1 mean's it'n not a "transaction accepted/not accepted" message, just a standard update
//                  0 means transaction was not accepted
//                  1 means transaction was accepted

bool CForTunaPool::StatusUpdate(int newState, int newEntriesCount, int newAccepted, std::string& error, int newSessionID){
    if(fFortunaStake) return false;
    if(state == POOL_STATUS_ERROR || state == POOL_STATUS_SUCCESS) return false;

    UpdateState(newState);
    entriesCount = newEntriesCount;

    if(error.size() > 0) strAutoDenomResult = _("Fortunastake:") + " " + error;

    if(newAccepted != -1) {
        lastEntryAccepted = newAccepted;
        countEntriesAccepted += newAccepted;
        if(newAccepted == 0){
            UpdateState(POOL_STATUS_ERROR);
            lastMessage = error;
        }

        if(newAccepted == 1) {
            sessionID = newSessionID;
            printf("CForTunaPool::StatusUpdate - set sessionID to %d\n", sessionID);
            sessionFoundFortunastake = true;
        }
    }

    if(newState == POOL_STATUS_ACCEPTING_ENTRIES){
        if(newAccepted == 1){
            printf("CForTunaPool::StatusUpdate - entry accepted! \n");
            sessionFoundFortunastake = true;
            //wait for other users. Fortunastake will report when ready
            UpdateState(POOL_STATUS_QUEUE);
        } else if (newAccepted == 0 && sessionID == 0 && !sessionFoundFortunastake) {
            printf("CForTunaPool::StatusUpdate - entry not accepted by fortunastake \n");
            UnlockCoins();
            UpdateState(POOL_STATUS_ACCEPTING_ENTRIES);
        }
        if(sessionFoundFortunastake) return true;
    }

    return true;
}

//
// After we receive the finalized transaction from the fortunastake, we must
// check it to make sure it's what we want, then sign it if we agree.
// If we refuse to sign, it's possible we'll be charged collateral
//
bool CForTunaPool::SignFinalTransaction(CTransaction& finalTransactionNew, CNode* node){
    if(fDebug) printf("CForTunaPool::AddFinalTransaction - Got Finalized Transaction\n");

    if(!finalTransaction.vin.empty()){
        printf("CForTunaPool::AddFinalTransaction - Rejected Final Transaction!\n");
        return false;
    }

    finalTransaction = finalTransactionNew;
    printf("CForTunaPool::SignFinalTransaction %s\n", finalTransaction.ToString().c_str());

    vector<CTxIn> sigs;

    //make sure my inputs/outputs are present, otherwise refuse to sign
    BOOST_FOREACH(const CForTunaEntry e, myEntries) {
        BOOST_FOREACH(const CForTunaEntryVin s, e.sev) {
            /* Sign my transaction and all outputs */
            int mine = -1;
            CScript prevPubKey = CScript();
            CTxIn vin = CTxIn();

            for(unsigned int i = 0; i < finalTransaction.vin.size(); i++){
                if(finalTransaction.vin[i] == s.vin){
                    mine = i;
                    prevPubKey = s.vin.prevPubKey;
                    vin = s.vin;
                }
            }

            if(mine >= 0){ //might have to do this one input at a time?
                int foundOutputs = 0;
                int64_t nValue1 = 0;
                int64_t nValue2 = 0;

                for(unsigned int i = 0; i < finalTransaction.vout.size(); i++){
                    BOOST_FOREACH(const CTxOut o, e.vout) {
                        if(finalTransaction.vout[i] == o){
                            foundOutputs++;
                            nValue1 += finalTransaction.vout[i].nValue;
                        }
                    }
                }

                BOOST_FOREACH(const CTxOut o, e.vout)
                    nValue2 += o.nValue;

                int targetOuputs = e.vout.size();
                if(foundOutputs < targetOuputs || nValue1 != nValue2) {
                    // in this case, something went wrong and we'll refuse to sign. It's possible we'll be charged collateral. But that's
                    // better then signing if the transaction doesn't look like what we wanted.
                    printf("CForTunaPool::Sign - My entries are not correct! Refusing to sign. %d entries %d target. \n", foundOutputs, targetOuputs);
                    return false;
                }

                if(fDebug) printf("CForTunaPool::Sign - Signing my input %i\n", mine);
                if(!SignSignature(*pwalletMain, prevPubKey, finalTransaction, mine, int(SIGHASH_ALL|SIGHASH_ANYONECANPAY))) { // changes scriptSig
                    if(fDebug) printf("CForTunaPool::Sign - Unable to sign my own transaction! \n");
                    // not sure what to do here, it will timeout...?
                }

                sigs.push_back(finalTransaction.vin[mine]);
                if(fDebug) printf(" -- dss %d %d %s\n", mine, (int)sigs.size(), finalTransaction.vin[mine].scriptSig.ToString().c_str());
            }

        }

        if(fDebug) printf("CForTunaPool::Sign - txNew:\n%s", finalTransaction.ToString().c_str());
    }

    // push all of our signatures to the fortunastake
    if(sigs.size() > 0 && node != NULL)
        node->PushMessage("dss", sigs);

    return true;
}

void CForTunaPool::NewBlock()
{
    if(fDebug) printf("CForTunaPool::NewBlock \n");

    //we we're processing lots of blocks, we'll just leave
    if(GetTime() - lastNewBlock < 10) return;
    lastNewBlock = GetTime();

    forTunaPool.CheckTimeout();

    if(!fFortunaStake){
        //denominate all non-denominated inputs every 50 blocks (25 minutes)
        if(pindexBest->nHeight % 50 == 0)
            UnlockCoins();
        // free up fortunastake connections every 30 blocks unless we are syncing
        if(pindexBest->nHeight % 60 == 0 && !IsInitialBlockDownload())
            ProcessFortunastakeConnections();
    }
}

void CForTunaPool::ClearLastMessage()
{
    lastMessage = "";
}

bool CForTunaPool::IsCompatibleWithSession(int64_t nDenom, CTransaction txCollateral, std::string& strReason)
{
    printf("CForTunaPool::IsCompatibleWithSession - sessionDenom %d sessionUsers %d\n", sessionDenom, sessionUsers);

    if (!unitTest && !IsCollateralValid(txCollateral)){
        if(fDebug) printf("CForTunaPool::IsCompatibleWithSession - collateral not valid!\n");
        strReason = _("Collateral not valid.");
        return false;
    }

    if(sessionUsers < 0) sessionUsers = 0;

    if(sessionUsers == 0) {
        sessionDenom = nDenom;
        sessionUsers++;
        lastTimeChanged = GetTimeMillis();
        entries.clear();

        if(!unitTest){
            //broadcast that I'm accepting entries, only if it's the first entry though
            CFortunaQueue dsq;
            dsq.nDenom = nDenom;
            dsq.vin = activeFortunastake.vin;
            dsq.time = GetTime();
            dsq.Sign();
            dsq.Relay();
        }

        UpdateState(POOL_STATUS_QUEUE);
        vecSessionCollateral.push_back(txCollateral);
        return true;
    }

    if((state != POOL_STATUS_ACCEPTING_ENTRIES && state != POOL_STATUS_QUEUE) || sessionUsers >= GetMaxPoolTransactions()){
        if((state != POOL_STATUS_ACCEPTING_ENTRIES && state != POOL_STATUS_QUEUE)) strReason = _("Incompatible mode.");
        if(sessionUsers >= GetMaxPoolTransactions()) strReason = _("Fortunastake queue is full.");
        printf("CForTunaPool::IsCompatibleWithSession - incompatible mode, return false %d %d\n", state != POOL_STATUS_ACCEPTING_ENTRIES, sessionUsers >= GetMaxPoolTransactions());
        return false;
    }

    if(nDenom != sessionDenom) {
        strReason = _("No matching denominations found for mixing.");
        return false;
    }

    printf("CForTunaPool::IsCompatibleWithSession - compatible\n");

    sessionUsers++;
    lastTimeChanged = GetTimeMillis();
    vecSessionCollateral.push_back(txCollateral);

    return true;
}

//create a nice string to show the denominations
void CForTunaPool::GetDenominationsToString(int nDenom, std::string& strDenom){
    // Function returns as follows:
    //
    // bit 0 - 100D+1 ( bit on if present )
    // bit 1 - 10D+1
    // bit 2 - 1D+1
    // bit 3 - .1D+1
    // bit 3 - non-denom

    strDenom = "";

    if(nDenom & (1 << 0)) {
        if(strDenom.size() > 0) strDenom += "+";
        strDenom += "100";
    }

    if(nDenom & (1 << 1)) {
        if(strDenom.size() > 0) strDenom += "+";
        strDenom += "10";
    }

    if(nDenom & (1 << 2)) {
        if(strDenom.size() > 0) strDenom += "+";
        strDenom += "1";
    }

    if(nDenom & (1 << 3)) {
        if(strDenom.size() > 0) strDenom += "+";
        strDenom += "0.1";
    }
}

// return a bitshifted integer representing the denominations in this list
int CForTunaPool::GetDenominations(const std::vector<CTxOut>& vout){
    std::vector<pair<int64_t, int> > denomUsed;

    // make a list of denominations, with zero uses
    BOOST_FOREACH(int64_t d, forTunaDenominations)
        denomUsed.push_back(make_pair(d, 0));

    // look for denominations and update uses to 1
    BOOST_FOREACH(CTxOut out, vout){
        bool found = false;
        BOOST_FOREACH (PAIRTYPE(int64_t, int)& s, denomUsed){
            if (out.nValue == s.first){
                s.second = 1;
                found = true;
            }
        }
        if(!found) return 0;
    }

    int denom = 0;
    int c = 0;
    // if the denomination is used, shift the bit on.
    // then move to the next
    BOOST_FOREACH (PAIRTYPE(int64_t, int)& s, denomUsed)
        denom |= s.second << c++;

    // Function returns as follows:
    //
    // bit 0 - 100D+1 ( bit on if present )
    // bit 1 - 10D+1
    // bit 2 - 1D+1
    // bit 3 - .1D+1

    return denom;
}

int CForTunaPool::GetDenominationsByAmounts(std::vector<int64_t>& vecAmount){
    CScript e = CScript();
    std::vector<CTxOut> vout1;

    // Make outputs by looping through denominations, from small to large
    BOOST_REVERSE_FOREACH(int64_t v, vecAmount){
        int nOutputs = 0;

        CTxOut o(v, e);
        vout1.push_back(o);
        nOutputs++;
    }

    return GetDenominations(vout1);
}

int CForTunaPool::GetDenominationsByAmount(int64_t nAmount, int nDenomTarget){
    CScript e = CScript();
    int64_t nValueLeft = nAmount;

    std::vector<CTxOut> vout1;

    // Make outputs by looping through denominations, from small to large
    BOOST_REVERSE_FOREACH(int64_t v, forTunaDenominations){
        if(nDenomTarget != 0){
            bool fAccepted = false;
            if((nDenomTarget & (1 << 0)) &&      v == ((100000*COIN)+100000000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 1)) && v == ((10000*COIN) +10000000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 2)) && v == ((1000*COIN)  +1000000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 3)) && v == ((100*COIN)   +100000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 4)) && v == ((10*COIN)    +10000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 5)) && v == ((1*COIN)     +1000)) {fAccepted = true;}
            else if((nDenomTarget & (1 << 6)) && v == ((.1*COIN)    +100)) {fAccepted = true;}
            if(!fAccepted) continue;
        }

        int nOutputs = 0;

        // add each output up to 10 times until it can't be added again
        while(nValueLeft - v >= 0 && nOutputs <= 10) {
            CTxOut o(v, e);
            vout1.push_back(o);
            nValueLeft -= v;
            nOutputs++;
        }
        printf("GetDenominationsByAmount --- %lu nOutputs %d\n", v, nOutputs);
    }

    //add non-denom left overs as change
    if(nValueLeft > 0){
        CTxOut o(nValueLeft, e);
        vout1.push_back(o);
    }

    return GetDenominations(vout1);
}

bool CForTunaSigner::IsVinAssociatedWithPubkey(CTxIn& vin, CPubKey& pubkey){
	bool fIsInitialDownload = IsInitialBlockDownload();
    if(fIsInitialDownload) return nullptr; // Needs to return a value, so returns null pointer v3.3.9.1 - macOS

    CScript payee2;
    payee2= GetScriptForDestination(pubkey.GetID());

    CTransaction txVin;
    uint256 hash;

    if(GetTransaction(vin.prevout.hash, txVin, hash)){
        CTxOut out = txVin.vout[vin.prevout.n];
		if ((out.nValue == GetMNCollateral()*COIN) && (out.scriptPubKey == payee2))
		{
			return true;
		}
    } else {
		if (fDebug) {
			printf("IsVinAssociatedWithPubKey:: GetTransaction failed for %s\n",vin.prevout.hash.ToString().c_str());
		}
	}

    CTxDestination address1;
    ExtractDestination(payee2, address1);
    CBitcoinAddress address2(address1);
	if (fDebug) {
		printf("IsVinAssociatedWithPubKey:: vin %s is not associated with pubkey %s for address %s\n",
			   vin.ToString().c_str(), pubkey.GetHash().ToString().c_str(), address2.ToString().c_str());
	}
    return false;
}

bool CForTunaSigner::SetKey(std::string strSecret, std::string& errorMessage, CKey& key, CPubKey& pubkey){
    CBitcoinSecret vchSecret;
    bool fGood = vchSecret.SetString(strSecret);

    if (!fGood) {
        errorMessage = _("Invalid private key.");
        return false;
    }

    key = vchSecret.GetKey();
    pubkey = key.GetPubKey();

    return true;
}

bool CForTunaSigner::SignMessage(std::string strMessage, std::string& errorMessage, vector<unsigned char>& vchSig, CKey key)
{
    CHashWriter ss(SER_GETHASH, 0);
    ss << strMessageMagic;
    ss << strMessage;

    if (!key.SignCompact(ss.GetHash(), vchSig)) {
        errorMessage = _("Signing failed.");
        return false;
    }

    return true;
}

bool CForTunaSigner::VerifyMessage(CPubKey pubkey, vector<unsigned char>& vchSig, std::string strMessage, std::string& errorMessage)
{
    CHashWriter ss(SER_GETHASH, 0);
    ss << strMessageMagic;
    ss << strMessage;

    CPubKey pubkey2;
    if (!pubkey2.RecoverCompact(ss.GetHash(), vchSig)) {
        errorMessage = _("Error recovering public key.");
        return false;
    }

    if (fDebug && pubkey2.GetID() != pubkey.GetID())
        printf("CForTunaSigner::VerifyMessage -- keys don't match: %s %s", pubkey2.GetID().ToString().c_str(), pubkey.GetID().ToString().c_str());

    return (pubkey2.GetID() == pubkey.GetID());
}

bool CFortunaQueue::Sign()
{
    if(!fFortunaStake) return false;

    std::string strMessage = vin.ToString() + boost::lexical_cast<std::string>(nDenom) + boost::lexical_cast<std::string>(time) + boost::lexical_cast<std::string>(ready);

    CKey key2;
    CPubKey pubkey2;
    std::string errorMessage = "";

    if(!forTunaSigner.SetKey(strFortunaStakePrivKey, errorMessage, key2, pubkey2))
    {
        printf("CFortunaQueue():Relay - ERROR: Invalid fortunastakeprivkey: '%s'\n", errorMessage.c_str());
        return false;
    }

    if(!forTunaSigner.SignMessage(strMessage, errorMessage, vchSig, key2)) {
        printf("CFortunaQueue():Relay - Sign message failed");
        return false;
    }

    if(!forTunaSigner.VerifyMessage(pubkey2, vchSig, strMessage, errorMessage)) {
        printf("CFortunaQueue():Relay - Verify message failed");
        return false;
    }

    return true;
}

bool CFortunaQueue::Relay()
{

    //LOCK(cs_vNodes);
    BOOST_FOREACH(CNode* pnode, vNodes){
        // always relay to everyone
        pnode->PushMessage("dsq", (*this));
    }

    return true;
}

bool CFortunaQueue::CheckSignature()
{
    BOOST_FOREACH(CFortunaStake& mn, vecFortunastakes) {

        if(mn.vin == vin) {
            std::string strMessage = vin.ToString() + boost::lexical_cast<std::string>(nDenom) + boost::lexical_cast<std::string>(time) + boost::lexical_cast<std::string>(ready);

            std::string errorMessage = "";
            if(!forTunaSigner.VerifyMessage(mn.pubkey2, vchSig, strMessage, errorMessage)){
                return error("CFortunaQueue::CheckSignature() - Got bad fortunastake address signature %s \n", vin.ToString().c_str());
            }

            return true;
        }
    }

    return false;
}

//TODO: Rename/move to core
void ThreadCheckForTunaPool(void* parg)
{
    // Make this thread recognisable as the wallet flushing thread
    RenameThread("denarius-mn");

    unsigned int c = 0;
    std::string errorMessage;

    while (true && !fShutdown)
    {
        c++;

        MilliSleep(1000);
        //printf("ThreadCheckForTunaPool::check timeout\n");
        //forTunaPool.CheckTimeout();

        int mnTimeout = 150; //2.5 minutes

        if(c % mnTimeout == 0){
            LOCK(cs_main);
            /*
                cs_main is required for doing fortunastake.Check because something
                is modifying the coins view without a mempool lock. It causes
                segfaults from this code without the cs_main lock.
            */
        {

        LOCK(cs_fortunastakes);
            vector<CFortunaStake>::iterator it = vecFortunastakes.begin();
            //check them separately
            while(it != vecFortunastakes.end()){
                (*it).Check();
                ++it;
            }

            //remove inactive
            it = vecFortunastakes.begin();
            while(it != vecFortunastakes.end()){
                if((*it).enabled == 4 || (*it).enabled == 3){
                    printf("Removing inactive fortunastake %s\n", (*it).addr.ToString().c_str());
                    it = vecFortunastakes.erase(it);
                    mnCount = mnCount-1;
                } else {
                    ++it;
                }
            }

        }
            fortunastakePayments.CleanPaymentList();
        }

        int mnRefresh = 30;

        //try to sync the fortunastake list and payment list every 30 seconds from at least 2 nodes until we have them all
        if(vNodes.size() > 1 && c % mnRefresh == 0 && (mnCount == 0 || vecFortunastakes.size() < mnCount)) {
            bool fIsInitialDownload = IsInitialBlockDownload();
            if(!fIsInitialDownload) {
                LOCK(cs_vNodes);
                BOOST_FOREACH(CNode* pnode, vNodes)
                {
                    if (pnode->nVersion >= forTunaPool.PROTOCOL_VERSION) {

                        // re-request from each node every 120 seconds
                        if(GetTime() - pnode->nLastDseg < 120)
                        {
                            continue;
                        } else {
                            printf("Asking for Fortunastake list from %s\n",pnode->addr.ToStringIPPort().c_str());
                            pnode->PushMessage("dseg", CTxIn()); //request full mn list
                            pnode->nLastDseg = GetTime();
                            pnode->PushMessage("getsporks"); //get current network sporks
                            RequestedFortunaStakeList++;
                        }
                    }
                }
            }
        }

        if(c % FORTUNASTAKE_PING_SECONDS == 0){
            activeFortunastake.ManageStatus();
        }

        //if(c % (60*5) == 0){
        if(c % 60 == 0 && vecFortunastakes.size()) {
            //let's connect to a random fortunastake every minute!
            int fs = rand() % vecFortunastakes.size();
            CService addr = vecFortunastakes[fs].addr;
            AddOneShot(addr.ToStringIPPort());
            if (fDebug) printf("added fortunastake at %s to connection attempts\n",addr.ToStringIPPort().c_str());

            //if we're low on peers, let's connect to some random ipv4 fortunastakes. ipv6 probably won't route anyway
            if (GetArg("-maxconnections", 125) > 16 && vNodes.size() < min(25, (int)GetArg("-maxconnections", 125)) && vecFortunastakes.size() > 25) {
                int x = 25 - vNodes.size();
                for (int i = x; i-- > 0; ) {
                    int fs = rand() % vecFortunastakes.size();
                    CService addr = vecFortunastakes[fs].addr;
                    if (addr.IsIPv4() && !addr.IsLocal()) {
                        AddOneShot(addr.ToStringIPPort());
                    }
                }

            }
            //if we've used 1/5 of the fortunastake list, then clear the list.
            if((int)vecFortunastakesUsed.size() > (int)vecFortunastakes.size() / 5)
                vecFortunastakesUsed.clear();
        }
    }
}

max_stars_repo_name: sandsunsurf/denarius

max_issues_repo_name: sandsunsurf/denarius

max_forks_repo_name: sandsunsurf/denarius


text:
#include <armadillo>

#include "evaluate_covariance.hpp"

using namespace arma;

/*
Get cross-covariance matrix between two location sets

Input:
    locationsX1: X coordinates of location set 1
    locationsY1: Y coordinates of location set 1
    locationsX2: X coordinates of location set 2
    locationsY2: Y coordinates of location set 2
    nLocations1: the number of locations in lcs1
    nLocations2: the number of locations in lcs2
    sill: sill parameter
    range: range parameter
    nugget: nugget parameter

Output:
    covarianceMatrix: obtained covariances, of dimension nLocations2 * nLocations1
*/
void evaluate_cross_covariance(double *covarianceMatrix, double *locationsX1, double *locationsY1, double *locationsX2, double *locationsY2, const unsigned long &nLocations1, const unsigned long &nLocations2, const double &sill, const double &range, const double &nugget)
{
    for(unsigned long index1 = 0; index1 < nLocations1; index1++)
        for(unsigned long index2 = 0; index2 < nLocations2; index2++)
        {
            double xDifference=locationsX1[index1]-locationsX2[index2];
            double yDifference=locationsY1[index1]-locationsY2[index2];

            double distance = sqrt( xDifference*xDifference + yDifference*yDifference );
            if (distance < 1e-14)
                covarianceMatrix[index1*nLocations2+index2] = sill + nugget;
            else
                covarianceMatrix[index1*nLocations2+index2] = sill*exp(-distance/range);
        }
}

/*
Get variance-covariance matrix for one location set

Input:
    locationsX: X coordinates
    locationsY: Y coordinates
    nLocations: the number of locations
    sill: sill parameter
    range: range parameter
    nugget: nugget parameter

Output:
    covarianceMatrix: obtained covariances, of dimension nLocations * nLocations
*/
void evaluate_variance_covariance(double *covarianceMatrix, double *locationsX, double *locationsY, const unsigned long &nLocations, const double &sill, const double &range, const double &nugget)
{
    double offDiagonal;
    double diagonal = sill + nugget;
    for(unsigned long index1 = 0; index1 < nLocations; index1++)
    {
        for(unsigned long index2 = 0; index2 < index1; index2++)
        {
            double xDifference=locationsX[index1]-locationsX[index2];
            double yDifference=locationsY[index1]-locationsY[index2];

            double distance = sqrt( xDifference*xDifference + yDifference*yDifference );
            if (distance < 1e-14)
                offDiagonal = sill + nugget;
            else
                offDiagonal = sill*exp(-distance/range);

            covarianceMatrix[index1*nLocations+index2] = offDiagonal;
            covarianceMatrix[index2*nLocations+index1] = offDiagonal;
        }
        covarianceMatrix[index1*nLocations+index1] = diagonal;
    }
}

ext: cpp


text:
// Copyright (c) 2019 Graphcore Ltd. All rights reserved.
#define BOOST_TEST_MODULE RecomputeTestPopxStandardCalls0

#include <boost/test/unit_test.hpp>
#include <iostream>
#include <vector>
#include <popart/testdevice.hpp>

#include <filereader.hpp>
#include <popart/builder.hpp>
#include <popart/op.hpp>
#include <popart/op/identity.hpp>
#include <popart/op/l1.hpp>
#include <popart/sgd.hpp>

#define private public
#define protected public
#include <popart/popx/devicex.hpp>
#include <popart/popx/irlowering.hpp>
#include <popart/session.hpp>
#undef private
#undef public

using namespace popart;

TensorId conv(Builder *b, TensorId act, ConstVoidData wdata) {
  auto aiOnnx  = b->aiOnnxOpset9();
  auto weights = b->addInitializedInputTensor(wdata);
  act = aiOnnx.conv({act, weights}, {1, 1}, 1, {}, {0, 0, 0, 0}, {1, 1});
  return act;
}

TensorId batchnormalization(Builder *b, TensorId act, ConstVoidData bndata) {
  auto aiOnnx = b->aiOnnxOpset9();
  auto scale  = b->addInitializedInputTensor(bndata);
  auto bias   = b->addInitializedInputTensor(bndata);
  auto mean   = b->addInitializedInputTensor(bndata);
  auto var    = b->addInitializedInputTensor(bndata);
  auto bn_out = aiOnnx.batchnormalization({act, scale, bias, mean, var}, 5);
  act         = bn_out.at(0);
  return act;
}

BOOST_AUTO_TEST_CASE(RecomputeTestPopxStandardCalls0) {

  auto builder = Builder::create();
  auto aiOnnx  = builder->aiOnnxOpset9();
  TensorInfo input_info{"FLOAT", std::vector<int64_t>{1, 4, 32, 32}};

  // convolution,  4 channels -> 4 channels
  TensorInfo weights_info{"FLOAT", std::vector<int64_t>{4, 4, 1, 1}};
  float weight_vals[4 * 4 * 1 * 1] = {0};
  ConstVoidData weight_data        = {weight_vals, weights_info};

  // batch-normalization
  TensorInfo bn_info{"FLOAT", std::vector<int64_t>{4}};
  float bn_vals[4]      = {0};
  ConstVoidData bn_data = {bn_vals, bn_info};

  int nLayers  = 8;
  TensorId act = builder->addInputTensor(input_info);
  for (int i = 0; i < nLayers; ++i) {
    TensorId skip = act;
    // residual 1:
    act           = conv(builder.get(), skip, weight_data);
    act           = batchnormalization(builder.get(), act, bn_data);
    TensorId res1 = aiOnnx.relu({act});

    // residual 2 (flip relu and batchnorm)
    act           = conv(builder.get(), skip, weight_data);
    act           = aiOnnx.relu({act});
    TensorId res2 = batchnormalization(builder.get(), act, bn_data);

    act = aiOnnx.add({res1, res2});
    act = aiOnnx.add({act, skip});
    act = aiOnnx.sigmoid({act});
  }
  auto l1 = builder->aiGraphcoreOpset1().l1loss({act}, 0.1);

  auto proto      = builder->getModelProto();
  auto modelProto = io::getModelFromString(proto);

  // Create the IR
  auto art       = AnchorReturnType("All");
  auto dataFlow  = DataFlow(1, {{act, art}});
  auto optimizer = ConstSGD(0.01);

  auto device = popart::createTestDevice(TEST_TARGET);

  auto opts              = SessionOptions();
  opts.autoRecomputation = RecomputationType::Standard;
  opts.enableOutlining   = false;

  auto session = popart::TrainingSession::createFromOnnxModel(
      proto,
      dataFlow,
      l1,
      optimizer,
      device,
      InputShapeInfo(),
      opts,
      Patterns::create({"PostNRepl", "ConvDataGrad"})
          .enableRuntimeAsserts(false));
  session->prepareDevice();

  popart::popx::Devicex *devicex =
      dynamic_cast<popart::popx::Devicex *>(session->device_.get());

  // we count how many times each op appears
  std::map<Op *, int, POpCmp> counts =
      devicex->lowering().getMainGraphOpCounts();

  int nOnes = 0;
  int nTwos = 0;
  // for norm only, we expect all Norm ops to appear twice, all others once
  for (auto op_count : counts) {
    if (op_count.second == 2) {
      ++nTwos;
    }
    if (op_count.second == 1) {
      ++nOnes;
    }
  }

  BOOST_CHECK(nTwos > 0);
  // more non-recomputed than recomputed (including backwards pass)
  BOOST_CHECK(nOnes > nTwos);
}


text:
/*
 * kaldi-io.cpp
 *
 *  Created on: Jul 29, 2014
 *      Author: chorows
 */

extern "C" {
#include "Python.h"
#include "numpy/arrayobject.h"
}

#include <boost/shared_ptr.hpp>
#include <boost/static_assert.hpp>

#include <boost/python.hpp>
#include <boost/python/operators.hpp>
#include <boost/python/stl_iterator.hpp>

#include <util/kaldi-io.h>
#include <util/kaldi-table.h>
#include <matrix/kaldi-matrix.h>
#include <matrix/kaldi-vector.h>
#include <util/table-types.h>

#include "python_wrappers.h"
#include "bp_converters.h"

using namespace std;

namespace bp = boost::python;

//keep a copy of the cPickle module in cache
struct PickleWrapper {
  PickleWrapper() {
    bp::object pickle = bp::import("cPickle");
    loads = pickle.attr("loads");
    dumps = pickle.attr("dumps");
  }

  bp::object loads, dumps;
};

//
// Holder for Python objects.
//
// In binary model uses Pickle to dump, the object is written as dump_length, pickled_string
// In text mode uses repr/eval (only single line), which works OK for simple types - lists, tuples, ints, but may fail for large arrays (as repr skips elemets for ndarray).
//
class PyObjectHolder {
 public:
  typedef bp::object T;

  PyObjectHolder() {
  }

  static bool Write(std::ostream &os, bool binary, const T &t) {
    kaldi::InitKaldiOutputStream(os, binary);  // Puts binary header if binary mode.
    try {
      if (binary) {  //pickle the object
        bp::object py_string = PW()->dumps(t,-1);
        int len = bp::extract<int>(py_string.attr("__len__")());
        const char* string = bp::extract<const char*>(py_string);
        kaldi::WriteBasicType(os, true, len);
        os.write(string, len);
      } else {  //use repr
        PyObject* repr = PyObject_Repr(t.ptr());
        os << PyString_AsString(repr) << '\n';
        Py_DECREF(repr);
      }
      return os.good();

    } catch (const std::exception &e) {
      KALDI_WARN<< "Exception caught writing Table object: " << e.what();
      if (!kaldi::IsKaldiError(e.what())) {std::cerr << e.what();}
      return false;  // Write failure.
    }
  }

  bool Read(std::istream &is) {
    bool is_binary;
    if (!kaldi::InitKaldiInputStream(is, &is_binary)) {
      KALDI_WARN << "Reading Table object [integer type], failed reading binary header\n";
      return false;
    }
    try {
      if (is_binary) {
        int len;
        kaldi::ReadBasicType(is, true, &len);
        std::auto_ptr<char> buf(new char[len]);
        is.read(buf.get(), len);
        bp::str py_string(buf.get(), len);
        t_ = PW()->loads(py_string);
      } else {
        std::string line;
        std::getline(is, line);
        bp::str repr(line);
        t_ = bp::eval(repr);
      }
      return true;
    } catch (std::exception &e) {
      KALDI_WARN << "Exception caught reading Table object";
      if (!kaldi::IsKaldiError(e.what())) {std::cerr << e.what();}
      return false;
    }
  }

  static bool IsReadInBinary() {return true;}

  const T &Value() const {return t_;}  // if t is a pointer, would return *t_;

  void Clear() {}

  ~PyObjectHolder() {}

private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(PyObjectHolder);
  T t_;  // t_ may alternatively be of type T*.
  static PickleWrapper *PW_;
  static PickleWrapper * PW() {
    if (!PW_) {
      PW_ = new PickleWrapper();
    }
    return PW_;
  }
};

PickleWrapper * PyObjectHolder::PW_ = 0;

template<class Real>
struct MatrixToNdArrayConverter {
  typedef kaldi::KaldiObjectHolder<kaldi::Matrix<Real> > HR;
  typedef kaldi::KaldiObjectHolder<kaldi::NpWrapperMatrix<Real> > HW;

  static inline bp::object kaldi_to_python(const kaldi::Matrix<Real>& mat) {
    npy_intp dims[2];
    dims[0] = mat.NumRows();
    dims[1] = mat.NumCols();
    int nd = 2;
    int arr_type = kaldi::get_np_type<Real>();
    PyObject* ao = PyArray_SimpleNew(nd, dims, arr_type);
    bp::object arr=bp::object(bp::handle<>(
        ao
        ));
    kaldi::NpWrapperMatrix<Real> arr_wrap((PyArrayObject*)arr.ptr());
    arr_wrap.CopyFromMat(mat);
    return arr;
  }

  static inline kaldi::NpWrapperMatrix<Real>* python_to_kaldi(bp::object o) {
    PyObject* raw_arr = PyArray_FromAny(o.ptr(),PyArray_DescrFromType(kaldi::get_np_type<Real>()), 2, 2, NPY_C_CONTIGUOUS | NPY_FORCECAST, NULL);
    //why does this fail: bp::object arr(bp::handle<>(raw_arr));
    bp::object arr=bp::object(bp::handle<>(raw_arr));
    return new kaldi::NpWrapperMatrix<Real>((PyArrayObject*)arr.ptr());
  }
};

template<class Real>
struct VectorToNdArrayConverter {
  typedef kaldi::KaldiObjectHolder<kaldi::Vector<Real> > HR;
  typedef kaldi::KaldiObjectHolder<kaldi::NpWrapperVector<Real> > HW;

  static inline bp::object kaldi_to_python(const kaldi::Vector<Real>& vec) {
    npy_intp dims[1];
    dims[0] = vec.Dim();
    int nd = 1;

    int arr_type = kaldi::get_np_type<Real>();
    PyObject* ao = PyArray_SimpleNew(nd, dims, arr_type);
    bp::object arr=bp::object(bp::handle<>(
        ao
        ));
    kaldi::NpWrapperVector<Real> vec_wrap((PyArrayObject*)arr.ptr());
    vec_wrap.CopyFromVec(vec);
    return arr;
  }

  static inline kaldi::NpWrapperVector<Real>* python_to_kaldi(bp::object o) {
    PyObject* raw_arr = PyArray_FromAny(o.ptr(),PyArray_DescrFromType(kaldi::get_np_type<Real>()), 1, 1, NPY_C_CONTIGUOUS | NPY_FORCECAST, NULL);
    //why does this fail: bp::object arr(bp::handle<>(raw_arr));
    bp::object arr=bp::object(bp::handle<>(raw_arr));
    return new kaldi::NpWrapperVector<Real>((PyArrayObject*)arr.ptr());
  }
};

template<typename T>
struct VectorToNDArrayBPConverter {
  static PyObject* convert(std::vector<T> const& vec) {
    npy_intp dims[1];
    dims[0] = vec.size();
    int nd = 1;
    int arr_type = kaldi::get_np_type<T>();
    PyObject* ao = PyArray_SimpleNew(nd, dims, arr_type);
    bp::object arr=bp::object(bp::handle<>(
        ao
        ));
    std::copy(vec.begin(), vec.end(), (T*)PyArray_DATA(ao));
    return bp::incref(arr.ptr());
  }
};

template<class Obj, class HW_, class HR_>
struct BoostPythonconverter {
  typedef HW_ HW;
  typedef HR_ HR;

  static inline bp::object kaldi_to_python(const Obj& o) {
      return bp::object(o);
    }

    static inline Obj * python_to_kaldi(bp::object o) {
      return new Obj(bp::extract<Obj >(o));
    }
};

template<class Converter>
class PythonToKaldiHolder {
 public:
  typedef bp::object T;
  typedef typename Converter::HR HR;
  typedef typename Converter::HW HW;

  PythonToKaldiHolder() : h_() {
  }

  static bool Write(std::ostream &os, bool binary, const T &t) {
    try {
      auto_ptr<typename HW::T> obj(Converter::python_to_kaldi(t));
      return HW::Write(os, binary, (*obj));
    } catch (std::exception &e) {
      KALDI_WARN << "Exception caught reading Table object";
      if (!kaldi::IsKaldiError(e.what())) {std::cerr << e.what();}
      return false;
    }
  }

  bool Read(std::istream &is) {
    if (!h_.Read(is))
      return false;
    t_ = Converter::kaldi_to_python(h_.Value());
    return true;
  }

  static bool IsReadInBinary() {return true;}

  const T &Value() const {return t_;}  // if t is a pointer, would return *t_;

  void Clear() {}

  ~PythonToKaldiHolder() {}

private:
  KALDI_DISALLOW_COPY_AND_ASSIGN(PythonToKaldiHolder);
  HR h_;
  T t_;  // t_ may alternatively be of type T*.
};

template<class T>
struct VectorHolder {
  typedef PythonToKaldiHolder<BoostPythonconverter<std::vector<T>,
      kaldi::BasicVectorHolder<T>, kaldi::BasicVectorHolder<T> > > type;

  static void register_converters() {
    bp::to_python_converter<std::vector<T>, kaldi::VectorToListBPConverter<T> >();
    kaldi::VectorFromListBPConverter<T>();
  }
};

template<class T>
struct VectorNDArrayHolder {
  typedef PythonToKaldiHolder<BoostPythonconverter<std::vector<T>,
      kaldi::BasicVectorHolder<T>, kaldi::BasicVectorHolder<T> > > type;

  static void register_converters() {
    bp::to_python_converter<std::vector<T>, VectorToNDArrayBPConverter<T> >();
    kaldi::VectorFromListBPConverter<T>();
  }
};

template<class T>
struct VectorVectorHolder {
  typedef PythonToKaldiHolder<BoostPythonconverter<std::vector<std::vector<T> > ,
      kaldi::BasicVectorVectorHolder<T>, kaldi::BasicVectorVectorHolder<T> > > type;

  static void register_converters() {
    bp::to_python_converter<std::vector<std::vector<kaldi::int32> >, kaldi::VectorToListBPConverter<std::vector<kaldi::int32> > >();
    kaldi::VectorFromListBPConverter<std::vector<kaldi::int32> >();
  }
};

template<class T>
struct PairVectorHolder {
  typedef PythonToKaldiHolder<BoostPythonconverter<std::vector<std::pair<T,T> > ,
      kaldi::BasicPairVectorHolder<T>, kaldi::BasicPairVectorHolder<T> > > type;

  static void register_converters() {
    //register the pair first
    bp::to_python_converter<std::pair<kaldi::int32, kaldi::int32>,
      kaldi::PairToTupleBPConverter<kaldi::int32, kaldi::int32> >();
    kaldi::PairFromTupleBPConverter<kaldi::int32, kaldi::int32>();

    //next register the pair vector
    bp::to_python_converter<std::vector<std::pair<kaldi::int32, kaldi::int32> >,
    kaldi::VectorToListBPConverter<std::pair<kaldi::int32, kaldi::int32> > >();
    kaldi::VectorFromListBPConverter<std::pair<kaldi::int32, kaldi::int32> >();
  }
};

template<class T>
const T& get_self_ref(const T& t) {
  return t;
}

template<class T>
void exit(T& t, const bp::object& type,
              const bp::object& value, const bp::object& traceback) {
  t.Close();
}

template<class T>
bp::object sequential_reader_next(T& reader) {
  if (!reader.IsOpen() || reader.Done()) {
    PyErr_SetString(PyExc_StopIteration, "No more data.");
    bp::throw_error_already_set();
  }
  //if not done, extract the contents
  bp::str key(reader.Key());
  bp::object val(reader.Value());
  //move the reading head, the contents will be read with the next call to next!
  reader.Next();
  return bp::make_tuple(key,val);
}

template <class Reader>
class RandomAccessWrapper: public bp::class_<Reader> {
public:
  template <class DerivedT>
  inline RandomAccessWrapper(char const* name, bp::init_base<DerivedT> const& i)
          : bp::class_<Reader>(name, i) {
          (*this)
          .def("close", &Reader::Close)
          .def("is_open", &Reader::IsOpen)
          .def("__contains__", &Reader::HasKey)
          .def("has_key", &Reader::HasKey)
          .def("__getitem__", &Reader::Value,
               bp::return_value_policy<bp::copy_const_reference>())
          .def("value", &Reader::Value,
               bp::return_value_policy<bp::copy_const_reference>())
          .def("__enter__", &get_self_ref<Reader>,
               bp::return_internal_reference<1>())
          .def("__exit__", &exit<Reader>)
          ;
      }
};

template <class Reader>
class SequentialReaderWrapper: public bp::class_<Reader> {
public:
  template <class DerivedT>
  inline SequentialReaderWrapper(char const* name, bp::init_base<DerivedT> const& i)
          : bp::class_<Reader>(name, i) {
          (*this)
          .def("close", &Reader::Close)
          .def("is_open", &Reader::IsOpen)
          .def("__enter__", &get_self_ref<Reader>,
               bp::return_internal_reference<1>())
          .def("__iter__", &get_self_ref<Reader>,
               bp::return_internal_reference<1>())
          .def("next", sequential_reader_next<Reader>)
          .def("__exit__", &exit<Reader>)
          .def("done", &Reader::Done)
          .def("_kaldi_value", &Reader::Value,
                     bp::return_value_policy<bp::copy_const_reference>())
          .def("_kaldi_next", &Reader::Next)
          .def("_kaldi_key", &Reader::Key)
          ;
      }
};

template <class Writer>
class WriterWrapper: public bp::class_<Writer> {
public:
  template <class DerivedT>
  inline WriterWrapper(char const* name, bp::init_base<DerivedT> const& i)
          : bp::class_<Writer>(name, i) {
          (*this)
          .def("close", &Writer::Close)
          .def("is_open", &Writer::IsOpen)
          .def("flush", &Writer::Flush)
          .def("write", &Writer::Write)
          .def("__setitem__", &Writer::Write)
          .def("__enter__", &get_self_ref<Writer>,
               bp::return_internal_reference<1>())
          .def("__exit__",&exit<Writer>)
          ;
      }
};

PyObject* KALDI_BASE_FLOAT() {
  return (PyObject*)PyArray_DescrFromType(kaldi::get_np_type<kaldi::BaseFloat>());
}

BOOST_PYTHON_MODULE(kaldi_io_internal)
{
  import_array();

  bp::def("KALDI_BASE_FLOAT", &KALDI_BASE_FLOAT);

  //Python objects
  RandomAccessWrapper<kaldi::RandomAccessTableReader<PyObjectHolder> >("RandomAccessPythonReader", bp::init<std::string>());
  RandomAccessWrapper<kaldi::RandomAccessTableReaderMapped<PyObjectHolder> >("RandomAccessPythonReaderMapped", bp::init<std::string, std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PyObjectHolder> >("SequentialPythonReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PyObjectHolder> >("PythonWriter", bp::init<std::string>());

  //Matrices as NdArrays
  RandomAccessWrapper<kaldi::RandomAccessTableReader<PythonToKaldiHolder<MatrixToNdArrayConverter<double> > > >("RandomAccessFloat64MatrixReader", bp::init<std::string>());
  RandomAccessWrapper<kaldi::RandomAccessTableReaderMapped<PythonToKaldiHolder<MatrixToNdArrayConverter<double> > > >("RandomAccessFloat64MatrixMapped",bp::init<std::string, std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PythonToKaldiHolder<MatrixToNdArrayConverter<double> > > >("SequentialFloat64MatrixReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PythonToKaldiHolder<MatrixToNdArrayConverter<double> > > >("Float64MatrixWriter", bp::init<std::string>());

  RandomAccessWrapper<kaldi::RandomAccessTableReader<PythonToKaldiHolder<MatrixToNdArrayConverter<float> > > >("RandomAccessFloat32MatrixReader", bp::init<std::string>());
  RandomAccessWrapper<kaldi::RandomAccessTableReaderMapped<PythonToKaldiHolder<MatrixToNdArrayConverter<float> > > >("RandomAccessFloat32MatrixMapped",bp::init<std::string, std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PythonToKaldiHolder<MatrixToNdArrayConverter<float> > > >("SequentialFloat32MatrixReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PythonToKaldiHolder<MatrixToNdArrayConverter<float> > > >("Float32MatrixWriter", bp::init<std::string>());

  //Vectors as NdArrays
  RandomAccessWrapper<kaldi::RandomAccessTableReader<PythonToKaldiHolder<VectorToNdArrayConverter<double> > > >("RandomAccessFloat64VectorReader", bp::init<std::string>());
  RandomAccessWrapper<kaldi::RandomAccessTableReaderMapped<PythonToKaldiHolder<VectorToNdArrayConverter<double> > > >("RandomAccessFloat64VectorReaderMapped",bp::init<std::string, std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PythonToKaldiHolder<VectorToNdArrayConverter<double> > > >("SequentialFloat64VectorReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PythonToKaldiHolder<VectorToNdArrayConverter<double> > > >("Float64VectorWriter", bp::init<std::string>());

  RandomAccessWrapper<kaldi::RandomAccessTableReader<PythonToKaldiHolder<VectorToNdArrayConverter<float> > > >("RandomAccessFloat32VectorReader", bp::init<std::string>());
  RandomAccessWrapper<kaldi::RandomAccessTableReaderMapped<PythonToKaldiHolder<VectorToNdArrayConverter<float> > > >("RandomAccessFloat32VectorReaderMapped",bp::init<std::string, std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PythonToKaldiHolder<VectorToNdArrayConverter<float> > > >("SequentialFloat32VectorReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PythonToKaldiHolder<VectorToNdArrayConverter<float> > > >("Float32VectorWriter", bp::init<std::string>());

  //Integers
  RandomAccessWrapper<kaldi::RandomAccessInt32Reader >("RandomAccessInt32Reader", bp::init<std::string>());
  SequentialReaderWrapper<kaldi::SequentialInt32Reader >("SequentialInt32Reader",bp::init<std::string>());
  WriterWrapper<kaldi::Int32Writer >("Int32Writer", bp::init<std::string>());

  // std::vector<int32> as ndarray
  VectorNDArrayHolder<kaldi::int32>::register_converters();
  RandomAccessWrapper<kaldi::RandomAccessTableReader<VectorNDArrayHolder<kaldi::int32>::type > >("RandomAccessInt32VectorReader", bp::init<std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<VectorNDArrayHolder<kaldi::int32>::type > >("SequentialInt32VectorReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<VectorNDArrayHolder<kaldi::int32>::type > >("Int32VectorWriter", bp::init<std::string>());

//  Vector of simple types as lists
//  VectorHolder<kaldi::int32>::register_converters();
//  RandomAccessWrapper<kaldi::RandomAccessTableReader<VectorHolder<kaldi::int32>::type > >("RandomAccessInt32VectorReader", bp::init<std::string>());
//  SequentialReaderWrapper<kaldi::SequentialTableReader<VectorHolder<kaldi::int32>::type > >("SequentialInt32VectorReader",bp::init<std::string>());
//  WriterWrapper<kaldi::TableWriter<VectorHolder<kaldi::int32>::type > >("Int32VectorWriter", bp::init<std::string>());

  // std::vector<std::vector<int32> >
  VectorVectorHolder<kaldi::int32>::register_converters();
  RandomAccessWrapper<kaldi::RandomAccessTableReader<VectorVectorHolder<kaldi::int32>::type > >("RandomAccessInt32VectorVectorReader", bp::init<std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<VectorVectorHolder<kaldi::int32>::type > >("SequentialInt32VectorVectorReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<VectorVectorHolder<kaldi::int32>::type > >("Int32VectorVectorWriter", bp::init<std::string>());

  // std::vector<std::pair<int32, int32> >
  PairVectorHolder<kaldi::int32>::register_converters();
  RandomAccessWrapper<kaldi::RandomAccessTableReader<PairVectorHolder<kaldi::int32>::type > >("RandomAccessInt32PairVectorReader", bp::init<std::string>());
  SequentialReaderWrapper<kaldi::SequentialTableReader<PairVectorHolder<kaldi::int32>::type > >("SequentialInt32PairVectorReader",bp::init<std::string>());
  WriterWrapper<kaldi::TableWriter<PairVectorHolder<kaldi::int32>::type > >("Int32PairVectorWriter", bp::init<std::string>());

}

max_stars_repo_name: jingyonghou/kaldi-python

max_issues_repo_name: jingyonghou/kaldi-python

max_forks_repo_name: jingyonghou/kaldi-python


text:
// -*- mode: c++; indent-tabs-mode: nil; -*-
//
// Copyright (c) 2010-2015 Illumina, Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:

// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.

// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

/**
 *  \brief Read a spacer-separated file and compute a ROC curve
 *
 *
 * \file roc.cpp
 * \author Peter Krusche
 * \email pkrusche@illumina.com
 *
 */

#include <boost/program_options.hpp>

#include "Version.hh"
#include "helpers/StringUtil.hh"

#include <iostream>
#include <limits>
#include <string>
#include <vector>
#include <fstream>
#include <cstdlib>
#include <map>
#include <set>
#include <boost/algorithm/string.hpp>

#include "Error.hh"

enum class tag_t { tp, fp, fn };

struct RocData {
    void add(double value, tag_t tag)
    {
        if(reverse) value = -value;
        data.push_back(std::make_pair(value, tag));
    }

    void print_roc(std::ostream & out) {
        if(data.empty())
        {
            return;
        }
        // pair gets sorted by first element, then second
        std::sort(data.begin(), data.end());
        int total_tp = 0;
        int total_fp = 0;
        int total_fn = 0;
        for (auto r : data) {
            switch(r.second)
            {
                case tag_t::tp: ++total_tp; break;
                case tag_t::fp: ++total_fp; break;
                case tag_t::fn: ++total_fn; break;
                default: break;
            }
        }

        out << label << "\t" << "tp" << "\t" << "fp" << "\t" << "fn" << "\t" << "precision" << "\t" << "recall" << "\n";
        double current_value = data[0].first - 1;
        double tp = total_tp;
        double fp = total_fp;
        double fn = total_fn;
        for (auto r : data) {
            if(r.first > current_value)
            {
                current_value = r.first;
                double precision = 1.0;
                double recall = 0;
                if(tp + fp > 0)
                {
                    precision = tp / (tp + fp);
                }
                if(tp + fn > 0)
                {
                    recall = tp / (tp + fn);
                }

                out << (reverse ? -current_value : current_value) << "\t" << tp << "\t" << fp << "\t" << fn << "\t" << precision << "\t" << recall << "\n";
            }

            switch(r.second)
            {
                case tag_t::fn:
                    break;
                case tag_t::tp:
                    --tp;
                    ++fn;
                    break;
                case tag_t::fp:
                    --fp;
                    break;
            }
        }
    }

    std::vector<std::pair<double, tag_t> > data;
    std::string label = "value";

    bool reverse = false;
};

int main(int argc, char* argv[]) {
    namespace po = boost::program_options;

    std::vector<std::string> files;
    std::string output = "-";
    std::string sep = "\t";
    int header_lines = 1;
    std::string value = "";
    int value_column = 0;
    std::string tag = "";
    int tag_column = 1;
    std::string filter = "";
    int filter_column = -1;
    std::string filter_name = "";

    bool verbose = false;
    bool reverse = false;

    try
    {
        // Declare the supported options.
        po::options_description desc("Allowed options");
        desc.add_options()
            ("help,h", "produce help message")
            ("version", "show version")
            ("verbose", "show verbose information (to stderr)")
            ("input-file", po::value<std::vector< std::string> >(), "The input files")
            ("output-file,o", po::value<std::string>(), "Output file name, defaults to - / write to stdout")
            ("separator,s", po::value<std::string>(), "separator character (default: '\\t' for reading tsv)")
            ("header-lines,H", po::value<int>(), "lines to skip before starting to read")
            ("value,v", po::value<std::string>(), "value column name")
            ("value-column", po::value<int>(), "value column number")
            ("reverse,R", po::value<bool>(), "Reverse counting for score (default: higher scores are better)")
            ("tag,t", po::value<std::string>(), "tag column name")
            ("tag-column", po::value<int>(), "tag column number. Tags must be TP/FP/FN, lines with different tags will be ignored")
            ("filter,f", po::value<std::string>(), "filter column name")
            ("filter-column", po::value<int>(), "filter column number. This is used if we the value we are varying is a threshold for a certain filter.")
            ("filter-name,n", po::value<std::string>(), "filter name if value is threshold for this filter")
        ;

        po::positional_options_description popts;
        popts.add("input-file", -1);

        po::options_description cmdline_options;
        cmdline_options
            .add(desc)
        ;

        po::variables_map vm;
        po::store(po::command_line_parser(argc, argv).
                  options(cmdline_options).positional(popts).run(), vm);
        po::notify(vm);

        if (vm.count("version"))
        {
            std::cout << "roc version " << HAPLOTYPES_VERSION << "\n";
            return 0;
        }

        if (vm.count("help"))
        {
            std::cout << desc << "\n";
            return 1;
        }

        if (vm.count("verbose"))
        {
            std::cerr << "Verbose mode enabled" << "\n";
            verbose = true;
        }

        if (vm.count("input-file"))
        {
            files = vm["input-file"].as< std::vector<std::string> >();
        }

        if (vm.count("output-file"))
        {
            output = vm["output-file"].as<std::string>();
        }

        if (vm.count("separator"))
        {
            sep = vm["separator"].as< std::string >();
        }

        if (vm.count("header-lines"))
        {
            header_lines = vm["header-lines"].as<int>();
        }

        if (vm.count("value-column"))
        {
            value_column = vm["value-column"].as<int>() - 1;
        }
        if (vm.count("value"))
        {
            value = vm["value"].as<std::string>();
        }
        if (vm.count("reverse"))
        {
            reverse = vm["reverse"].as<bool>();
        }

        if (vm.count("tag-column"))
        {
            tag_column = vm["tag-column"].as<int>() - 1;
        }
        if (vm.count("tag"))
        {
            tag = vm["tag"].as<std::string>();
        }

        if (vm.count("filter-column"))
        {
            filter_column = vm["filter-column"].as<int>() - 1;
        }
        if (vm.count("filter"))
        {
            filter = vm["filter"].as<std::string>();
        }
        if (vm.count("filter-name"))
        {
            filter_name = vm["filter-name"].as<std::string>();
        }
    }
    catch (po::error & e)
    {
        std::cerr << e.what() << "\n";
        return 1;
    }

    RocData data;
    data.reverse = reverse;
    if(!value.empty())
    {
        data.label = value;
    }

    if(files.size() == 0) {
        files.push_back("-");
    }

    int total_tp = 0;
    int total_fp = 0;
    int total_fn = 0;
    int total_filtered = 0;
    int total_ignored = 0;
    std::map<std::string, int> filter_stats;
    std::map<std::string, int> filter_stats_tp;
    std::map<std::string, int> filter_stats_fp;
    std::map<std::string, int> filter_stats_fn;

    for (auto i : files) {
        std::istream * in = NULL;
        bool deleteme = false;
        if(i == "-") {
            in = &std::cin;
        } else {
            in = new std::ifstream(i.c_str());
            deleteme = true;
        }

        // read input data
        int min_columns = std::max(value_column, tag_column);
        min_columns = std::max(filter_column, min_columns);

        int hc = header_lines;
        while(in->good()) {
            std::string line;
            std::getline(*in, line);
            std::vector<std::string> v;
            stringutil::split(line, v, sep, true);

            if(hc > 0) {
                if (!value.empty() || !tag.empty() || !filter.empty())
                {
                    for (size_t i = 0; i < v.size(); ++i)
                    {
                        v[i] = stringutil::replaceAll(v[i], "\r", "");
                        if(!v[i].empty() && v[i] == value) {
                            value_column = i;
                        }
                        if(!v[i].empty() && v[i] == tag) {
                            tag_column = i;
                        }
                        if(!v[i].empty() && v[i] == filter) {
                            filter_column = i;
                        }
                    }
                }
                min_columns = std::max(value_column, tag_column);
                min_columns = std::max(filter_column, min_columns);
                --hc;
                continue;
            }

            if(((int)v.size()) < min_columns + 1) {
                ++total_ignored;
                continue;
            }

            std::string & ltag = v[tag_column];
            boost::algorithm::to_lower(ltag);

            // true if filtered by other filter than the one we're looking at. These go to the beginning.
            bool filtered_other = false;

            if (filter_column >= 0)
            {
                std::vector<std::string> filters;
                std::string fcol = v[filter_column];
                if(fcol == "." || fcol == "PASS") {
                    fcol = "";
                }
                stringutil::split(fcol, filters, ";,");
                for (auto f : filters)
                {
                    auto q = filter_stats.find(f);
                    if(q != filter_stats.end()) {
                        q->second++;
                    } else {
                        filter_stats[f] = 1;
                    }
                    if (ltag == "tp")
                    {
                        auto q = filter_stats_tp.find(f);
                        if(q != filter_stats_tp.end()) {
                            q->second++;
                        } else {
                            filter_stats_tp[f] = 1;
                        }
                    } else if (ltag == "fp") {
                        auto q = filter_stats_fp.find(f);
                        if(q != filter_stats_fp.end()) {
                            q->second++;
                        } else {
                            filter_stats_fp[f] = 1;
                        }
                    } else if (ltag == "fn") {
                        auto q = filter_stats_fn.find(f);
                        if(q != filter_stats_fn.end()) {
                            q->second++;
                        } else {
                            filter_stats_fn[f] = 1;
                        }
                    }
                }

                if(!filter_name.empty())
                {
                    auto it = filters.begin();
                    std::vector<std::string> _filters_to_remove;
                    stringutil::split(filter_name, _filters_to_remove, ";,");
                    std::set<std::string> filters_to_remove;
                    for (auto f : _filters_to_remove) {
                        filters_to_remove.insert(f);
                        if(f == "*")
                        {
                            filters.clear();
                            break;
                        }
                    }
                    while(it != filters.end())
                    {
                        if(filters_to_remove.count(*it) > 0) {
                            filters.erase(it);
                            it = filters.begin();
                        } else {
                            ++it;
                        }
                    }
                }
                if (!filters.empty())
                {
                    ++total_filtered;
                    filtered_other = true;
                }
            }

            tag_t ttag = tag_t::fn;

            if(ltag.substr(0, 2) == "tp") {
                ttag = tag_t::tp;
                ++total_tp;
            } else if(ltag.substr(0, 2) == "fp") {
                ttag = tag_t::fp;
                ++total_fp;
            } else if(ltag.substr(0, 2) == "fn") {
                ttag = tag_t::fn;
                ++total_fn;
            } else {
                ++total_ignored;
                continue;
            }

            double xvalue = atof(v[value_column].c_str());
            if(v[value_column] == "")
            {
                xvalue = 0;
            }
            if (filtered_other)
            {
                xvalue = std::numeric_limits<double>::min();
            }
            data.add(xvalue, ttag);
        }
        if (deleteme)
        {
            delete in;
        }
    }
    if(verbose) {
        std::cerr << "tp: " << total_tp << " fp: " << total_fp << " fn: " << total_fn
                  << " filtered: " << total_filtered << " ignored: " << total_ignored << "\n";
        for (auto i : filter_stats) {
            int f_tp = 0;
            if (filter_stats_tp.find(i.first) != filter_stats_tp.end())
            {
                f_tp = filter_stats_tp[i.first];
            }
            int f_fp = 0;
            if (filter_stats_fp.find(i.first) != filter_stats_fp.end())
            {
                f_fp = filter_stats_fp[i.first];
            }
            std::cerr << "F:" << i.first << " : " << i.second
                      << " -- tp: " << f_tp
                      << " fp: " << f_fp
                      << "\n";
        }
    }
    if(output == "-") {
        data.print_roc(std::cout);
    } else {
        std::ofstream o(output);
        data.print_roc(o);
    }

    return 0;
}


text:
//
//  nDijkstra.hpp
//  evoNik
//
//  Created by Nikhil Joshi on 4/28/12.
//  Copyright (c) 2012 California Institute of Technology. All rights reserved.
//

#ifndef evoNik_nDijkstra_hpp
#define evoNik_nDijkstra_hpp

# include <cmath>
#include <iostream>

#include <boost/config.hpp>
#include <boost/graph/graph_traits.hpp>
#include <boost/graph/adjacency_list.hpp>
#include <boost/graph/dijkstra_shortest_paths.hpp>

#include "constants.hpp"

class nDijkstra_map2d{
public:
    std::map< std::pair< unsigned int, unsigned int > , int > sectorMap;
    int add(unsigned int x, unsigned int y);
    int get(unsigned int x, unsigned int y);
    void clear(void)                           {  sectorMap.clear();  }
};

class nDijkstra{
public:
    nDijkstra_map2d sectorMap;
    size_t height;
    size_t width;
    int old_goal_w;
    int old_goal_h;

    typedef boost::adjacency_list <boost::vecS, boost::vecS, boost::undirectedS, boost::no_property, boost::property <boost::edge_weight_t, double> > graph_t;
    typedef boost::graph_traits < graph_t >::vertex_descriptor vertex_descriptor;
    typedef boost::graph_traits < graph_t >::edge_descriptor edge_descriptor;

    graph_t test_graph;

    // constructor
    nDijkstra(){
        old_goal_w = old_goal_h = -1;
    }

    void buildGraph(const std::vector<std::vector<unsigned int> > & freeSpaceArray,
                    bool diag);
    void computeFitnessArray(std::vector<std::vector<double> > & fitnessArray,
                             int goal_w,
                             int goal_h,
                             bool cache=true);

};

#endif

ext: hpp

max_stars_repo_name: nikhiljjoshi/evoNik

max_issues_repo_name: nikhiljjoshi/evoNik

max_forks_repo_name: nikhiljjoshi/evoNik


text:
/*=============================================================================
    Copyright (c) 1999-2003 Jaakko Jarvi
    Copyright (c) 2001-2011 Joel de Guzman

    Distributed under the Boost Software License, Version 1.0. (See accompanying
    file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
==============================================================================*/
#if !defined(FUSION_NOT_EQUAL_TO_05052005_0431)
#define FUSION_NOT_EQUAL_TO_05052005_0431

#include <boost/fusion/sequence/comparison/enable_comparison.hpp>
#include <boost/fusion/sequence/intrinsic/begin.hpp>
#include <boost/fusion/sequence/intrinsic/end.hpp>
#include <boost/fusion/sequence/intrinsic/size.hpp>
#include <boost/fusion/support/config.hpp>

#if defined(FUSION_DIRECT_OPERATOR_USAGE)
#include <boost/fusion/sequence/comparison/detail/not_equal_to.hpp>
#else
#include <boost/fusion/sequence/comparison/equal_to.hpp>
#endif

namespace boost {
namespace fusion {
template <typename Seq1, typename Seq2>
BOOST_CONSTEXPR BOOST_FUSION_GPU_ENABLED inline bool
not_equal_to(Seq1 const &a, Seq2 const &b) {
#if defined(FUSION_DIRECT_OPERATOR_USAGE)
  return result_of::size<Seq1>::value != result_of::size<Seq2>::value ||
         detail::sequence_not_equal_to<
             Seq1 const, Seq2 const,
             result_of::size<Seq1>::value ==
                 result_of::size<Seq2>::value>::call(fusion::begin(a),
                                                     fusion::begin(b));
#else
  return !(a == b);
#endif
}

namespace operators {
template <typename Seq1, typename Seq2>
BOOST_CONSTEXPR BOOST_FUSION_GPU_ENABLED inline
    typename boost::enable_if<traits::enable_equality<Seq1, Seq2>, bool>::type
    operator!=(Seq1 const &a, Seq2 const &b) {
  return fusion::not_equal_to(a, b);
}
} // namespace operators
using operators::operator!=;
} // namespace fusion
} // namespace boost

#endif

max_stars_repo_name: henrywarhurst/matrix

max_issues_repo_name: henrywarhurst/matrix

max_forks_repo_name: henrywarhurst/matrix


text:
/*
 * Copyright (c) 2008-2019, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <gtest/gtest.h>

#include <boost/foreach.hpp>

#include <hazelcast/client/exception/IOException.h>
#include <hazelcast/client/serialization/pimpl/DataInput.h>
#include "hazelcast/util/UTFUtil.h"

namespace hazelcast {
    namespace client {
        namespace test {
            namespace util {
                class UTFUtilTest : public ::testing::Test {
                protected:
                    // includes one, two three and 4 byte code points
                    static const std::string VALID_UTF_STRING;

                    static const std::string INVALID_UTF_STRING_INSUFFICIENT_BYTES;
                };

                const std::string UTFUtilTest::VALID_UTF_STRING = "a \xc3\xa9 \xe5\x92\xa7 \xf6\xa7\x93\xb5";
                const std::string UTFUtilTest::INVALID_UTF_STRING_INSUFFICIENT_BYTES = "a \xc3\xa9 \xe5\x92 \xf6\xa7\x93\xb5";

                TEST_F(UTFUtilTest, validUTF8) {
                    ASSERT_GT(hazelcast::util::UTFUtil::isValidUTF8(VALID_UTF_STRING), 0);
                }

                TEST_F(UTFUtilTest, invalidUTF8) {
                    ASSERT_EQ(-1, hazelcast::util::UTFUtil::isValidUTF8(INVALID_UTF_STRING_INSUFFICIENT_BYTES));
                }

                TEST_F(UTFUtilTest, readValidUTF8) {
                    std::vector<byte> strBytes;
                    BOOST_FOREACH(char b, VALID_UTF_STRING) {
                                    strBytes.push_back((byte) b);
                                }
                    serialization::pimpl::DataInput in(strBytes);
                    std::vector<char> utfBuffer;
                    utfBuffer.reserve(
                            client::serialization::pimpl::DataInput::MAX_UTF_CHAR_SIZE * VALID_UTF_STRING.size());
                    int numberOfUtfChars = hazelcast::util::UTFUtil::isValidUTF8(VALID_UTF_STRING);
                    for (int i = 0; i < numberOfUtfChars; ++i) {
                        byte c = in.readByte();
                        hazelcast::util::UTFUtil::readUTF8Char(in, c, utfBuffer);
                    }

                    std::string result(utfBuffer.begin(), utfBuffer.end());
                    ASSERT_EQ(VALID_UTF_STRING, result);
                }

                TEST_F(UTFUtilTest, readInvalidUTF8) {
                    std::vector<byte> strBytes;
                    BOOST_FOREACH(char b, INVALID_UTF_STRING_INSUFFICIENT_BYTES) {
                                    strBytes.push_back((byte) b);
                                }
                    serialization::pimpl::DataInput in(strBytes);
                    std::vector<char> utfBuffer;
                    utfBuffer.reserve(
                            client::serialization::pimpl::DataInput::MAX_UTF_CHAR_SIZE * VALID_UTF_STRING.size());
                    for (int i = 0; i < 5; ++i) {
                        byte c = in.readByte();
                        // The 4th utf character is missing one byte intentionally in the invalid utf string
                        if (i == 4) {
                            ASSERT_THROW(hazelcast::util::UTFUtil::readUTF8Char(in, c, utfBuffer),
                                         exception::UTFDataFormatException);
                        } else {
                            hazelcast::util::UTFUtil::readUTF8Char(in, c, utfBuffer);
                        }
                    }
                }
            }
        }
    }
}

max_stars_repo_name: sbrown-uhd/hazelcast-cpp-client

max_issues_repo_name: sbrown-uhd/hazelcast-cpp-client

max_forks_repo_name: sbrown-uhd/hazelcast-cpp-client


text:
//
// Created by nvidia on 6/22/18.
//

//
// Created by nvidia on 6/22/18.
//

#include <iostream>
#include <opencv2/opencv.hpp>
#include <opencv2/ximgproc.hpp>
#include <algorithm> // for copy
#include <iterator> // for ostream_iterator
#include <boost/graph/graph_utility.hpp>
#include "../lib/SpVec.hpp"
#include "../lib/superpixel_seg.hpp"
#include "../lib/distance.hpp"
#include "../lib/FloodFillSsdSuperpixelAlgorithm.hpp"
#include "../lib/RTreesClassifier.hpp"
#include "../lib/util.hpp"

using namespace cv;
using namespace std;
using namespace cv::ximgproc;

using namespace cove;

inline TermCriteria TC(int iters, double eps)
{
    return TermCriteria(TermCriteria::MAX_ITER + (eps > 0 ? TermCriteria::EPS : 0), iters, eps);
}

int main(int argc, char** argv) {
    int idx = 1;
#ifdef DEBUG
    namedWindow("params", 1);
#endif
    ALGORITHM_PARAM(blurKernel, 3, 10)
    ALGORITHM_PARAM(iterations, 10, 20)
    ALGORITHM_PARAM(blurDev, 0, 100)
    ALGORITHM_PARAM(regionSize, 7, 50)
    ALGORITHM_PARAM(ratio, 70, 200)
    ALGORITHM_PARAM(cannyLow, 70, 150)
    ALGORITHM_PARAM(ssdEpsilonSlider, 10, 250)

    int frameno=0;
    Mat frame;
    Mat label;

    FloodFillSsdSuperpixelAlgorithm algorithm(80, 60, regionSize, static_cast<float>(ssdEpsilonSlider) / 10, blurKernel, blurDev, iterations,
                                              static_cast<float>(ratio) / 1000, cannyLow, cannyLow*2.5);

    vector<double> ius;

    Mat samples = Mat::zeros(0, 10, CV_32F);
    Mat responses = Mat::zeros(0, 1, CV_32S);

    for (int idx = 1; idx < argc*0.8; idx++) {
        clock_t start = CLOCK();

#ifdef DEBUG
        if (blurKernel % 2 == 0) {
            blurKernel++;
        }
#endif

        string path(argv[idx]);
        string filename = util::getFileName(path, true);
        string labelPath = "img/labels/" + filename;
        frame = imread(path);
        label = imread(labelPath);
//        SHOW_ALWAYS("frame", frame)
//        SHOW_ALWAYS("label", label)
        if (frame.empty() || label.empty()) {
            cout << "frame empty" << endl;
            break;
        }

        Mat floorMask;
        Mat resizedLabel;
        algorithm.Run(frame, floorMask, resizedLabel);
        resize(label, resizedLabel, resizedLabel.size(), 0, 0, INTER_NEAREST);
//        SHOW_ALWAYS("resize", resizedLabel)
//        SHOW_ALWAYS("calced floor", floorMask)
        Mat carpet;
        Mat hardfloor;
        Mat lab;
        Mat sizedFrame;
        inRange(resizedLabel, Scalar(0, 0, 255), Scalar(0, 0, 255), carpet);
        inRange(resizedLabel, Scalar(0, 255, 0), Scalar(0, 255, 0), hardfloor);

        auto super = algorithm.getSuperPixels(frame, lab, sizedFrame);
        cvtColor(sizedFrame, sizedFrame, COLOR_BGR2Lab);
        super->iterate(iterations);
        super->enforceLabelConnectivity(15);
        Mat spxLabels;
        super->getLabels(spxLabels);
        for (int i = 0; i < super->getNumberOfSuperpixels(); i++) {
            Mat mask;
            inRange(spxLabels, Scalar(i), Scalar(i), mask);
            // classify on label
            int label;
            Mat temp;
            bitwise_and(carpet, mask, temp);
            if (countNonZero(temp) > 7 * 7 / 2) {
                // is carpet
                label = 1;
                goto add_data;
            }
            bitwise_and(hardfloor, mask, temp);
            if (countNonZero(temp) > 7 * 7 / 2) {
                // is hardfloor
                label = 2;
                goto add_data;
            }
            // is bg
            label = 3;
            goto add_data;

            add_data:
            samples.push_back(RTreesClassifier::getData(mask, sizedFrame));;

            Mat thisResponse = Mat::zeros(1, 1, CV_32S);
            thisResponse.at<int>(0, 0) = label;
            responses.push_back(thisResponse);

        }

        // =========== time fps and exit ===========
        double dur = CLOCK() - start;
        printf("avg time per frame %f ms. fps %f. frameno = %d\n", avgdur(dur), avgfps(), frameno++);
        cout << argv[idx] << endl;

    }
    auto data = ml::TrainData::create(samples, ml::ROW_SAMPLE, responses);

    auto rtrees = ml::RTrees::create();
    rtrees->setMaxDepth(20);
    rtrees->setMinSampleCount(10);
    rtrees->setRegressionAccuracy(0);
    rtrees->setUseSurrogates(false);
    rtrees->setMaxCategories(15);
    rtrees->setPriors(Mat());
    rtrees->setCalculateVarImportance(true);
    rtrees->setActiveVarCount(0);
    rtrees->setTermCriteria(TC(100, 0.01f));
    rtrees->train(data);
    rtrees->save("RTREES");

    return 0;
}

max_stars_repo_name: NVIDIA-Jetson/Foursee-Navigation

max_issues_repo_name: NVIDIA-Jetson/Foursee-Navigation

max_forks_repo_name: NVIDIA-Jetson/Foursee-Navigation