text:
#include "QueryBuilder.h"
#include "TermDocumentIterator.h"
#include "SearchTermDocumentIterator.h"
#include "RankTermDocumentIterator.h"
#include "ANDDocumentIterator.h"
#include "ORDocumentIterator.h"
#include "NOTDocumentIterator.h"
#include "WildcardDocumentIterator.h"
#include "WildcardPhraseDocumentIterator.h"
#include "ExactPhraseDocumentIterator.h"
#include "OrderedPhraseDocumentIterator.h"
#include "NearbyPhraseDocumentIterator.h"
#include "PersonalSearchDocumentIterator.h"
#include "VirtualTermDocumentIterator.h"
#include "FilterCache.h"

#include <common/TermTypeDetector.h>

#include <ir/index_manager/utility/Bitset.h>

#include <util/get.h>

#include <vector>
#include <string>
#include <memory>
#include <algorithm>

#include <boost/token_iterator.hpp>
#include <boost/scoped_ptr.hpp>

//#define VERBOSE_SERACH_MANAGER 1

using namespace std;

using izenelib::ir::indexmanager::TermDocFreqs;
using izenelib::ir::indexmanager::TermReader;

namespace sf1r
{

QueryBuilder::QueryBuilder(
    const boost::shared_ptr<DocumentManager> documentManager,
    const boost::shared_ptr<InvertedIndexManager> indexManager,
    const schema_map& schemaMap,
    size_t filterCacheNum
)
    :documentManagerPtr_(documentManager)
    ,indexManagerPtr_(indexManager)
    ,schemaMap_(schemaMap)
    ,filterCache_(new FilterCache(filterCacheNum))
{
    if (indexManager)
        pIndexReader_ = (indexManager->pIndexReader_);
}

QueryBuilder::~QueryBuilder()
{
}

void QueryBuilder::reset_cache()
{
    filterCache_->clear();
}

bool QueryBuilder::do_process_filtertree(
    const ConditionsNode& conditionsTree_,
    boost::shared_ptr<InvertedIndexManager::FilterBitmapT>& pFilterBitmap)
{
    if (conditionsTree_.conditionLeafList_.size() == 1
            && conditionsTree_.conditionsNodeList_.size() == 0)
    {
        if (!filterCache_->get(conditionsTree_.conditionLeafList_[0], pFilterBitmap))
        {
            pFilterBitmap.reset(new InvertedIndexManager::FilterBitmapT);

            QueryFiltering::FilteringOperation filterOperation = conditionsTree_.conditionLeafList_[0].operation_;
            const std::string& property = conditionsTree_.conditionLeafList_[0].property_;
            const std::vector<PropertyValue>& filterParam = conditionsTree_.conditionLeafList_[0].values_;

            indexManagerPtr_->makeRangeQuery(filterOperation, property, filterParam, pFilterBitmap);

            filterCache_->set(conditionsTree_.conditionLeafList_[0], pFilterBitmap);
        }
        return true;
    }
    /// not leaf node;
    std::string relation = conditionsTree_.relation_;
    std::vector<boost::shared_ptr<InvertedIndexManager::FilterBitmapT> > filterBitmapTList1;
    filterBitmapTList1.resize(conditionsTree_.conditionLeafList_.size());
    for (unsigned int i = 0; i < conditionsTree_.conditionLeafList_.size(); ++i)
    {
        if (!filterCache_->get(conditionsTree_.conditionLeafList_[i], filterBitmapTList1[i]))
        {
            filterBitmapTList1[i].reset(new InvertedIndexManager::FilterBitmapT);

            QueryFiltering::FilteringOperation filterOperation = conditionsTree_.conditionLeafList_[i].operation_;
            const std::string& property = conditionsTree_.conditionLeafList_[i].property_;
            const std::vector<PropertyValue>& filterParam = conditionsTree_.conditionLeafList_[i].values_;

            indexManagerPtr_->makeRangeQuery(filterOperation, property, filterParam, filterBitmapTList1[i]);

            filterCache_->set(conditionsTree_.conditionLeafList_[i], filterBitmapTList1[i]);
        }
    }

    std::vector<boost::shared_ptr<InvertedIndexManager::FilterBitmapT> > filterBitmapTList2;
    filterBitmapTList2.resize(conditionsTree_.conditionsNodeList_.size());
    for (unsigned int j = 0; j < conditionsTree_.conditionsNodeList_.size(); ++j)
    {
        if (!do_process_filtertree(conditionsTree_.conditionsNodeList_[j], filterBitmapTList2[j]))
            return false;
    }

    const unsigned int bitsNum = documentManagerPtr_->getMaxDocId() + 1;
    const unsigned int wordBitNum = sizeof(InvertedIndexManager::FilterWordT) << 3;
    const unsigned int wordsNum = (bitsNum - 1) / wordBitNum + 1;

    pFilterBitmap.reset(new InvertedIndexManager::FilterBitmapT);

    boost::shared_ptr<InvertedIndexManager::FilterBitmapT> dest;
    if (relation == "and")
    {
        pFilterBitmap->addStreamOfEmptyWords(true, wordsNum);
        for (unsigned int k = 0; k < filterBitmapTList1.size(); ++k)
        {
            dest.reset(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicaland(*(filterBitmapTList1[k]), *dest);
            (*pFilterBitmap).swap(*dest);
        }
        for (unsigned int k = 0; k < filterBitmapTList2.size(); ++k)
        {
            dest.reset(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicaland(*(filterBitmapTList2[k]), *dest);
            (*pFilterBitmap).swap(*dest);
        }
    }
    else if (relation == "or")
    {
        pFilterBitmap->addStreamOfEmptyWords(false, wordsNum);
        for (unsigned int k = 0; k < filterBitmapTList1.size(); ++k)
        {
            dest.reset(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicalor(*(filterBitmapTList1[k]), *dest);
            (*pFilterBitmap).swap(*dest);
        }
        for (unsigned int k = 0; k < filterBitmapTList2.size(); ++k)
        {
            dest.reset(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicalor(*(filterBitmapTList2[k]), *dest);
            (*pFilterBitmap).swap(*dest);
        }
    }
    return true;
}

bool QueryBuilder::prepare_filter(
        const ConditionsNode& conditionsTree_,
        boost::shared_ptr<InvertedIndexManager::FilterBitmapT>& pFilterBitmapx)
{
    return do_process_filtertree(conditionsTree_, pFilterBitmapx);
}
/*
void QueryBuilder::do_process_node(
    QueryFiltering::FilteringTreeValue &filteringTreeRules
    , std::stack<boost::shared_ptr<InvertedIndexManager::FilterBitmapT> >& BitMapSetStack
    , boost::shared_ptr<InvertedIndexManager::FilterBitmapT>& pFilterBitmap)
{
    std::string relation = filteringTreeRules.relation_;
    std::vector<boost::shared_ptr<InvertedIndexManager::FilterBitmapT> > pFilterBitmaplist;
    pFilterBitmaplist.resize(filteringTreeRules.childNum_);

    for (unsigned int j = 0; j < filteringTreeRules.childNum_; ++j)
    {
        pFilterBitmaplist[j] = BitMapSetStack.top();
        BitMapSetStack.pop();
    }
    //get new FitlerBitMap by the relation
    pFilterBitmap = pFilterBitmaplist[0];
    if (relation == "and")
    {
        for (unsigned int k = 1; k < pFilterBitmaplist.size(); ++k)
        {
            boost::shared_ptr<InvertedIndexManager::FilterBitmapT> dest(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicaland(*pFilterBitmaplist[k], *dest);
            pFilterBitmap = dest;
        }
    }
    else if (relation == "or")
    {
        for (unsigned int k = 1; k < pFilterBitmaplist.size(); ++k)
        {
            boost::shared_ptr<InvertedIndexManager::FilterBitmapT> dest(new InvertedIndexManager::FilterBitmapT);
            pFilterBitmap->logicalor(*pFilterBitmaplist[k], *dest);
            pFilterBitmap = dest;
        }
    }
}

void QueryBuilder::prepare_filter(
    std::vector<QueryFiltering::FilteringTreeValue> filteringTreeRules_,
    boost::shared_ptr<InvertedIndexManager::FilterBitmapT>& pFilterBitmapx)
{
    // use stack to deal with filteringTreeRules_
    std::stack<boost::shared_ptr<InvertedIndexManager::FilterBitmapT> > BitMapSetStack;
    //for buttom to top
    for (int i = filteringTreeRules_.size() - 1; i >= 0; i--)
    {
        if (filteringTreeRules_[i].isRelationNode_)
        {
            if (i != 0)
            {
                boost::shared_ptr<InvertedIndexManager::FilterBitmapT> pFilterBitmap;
                pFilterBitmap.reset(new InvertedIndexManager::FilterBitmapT);

                do_process_node(filteringTreeRules_[i]
                                , BitMapSetStack
                                , pFilterBitmap);

                BitMapSetStack.push(pFilterBitmap);
            }
            else
            {
                do_process_node(filteringTreeRules_[i]
                                , BitMapSetStack
                                , pFilterBitmapx);
            }

        }
        else
        {
            boost::shared_ptr<InvertedIndexManager::FilterBitmapT> pFilterBitmap;

            if (!filterCache_->get(filteringTreeRules_[i].fitleringType_, pFilterBitmap))
            {
                pFilterBitmap.reset(new InvertedIndexManager::FilterBitmapT);
                QueryFiltering::FilteringOperation filterOperation = filteringTreeRules_[i].fitleringType_.operation_;
                const std::string& property = filteringTreeRules_[i].fitleringType_.property_;
                const std::vector<PropertyValue>& filterParam = filteringTreeRules_[i].fitleringType_.values_;
                indexManagerPtr_->makeRangeQuery(filterOperation, property, filterParam, pFilterBitmap);
                filterCache_->set(filteringTreeRules_[i].fitleringType_, pFilterBitmap);
            }

            BitMapSetStack.push(pFilterBitmap);
        }
    }
}
*/

WANDDocumentIterator* QueryBuilder::prepare_wand_dociterator(
    const SearchKeywordOperation& actionOperation,
    collectionid_t colID,
    const property_weight_map& propertyWeightMap,
    const std::vector<std::string>& properties,
    const std::vector<unsigned int>& propertyIds,
    bool readPositions,
    const std::vector<std::map<termid_t, unsigned> >& termIndexMaps
)
{
    /*size_t size_of_properties = propertyIds.size();

    WANDDocumentIterator* pWandScorer = new WANDDocumentIterator(
        propertyWeightMap,
        propertyIds,
        properties);
    if (pIndexReader_->isDirty())
    {
        pIndexReader_ = indexManagerPtr_->getIndexReader();
    }
    try
    {
        size_t success_properties = 0;
        for (size_t i = 0; i < size_of_properties; i++)
        {
            prepare_for_wand_property_(
                pWandScorer,
                success_properties,
                actionOperation,
                colID,
                properties[i],
                propertyIds[i],
                readPositions,
                termIndexMaps[i]
            );
        }

        if (success_properties)
            return pWandScorer;
        else
            delete pWandScorer;

        return NULL;
    }
    catch(std::exception& e)
    {
        delete pWandScorer;
        throw std::runtime_error("Failed to prepare wanddociterator");
        return NULL;
    }*/
    return NULL;
}

void QueryBuilder::prepare_for_wand_property_(
    WANDDocumentIterator* pWandScorer,
    size_t & success_properties,
    const SearchKeywordOperation& actionOperation,
    collectionid_t colID,
    const std::string& property,
    unsigned int propertyId,
    bool readPositions,
    const std::map<termid_t, unsigned>& termIndexMapInProperty
)
{
    typedef std::map<termid_t, unsigned>::const_iterator const_iter;
    schema_iterator found = schemaMap_.find(property);
    if (found == schemaMap_.end())
        return;

    TermReader* pTermReader = NULL;
    pTermReader = pIndexReader_->getTermReader(colID);
    if (!pTermReader)
        return;

    const_iter iter = termIndexMapInProperty.begin();
    for ( ; iter != termIndexMapInProperty.end(); ++iter)
    {
        termid_t termId = iter->first;
        unsigned int termIndex = iter->second;
        Term term(property.c_str(),termId);
        bool find = pTermReader->seek(&term);

        if (find)
        {
            TermDocFreqs* pTermDocReader = 0;
            if (readPositions)
                pTermDocReader = pTermReader->termPositions();
            else
                pTermDocReader = pTermReader->termDocFreqs();
            if(pTermDocReader) ///NULL when exception
            {
                TermDocumentIterator* pIterator = NULL;
                pIterator =
                    new TermDocumentIterator(termId,
                                             colID,
                                             pIndexReader_,
                                             property,
                                             propertyId,
                                             termIndex,
                                             readPositions);
                pIterator->set( pTermDocReader );
                pWandScorer->add(pIterator);
                ++success_properties;
            }
        }
    }
    if (pTermReader)
        delete pTermReader;
}

MultiPropertyScorer* QueryBuilder::prepare_dociterator(
    const SearchKeywordOperation& actionOperation,
    collectionid_t colID,
    const property_weight_map& propertyWeightMap,
    const std::vector<std::string>& properties,
    const std::vector<unsigned int>& propertyIds,
    bool readPositions,
    const std::vector<std::map<termid_t, unsigned> >& termIndexMaps
)
{
    size_t size_of_properties = propertyIds.size();
    std::auto_ptr<MultiPropertyScorer> docIterPtr(new MultiPropertyScorer(propertyWeightMap, propertyIds));
    if (pIndexReader_->isDirty())
    {
        pIndexReader_ = indexManagerPtr_->getIndexReader();
    }

    size_t success_properties = 0;

    for (size_t i = 0; i < size_of_properties; i++)
    {
        schema_iterator found = schemaMap_.find(properties[i]);
        if (found == schemaMap_.end())
            continue;
        if(found->second.subProperties_.empty())
            prepare_for_property_(
                docIterPtr.get(),
                success_properties,
                actionOperation,
                colID,
                found->second,
                readPositions,
                termIndexMaps[i]
            );
        else
            prepare_for_virtual_property_(
                docIterPtr.get(),
                success_properties,
                actionOperation,
                colID,
                found->second,
                readPositions,
                termIndexMaps[i],
                propertyWeightMap
            );
    }

    if (success_properties)
        return docIterPtr.release();

    return NULL;
}

void QueryBuilder::prefetch_term_doc_readers_(
    const std::vector<pair<termid_t, string> >& termList,
    collectionid_t colID,
    const PropertyConfig& properyConfig,
    bool readPositions,
    bool& isNumericFilter,
    std::map<termid_t, std::vector<TermDocFreqs*> > & termDocReaders
)
{
    sf1r::PropertyDataType dataType;

    dataType = properyConfig.getType();
    if( properyConfig.isIndex() &&
            properyConfig.getIsFilter() &&
            properyConfig.getType() != STRING_PROPERTY_TYPE)
        isNumericFilter = true;

    const string& property = properyConfig.getName();

#if PREFETCH_TERMID
    boost::scoped_ptr<TermReader> pTermReader;
    if (!isNumericFilter)
    {
        pTermReader.reset(pIndexReader_->getTermReader(colID));
        if (!pTermReader)
            return;
    }

    for (std::vector<pair<termid_t, string> >::const_iterator it = termList.begin();
            it != termList.end(); ++it)
    {
        termid_t termId = (*it).first;
        const string& termStr = (*it).second;
        if(!isNumericFilter)
        {
            Term term(property.c_str(),termId);
            bool find = pTermReader->seek(&term);

            if (find)
            {
                TermDocFreqs* pTermDocReader = 0;
                if (readPositions)
                    pTermDocReader = pTermReader->termPositions();
                else
                    pTermDocReader = pTermReader->termDocFreqs();
                if(pTermDocReader) ///NULL when exception
                    termDocReaders[termId].push_back(pTermDocReader);
            }
        }
        else
        {
            boost::shared_ptr<InvertedIndexManager::FilterBitmapT> pFilterBitmap;
            boost::shared_ptr<Bitset> pBitset;

            if (!TermTypeDetector::isTypeMatch(termStr, dataType))
                continue;

            PropertyType value;
            PropertyValue2IndexPropertyType converter(value);
            boost::apply_visitor(converter, TermTypeDetector::propertyValue_.getVariant());
            bool find = indexManagerPtr_->seekTermFromBTreeIndex(colID, property, value);

            if (find)
            {
                QueryFiltering::FilteringType filteringRule;
                filteringRule.operation_ = QueryFiltering::EQUAL;
                filteringRule.property_ = property;
                std::vector<PropertyValue> filterParam;
                filterParam.push_back(TermTypeDetector::propertyValue_);
                filteringRule.values_ = filterParam;
                if(!filterCache_->get(filteringRule, pFilterBitmap))
                {
                    pFilterBitmap.reset(new InvertedIndexManager::FilterBitmapT);
                    pBitset.reset(new Bitset(pIndexReader_->maxDoc() + 1));

                    indexManagerPtr_->getDocsByNumericValue(colID, property, value, *pBitset);
                    pBitset->compress(*pFilterBitmap);
                    filterCache_->set(filteringRule, pFilterBitmap);
                }
                TermDocFreqs* pTermDocReader = new InvertedIndexManager::FilterTermDocFreqsT(pFilterBitmap);
                termDocReaders[termId].push_back(pTermDocReader);
            }
        }
    }
#endif
}

void QueryBuilder::prepare_for_property_(
    MultiPropertyScorer* pScorer,
    size_t & success_properties,
    const SearchKeywordOperation& actionOperation,
    collectionid_t colID,
    const PropertyConfig& properyConfig,
    bool readPositions,
    const std::map<termid_t, unsigned>& termIndexMapInProperty
)
{
    std::map<termid_t, std::vector<TermDocFreqs*> > termDocReaders;
    bool isNumericFilter = false;
    std::vector<pair<termid_t, string> > termList;
    actionOperation.getQueryTermInfoList(properyConfig.getName(), termList);
    std::sort(termList.begin(), termList.end());
    prefetch_term_doc_readers_(termList,colID,properyConfig,readPositions,isNumericFilter,termDocReaders);
    sf1r::PropertyDataType dataType = properyConfig.getType();
    const string& property = properyConfig.getName();
    unsigned propertyId = properyConfig.getPropertyId();
    std::map<termid_t, VirtualPropertyTermDocumentIterator* > virtualTermIters;
    DocumentIterator* pIter = NULL;
    QueryTreePtr queryTree;
    if ( actionOperation.getQueryTree(property, queryTree) )
        do_prepare_for_property_(
            queryTree,
            colID,
            property,
            propertyId,
            dataType,
            isNumericFilter,
            readPositions,
            termIndexMapInProperty,
            pIter,
            virtualTermIters,
            termDocReaders,
            actionOperation.hasUnigramProperty_,
            actionOperation.isUnigramSearchMode_
        );

    if (pIter)
    {
        pScorer->add(propertyId, pIter, termDocReaders);
        ++success_properties;
    }
    else
    {
        for (std::map<termid_t, std::vector<TermDocFreqs*> >::iterator
                it = termDocReaders.begin(); it != termDocReaders.end(); ++it)
        {
            for (size_t j =0; j<it->second.size(); j ++ )
            {
                delete it->second[j];
            }
            it->second.clear();
        }
    }
}

void QueryBuilder::post_prepare_ranker_(
    const std::string& virtualProperty,
    const std::vector<std::string>& indexPropertyList,
    unsigned indexPropertySize,
    const property_term_info_map& propertyTermInfoMap,
    DocumentFrequencyInProperties& dfmap,
    CollectionTermFrequencyInProperties& ctfmap,
    MaxTermFrequencyInProperties& maxtfmap,
    bool readTermPosition,
    std::vector<RankQueryProperty>& rankQueryProperties,
    std::vector<boost::shared_ptr<PropertyRanker> >& propertyRankers)
{
    static const PropertyTermInfo emptyPropertyTermInfo;
    for (unsigned i = 0; i < indexPropertySize; ++i)
    {
        const std::string& currentProperty = indexPropertyList[i];
        schema_iterator found = schemaMap_.find(currentProperty);
        if (found == schemaMap_.end())
            continue;
        if (found->second.subProperties_.empty())
        {

            rankQueryProperties[i].setNumDocs(
                indexManagerPtr_->numDocs()
            );

            rankQueryProperties[i].setTotalPropertyLength(
                documentManagerPtr_->getTotalPropertyLength(currentProperty)
            );
            const PropertyTermInfo::id_uint_list_map_t& termPositionsMap = virtualProperty.empty()?
                    izenelib::util::getOr(
                        propertyTermInfoMap,
                        currentProperty,
                        emptyPropertyTermInfo
                    ).getTermIdPositionMap()
                    :
                    izenelib::util::getOr(
                        propertyTermInfoMap,
                        virtualProperty,
                        emptyPropertyTermInfo
                    ).getTermIdPositionMap()
                    ;

            typedef PropertyTermInfo::id_uint_list_map_t::const_iterator
            term_id_position_iterator;
            unsigned queryLength = 0;
            unsigned index = 0;
            for (term_id_position_iterator termIt = termPositionsMap.begin();
                    termIt != termPositionsMap.end(); ++termIt, ++index)
            {
                rankQueryProperties[i].addTerm(termIt->first);
                rankQueryProperties[i].setTotalTermFreq(
                    ctfmap[currentProperty][termIt->first]
                );
                rankQueryProperties[i].setDocumentFreq(
                    dfmap[currentProperty][termIt->first]
                );

                rankQueryProperties[i].setMaxTermFreq(
                    maxtfmap[currentProperty][termIt->first]
                );

                queryLength += termIt->second.size();
                if (readTermPosition)
                {
                    typedef PropertyTermInfo::id_uint_list_map_t::mapped_type
                    uint_list_map_t;
                    typedef uint_list_map_t::const_iterator uint_list_iterator;
                    for (uint_list_iterator posIt = termIt->second.begin();
                            posIt != termIt->second.end(); ++posIt)
                    {
                        rankQueryProperties[i].pushPosition(*posIt);
                    }
                }
                else
                {
                    rankQueryProperties[i].setTermFreq(termIt->second.size());
                }
            }

            rankQueryProperties[i].setQueryLength(queryLength);
        }
        else
        {
            rankQueryProperties[i].setNumDocs(
                indexManagerPtr_->numDocs());
///cout<<"indexManagerPtr_->numDocs()"<<indexManagerPtr_->numDocs()<<endl;
            const PropertyTermInfo::id_uint_list_map_t& termPositionsMap = izenelib::util::getOr(
                        propertyTermInfoMap,
                        currentProperty,
                        emptyPropertyTermInfo
                    ).getTermIdPositionMap();
            uint32_t DocumentFreq = 0;
            uint32_t MaxTermFreq = 0;
            uint32_t index = 0;
            uint32_t queryLength = 0;
            uint32_t ctf = 0;
            uint32_t TotalPropertyLength = 0;
            typedef PropertyTermInfo::id_uint_list_map_t::const_iterator
            term_id_position_iterator;
            for (term_id_position_iterator termIt = termPositionsMap.begin();
                termIt != termPositionsMap.end(); ++termIt)
            {
                rankQueryProperties[i].addTerm(termIt->first);
                queryLength += termIt->second.size();
                for (uint32_t j = 0; j < found->second.subProperties_.size(); ++j, ++index)
                {
                    uint32_t dftmp = dfmap[found->second.subProperties_[index]][termIt->first];// subString and terid;
                    uint32_t ctftmp = ctfmap[found->second.subProperties_[index]][termIt->first];
                    uint32_t maxtf =  maxtfmap[found->second.subProperties_[index]][termIt->first];
                    DocumentFreq += dftmp;
                    ctf += ctftmp;
                    if ( maxtf > MaxTermFreq)
                    {
                        MaxTermFreq = maxtf;
                    }
                    if (readTermPosition)
                    {
                        /**
                        code
                        */
                    }
                    if (termIt == termPositionsMap.begin())
                    {
                        TotalPropertyLength += documentManagerPtr_->getTotalPropertyLength(found->second.subProperties_[j]);
                    }
                }
                rankQueryProperties[i].setTermFreq(termIt->second.size());
///cout<<"MaxTermFreq"<<MaxTermFreq<<endl;
///cout<<"DocumentFreq"<<DocumentFreq<<endl;
///cout<<"ctf"<<ctf<<endl;//该属性所有包含该TermID的频率值;
                rankQueryProperties[i].setMaxTermFreq(MaxTermFreq);
                MaxTermFreq = 0;
                rankQueryProperties[i].setDocumentFreq(DocumentFreq);
                DocumentFreq = 0;
                rankQueryProperties[i].setTotalTermFreq(ctf);
                ctf = 0;
                index = 0;
            }

            rankQueryProperties[i].setTotalPropertyLength(TotalPropertyLength);
            TotalPropertyLength = 0;
///cout<<"virtual queryLength"<<queryLength<<endl;
            rankQueryProperties[i].setQueryLength(queryLength);
            queryLength = 0;
        }
    }

    for (size_t i = 0; i < indexPropertySize; ++i )
    {
        propertyRankers[i]->setupStats(rankQueryProperties[i]);
    }
}

void QueryBuilder::prepare_for_virtual_property_(
    MultiPropertyScorer* pScorer,
    size_t & success_properties,
    const SearchKeywordOperation& actionOperation,
    collectionid_t colID,
    const PropertyConfig& properyConfig,
    bool readPositions,
    const std::map<termid_t, unsigned>& termIndexMap,
    const property_weight_map& propertyWeightMap
    )
{
    LOG(INFO)<<"start prepare_for_virtual_property_"<<endl;
    QueryTreePtr queryTree;
    if (! actionOperation.getQueryTree(properyConfig.getName(), queryTree) )
        return;
    DocumentIterator* pIter = NULL;

    std::vector<pair<termid_t, string> > termList;
    actionOperation.getQueryTermInfoList(properyConfig.getName(), termList);
    std::sort(termList.begin(), termList.end());
    std::vector<std::map<termid_t, std::vector<izenelib::ir::indexmanager::TermDocFreqs*> > > termDocReadersList;
    termDocReadersList.resize(properyConfig.subProperties_.size());

    std::vector<std::string> properties;
    std::vector<unsigned int> propertyIds;
    std::vector<PropertyDataType> propertyDataTypes;
    bool isNumericFilter = false;

    for(unsigned j = 0; j < properyConfig.subProperties_.size(); ++j)
    {
        schema_iterator p = schemaMap_.find(properyConfig.subProperties_[j]);
        prefetch_term_doc_readers_(termList, colID, p->second, readPositions, isNumericFilter, termDocReadersList[j]);
        properties.push_back(p->second.getName());
        propertyIds.push_back(p->second.getPropertyId());
        propertyDataTypes.push_back(p->second.getType());
    }

    bool ret = do_prepare_for_virtual_property_(
                queryTree,
                colID,
                properties,
                propertyIds,
                propertyDataTypes,
                isNumericFilter,
                readPositions,
                termIndexMap,
                pIter,
                termDocReadersList,
                actionOperation.hasUnigramProperty_,
                actionOperation.isUnigramSearchMode_
            );

    if (!ret)
    {
        for(unsigned j = 0; j < properyConfig.subProperties_.size(); ++j)
        {
            for (std::map<termid_t, std::vector<TermDocFreqs*> >::iterator
                    it = termDocReadersList[j].begin(); it != termDocReadersList[j].end(); ++it)
            {
                for (size_t i =0; i < it->second.size(); ++i )
                {
                    delete it->second[i];
                }
                it->second.clear();
            }
        }
        return;
    }

    if (pIter)
    {
        pScorer->add(properyConfig.getPropertyId(), pIter);
        pScorer->add(termDocReadersList);
        success_properties++;
    }
    else
    {
        for(unsigned j = 0; j < properyConfig.subProperties_.size(); ++j)
        {
            for (std::map<termid_t, std::vector<TermDocFreqs*> >::iterator
                    it = termDocReadersList[j].begin(); it != termDocReadersList[j].end(); ++it)
            {
                for (size_t i =0; i < it->second.size(); ++i )
                {
                    delete it->second[i];
                }
                it->second.clear();
            }
        }
    }
}

bool QueryBuilder::do_prepare_for_virtual_property_(
    QueryTreePtr& queryTree,
    collectionid_t colID,
    const std::vector<std::string>& properties,
    std::vector<unsigned int>& propertyIds,
    std::vector<PropertyDataType>& propertyDataTypes,
    bool isNumericFilter,
    bool isReadPosition,
    const std::map<termid_t, unsigned>& termIndexMap,
    DocumentIteratorPointer& pDocIterator,
    std::vector<std::map<termid_t, std::vector<izenelib::ir::indexmanager::TermDocFreqs*> > >& termDocReadersList,
    bool hasUnigramProperty,
    bool isUnigramSearchMode,
    int parentAndOrFlag
)
{
    switch (queryTree->type_)
    {
    case QueryTree::KEYWORD:
    {
        string keyword = queryTree->keyword_;
        termid_t keywordId = queryTree->keywordId_;

        unsigned termIndex = izenelib::util::getOr(
                                     termIndexMap,
                                     keywordId,
                                     (std::numeric_limits<unsigned>::max) ()
                                 );

        VirtualTermDocumentIterator* pVirtualTermDocIter = new VirtualTermDocumentIterator(
            keywordId,
            keyword,
            colID,
            pIndexReader_,
            properties,
            propertyIds,
            propertyDataTypes,
            termIndex,
            isReadPosition
        );
        DocumentIterator* pIterator = new ORDocumentIterator();
        try
        {
            for( uint32_t i = 0; i < properties.size(); ++i)
            {
                TermDocumentIterator* pTermIterator = NULL;

                if (!isUnigramSearchMode)
                {
                    pTermIterator = new TermDocumentIterator(
                        keywordId,
                        keyword,
                        colID,
                        pIndexReader_,
                        indexManagerPtr_,
                        properties[i],
                        propertyIds[i],
                        propertyDataTypes[i],
                        isNumericFilter,
                        termIndex,
                        isReadPosition);
                }
                else
                {
                    if (queryTree->type_ == QueryTree::KEYWORD)
                    {
                        pTermIterator = new SearchTermDocumentIterator(keywordId,
                                colID,
                                pIndexReader_,
                                properties[i],
                                propertyIds[i],
                                termIndex,
                                isReadPosition);
                    }
                }

                std::map<termid_t, std::vector<TermDocFreqs*> >::iterator constIt
                = termDocReadersList[i].find(keywordId);

                if (constIt != termDocReadersList[i].end() && !constIt->second.empty() )
                {
                    pVirtualTermDocIter->set(constIt->second.back() );
                    pTermIterator->set(constIt->second.back() );
                    if(pIterator == NULL)
                        pIterator = pTermIterator;
                    else
                        pIterator->add(pTermIterator);

                    constIt->second.pop_back();
                }
                else
                {
                    delete pTermIterator;
                    pTermIterator = NULL;
                    pVirtualTermDocIter->set(0);
                }
            }

            pVirtualTermDocIter->add(pIterator);
            if (pDocIterator == NULL)
            {
                pDocIterator = pVirtualTermDocIter;
            }
            else
                pDocIterator->add(pVirtualTermDocIter);
        }
        catch(std::exception& e)
        {
            LOG (ERROR) << "exception in virtual keyword search...";
            delete pIterator;
            return false;
        }
        return true;
        break;
    }

    case QueryTree::AND:
    {
        DocumentIterator* pIterator = new ANDDocumentIterator();
        try
        {
            for (QTIter andChildIter = queryTree->children_.begin();
                    andChildIter != queryTree->children_.end(); ++andChildIter)
            {
                bool ret = do_prepare_for_virtual_property_(
                            *andChildIter,
                            colID,
                            properties,
                            propertyIds,
                            propertyDataTypes,
                            isNumericFilter,
                            isReadPosition,
                            termIndexMap,
                            pIterator,
                            termDocReadersList,
                            hasUnigramProperty,
                            isUnigramSearchMode,
                            1
                            );
                if (!ret)
                {
                    delete pIterator;
                    return false;
                }
            }
            if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);

        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    }

    case QueryTree::NOT:
    {
        if ( queryTree->children_.size() != 1 )
            return false;
        if (NULL == pDocIterator)
            ///only NOT is not permitted
            return false;
        try
        {
            QTIter notChildIter = queryTree->children_.begin();// more is a xunhuan...
            if (notChildIter == queryTree->children_.end())
                return false;
            DocumentIterator* pIterator = NULL;
            if ((*notChildIter)->type_ == QueryTree::NOT)
            {
                return false;
            }// do not support: NOT NOT like !(!())
            do_prepare_for_virtual_property_(
                            *notChildIter,
                            colID,
                            properties,
                            propertyIds,
                            propertyDataTypes,
                            isNumericFilter,
                            isReadPosition,
                            termIndexMap,
                            pIterator,
                            termDocReadersList,
                            hasUnigramProperty,
                            isUnigramSearchMode,
                            1
                        );
            if (pIterator)
            {
                pIterator->setNot(true);
                pDocIterator->add(pIterator);
            }
        }
        catch(std::exception& e)
        {
            //delete pIterator;
            return true;
        }
        break;
    }

    case QueryTree::OR:
    {
        DocumentIterator* pIterator = new ORDocumentIterator();
        try
        {
            for (QTIter andChildIter = queryTree->children_.begin();
                    andChildIter != queryTree->children_.end(); ++andChildIter)
            {
                bool ret = do_prepare_for_virtual_property_(
                            *andChildIter,
                            colID,
                            properties,
                            propertyIds,
                            propertyDataTypes,
                            isNumericFilter,
                            isReadPosition,
                            termIndexMap,
                            pIterator,
                            termDocReadersList,
                            hasUnigramProperty,
                            isUnigramSearchMode,
                            1
                            );
                if (!ret)
                {
                    delete pIterator;
                    return false;
                }
            }
            //parentAndorFlag
            if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    }

    default:
        break;
    }
    return true;
}

bool QueryBuilder::do_prepare_for_property_(
    QueryTreePtr& queryTree,
    collectionid_t colID,
    const std::string& property,
    unsigned int propertyId,
    PropertyDataType propertyDataType,
    bool isNumericFilter,
    bool readPositions,
    const std::map<termid_t, unsigned>& termIndexMapInProperty,
    DocumentIteratorPointer& pDocIterator,
    std::map<termid_t, VirtualPropertyTermDocumentIterator* >& virtualTermIters,
    std::map<termid_t, std::vector<izenelib::ir::indexmanager::TermDocFreqs*> >& termDocReaders,
    bool hasUnigramProperty,
    bool isUnigramSearchMode,
    const std::string& virtualProperty,
    int parentAndOrFlag
)
{
    switch (queryTree->type_)
    {
    case QueryTree::KEYWORD:
    case QueryTree::RANK_KEYWORD:
    {
        string keyword = queryTree->keyword_;
        termid_t keywordId = queryTree->keywordId_;
        unsigned termIndex = izenelib::util::getOr(
                                 termIndexMapInProperty,
                                 keywordId,
                                 (std::numeric_limits<unsigned>::max) ()
                             );

        TermDocumentIterator* pIterator = NULL;
        try
        {
            if (!isUnigramSearchMode)
            {
                pIterator = new TermDocumentIterator(
                    keywordId,
                    keyword,
                    colID,
                    pIndexReader_,
                    indexManagerPtr_,
                    property,
                    propertyId,
                    propertyDataType,
                    isNumericFilter,
                    termIndex,
                    readPositions);
            }
            else
            {
                // term for searching
                if (queryTree->type_ == QueryTree::KEYWORD)
                {
                    // termIndex is invalid in this case, it will not be used
                    pIterator = new SearchTermDocumentIterator(keywordId,
                            colID,
                            pIndexReader_,
                            property,
                            propertyId,
                            termIndex,
                            readPositions);
                }
                // term for ranking
                else //if(queryTree->type_ == QueryTree::RANK_KEYWORD)
                {
                    pIterator = new RankTermDocumentIterator(keywordId,
                            colID,
                            pIndexReader_,
                            property,
                            propertyId,
                            termIndex,
                            readPositions,
                            (parentAndOrFlag == 1));
                }
            }

#if PREFETCH_TERMID
            std::map<termid_t, std::vector<TermDocFreqs*> >::iterator constIt
            = termDocReaders.find(keywordId);
            if (constIt != termDocReaders.end() && !constIt->second.empty() )
            {
#ifdef VERBOSE_SERACH_MANAGER
                cout<<"have term  property "<<property<<" keyword ";
                queryTree->keywordUString_.displayStringValue(izenelib::util::UString::UTF_8);
                cout<<endl;
#endif
                pIterator->set(constIt->second.back() );
                if(virtualProperty.empty())
                {
                    ///General keyword search
                    if (NULL == pDocIterator)
                        pDocIterator = pIterator;
                    else
                        pDocIterator->add(pIterator);
                }
                else
                {
                    ///Keyword search over virtual property
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vpIt =
                        virtualTermIters.find(keywordId);
                    if(vpIt != virtualTermIters.end())
                    {
                        vpIt->second->add(pIterator);
                    }
                    else
                    {
                        VirtualPropertyTermDocumentIterator* pVirtualTermDocIter =
                            new VirtualPropertyTermDocumentIterator(virtualProperty);
                        pVirtualTermDocIter->add(pIterator);
                        virtualTermIters[keywordId] = pVirtualTermDocIter;
                        if (NULL == pDocIterator)
                            pDocIterator = pVirtualTermDocIter;
                    }
                }

                constIt->second.pop_back();
            }
            else
#endif
            {
                if (pIterator->accept())
                {
#ifdef VERBOSE_SERACH_MANAGER
                    cout<<"have term  property "<<property<<" keyword ";
                    queryTree->keywordUString_.displayStringValue(izenelib::util::UString::UTF_8);
                    cout<<endl;
#endif
                    if(virtualProperty.empty())
                    {
                        ///General keyword search
                        if (NULL == pDocIterator)
                            pDocIterator = pIterator;
                        else
                            pDocIterator->add(pIterator);
                    }
                    else
                    {
                        ///Keyword search over virtual property
                        std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vpIt =
                            virtualTermIters.find(keywordId);
                        if(vpIt != virtualTermIters.end())
                        {
                            vpIt->second->add(pIterator);
                        }
                        else
                        {
                            VirtualPropertyTermDocumentIterator* pVirtualTermDocIter =
                                new VirtualPropertyTermDocumentIterator(virtualProperty);
                            pVirtualTermDocIter->add(pIterator);
                            virtualTermIters[keywordId] = pVirtualTermDocIter;
                            if (NULL == pDocIterator)
                                pDocIterator = pVirtualTermDocIter;
                        }
                    }

                }
                else
                {
#ifdef VERBOSE_SERACH_MANAGER
                    cout<<"do not have term property "<<property<<" ";
                    queryTree->keywordUString_.displayStringValue(izenelib::util::UString::UTF_8);
                    cout<<endl;
#endif
                    delete pIterator;
                    pIterator = NULL;
                    if (!isNumericFilter)
                        return false;
                }
            }
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        return true;
        break;
    } // end - QueryTree::KEYWORD
    case QueryTree::NOT:
    {
        if ( queryTree->children_.size() != 1 )
            return false;
        if (NULL == pDocIterator)
            ///only NOT is not permitted
            return false;
        try
        {
            QTIter notChildIter = queryTree->children_.begin();// more is a xunhuan...
            if (notChildIter == queryTree->children_.end())
                return false;
            DocumentIterator* pIterator = NULL;
            if ((*notChildIter)->type_ == QueryTree::NOT)
            {
                return false;
            }
            do_prepare_for_property_(
                          *notChildIter,
                          colID,
                          property,
                          propertyId,
                          propertyDataType,
                          isNumericFilter,
                          readPositions,
                          termIndexMapInProperty,
                          pIterator,
                          virtualTermIters,
                          termDocReaders,
                          hasUnigramProperty,
                          isUnigramSearchMode,
                          virtualProperty,
                          1
                      );
            if (pIterator)
            {
                pIterator->setNot(true);
                pDocIterator->add(pIterator);
            }
        }
        catch(std::exception& e)
        {
            //delete pIterator;
            return true;
        }
        break;
    } // end QueryTree::NOT
    case QueryTree::UNIGRAM_WILDCARD:
    {
        vector<termid_t> termIds;
        vector<unsigned> termIndexes;
        vector<size_t> asterisk_pos;
        vector<size_t> question_mark_pos;
        for (std::list<QueryTreePtr>::iterator childIter = queryTree->children_.begin();
                childIter != queryTree->children_.end(); ++childIter)
        {
            if ((*childIter)->type_ == QueryTree::ASTERISK)
            {
                asterisk_pos.push_back(termIds.size());
            }
            else if ((*childIter)->type_ == QueryTree::QUESTION_MARK)
            {
                question_mark_pos.push_back(termIds.size());
            }
            else
            {
                termid_t keywordId((*childIter)->keywordId_);
                unsigned termIndex = izenelib::util::getOr(
                                         termIndexMapInProperty,
                                         keywordId,
                                         std::numeric_limits<unsigned>::max()
                                     );
                termIds.push_back(keywordId);
                termIndexes.push_back(termIndex);
            }
        }

        // if there is no unigram alias-ed property, perform on original property
        std::string unigramProperty = property;
        if (hasUnigramProperty)
        {
            unigramProperty += "_unigram";
        }
        unsigned int unigramPropertyId = 0;
        schema_iterator found = schemaMap_.find(unigramProperty);
        if (found != schemaMap_.end())
        {
            unigramPropertyId = found->second.getPropertyId();
        }
        WildcardPhraseDocumentIterator* pIterator =
            new WildcardPhraseDocumentIterator(
            termIds,
            termIndexes,
            asterisk_pos,
            question_mark_pos,
            colID,
            pIndexReader_,
            unigramProperty,
            unigramPropertyId,
            documentManagerPtr_
        );
        if(NULL == pDocIterator)
            pDocIterator = pIterator;
        else
            pDocIterator->add(pIterator);

        break;
    } // end - QueryTree::UNIGRAM_WILDCARD
    case QueryTree::TRIE_WILDCARD:
    {
        if (queryTree->children_.size() == 0)
            return false;

        //5 top frequent terms
        static const int kWildcardTermThreshold = 5;
        WildcardDocumentIterator* pWildCardDocIter =
            new WildcardDocumentIterator(
            colID,
            pIndexReader_,
            property,
            propertyId,
            readPositions,
            kWildcardTermThreshold
        );
        for (QTIter iter = queryTree->children_.begin();
                iter != queryTree->children_.end(); ++iter)
        {
            termid_t termId = (*iter)->keywordId_;
            unsigned termIndex = izenelib::util::getOr(
                                     termIndexMapInProperty,
                                     termId,
                                     (std::numeric_limits<unsigned>::max) ()
                                 );

            pWildCardDocIter->add(termId, termIndex, termDocReaders);
        }
        if (0 == pWildCardDocIter->numIterators())
        {
            delete pWildCardDocIter;
            return false;
        }
        if(NULL == pDocIterator)
            pDocIterator = pWildCardDocIter;
        else
            pDocIterator->add(pWildCardDocIter);
        break;
    } // end - QueryTree::TRIE_WILDCARD
    case QueryTree::AND:
    {
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"AND query "<<property<<endl;
#endif
        DocumentIterator* pIterator = new ANDDocumentIterator();
        pIterator->setMissRate(queryTree->children_.size());
        bool ret = false;
        try
        {
            for (QTIter andChildIter = queryTree->children_.begin();
                    andChildIter != queryTree->children_.end(); ++andChildIter)
            {
                ret = do_prepare_for_property_(
                          *andChildIter,
                          colID,
                          property,
                          propertyId,
                          propertyDataType,
                          isNumericFilter,
                          readPositions,
                          termIndexMapInProperty,
                          pIterator,
                          virtualTermIters,
                          termDocReaders,
                          hasUnigramProperty,
                          isUnigramSearchMode,
                          virtualProperty,
                          1
                      );
                if (virtualProperty.empty()&&!ret)
                {
                    delete pIterator;
                    return false;
                }
            }
            if(!virtualProperty.empty())
            {
                if(!parentAndOrFlag)
                {
                    if(NULL == pDocIterator)
                    {
                        pDocIterator = pIterator;
                    }
                    else delete pIterator;
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vit = virtualTermIters.begin();
                    for(; vit != virtualTermIters.end(); ++vit)
                        pDocIterator->add(vit->second);
                }
            }
            else
                ///General AND query
                if (NULL == pDocIterator)
                    pDocIterator = pIterator;
                else
                    pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    } // end - QueryTree::AND
    case QueryTree::WAND:
    {
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"WAND query "<<property<<endl;
#endif
        DocumentIterator* pIterator = new WANDDocumentIterator();
        pIterator->setMissRate(queryTree->children_.size());
        bool ret = false;
        try
        {
            for (QTIter wandChildIter = queryTree->children_.begin();
                    wandChildIter != queryTree->children_.end(); ++wandChildIter)
            {
                ret |= do_prepare_for_property_(
                           *wandChildIter,
                           colID,
                           property,
                           propertyId,
                           propertyDataType,
                           isNumericFilter,
                           readPositions,
                           termIndexMapInProperty,
                           pIterator,
                           virtualTermIters,
                           termDocReaders,
                           hasUnigramProperty,
                           isUnigramSearchMode,
                           virtualProperty,
                           1
                       );
            }
            if (!ret)
            {
                delete pIterator;
                return false;
            }

            if(!virtualProperty.empty())
            {
                if(!parentAndOrFlag)
                {
                    if(NULL == pDocIterator)
                    {
                        pDocIterator = pIterator;
                    }
                    else delete pIterator;
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vit = virtualTermIters.begin();
                    for(; vit != virtualTermIters.end(); ++vit)
                        pDocIterator->add(vit->second);
                }
            }
            else if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    } // end - QueryTree::WAND
    case QueryTree::OR:
    {
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"OR query "<<property<<endl;
#endif
        DocumentIterator* pIterator = new ORDocumentIterator();
        pIterator->setMissRate(queryTree->children_.size());
        bool ret = false;
        try
        {
            for (QTIter orChildIter = queryTree->children_.begin();
                    orChildIter != queryTree->children_.end(); ++orChildIter)
            {
                ret |= do_prepare_for_property_(
                           *orChildIter,
                           colID,
                           property,
                           propertyId,
                           propertyDataType,
                           isNumericFilter,
                           readPositions,
                           termIndexMapInProperty,
                           pIterator,
                           virtualTermIters,
                           termDocReaders,
                           hasUnigramProperty,
                           isUnigramSearchMode,
                           virtualProperty,
                           1
                       );
            }
            if (!ret)
            {
                delete pIterator;
                return false;
            }

            if(!virtualProperty.empty())
            {
                if(!parentAndOrFlag)
                {
                    if(NULL == pDocIterator)
                    {
                        pDocIterator = pIterator;
                    }
                    else delete pIterator;
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vit = virtualTermIters.begin();
                    for(; vit != virtualTermIters.end(); ++vit)
                        pDocIterator->add(vit->second);
                }
            }
            else if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    } // end - QueryTree::OR
    case QueryTree::AND_PERSONAL:
    {
        /// always return true in this case,
        /// it's tolerable that personal items are not indexed.
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"AND-Personal query "<<property<<endl;
#endif
        DocumentIterator* pIterator = new PersonalSearchDocumentIterator(true);
        bool ret = false;
        try
        {
            for (QTIter andChildIter = queryTree->children_.begin();
                    andChildIter != queryTree->children_.end(); ++andChildIter)
            {
                ret = do_prepare_for_property_(
                          *andChildIter,
                          colID,
                          property,
                          propertyId,
                          propertyDataType,
                          isNumericFilter,
                          readPositions,
                          termIndexMapInProperty,
                          pIterator,
                          virtualTermIters,
                          termDocReaders,
                          hasUnigramProperty,
                          isUnigramSearchMode,
                          virtualProperty
                      );
                if (virtualProperty.empty()&&!ret)
                {
                    delete pIterator;
                    ///return false;
                    return true;
                }
            }
            if(!virtualProperty.empty())
            {
                if(!parentAndOrFlag)
                {
                    if(NULL == pDocIterator)
                    {
                        pDocIterator = pIterator;
                    }
                    else delete pIterator;
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vit = virtualTermIters.begin();
                    for(; vit != virtualTermIters.end(); ++vit)
                        pDocIterator->add(vit->second);
                }
            }
            else if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    } // end - QueryTree::AND_PERSONAL
    case QueryTree::OR_PERSONAL:
    {
        /// always return true in this case,
        /// it's tolerable that personal items are not indexed.
        /// OR_PERSONAL iterator is an AND semantic, but a child of OR.
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"OR-Personal query "<<property<<endl;
#endif
        DocumentIterator* pIterator = new PersonalSearchDocumentIterator(false);
        bool ret = false;
        try
        {
            for (QTIter orChildIter = queryTree->children_.begin();
                    orChildIter != queryTree->children_.end(); ++orChildIter)
            {
                ret = do_prepare_for_property_(
                          *orChildIter,
                          colID,
                          property,
                          propertyId,
                          propertyDataType,
                          isNumericFilter,
                          readPositions,
                          termIndexMapInProperty,
                          pIterator,
                          virtualTermIters,
                          termDocReaders,
                          hasUnigramProperty,
                          isUnigramSearchMode,
                          virtualProperty
                      );
                if (!ret)
                {
                    delete pIterator;
                    ///return false;
                    return true;
                }
            }
            if(!virtualProperty.empty())
            {
                if(!parentAndOrFlag)
                {
                    if(NULL == pDocIterator)
                    {
                        pDocIterator = pIterator;
                    }
                    else delete pIterator;
                    std::map<termid_t, VirtualPropertyTermDocumentIterator* >::iterator vit = virtualTermIters.begin();
                    for(; vit != virtualTermIters.end(); ++vit)
                        pDocIterator->add(vit->second);
                }
            }
            else if (NULL == pDocIterator)
                pDocIterator = pIterator;
            else
                pDocIterator->add(pIterator);
        }
        catch(std::exception& e)
        {
            delete pIterator;
            return false;
        }
        break;
    } // end - QueryTree::OR_PERSONAL
    case QueryTree::EXACT:
    {
#ifdef VERBOSE_SERACH_MANAGER
        cout<<"EXACT query "<<property<<endl;
#endif

        vector<termid_t> termIds;
        vector<unsigned> termIndexes;
        getTermIdsAndIndexesOfSiblings(
            queryTree,
            termIndexMapInProperty,
            property,
            termIds,
            termIndexes
        );

        // if there is no unigram alias-ed property, perform on original property
        std::string unigramProperty = property;
        if (hasUnigramProperty)
        {
            unigramProperty += "_unigram";
        }
        unsigned int unigramPropertyId = 0;
        schema_iterator found = schemaMap_.find(unigramProperty);
        if (found != schemaMap_.end())
        {
            unigramPropertyId = found->second.getPropertyId();
        }
        ExactPhraseDocumentIterator* pIterator =
            new ExactPhraseDocumentIterator(
            termIds,
            termIndexes,
            colID,
            pIndexReader_,
            unigramProperty,
            unigramPropertyId
        );
        pIterator->setOrigProperty(property);
        if (NULL == pDocIterator)
            pDocIterator = pIterator;
        else
            pDocIterator->add(pIterator);

        break;
    }  // end - QueryTree::EXACT
    case QueryTree::NEARBY:
    {
        vector<termid_t> termIds;
        vector<unsigned> termIndexes;
        getTermIdsAndIndexesOfSiblings(
            queryTree,
            termIndexMapInProperty,
            property,
            termIds,
            termIndexes
        );

        NearbyPhraseDocumentIterator* pIterator =
            new NearbyPhraseDocumentIterator(
            termIds,
            termIndexes,
            colID,
            pIndexReader_,
            property,
            propertyId,
            queryTree->distance_
        );

        if (NULL == pDocIterator)
            pDocIterator = pIterator;
        else
            pDocIterator->add(pIterator);
        break;
    } // end - QueryTree::NEARBY
    case QueryTree::ORDER:
    {
        vector<termid_t> termIds;
        vector<unsigned> termIndexes;
        getTermIdsAndIndexesOfSiblings(
            queryTree,
            termIndexMapInProperty,
            property,
            termIds,
            termIndexes
        );

        OrderedPhraseDocumentIterator* pIterator =
            new OrderedPhraseDocumentIterator(
            termIds,
            termIndexes,
            colID,
            pIndexReader_,
            property,
            propertyId
        );
        if (NULL == pDocIterator)
            pDocIterator = pIterator;
        else
            pDocIterator->add(pIterator);
        break;
    } // end - QueryTree::ORDER
    default:
        break;
    } // end - switch

    return true;
}

void QueryBuilder::getTermIdsAndIndexesOfSiblings(
    QueryTreePtr& queryTree,
    const std::map<termid_t, unsigned>& termIndexMapInProperty,
    const std::string& property,
    std::vector<termid_t>& outTermIds,
    std::vector<unsigned>& outTermIndexes
)
{
    std::vector<termid_t> termIds;
    std::vector<unsigned> termIndexes;

    for (QTIter childIter = queryTree->children_.begin();
            childIter != queryTree->children_.end(); ++childIter)
    {
        termid_t keywordId( (*childIter)->keywordId_ );
        unsigned termIndex = izenelib::util::getOr(
                                 termIndexMapInProperty,
                                 keywordId,
                                 (std::numeric_limits<unsigned>::max) ()
                             );

        termIds.push_back(keywordId);
        termIndexes.push_back(termIndex);
    }

    termIds.swap(outTermIds);
    termIndexes.swap(outTermIndexes);
}

propertyid_t QueryBuilder::getPropertyIdByName(const std::string& name) const
{
    schema_iterator found = schemaMap_.find(name);
    if (found != schemaMap_.end())
    {
        return found->second.getPropertyId();
    }
    else
    {
        return 0;
    }
}

} // namespace sf1r


text:
/*
 * ConvertedPdfMap.hpp
 *
 *  Created on: 20/apr/2013
 *      Author: alessandro
 */

#ifndef CONVERTEDPDFMAP_HPP_
#define CONVERTEDPDFMAP_HPP_

#include <map>
#include <boost/serialization/map.hpp>

using namespace std;

/*
 * Class ConvertedPdfMap
 *
 * Indicates which pdf is already converted, with number of pages.
 * Implements the serialize function for boost serialization method of std::map<KEY, VALUE>
 * KEY is the pdf filename (without path)
 * VALUE is the pdf path
 */
class ConvertedPdfMap {
private:
	map<string, pair<string,int>> convertedPdfMap;

public:
	typedef map<string, pair<string,int>>::const_iterator const_iterator;
	typedef map<string, pair<string,int>>::iterator iterator;

	ConvertedPdfMap() {
	}

	/*
	 * Capacity
	 */
	size_t size() const {
		return convertedPdfMap.size();
	}

	bool empty() const {
		return convertedPdfMap.empty();
	}

	/*
	 * Iterators
	 */
	const_iterator find(const string& __x) const {
		return convertedPdfMap.find(__x);
	}

	iterator begin() {
		return convertedPdfMap.begin();
	}

	const_iterator begin() const {
		return convertedPdfMap.begin();
	}

	const_iterator end() const {
		return convertedPdfMap.end();
	}

	/*
	 * Operators
	 */
	pair<string,int>&
	operator[](const string& __k) {
		return convertedPdfMap[__k];
	}

	/*
	 * Modifiers
	 */
	pair<iterator, bool> insert(const pair<string, pair<string,int>>& __pair) {
		return convertedPdfMap.insert(__pair);
	}

	/*
	 * Boost Serialization method
	 */
	template<class Archive>
	void serialize(Archive & ar, const unsigned int version) {
		ar & BOOST_SERIALIZATION_NVP(convertedPdfMap);
	}
};

#endif /* CONVERTEDPDFMAP_HPP_ */

ext: hpp

max_stars_repo_name: marcorighini/tbdannotation

max_issues_repo_name: marcorighini/tbdannotation

max_forks_repo_name: marcorighini/tbdannotation


text:
//==================================================================================================
/*!

  Copyright 2016 NumScale SAS
  Copyright 2016 J.T Lapreste

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#include <boost/simd/constant/true.hpp>
#include <boost/simd/logical.hpp>
#include <boost/simd/pack.hpp>
#include <simd_test.hpp>

STF_CASE_TPL( "Check true behavior", STF_NUMERIC_TYPES )
{
  using boost::simd::as;
  using boost::simd::logical;
  using boost::simd::pack;
  using boost::simd::True;

  STF_TYPE_IS(decltype(True<pack<logical<T>>>()), pack<logical<T>> );

  pack<logical<T>> ref;
  for(auto e : ref) e = true;

  STF_ALL_EQUAL( True<pack<logical<T>>>(), ref);
}

ext: cpp


text:
// Copyright (c) 2019 The Bitcoin SV developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "config.h"
#include "key.h"
#include "pubkey.h"
#include "test/test_bitcoin.h"
#include "txmempool.h"
#include "txn_validator.h"

#include <boost/test/unit_test.hpp>

namespace {
    // Support for P2P node.
    CService ip(uint32_t i) {
        struct in_addr s;
        s.s_addr = i;
        return CService(CNetAddr(s), Params().GetDefaultPort());
    }
    // Create scriptPubKey from a given key
    CScript GetScriptPubKey(CKey& key) {
        CScript scriptPubKey = CScript() << ToByteVector(key.GetPubKey())
                                         << OP_CHECKSIG;
        return scriptPubKey;
    }
    // Create a spend txn
    CMutableTransaction CreateSpendTxn(CTransaction& foundTxn,
                                       CKey& key,
                                       CScript& scriptPubKey) {
        CMutableTransaction spend_txn;
        spend_txn.nVersion = 1;
        spend_txn.vin.resize(1);
        spend_txn.vin[0].prevout = COutPoint(foundTxn.GetId(), 0);
        spend_txn.vout.resize(1);
        spend_txn.vout[0].nValue = 11 * CENT;
        spend_txn.vout[0].scriptPubKey = scriptPubKey;
        // Sign:
        std::vector<uint8_t> vchSig {};
        uint256 hash = SignatureHash(scriptPubKey, CTransaction(spend_txn), 0,
                                     SigHashType().withForkId(),
                                     foundTxn.vout[0].nValue);
        BOOST_CHECK(key.Sign(hash, vchSig));
        vchSig.push_back(uint8_t(SIGHASH_ALL | SIGHASH_FORKID));
        spend_txn.vin[0].scriptSig << vchSig;
        return spend_txn;
    }
    // Make N unique large (but rubbish) transactions
    std::vector<CMutableTransaction> MakeNLargeTxns(size_t nNumTxns,
                                                    CTransaction& foundTxn,
                                                    CScript& scriptPubKey) {
        std::vector<CMutableTransaction> res {};
        for (size_t i=0; i<nNumTxns; i++) {
            CMutableTransaction txn;
            txn.nVersion = 1;
            txn.vin.resize(1);
            txn.vin[0].prevout = COutPoint(foundTxn.GetId(), i);
            txn.vout.resize(1000);
            for(size_t j=0; j<1000; ++j) {
                txn.vout[j].nValue = 11 * CENT;
                txn.vout[j].scriptPubKey = scriptPubKey;
            }
            res.emplace_back(txn);
        }
        return res;
    }
    // Create N spend txns from a given found txn
    std::vector<CMutableTransaction> CreateNSpendTxns(size_t nSpendTxns,
                                                      CTransaction& foundTxn,
                                                      CKey& key,
                                                      CScript& scriptPubKey) {
        // Create txns spending the same coinbase txn.
        std::vector<CMutableTransaction> spends {};
        for (size_t i=0; i<nSpendTxns; i++) {
            spends.emplace_back(CreateSpendTxn(foundTxn, key, scriptPubKey));
        };
        return spends;
    }
    // Create txn input data for a given txn and source
    TxInputDataSPtr TxInputData(TxSource source,
                                CMutableTransaction& spend,
                                std::shared_ptr<CNode> pNode = nullptr) {
        // Return txn's input data
        return std::make_shared<CTxInputData>(
                                   source,   // tx source
                                   TxValidationPriority::normal, // tx validation priority
                                   MakeTransactionRef(spend),// a pointer to the tx
                                   GetTime(),// nAcceptTime
                                   false,    // mfLimitFree
                                   Amount(0),// nAbsurdFee
                                   pNode);   // pNode
    }
    // Create a vector with input data for a given txn and source
    std::vector<TxInputDataSPtr> TxInputDataVec(TxSource source,
                                                std::vector<CMutableTransaction>& spends,
                                                std::shared_ptr<CNode> pNode = nullptr) {
        std::vector<TxInputDataSPtr> vTxInputData {};
        for (auto& elem : spends) {
            vTxInputData.
                emplace_back(
                        std::make_shared<CTxInputData>(
                                            source,   // tx source
                                            TxValidationPriority::normal, // tx validation priority
                                            MakeTransactionRef(elem),  // a pointer to the tx
                                            GetTime(),// nAcceptTime
                                            false,    // mfLimitFree
                                            Amount(0),// nAbsurdFee
                                            pNode));   // pNode
        }
        return vTxInputData;
    }
    // Validate txn using asynchronous validation interface
    void ProcessTxnsAsynchApi(std::vector<CMutableTransaction>& spends,
                              TxSource source,
                              std::shared_ptr<CNode> pNode = nullptr) {
        // Create txn validator
        std::shared_ptr<CTxnValidator> txnValidator {
            std::make_shared<CTxnValidator>(
                    GlobalConfig::GetConfig(),
                    mempool,
                    std::make_shared<CTxnDoubleSpendDetector>())
        };
        // Clear mempool before validation
        mempool.Clear();
        // Schedule txns for processing.
        txnValidator->newTransaction(TxInputDataVec(source, spends, pNode));
        // Wait for the Validator to process all queued txns.
        txnValidator->waitForEmptyQueue();
    }
    // Validate txn using synchronous validation interface
    void ProcessTxnsSynchApi(std::vector<CMutableTransaction>& spends,
                             TxSource source,
                             std::shared_ptr<CNode> pNode = nullptr) {
        // Create txn validator
        std::shared_ptr<CTxnValidator> txnValidator {
            std::make_shared<CTxnValidator>(
                    GlobalConfig::GetConfig(),
                    mempool,
                    std::make_shared<CTxnDoubleSpendDetector>())
        };
        // Clear mempool before validation
        mempool.Clear();
        // Mempool Journal ChangeSet
        mining::CJournalChangeSetPtr changeSet {nullptr};
        // Validate the first txn
        CValidationState result {
            txnValidator->processValidation(TxInputData(source, spends[0], pNode), changeSet)
        };
        BOOST_CHECK(result.IsValid());
        // Validate the second txn
        // spends1 should be rejected if spend0 is in the mempool
        result = txnValidator->processValidation(TxInputData(source, spends[1], pNode), changeSet);
        BOOST_CHECK(!result.IsValid());
    }
    // Validate txns using synchronous batch validation interface
    void ProcessTxnsSynchBatchApi(std::vector<CMutableTransaction>& spends,
                                  TxSource source,
                                  std::shared_ptr<CNode> pNode = nullptr) {
        // Create txn validator
        std::shared_ptr<CTxnValidator> txnValidator {
            std::make_shared<CTxnValidator>(
                    GlobalConfig::GetConfig(),
                    mempool,
                    std::make_shared<CTxnDoubleSpendDetector>())
        };
        // Clear mempool before validation
        mempool.Clear();
        // Mempool Journal ChangeSet
        mining::CJournalChangeSetPtr changeSet {nullptr};
        // Validate the first txn
        txnValidator->processValidation(TxInputDataVec(source, spends, pNode), changeSet);
    }
    struct TestChain100Setup2 : TestChain100Setup {
        CScript scriptPubKey {
            GetScriptPubKey(coinbaseKey)
        };
        // spends contains two txns spending the same coinbase txn
        std::vector<CMutableTransaction> spends2 {
            CreateSpendTxn(coinbaseTxns[0], coinbaseKey, scriptPubKey),
            CreateSpendTxn(coinbaseTxns[0], coinbaseKey, scriptPubKey)
        };
        // spends contains N txns spending the same coinbase txn
        std::vector<CMutableTransaction> spendsN {
            CreateNSpendTxns(10, coinbaseTxns[0], coinbaseKey, scriptPubKey)
        };
    };
}

BOOST_FIXTURE_TEST_SUITE(test_txnvalidator, TestChain100Setup2)

BOOST_AUTO_TEST_CASE(txn_validator_creation) {
    // Create txn validator
    std::shared_ptr<CTxnValidator> txnValidator {
        std::make_shared<CTxnValidator>(
                GlobalConfig::GetConfig(),
                mempool,
                std::make_shared<CTxnDoubleSpendDetector>())
    };
    // Check if the Validator was created
    BOOST_REQUIRE(txnValidator);
    // Check if orphan txns buffer was created
    BOOST_REQUIRE(txnValidator->getOrphanTxnsPtr());
    // Check if txn recent rejects buffer was created
    BOOST_REQUIRE(txnValidator->getTxnRecentRejectsPtr());
}

BOOST_AUTO_TEST_CASE(txn_validator_set_get_frequency) {
    // Create txn validator
    std::shared_ptr<CTxnValidator> txnValidator {
        std::make_shared<CTxnValidator>(
                GlobalConfig::GetConfig(),
                mempool,
                std::make_shared<CTxnDoubleSpendDetector>())
    };
    auto defaultfreq = std::chrono::milliseconds(CTxnValidator::DEFAULT_ASYNCH_RUN_FREQUENCY_MILLIS);
    BOOST_CHECK(defaultfreq == txnValidator->getRunFrequency());
    txnValidator->setRunFrequency(++defaultfreq);
    BOOST_CHECK(defaultfreq == txnValidator->getRunFrequency());
}

BOOST_AUTO_TEST_CASE(txn_validator_istxnknown) {
    // Create txn validator
    std::shared_ptr<CTxnValidator> txnValidator {
        std::make_shared<CTxnValidator>(
                GlobalConfig::GetConfig(),
                mempool,
                std::make_shared<CTxnDoubleSpendDetector>())
    };
    // Schedule txns for processing.
    txnValidator->newTransaction(TxInputDataVec(TxSource::p2p, spendsN));
    BOOST_CHECK(txnValidator->isTxnKnown(spendsN[0].GetId()));
    // Wait for the Validator to process all queued txns.
    txnValidator->waitForEmptyQueue();
    BOOST_CHECK(!txnValidator->isTxnKnown(spendsN[0].GetId()));
}

/**
 * TxnValidator: Test synch interface.
 */
BOOST_AUTO_TEST_CASE(txnvalidator_doublespend_synch_api) {
    // Test: Txns from wallet.
    ProcessTxnsSynchApi(spends2, TxSource::wallet);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from rpc.
    ProcessTxnsSynchApi(spends2, TxSource::rpc);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from file.
    ProcessTxnsSynchApi(spends2, TxSource::file);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from p2p.
    {
        // Create a dummy address
        CAddress dummy_addr(ip(0xa0b0c001), NODE_NONE);
        CConnman::CAsyncTaskPool asyncTaskPool{GlobalConfig::GetConfig()};
        CNodePtr pDummyNode =
            CNode::Make(
                0,
                NODE_NETWORK,
                0,
                INVALID_SOCKET,
                dummy_addr,
                0u,
                0u,
                asyncTaskPool,
                "",
                true);
        ProcessTxnsSynchApi(spends2, TxSource::p2p, pDummyNode);
        BOOST_CHECK_EQUAL(mempool.Size(), 1);
    }
    // Process txn if it is valid.
    ProcessTxnsSynchApi(spends2, TxSource::p2p);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from reorg.
    ProcessTxnsSynchApi(spends2, TxSource::reorg);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Process txn if it is valid.
    ProcessTxnsSynchApi(spends2, TxSource::unknown);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_doublespend_synch_batch_api) {
    // Test: Txns from wallet.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::wallet);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from rpc.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::rpc);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from file.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::file);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from p2p.
    {
        // Create a dummy address
        CAddress dummy_addr(ip(0xa0b0c001), NODE_NONE);
        CConnman::CAsyncTaskPool asyncTaskPool{GlobalConfig::GetConfig()};
        CNodePtr pDummyNode =
            CNode::Make(
                0,
                NODE_NETWORK,
                0,
                INVALID_SOCKET,
                dummy_addr,
                0u,
                0u,
                asyncTaskPool,
                "",
                true);
        ProcessTxnsSynchBatchApi(spendsN, TxSource::p2p, pDummyNode);
        BOOST_CHECK_EQUAL(mempool.Size(), 1);
    }
    // Process txn if it is valid.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::p2p);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Test: Txns from reorg.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::reorg);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
    // Process txn if it is valid.
    ProcessTxnsSynchBatchApi(spendsN, TxSource::unknown);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

/**
 * TxnValidator: Test asynch interface.
 */
BOOST_AUTO_TEST_CASE(txnvalidator_wallet_doublespend_via_asynch_api) {
    // Test: Txns from wallet.
    ProcessTxnsAsynchApi(spendsN, TxSource::wallet);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_rpc_doublespend_via_asynch_api) {
    // Test: Txns from rpc.
    ProcessTxnsAsynchApi(spendsN, TxSource::rpc);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_file_doublespend_via_asynch_api) {
    // Test: Txns from file.
    ProcessTxnsAsynchApi(spendsN, TxSource::file);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_p2p_doublespend_via_asynch_api) {
    // Test: Txns from p2p.
    {
        // Create a dummy address
        CAddress dummy_addr(ip(0xa0b0c001), NODE_NONE);
        CConnman::CAsyncTaskPool asyncTaskPool{GlobalConfig::GetConfig()};
        CNodePtr pDummyNode =
            CNode::Make(
                0,
                NODE_NETWORK,
                0,
                INVALID_SOCKET,
                dummy_addr,
                0u,
                0u,
                asyncTaskPool,
                "",
                true);
        ProcessTxnsAsynchApi(spendsN, TxSource::p2p, pDummyNode);
        BOOST_CHECK_EQUAL(mempool.Size(), 1);
    }
    // Process txn if it is valid.
    ProcessTxnsAsynchApi(spendsN, TxSource::p2p);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_reorg_doublespend_via_asynch_api) {
    // Test: Txns from reorg.
    ProcessTxnsAsynchApi(spendsN, TxSource::reorg);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_dummy_doublespend_via_asynch_api) {
    // Process txn if it is valid.
    ProcessTxnsAsynchApi(spendsN, TxSource::unknown);
    BOOST_CHECK_EQUAL(mempool.Size(), 1);
}

BOOST_AUTO_TEST_CASE(txnvalidator_limit_memory_usage)
{
    // Make sure validation thread won't run during this test
    gArgs.ForceSetArg("-txnvalidationasynchrunfreq", "10000");
    gArgs.ForceSetArg("-txnvalidationqueuesmaxmemory", "1");

    // Create a larger number of txns than will fit in a 1Mb queue
    std::vector<CMutableTransaction> txns { MakeNLargeTxns(25, coinbaseTxns[0], scriptPubKey) };
    auto txnsInputs { TxInputDataVec(TxSource::p2p, txns) };

    // Create txn validator
    std::shared_ptr<CTxnValidator> txnValidator {
        std::make_shared<CTxnValidator>(
                GlobalConfig::GetConfig(),
                mempool,
                std::make_shared<CTxnDoubleSpendDetector>())
    };

    // Attempt to enqueue all txns and verify that we stopped when we hit the max size limit
    txnValidator->newTransaction(txnsInputs);
    BOOST_CHECK(txnValidator->GetTransactionsInQueueCount() < txns.size());
    BOOST_CHECK(txnValidator->GetStdQueueMemUsage() <= 1*1024*1024);
    BOOST_CHECK_EQUAL(txnValidator->GetNonStdQueueMemUsage(), 0);
}

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: xloem/bitcoin-sv

max_issues_repo_name: xloem/bitcoin-sv

max_forks_repo_name: xloem/bitcoin-sv


text:
/*!
  @file   FaceTrackerFactoryJson.cpp
  @author David Hirvonen
  @brief  Utility class implementation for loading FaceTracker from a JSON descriptor.

  \copyright Copyright 2017 Elucideye, Inc. All rights reserved.
  \license{This project is released under the 3 Clause BSD License.}

*/

#include "FaceTrackerFactoryJson.h"

#include <drishti/drishti_cv.hpp>

// Need std:: extensions for android targets
#if !defined(DRISHTI_SDK_TEST_HAVE_TO_STRING)
#include "stdlib_string.h"
#endif

#include <nlohmann/json.hpp> // nlohman-json

#include <boost/filesystem.hpp> // for portable path (de)construction

namespace bfs = boost::filesystem;

static std::string cat(const std::string& a, const std::string& b) { return a + b; }

FaceTrackerFactoryJson::FaceTrackerFactoryJson(const std::string& sModels, const std::string& logger)
{
    std::ifstream ifs(sModels);
    if (!ifs)
    {
        throw std::runtime_error(cat("FactoryLoader::FactoryLoader() failed to open ", sModels));
    }

    nlohmann::json json;
    ifs >> json;

    factory.logger = logger; // logger name
    std::vector<std::pair<const char*, std::istream**>> bindings = {
        { "face_detector", &factory.sFaceDetector },
        { "eye_model_regressor", &factory.sEyeRegressor },
        { "face_landmark_regressor", &factory.sFaceRegressor },
        { "face_detector_mean", &factory.sFaceModel }
    };

    // Get the directory name:
    auto path = bfs::path(sModels);
    for (auto& binding : bindings)
    {
        auto filename = path.parent_path() / json[binding.first].get<std::string>();
        std::shared_ptr<std::istream> stream = std::make_shared<std::ifstream>(filename.string(), std::ios_base::binary | std::ios::in);
        if (!stream || !stream->good())
        {
            throw std::runtime_error(cat("FactoryLoader::FactoryLoader() failed to open ", binding.first));
        }

        (*binding.second) = stream.get();
        streams.push_back(stream);
    }

    good = true;
}