text:
/*
 *    Copyright (c) 2020-2021 Grant Erickson
 *    All rights reserved.
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing,
 *    software distributed under the License is distributed on an "AS
 *    IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *    express or implied.  See the License for the specific language
 *    governing permissions and limitations under the License.
 *
 */

/**
 *    @file
 *      This file defines an object for effecting a HLX simulated
 *      server controller.
 *
 */

#ifndef OPENHLXSIMULATORAPPLICATIONCONTROLLER_HPP
#define OPENHLXSIMULATORAPPLICATIONCONTROLLER_HPP

#include <map>

#include <boost/filesystem.hpp>

#include <CoreFoundation/CFRunLoop.h>

#include <OpenHLX/Common/ApplicationControllerBasis.hpp>
#include <OpenHLX/Common/ApplicationObjectControllerContainerTemplate.hpp>
#include <OpenHLX/Common/Errors.hpp>
#include <OpenHLX/Common/RunLoopParameters.hpp>
#include <OpenHLX/Common/Timer.hpp>
#include <OpenHLX/Common/TimerDelegate.hpp>
#include <OpenHLX/Server/ApplicationControllerBasis.hpp>
#include <OpenHLX/Server/CommandManager.hpp>
#include <OpenHLX/Server/CommandManagerDelegate.hpp>
#include <OpenHLX/Server/ConnectionManager.hpp>
#include <OpenHLX/Server/ConnectionManagerDelegate.hpp>

#include "ApplicationControllerDelegate.hpp"
#include "ConfigurationController.hpp"
#include "ConfigurationControllerDelegate.hpp"
#include "EqualizerPresetsController.hpp"
#include "FavoritesController.hpp"
#include "FrontPanelController.hpp"
#include "GroupsController.hpp"
#include "GroupsControllerDelegate.hpp"
#include "InfraredController.hpp"
#include "NetworkController.hpp"
#include "SourcesController.hpp"
#include "ZonesController.hpp"

namespace HLX
{

namespace Simulator
{

namespace Application
{

/**
 *  @brief
 *    An object for effecting a HLX server controller.
 *
 *  @ingroup server
 *
 */
class Controller :
    public Common::Application::ControllerBasis,
    public Server::Application::ControllerBasis,
    public Common::Application::ObjectControllerContainerTemplate<Simulator::ObjectControllerBasis>,
    public Server::ConnectionManagerDelegate,
    public Server::CommandManagerDelegate,
    public Simulator::ObjectControllerBasisDelegate,
    public Common::TimerDelegate,
    public ConfigurationControllerDelegate,
    public GroupsControllerDelegate
{
public:
    Controller(void);
    virtual ~Controller(void);

    // Initializer(s)

    Common::Status Init(const Common::RunLoopParameters &aRunLoopParameters, const boost::filesystem::path &aConfigurationPath);

    ControllerDelegate *GetDelegate(void) const;

    Common::Status SetDelegate(ControllerDelegate *aDelegate);

    Common::Status RegisterRequestHandler(Server::Command::RequestBasis &aRequest, void *aContext, Server::CommandManager::OnRequestReceivedFunc aOnRequestReceivedHandler);

    // Configuration Controller Delegate Methods

    Common::Status LoadFromBackupConfiguration(ConfigurationController &aController, CFDictionaryRef aBackupDictionary) final;
    Common::Status LoadFromBackupConfigurationStorage(ConfigurationController &aController, CFDictionaryRef &aBackupDictionary) final;
    void QueryCurrentConfiguration(ConfigurationController &aController, Server::ConnectionBasis &aConnection, Common::ConnectionBuffer::MutableCountedPointer &aBuffer) final;
    void ResetToDefaultConfiguration(ConfigurationController &aController) final;
    void SaveToBackupConfiguration(ConfigurationController &aController, CFMutableDictionaryRef aBackupDictionary) final;
    Common::Status SaveToBackupConfigurationStorage(ConfigurationController &aController, CFDictionaryRef aBackupDictionary) final;

    // Controller Delegate Methods

    void ControllerConfigurationIsDirty(Simulator::ObjectControllerBasis &aController) final;

    // Connection Manager Delegate Methods

    // Resolve

    void ConnectionManagerWillResolve(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, const char *aHost) final;
    void ConnectionManagerIsResolving(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, const char *aHost) final;
    void ConnectionManagerDidResolve(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, const char *aHost, const Common::IPAddress &aIPAddress) final;
    void ConnectionManagerDidNotResolve(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, const char *aHost, const Common::Error &aError) final;

    // Listen

    void ConnectionManagerWillListen(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerIsListening(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerDidListen(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerDidNotListen(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef, const Common::Error &aError) final;

    // Accept Delegate Methods

    void ConnectionManagerWillAccept(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerIsAccepting(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerDidAccept(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef) final;
    void ConnectionManagerDidNotAccept(Server::ConnectionManager &aConnectionManager, CFURLRef aURLRef, const Common::Error &aError) final;

    // Disconnect Delegate Methods

    void ConnectionManagerWillDisconnect(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, CFURLRef aURLRef) final;
    void ConnectionManagerDidDisconnect(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, CFURLRef aURLRef, const Common::Error &aError) final;
    void ConnectionManagerDidNotDisconnect(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, CFURLRef aURLRef, const Common::Error &aError) final;

    // Error Delegate Method

    void ConnectionManagerError(Common::ConnectionManagerBasis &aConnectionManager, const Common::ConnectionManagerBasis::Roles &aRoles, const Common::Error &aError) final;

    // Groups Controller Delegate Methods

    Common::Status ShouldAdjustVolume(GroupsController &aController, const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel, const Model::VolumeModel::LevelType &aAdjustment) final;
    Common::Status ShouldSetMute(GroupsController &aController, const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel, const Model::VolumeModel::MuteType &aMute) final;
    Common::Status ShouldSetSource(GroupsController &aController, const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel, const Model::SourceModel::IdentifierType &aSourceIdentifier) final;
    Common::Status ShouldSetVolume(GroupsController &aController, const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel, const Model::VolumeModel::LevelType &aVolume) final;
    Common::Status ShouldToggleMute(GroupsController &aController, const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel) final;

    // Timer Delegate Method

    void TimerDidFire(Common::Timer &aTimer) final;

private:
    typedef Common::Application::ObjectControllerContainerTemplate<Simulator::ObjectControllerBasis> SimulatorObjectControllerContainer;

private:
    class ShouldDoForGroupZonesFunctorBasis
    {
    protected:
        ShouldDoForGroupZonesFunctorBasis(ZonesController &aZonesController);

    public:
        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) = 0;

    protected:
        ZonesController &                           mZonesController;
    };

    class ShouldAdjustVolumeFunctor :
        public ShouldDoForGroupZonesFunctorBasis
    {
    public:
        ShouldAdjustVolumeFunctor(ZonesController &aZonesController, const Model::VolumeModel::LevelType &aAdjustment);

        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) final;

    private:
        const Model::VolumeModel::LevelType &      mAdjustment;
    };

    class ShouldSetMuteFunctor :
        public ShouldDoForGroupZonesFunctorBasis
    {
    public:
        ShouldSetMuteFunctor(ZonesController &aZonesController, const Model::VolumeModel::MuteType &aMute);

        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) final;

    private:
        const Model::VolumeModel::MuteType &        mMute;
    };

    class ShouldSetSourceFunctor :
        public ShouldDoForGroupZonesFunctorBasis
    {
    public:
        ShouldSetSourceFunctor(ZonesController &aZonesController, const Model::SourceModel::IdentifierType &aSourceIdentifier);

        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) final;

    private:
        const Model::SourceModel::IdentifierType &  mSourceIdentifier;
    };

    class ShouldSetVolumeFunctor :
        public ShouldDoForGroupZonesFunctorBasis
    {
    public:
        ShouldSetVolumeFunctor(ZonesController &aZonesController, const Model::VolumeModel::LevelType &aVolume);

        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) final;

    private:
        const Model::VolumeModel::LevelType &      mVolume;
    };

    class ShouldToggleMuteFunctor :
        public ShouldDoForGroupZonesFunctorBasis
    {
    public:
        ShouldToggleMuteFunctor(ZonesController &aZonesController);

        virtual Common::Status operator ()(const Model::ZoneModel::IdentifierType &aZoneIdentifier) final;
    };

    Common::Status ShouldDoForGroupZones(const Model::GroupModel::IdentifierType &aGroupIdentifier, const Model::GroupModel &aGroupModel, ShouldDoForGroupZonesFunctorBasis &aFunctorBasis);

private:
    Common::Status InitControllers(const Common::RunLoopParameters &aRunLoopParameters);
    Common::Status InitConfiguration(const Common::RunLoopParameters &aRunLoopParameters, const boost::filesystem::path &aPath);

private:
    // Sub-controller order is important since this is the order that
    // most closely matches the order in which the actual HLX hardware
    // responds to for the 'query current configuration' command.

    Common::RunLoopParameters       mRunLoopParameters;
    boost::filesystem::path         mConfigurationPath;
    ConfigurationController         mConfigurationController;
    NetworkController               mNetworkController;
    FavoritesController             mFavoritesController;
    GroupsController                mGroupsController;
    FrontPanelController            mFrontPanelController;
    InfraredController              mInfraredController;
    EqualizerPresetsController      mEqualizerPresetsController;
    SourcesController               mSourcesController;
    ZonesController                 mZonesController;
    ControllerDelegate *            mDelegate;
    Common::Timer                   mConfigurationAutoSaveTimer;
    bool                            mConfigurationIsDirty;
};

}; // namespace Application

}; // namespace Simulator

}; // namespace HLX

#endif // OPENHLXSIMULATORAPPLICATIONCONTROLLER_HPP

max_stars_repo_name: gerickson/openhlx

max_issues_repo_name: gerickson/openhlx

max_forks_repo_name: gerickson/openhlx


text:
/*
 Copyright (C) GridGain Systems. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

/*  _________        _____ __________________        _____
 *  __  ____/___________(_)______  /__  ____/______ ____(_)_______
 *  _  / __  __  ___/__  / _  __  / _  / __  _  __ `/__  / __  __ \
 *  / /_/ /  _  /    _  /  / /_/ /  / /_/ /  / /_/ / _  /  _  / / /
 *  \____/   /_/     /_/   \_,__/   \____/   \__,_/  /_/   /_/ /_/
 */

#ifndef GRIDCLIENT_JSON_MARSHALLER_HPP_INCLUDED
#define GRIDCLIENT_JSON_MARSHALLER_HPP_INCLUDED

#include <vector>
#include <boost/property_tree/json_parser.hpp>

#include "gridgain/impl/cmd/gridclientmessageauthrequestcommand.hpp"
#include "gridgain/impl/cmd/gridclientmessageauthresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagelogrequestcommand.hpp"
#include "gridgain/impl/cmd/gridclientmessagelogresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagetopologyrequestcommand.hpp"
#include "gridgain/impl/cmd/gridclientmessagetopologyresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagecacheresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagecacherequestcommand.hpp"
#include "gridgain/impl/cmd/gridclientmessagecachemodifyresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagecachemetricsresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagecachegetresult.hpp"
#include "gridgain/impl/cmd/gridclientmessagetaskrequestcommand.hpp"
#include "gridgain/impl/cmd/gridclientmessagetaskresult.hpp"

#include "gridgain/gridclientexception.hpp"

/** Type definition for HTTP request key/value pairs. */
typedef std::map<std::string, std::string> TRequestParams;

/** Type definition for parsed JSON response. */
typedef boost::property_tree::ptree TJson;

/**
 * Grid JSON marshaler.
 */
class GridClientJsonMarshaller {
private:
    /**
     * Fills common request parameters based on common message data.
     *
     * @param req Message to take data from.
     * @param params Request key/value map to fill.
     */
    static void fillRequestHeader(const GridClientMessage& req, TRequestParams& params);

    /**
     * Fills common response message result data from JSON response.
     *
     * @param json Parsed HTTP output.
     * @param clientMsg Response to be returned to client.
     */
    static void fillResponseHeader(const TJson& json, GridClientMessageResult& clientMsg) throw (GridClientException);
public:
    /**
     * Sets HTTP request parameters based on data from authentication request.
     *
     * @param req Authentication request.
     * @param params Request key/value map.
     */
    static void wrap(const GridAuthenticationRequestCommand& req, TRequestParams& params);

    /**
     * Parses HTTP response into authentication result.
     *
     * @param json Data received from server.
     * @param resp Authentication response.
     */
    static void unwrap(const TJson& json, GridClientMessageAuthenticationResult& resp);

    /**
     * Sets HTTP request parameters based on data from log request.
     *
     * @param req Log request.
     * @param params Request key/value map.
     */
    static void wrap(const GridLogRequestCommand& req, TRequestParams& params);

    /**
     * Parses HTTP response into log result.
     *
     * @param json Data received from server.
     * @param resp Log response.
     */
    static void unwrap(const TJson& json, GridClientMessageLogResult& resp);

    /**
     * Sets HTTP request parameters based on data topology log request.
     *
     * @param req Topology request.
     * @param params Request key/value map.
     */
    static void wrap(const GridTopologyRequestCommand& req, TRequestParams& params);

    /**
     * Parses HTTP response into topology result.
     *
     * @param json Data received from server.
     * @param resp Topology response.
     */
    static void unwrap(const TJson& json, GridClientMessageTopologyResult& resp);

    /**
     * Sets HTTP request parameters based on data from cache request.
     *
     * @param req Cache request.
     * @param params Request key/value map.
     */
    static void wrap(const GridCacheRequestCommand& req, TRequestParams& params);

    /**
     * Parses HTTP response into cache modify result.
     *
     * @param json Data received from server.
     * @param resp Cache modify response.
     */
    static void unwrap(const TJson& json, GridClientMessageCacheModifyResult& resp);

    /**
     * Parses HTTP response into cache metrics result.
     *
     * @param json Data received from server.
     * @param resp Cache metrics response.
     */
    static void unwrap(const TJson& json, GridClientMessageCacheMetricResult& resp);

    /**
     * Parses HTTP response into cache get result.
     *
     * @param json Data received from server.
     * @param resp Cache get response.
     */
    static void unwrap(const TJson& json, GridClientMessageCacheGetResult& resp);

    /**
     * Sets HTTP request parameters based on data from task call request.
     *
     * @param req Topology request.
     * @param params Request key/value map.
     */
    static void wrap(const GridTaskRequestCommand& req, TRequestParams& params);

    /**
     * Parses HTTP response into task call result.
     *
     * @param json Data received from server.
     * @param resp Task call response.
     */
    static void unwrap(const TJson& json, GridClientMessageTaskResult& resp);
};

#endif

ext: hpp

max_stars_repo_name: cybernetics/gridgain

max_issues_repo_name: cybernetics/gridgain

max_forks_repo_name: cybernetics/gridgain


text:
#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<boost::int64_t, 4, 5> test_long_4_5(const Eigen::Matrix<boost::int64_t, 4, 5>& M) { return M; }
void export_long_4_5() { boost::python::def("test_long_4_5", test_long_4_5); }

ext: cpp

max_stars_repo_name: chengfzy/kalibr

max_issues_repo_name: chengfzy/kalibr

max_forks_repo_name: chengfzy/kalibr


text:
/*
 * Copyright (C) 2019-2020, Xilinx Inc - All rights reserved
 * Xilinx Resouce Management
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You may
 * not use this file except in compliance with the License. A copy of the
 * License is located at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

#ifndef _XRM_SYSTEM_HPP_
#define _XRM_SYSTEM_HPP_

#include <vector>
#include <map>
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <pthread.h>
#include <fstream>
#include <iostream>
#include <dlfcn.h>
#include <boost/format.hpp>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/serialization/serialization.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/map.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <syslog.h>
#include <xrt.h>

#include "xrm_limits.h"
#include "xrm_error.h"
#include "xrm.h"

namespace pt = boost::property_tree;

namespace xrm {

/* cu status */
typedef struct cuStatus {
    bool isBusy;
    int32_t usedLoad;    // Percentage: 0 - 100
    uint8_t extData[64]; // for future extension
} cuStatus;

/* request compute unit resource property */
typedef struct cuProperty {
    char kernelName[XRM_MAX_NAME_LEN];  // unique kernel name, not instance name
    char kernelAlias[XRM_MAX_NAME_LEN]; // unique alias of kernel name
    char cuName[XRM_MAX_NAME_LEN];      // unique cu name (kernelName:instanceName)
    bool devExcl;                       // request exclusive device usage for this client
    int32_t requestLoad;                // request load percentage of the CU: 1 - 100
    uint64_t clientId;
    uint64_t poolId;     /* request to allocate resource from specified resource pool,
                          * the system default resource pool id is 0.
                          */
    uint8_t extData[64]; // for future extension

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& kernelName& kernelAlias& cuName& devExcl& requestLoad& clientId& poolId;
    }
} cuProperty;

/* list of request compute unit resource property */
typedef struct cuListProperty {
    cuProperty cuProps[XRM_MAX_LIST_CU_NUM];
    int32_t cuNum;
    bool sameDevice;
    uint8_t extData[64]; // for future extension

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& cuProps& cuNum& sameDevice;
    }
} cuListProperty;

/* user defined compute unit resource group information */
typedef struct udfCuGroupInformation {
    std::string udfCuGroupName; // name of user defined cu group
    cuListProperty optionUdfCuListProps[XRM_MAX_UDF_CU_GROUP_OPTION_LIST_NUM];
    int32_t optionUdfCuListNum;
    uint8_t extData[64]; // for future extension

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& udfCuGroupName& optionUdfCuListProps& optionUdfCuListNum;
    }
} udfCuGroupInformation;

/* compute resource group property */
typedef struct cuGroupProperty {
    std::string udfCuGroupName; // name of user defined cu group
    uint64_t clientId;
    uint64_t poolId;     /* request to allocate resource from specified resource pool,
                          * 0 means to allocate resource from system default resource pool.
                          */
    uint8_t extData[64]; // for future extension
} cuGroupProperty;

/* pool of request compute unit resource property */
typedef struct cuPoolProperty {
    cuListProperty cuListProp;
    int32_t cuListNum; // number of such cu list
    uuid_t xclbinUuid;
    int32_t xclbinNum;   // number of such xclbin
    uint64_t clientId;   // client ID
    uint8_t extData[64]; // for future extension
} cuPoolProperty;

/* cu type */
typedef enum cuTypes {
    CU_NULL = 0,
    CU_IPKERNEL = 1,   // IP kernel
    CU_SOFTKERNEL = 2, // soft kernel
} cuTypes;

/* allocated/released compute unit resource */
typedef struct cuResource {
    char xclbinFileName[XRM_MAX_PATH_NAME_LEN];       // include path and name
    char kernelPluginFileName[XRM_MAX_PATH_NAME_LEN]; // just the name
    char kernelName[XRM_MAX_NAME_LEN];                // kernel name, not instance name
    char kernelAlias[XRM_MAX_NAME_LEN];               // unique alias of kernel name
    char instanceName[XRM_MAX_NAME_LEN];              // instance name
    char cuName[XRM_MAX_NAME_LEN];                    // cu name
    char uuidStr[XRM_MAX_NAME_LEN];
    int32_t deviceId;
    int32_t cuId;
    int32_t channelId;
    cuTypes cuType;
    uint64_t baseAddr;        // base address of the CU
    uint32_t membankId;       // connected memory bank id
    uint32_t membankType;     // connected memory bank type
    uint64_t membankSize;     // connected memory bank size
    uint64_t membankBaseAddr; // connected memory bank base address
    uint64_t allocServiceId;  // unique service id of the allocation
    uint64_t clientId;
    int32_t channelLoad; // load percentage of the CU for this channel: 1 - 100
    uint64_t poolId;     // id of the cu pool this cu comes from, default pool id is 0
    uint8_t extData[64]; // for future extension
} cuResource;

/*
 * allocated/released compute unit resource list
 */
typedef struct cuListResource {
    cuResource cuResources[XRM_MAX_LIST_CU_NUM];
    int32_t cuNum;
    uint8_t extData[64]; // for future extension
} cuListResource;

/*
 * allocated/released compute unit resource group
 */
typedef struct cuGroupResource {
    cuResource cuResources[XRM_MAX_GROUP_CU_NUM];
    int32_t cuNum;
    uint8_t extData[64]; // for future extension
} cuGroupResource;

/*
 * allocated/released compute unit resource pool
 */
typedef struct cuPoolResource {
    cuResource cuResources[XRM_MAX_POOL_CU_NUM];
    int32_t cuNum;
    uint8_t extData[64]; // for future extension
} cuPoolResource;

/* Alloction information for querying */
typedef struct allocationQueryInfo {
    uint64_t allocServiceId;            // the service id returned from allocation
    char kernelName[XRM_MAX_NAME_LEN];  // kernel name, not instance name
    char kernelAlias[XRM_MAX_NAME_LEN]; // unique alias of kernel name
    uint8_t extData[64];                // for future extension
} allocationQueryInfo;

/* channel data */
typedef struct channelData {
    int32_t channelId;
    uint64_t allocServiceId;
    uint64_t poolId;
    uint64_t clientId;
    int32_t channelLoad; // load percentage of the CU: 0 - 100

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& channelId& allocServiceId& poolId& clientId& channelLoad;
    }
} channelData;

/* reserve related data */
typedef struct reserveData {
    uint64_t reservePoolId;
    int32_t reserveLoad;     // load percentage of the CU: 0 - 100
    int32_t reserveUsedLoad; // load percentage of the CU: 0 - 100
    bool clientIsActive;
    uint64_t clientId;

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& reservePoolId& reserveLoad& reserveUsedLoad& clientIsActive& clientId;
    }
} reserveData;

/* compute unit data */
typedef struct cuData {
    int32_t cuId; // index on one device, start from 0
    cuTypes cuType;
    std::string kernelName;   // kernel name, not instance name
    std::string kernelAlias;  // alias of kernel name
    std::string cuName;       // kernel_name:instance_name (for IP kernel)
    std::string instanceName; // instance name
    std::string kernelPluginFileName;
    uint64_t maxCapacity;
    uint64_t baseAddr;        // the base address of the CU
    uint32_t membankId;       // connected memory bank id
    uint32_t membankType;     // connected memory bank type
    uint64_t membankSize;     // connected memory bank size
    uint64_t membankBaseAddr; // connected memory bank base address
    channelData channels[XRM_MAX_KERNEL_CHANNELS];
    int32_t numChanInuse;
    uint64_t clients[XRM_MAX_KERNEL_CHANNELS]; // client id attached to cu
    int32_t numClient;                         // current number of processes attached to cu
    reserveData reserves[XRM_MAX_KERNEL_RESERVES];
    int32_t numReserve; // number of reserves on this cu
    int32_t usedLoad;   // percentage: 0 - 100, allocated or reserved load

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& cuId& cuType& kernelName& kernelAlias& cuName& instanceName& kernelPluginFileName& maxCapacity& baseAddr&
            baseAddr& channels& numChanInuse& clients& numClient& reserves& numReserve& usedLoad;
    }
} cuData;

/* memory topology information */
typedef struct memTopologyData {
    uint8_t memType;
    uint8_t memUsed;
    uint64_t memSize;
    uint64_t memBaseAddress;
    unsigned char memTag[16];

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& memType& memUsed& memSize& memBaseAddress& memTag;
    }
} memTopologyData;

/* connectivity information */
typedef struct connectData {
    int32_t argIndex;
    int32_t ipLayoutIndex;
    int32_t memDataIndex;

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& argIndex& ipLayoutIndex& memDataIndex;
    }
} connectData;

/* xclbin information */
typedef struct xclbinInformation {
    uuid_t uuid;
    std::string uuidStr;
    int32_t numCu;
    int32_t numHardwareCu;
    int32_t numSoftwareCu;
    cuData cuList[XRM_MAX_XILINX_KERNELS];
    int32_t numMemBank;
    int32_t numConnect;
    memTopologyData memTopologyList[XRM_MAX_DDR_MAP];
    connectData connectList[XRM_MAX_CONNECTION_ENTRIES];

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& uuid& uuidStr& numCu& numHardwareCu& numSoftwareCu& cuList& numMemBank& numConnect& memTopologyList&
            connectList;
    }
} xclbinInformation;

typedef struct clientData {
    uint64_t clientId;
    int32_t ref;

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& clientId& ref;
    }
} clientData;

typedef struct deviceData {
    std::string dsaName;
    std::string xclbinName;
    int32_t devId;
    bool isLoaded; // xclbin loaded or not
    bool isExcl;   // exclusive access to the device
    char* memBuffer;
    xclDeviceHandle deviceHandle;
    xclDeviceInfo2 deviceInfo;
    xclbinInformation xclbinInfo;
    clientData clientProcs[XRM_MAX_DEV_CLIENTS]; // processes using device (allocation but NOT reservation)

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& dsaName& xclbinName& devId& isLoaded& isExcl& xclbinInfo& clientProcs;
    }
} deviceData;

typedef struct pluginInformation {
    std::string xrmPluginName;
    std::string xrmPluginFileName;
    xrmPluginData pluginData;
} pluginInformation;

/* version depended functions from Lib */

/**
 * In xrt 2019.2 (xrt 2.3.xxx)
 * XRT_VERSION_RELEASE 201920
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 3
 *
 * and
 *
 * In xrt 2019.2_pu1 (xrt 2.4.xxx)
 * XRT_VERSION_RELEASE 201920_PU1
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 4
 *
 * int xclCuName2Index(xclDeviceHandle handle, const char *cu_name, uint32_t *cu_index);
 */
typedef int32_t (*libVersionDepFuncXclCuName2IndexType)(xclDeviceHandle handle,
                                                        const char* cu_name,
                                                        uint32_t* cu_index);

/**
 * In xrt 2019.2_pu2 (xrt 2.5.xxx)
 * XRT_VERSION_RELEASE 201920_PU2
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 5
 *
 * int xclIPName2Index(xclDeviceHandle handle, const char *ipName, uint32_t *ipIndex);
 */
typedef int32_t (*libVersionDepFuncXclIPName2IndexType25)(xclDeviceHandle handle,
                                                          const char* ipName,
                                                          uint32_t* ipIndex);

/**
 * In xrt 2020.1 and newer version (xrt >= 2.6.xxx)
 * XRT_VERSION_RELEASE 202010
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 6
 *
 * int xclIPName2Index(xclDeviceHandle handle, const char *ipName);
 */
typedef int32_t (*libVersionDepFuncXclIPName2IndexType26)(xclDeviceHandle handle, const char* ipName);

/**
 * In xrt 2020.1 and older version (xrt <= 2.6.xxx)
 * XRT_VERSION_RELEASE 202010
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 6
 *
 * int xclLockDevice(xclDeviceHandle handle);
 */
typedef int32_t (*libVersionDepFuncXclLockDeviceType)(xclDeviceHandle handle);

/**
 * In xrt 2020.1 and older version (xrt <= 2.6.xxx)
 * XRT_VERSION_RELEASE 202010
 * XRT_VERSION_MAJOR 2
 * XRT_VERSION_MINOR 6
 *
 * int xclUnlockDevice(xclDeviceHandle handle);
 */
typedef int32_t (*libVersionDepFuncXclUnlockDeviceType)(xclDeviceHandle handle);

typedef struct libVersionDepFunctionsData {
    libVersionDepFuncXclCuName2IndexType libVersionDepFuncXclCuName2Index;
    libVersionDepFuncXclIPName2IndexType25 libVersionDepFuncXclIPName2Index25;
    libVersionDepFuncXclIPName2IndexType26 libVersionDepFuncXclIPName2Index26;
    libVersionDepFuncXclLockDeviceType libVersionDepFuncXclLockDevice;
    libVersionDepFuncXclUnlockDeviceType libVersionDepFuncXclUnlockDevice;
} libVersionDepFunctionsData;

struct Version {
    int major = 0, minor = 0, revision = 0;
    Version(std::string version) { std::sscanf(version.c_str(), "%d.%d.%d", &major, &minor, &revision); }

    bool operator<(const Version& other) {
        if (major < other.major) return true;
        if (minor < other.minor) return true;
        if (revision < other.revision) return true;
        return false;
    }

    bool operator==(const Version& other) {
        return major == other.major && minor == other.minor && revision == other.revision;
    }

    friend std::ostream& operator<<(std::ostream& stream, const Version& ver) {
        stream << ver.major;
        stream << '.';
        stream << ver.minor;
        stream << '.';
        stream << ver.revision;
        return stream;
    }
};

class system {
   public:
    system() {}

    void initSystem();
    void initConfig();
    void initLibVersionDepFunctions();
    void initDevices();
    void initXrmPlugins();
    void initUdfCuGroups();
    int32_t loadDevices(pt::ptree& loadTree, std::string& errmsg);
    int32_t loadOneDevice(pt::ptree& loadTree, std::string& errmsg);
    bool unloadOneDevice(const int32_t& devId, std::string& errmsg);
    void* listDevice(int32_t devId);
    int32_t getDeviceNumber();
    int32_t getLogLevel();
    uint64_t getNewClientId();
    void* listXrmPlugin(int32_t pluginId);
    int32_t getXrmPluginNumber();

    void binToHexstr(unsigned char* in, int32_t insz, std::string& outStr);
    void hexstrToBin(std::string& inStr, int32_t insz, unsigned char* out);

    void initLock();
    inline void enterLock();
    inline void exitLock();
    void logMsg(xrmLogLevelType logLevel, const char* format, ...);

    void save();
    bool restore();

    void recycleResource(uint64_t clientId);

    uint64_t getCuMaxCapacity(cuProperty* cuProp);
    int32_t checkCuStat(cuResource* crRes, cuStatus* cuStat);

    /* following alloc and free function need to do lock protect */
    int32_t resAllocCu(cuProperty* cuProp, cuResource* cuRes, bool updateId);
    int32_t resAllocCuList(cuListProperty* cuListProp, cuListResource* cuListRes);
    int32_t resAllocationQuery(allocationQueryInfo* allocQuery, cuListResource* cuListRes);
    int32_t resReleaseCu(cuResource* cuRes);
    int32_t resReleaseCuList(cuListResource* cuListRes);
    int32_t resUdfCuGroupDeclare(udfCuGroupInformation* udfCuGroupInfo);
    int32_t resUdfCuGroupUndeclare(udfCuGroupInformation* udfCuGroupInfo);
    int32_t resAllocCuGroup(cuGroupProperty* cuGroupProp, cuGroupResource* cuGroupRes);
    int32_t resReleaseCuGroup(cuGroupResource* cuGroupRes);
    void flushUdfCuGroupInfo(uint32_t udfCuGroupIdx);

    int32_t resReserveCu(uint64_t reservePoolId, cuProperty* cuProp);
    int32_t resReserveCuList(uint64_t reservePoolId, cuListProperty* cuListProp);
    int32_t resReserveCuOfXclbin(uint64_t reservePoolId, uuid_t uuid, uint64_t clientId);
    int32_t resRelinquishCuList(uint64_t reservePoolId);
    uint64_t resReserveCuPool(cuPoolProperty* cuPoolProp);
    int32_t resRelinquishCuPool(uint64_t reservePoolId);
    int32_t resReservationQuery(uint64_t reservePoolId, cuPoolResource* cuPoolRes);

    int32_t resAllocCuWithLoad(cuProperty* cuProp, std::string xclbin, cuResource* cuRes, bool updateId);
    int32_t resLoadAndAllocAllCu(std::string xclbin, uint64_t clientId, cuListResource* cuListRes);

    /* xrm plugin related functions */
    void flushXrmPluginInfo(uint32_t xrmPluginId);
    int32_t loadOneXrmPlugin(std::string& xrmPluginFileName, std::string& xrmPluginName, std::string& errmsg);
    int32_t loadXrmPlugins(pt::ptree& loadPluginsTree, std::string& errmsg);
    int32_t unloadOneXrmPlugin(std::string& xrmPluginName, std::string& errmsg);
    int32_t unloadXrmPlugins(pt::ptree& loadPluginsTree, std::string& errmsg);
    int32_t execXrmPluginFunc(const char* xrmPluginName,
                              uint32_t funcId,
                              xrmPluginFuncParam* param,
                              std::string& errmsg);

    /* wrap function from lib */
    int32_t wrapIPName2Index(xclDeviceHandle handle, const char* ipName);
    int32_t wrapLockDevice(xclDeviceHandle handle);
    int32_t wrapUnlockDevice(xclDeviceHandle handle);

   private:
    int32_t xclbinLoadToDevice(int32_t devId, std::string& errmsg);
    int32_t openDevice(int32_t devId);
    int32_t xclbinReadFile(int32_t devId, std::string& name, std::string& errmsg);
    int32_t deviceLoadXclbin(int32_t devId, std::string& xclbin, std::string& errmsg);
    int32_t loadAnyDevice(int32_t* devId, std::string& xclbin, std::string& errmsg);
    void flushDevData(int32_t devId);
    bool deviceFreeBuffer(int32_t devId);
    bool deviceClearInfo(int32_t devId);
    bool isDeviceLoaded(int32_t devId);
    bool isDeviceLoadedXclbinWithUuid(int32_t devId, uuid_t uuid);
    bool isDeviceBusy(int32_t devId);

    int32_t xclbinGetInfo(int32_t devId, std::string& errmsg);
    int32_t xclbinGetKeyvalues(int32_t devId, std::string& errmsg);
    int32_t xclbinGetUuid(int32_t devId, std::string& errmsg);
    int32_t xclbinGetLayout(int32_t devId, std::string& errmsg);
    int32_t xclbinGetConnectivity(int32_t devId, std::string& errmsg);
    int32_t xclbinGetMemTopology(int32_t devId, std::string& errmsg);

    void deviceDumpResource(int32_t devId);
    int32_t deviceLockXclbin(int32_t devId);
    int32_t deviceUnlockXclbin(int32_t devId);

    int32_t cuFindFreeChannelId(cuData* cu);
    void cuInitChannels(cuData* cu);

    int32_t allocDevForClient(int32_t* devId, cuProperty* cuProp);
    int32_t getNextFreeDevForClient(int32_t* devId, cuProperty* cuProp);
    int32_t verifyProcess(pid_t pid);
    int32_t allocClientFromDev(int32_t devId, cuProperty* cuProp);
    int32_t allocCuFromDev(int32_t devId, cuProperty* cuProp, cuResource* cuRes);
    int32_t allocChanClientFromCu(cuData* cu, cuProperty* cuProp, cuResource* cuRes);
    void addClientToCu(cuData* cu, uint64_t clientId);
    int isClientUsingCu(cuData* cu, uint64_t clientId);

    int32_t releaseClientOnDev(int32_t devId, uint64_t clientId);
    void releaseAllCuChanClientOnDev(deviceData* dev, uint64_t clientId);
    void removeClientOnCu(cuData* cu, uint64_t clientId);

    uint64_t getNextAllocServiceId();
    void updateAllocServiceId();
    uint64_t getNextReservePoolId();
    void updateReservePoolId();

    int32_t isReservePoolUsingCu(cuData* cu, uint64_t reservePoolId);
    int32_t reserveLoadFromCu(uint64_t reservePoolId, cuData* cu, cuProperty* cuProp);
    int32_t reserveCuFromDev(uint64_t reservePoolId, int32_t devId, cuProperty* cuProp);

    deviceData m_devList[XRM_MAX_XILINX_DEVICES];
    uint32_t m_logLevel;
    uint64_t m_clientId;
    int32_t m_numDevice;
    pluginInformation m_xrmPluginList[XRM_MAX_PLUGIN_NUM];
    uint32_t m_numXrmPlugin;
    udfCuGroupInformation m_udfCuGroups[XRM_MAX_UDF_CU_GROUP_NUM];
    uint32_t m_numUdfCuGroup;
    libVersionDepFunctionsData m_libVersionDepFuncs;
    std::string m_xrtVersionFileFullPathName;
    std::string m_libXrtCoreFileFullPathName;
    uint64_t m_allocServiceId;
    uint64_t m_reservePoolId;
    pthread_mutex_t m_lock;

    friend class boost::serialization::access;

    template <class Archive>
    void serialize(Archive& ar, const unsigned int version) {
        std::ignore = version;
        ar& m_logLevel& m_clientId& m_devList& m_numDevice& m_udfCuGroups& m_allocServiceId& m_reservePoolId;
    }
};
} // namespace xrm

#endif // _XRM_SYSTEM_HPP_

max_stars_repo_name: yuanjiehuang/XRM

max_issues_repo_name: yuanjiehuang/XRM

max_forks_repo_name: yuanjiehuang/XRM


text:
/* Copyright (C) 2019 Neo Huang - All Rights Reserved */
#include "flashroute/single_host.h"

#include <memory>
#include <string>

#include <boost/format.hpp>
#include "glog/logging.h"

#include "flashroute/utils.h"
#include "flashroute/udp_prober.h"
#include "flashroute/udp_prober_v6.h"

namespace flashroute {

SingleHost::SingleHost(const uint16_t srcPort, const uint16_t dstPort,
                       const uint8_t ttlOffset)
    : srcPort_(srcPort), dstPort_(dstPort), ttlOffset_(ttlOffset) {
  results_ = new std::unordered_map<
      uint8_t, std::tuple<std::shared_ptr<IpAddress>, uint32_t>>();
}

SingleHost::~SingleHost() {
  delete results_;
}

void SingleHost::startScan(const std::string& target,
                           const std::string& interface) {
  auto remoteHost =
      std::unique_ptr<IpAddress>(parseIpFromStringToIpAddress(target));

  std::string localIpAddress =
      getAddressByInterface(interface, remoteHost->isIpv4());

  PacketReceiverCallback response_handler =
      [this](const IpAddress& destination, const IpAddress& responder,
             uint8_t distance, uint32_t rtt, bool fromDestination, bool ipv4,
             void* packetBuffer, uint32_t packetLen) {
        parseIcmpProbing(destination, responder, distance, rtt, fromDestination,
                         ipv4, packetBuffer, packetLen);
      };

  Prober* prober;
  if (remoteHost->isIpv4()) {
    prober = new UdpProber(&response_handler, 0, 0, dstPort_, "test", true,
                           ttlOffset_);
  } else {
    prober = new UdpProberIpv6(&response_handler, 0, 0, dstPort_, "test",
                               ttlOffset_);
  }
  NetworkManager networkManager(prober, interface, 100, remoteHost->isIpv4());
  networkManager.startListening();

  for (uint8_t i = 1 + ttlOffset_; i <= 32 + ttlOffset_; i ++) {
    networkManager.scheduleProbeRemoteHost(*remoteHost, i);
  }

  sleep(3);

  for (uint8_t i = 1 + ttlOffset_; i <= 32 + ttlOffset_; i++) {
    if (results_->find(i) == results_->end()) {
      LOG(INFO) << boost::format("%1% %|5t|*") % static_cast<int>(i);
    } else {
      LOG(INFO) << boost::format("%1% %|5t|%2% %|5t|%3% ms") %
                       static_cast<int>(i) %
                       parseIpFromIpAddressToString(
                           *std::get<0>(results_->find(i)->second)) %
                       std::get<1>(results_->find(i)->second);
    }
  }

  LOG(INFO) << " =============================";

  LOG(INFO) << "Checksum Mismatches: " << prober->getChecksumMismatches();
  LOG(INFO) << "Distance Abnormalities: " << prober->getDistanceAbnormalities();
  LOG(INFO) << "Other Mismatches: " << prober->getOtherMismatches();
}

bool SingleHost::parseIcmpProbing(const IpAddress& destination,
                                  const IpAddress& responder, uint8_t distance,
                                  uint32_t rtt, bool fromDestination, bool ipv4,
                                  void* packetBuffer, uint32_t packetLen) {
  results_->insert(
      {distance,
       std::make_tuple(std::shared_ptr<IpAddress>(responder.clone()), rtt)});
  return true;
}

}  // namespace flashroute

ext: cc

max_stars_repo_name: maxmouchet/FlashRoute

max_issues_repo_name: maxmouchet/FlashRoute

max_forks_repo_name: maxmouchet/FlashRoute


text:
#include <iostream>
#include <functional>
#include <unordered_map>
#include <thread>
#include <mutex>

#include <boost/unordered_map.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>

#include <succinct/mapper.hpp>

#include "index_types.hpp"
#include "wand_data_compressed.hpp"
#include "wand_data_raw.hpp"
#include "queries.hpp" // BOW queries
#include "weighted_queries.hpp" // RM queries
#include "util.hpp"
#include "docvector/document_index.hpp"

namespace {
void printUsage(const std::string &programName) {
  std::cerr << "Usage: " << programName
            << " index_type query_algorithm index_filename forward_index_filename ext_index_filename ext_forward_index_filename --map map_filename --ext_map ext_map_filename --output out_name --wand wand_data_filename --ext_wand ext_wand_data_filename"
            << " [--compressed-wand] --query query_filename --kexp no_docs_for_expansion --texp no_terms_to_expand"
            << " --rweight rm_weight_original_query [0, 1] --kfinal no_docs_for_final --lexicon lexicon_file " << std::endl;
}
} // namespace

using namespace ds2i;

template<typename Functor>
void op_dump_trec(Functor query_func, // XXX!!!
                 std::vector<std::pair<uint32_t, ds2i::term_id_vec>> const &queries,
                 std::vector<std::string>& id_map,
                 std::string const &query_type,
                 std::ofstream& output) {
    using namespace ds2i;

    // Run queries
    for (auto const &query: queries) {
      std::vector<std::pair<double, uint64_t>> top_k;
      auto tick = get_time_usecs();
      top_k = query_func(query.second); // All stages
      auto tock = get_time_usecs();
      double elapsedms = (tock-tick)/1000;
      std::cerr << query.first << " took ~ " << elapsedms << " ms\n";
      output_trec(top_k, query.first, id_map, query_type, output);
    }
}

typedef std::vector<std::pair<double, uint64_t>> top_k_list;

template<typename IndexType, typename WandType>
void rm_three_expansion_external(const char *index_filename,
              const char *ext_index_filename,
              const char *wand_data_filename,
              const char *ext_wand_data_filename,
              const char *forward_index_filename,
              const char *ext_forward_index_filename,
              std::vector<std::pair<uint32_t, ds2i::term_id_vec>> const &queries,
              std::vector<std::pair<uint32_t, ds2i::term_id_vec>> const &ext_queries,
              std::string const &type,
              std::string const &query_type,
              const char *map_filename,
              const char *ext_map_filename,
              const char *output_filename,
              const uint64_t m_k,
              const uint64_t exp_k,
              const uint64_t expand_term_count,
              const double r_weight,
              std::unordered_map<term_id_type, term_id_type>& back_map) {
    using namespace ds2i;

    /* Target Corpus Init */

    IndexType index;
    logger() << "Loading target index from " << index_filename << std::endl;
    boost::iostreams::mapped_file_source m(index_filename);
    succinct::mapper::map(index, m);

    document_index forward_index;
    logger() << "Loading target forward index from " << forward_index_filename << std::endl;
    forward_index.load(std::string(forward_index_filename));

    std::vector<std::string> doc_map;
    logger() << "Loading target map file from " << map_filename << std::endl;
    std::ifstream map_in(map_filename);
    std::string t_docid;
    while (map_in >> t_docid) {
      doc_map.emplace_back(t_docid);
    }
    logger() << "Loaded " << doc_map.size() << " DocID's from target corpus" << std::endl;

    logger() << "Warming up posting lists on target collection" << std::endl;
    std::unordered_set<term_id_type> warmed_up;
    for (auto const &q: queries) {
        for (auto t: q.second) {
            if (!warmed_up.count(t)) {
                index.warmup(t);
                warmed_up.insert(t);
            }
        }
    }

    /* External Corpus Init */

    IndexType ext_index;
    logger() << "Loading external index from " << ext_index_filename << std::endl;
    boost::iostreams::mapped_file_source ext_m(ext_index_filename);
    succinct::mapper::map(ext_index, ext_m);

    document_index ext_forward_index;
    logger() << "Loading external forward index from " << ext_forward_index_filename << std::endl;
    ext_forward_index.load(std::string(ext_forward_index_filename));

    std::vector<std::string> ext_doc_map;
    logger() << "Loading external map file from " << ext_map_filename << std::endl;
    std::ifstream ext_map_in(ext_map_filename);
    std::string ext_t_docid;
    while (ext_map_in >> ext_t_docid) {
      ext_doc_map.emplace_back(ext_t_docid);
    }
    logger() << "Loaded " << ext_doc_map.size() << " DocID's from external corpus" << std::endl;

    logger() << "Warming up posting lists on external collection" << std::endl;
    std::unordered_set<term_id_type> ext_warmed_up;
    for (auto const &ext_q: ext_queries) {
        for (auto ext_t: ext_q.second) {
            if (!ext_warmed_up.count(ext_t)) {
                ext_index.warmup(ext_t);
                ext_warmed_up.insert(ext_t);
            }
        }
    }

    /* Source Corpus Wand Init */

    WandType wdata;

    std::vector<std::string> query_types;
    boost::algorithm::split(query_types, query_type, boost::is_any_of(":"));
    boost::iostreams::mapped_file_source md;
    if (wand_data_filename) {
        md.open(wand_data_filename);
        succinct::mapper::map(wdata, md, succinct::mapper::map_flags::warmup);
    }

    /* External Corpus Wand Init */

    WandType ext_wdata;

    std::vector<std::string> ext_query_types;
    boost::algorithm::split(ext_query_types, query_type, boost::is_any_of(":"));
    boost::iostreams::mapped_file_source ext_md;
    if (ext_wand_data_filename) {
        ext_md.open(ext_wand_data_filename);
        succinct::mapper::map(ext_wdata, ext_md, succinct::mapper::map_flags::warmup);
    }

    /* Our output file */

    std::ofstream output_handle(output_filename);

    // Init our source ranker
    std::unique_ptr<doc_scorer> ranker = build_ranker(wdata.average_doclen(),
                                                      wdata.num_docs(),
                                                      wdata.terms_in_collection(),
                                                      wdata.ranker_id());

    // Init our external ranker
    std::unique_ptr<doc_scorer> ext_ranker = build_ranker(ext_wdata.average_doclen(),
                                                      ext_wdata.num_docs(),
                                                      ext_wdata.terms_in_collection(),
                                                      ext_wdata.ranker_id());

    // Final k documents to retrieve
    uint64_t k_final = m_k;

    // Initial k docs to retrieve
    uint64_t k_expand = exp_k;

    logger() << "Performing " << type << " queries" << std::endl;

    for (auto const &t: query_types) {
        logger() << "Query type: " << t << std::endl;

        std::function<std::vector<std::pair<double, uint64_t>>(ds2i::term_id_vec)> query_fun;
        if (t == "wand" && wand_data_filename) {
            query_fun = [&](ds2i::term_id_vec query) {
              // Default returns count of top-k, but we want the vector
              auto tmp = wand_query<WandType>(ext_wdata, k_expand);
              tmp(ext_index, query, ext_ranker);
              auto tk = tmp.topk();
              auto weighted_query = ext_forward_index.rm_expander(tk, expand_term_count);
              normalize_weighted_query_ext(weighted_query, back_map);
              query_from_ext_to_src(query, back_map);
              add_original_query(r_weight, weighted_query, query);
              auto final_traversal = weighted_maxscore_query<WandType>(wdata, k_final);
              final_traversal(index, weighted_query, ranker);
              return final_traversal.topk();
            };
        } else if (t == "block_max_wand" && wand_data_filename) {
            query_fun = [&](ds2i::term_id_vec query) {
              auto tmp = block_max_wand_query<WandType>(ext_wdata, k_expand);
              tmp(ext_index, query, ext_ranker);
              auto tk = tmp.topk();
              auto weighted_query = ext_forward_index.rm_expander(tk, expand_term_count);
              normalize_weighted_query_ext(weighted_query, back_map);
              query_from_ext_to_src(query, back_map);
              add_original_query(r_weight, weighted_query, query);
              auto final_traversal = weighted_maxscore_query<WandType>(wdata, k_final);
              final_traversal(index, weighted_query, ranker);
              return final_traversal.topk();
            };
        }  else if (t == "ranked_or" && wand_data_filename) {
            query_fun = [&](ds2i::term_id_vec query) {
              auto tmp = ranked_or_query<WandType>(ext_wdata, k_expand);
              tmp(ext_index, query, ext_ranker);
              auto tk = tmp.topk();
              auto weighted_query = ext_forward_index.rm_expander(tk, expand_term_count);
              normalize_weighted_query_ext(weighted_query, back_map);
              query_from_ext_to_src(query, back_map);
              add_original_query(r_weight, weighted_query, query);
              auto final_traversal = weighted_maxscore_query<WandType>(wdata, k_final);
              final_traversal(index, weighted_query, ranker);
              return final_traversal.topk();
          };
        } else if (t == "maxscore" && wand_data_filename) {
            /* Our favorite 1 */
            query_fun = [&](ds2i::term_id_vec query) {
              auto tmp = block_max_wand_query<WandType>(ext_wdata, k_expand); // USE BMW FOR STAGE 1
              tmp(ext_index, query, ext_ranker);
              auto tk = tmp.topk();
              auto weighted_query = ext_forward_index.rm_expander(tk, expand_term_count);
              normalize_weighted_query_ext(weighted_query, back_map);
              query_from_ext_to_src(query, back_map);
              add_original_query(r_weight, weighted_query, query);
              auto final_traversal = weighted_maxscore_query<WandType>(wdata, k_final);
              final_traversal(index, weighted_query, ranker);
              return final_traversal.topk();
            };
        } else {
            logger() << "Unsupported query type: " << t << std::endl;
            break;
        }

        op_dump_trec(query_fun, ext_queries, doc_map, t, output_handle);
    }
}

typedef wand_data<wand_data_raw> wand_raw_index;
typedef wand_data<wand_data_compressed<uniform_score_compressor>> wand_uniform_index;

int main(int argc, const char **argv) {
    using namespace ds2i;

    std::string programName = argv[0];
    if (argc < 4) {
    printUsage(programName);
    return 1;
    }

    document_index forward_index;

    std::string type = argv[1];
    std::string query_type = argv[2];

    const char *index_filename = argv[3];
    const char *forward_filename = argv[4];
    const char *wand_data_filename = nullptr;
    const char *map_filename = nullptr;
    const char *lexicon_filename = nullptr;

    const char *ext_index_filename = argv[5];
    const char *ext_forward_filename = argv[6];
    const char *ext_wand_data_filename = nullptr;
    const char *ext_map_filename = nullptr;
    const char *ext_lexicon_filename = nullptr;

    const char *out_filename = nullptr;
    const char *query_filename = nullptr;

    uint64_t m_k = 0;
    uint64_t exp_k = 0;
    uint64_t exp_t = 0;
    double r_weight = 0;
    bool compressed = false;

    std::vector<std::pair<uint32_t, term_id_vec>> queries;
    std::vector<std::pair<uint32_t, term_id_vec>> ext_queries;

    for (int i = 5; i < argc; ++i) {
        std::string arg = argv[i];

        if(arg == "--wand"){
            wand_data_filename = argv[++i];;
        }

        if(arg == "--ext_wand"){
            ext_wand_data_filename = argv[++i];;
        }

        if (arg == "--map") {
            map_filename = argv[++i];
        }

        if (arg == "--ext_map") {
            ext_map_filename = argv[++i];
        }

        if (arg == "--lexicon") {
          lexicon_filename = argv[++i];
        }

        if (arg == "--ext_lexicon") {
          ext_lexicon_filename = argv[++i];
        }

        if (arg == "--output") {
          out_filename = argv[++i];
        }

        if(arg == "--compressed-wand"){
            compressed = true;
        }

        if (arg == "--query") {
            query_filename = argv[++i];
        }

        if (arg == "--kfinal") {
          m_k = std::stoull(argv[++i]);
        }

        if (arg == "--kexp") {
          exp_k = std::stoull(argv[++i]);
        }

        if (arg == "--texp") {
          exp_t = std::stoull(argv[++i]);
        }

        if (arg == "--rweight") {
          r_weight = std::stod(argv[++i]);
        }
    }

    if (exp_k == 0 || m_k == 0 || exp_t == 0) {
        std::cerr << "Must set --texp --kexp and --kfinal" << std::endl;
        return EXIT_FAILURE;
    }

    if (out_filename == nullptr || map_filename == nullptr) {
      std::cerr << "ERROR: Must provide map and output file. Quitting."
                << std::endl;
      return EXIT_FAILURE;
    }

    if (lexicon_filename == nullptr || ext_lexicon_filename == nullptr || ext_map_filename == nullptr) {
      std::cerr << "ERROR: Must provide external map and source + target lexicon files. Quitting."
                << std::endl;
      return EXIT_FAILURE;
    }

    if (r_weight < 0 || r_weight > 1) {
      std::cerr << "Weight must be between 0 and 1/"
                << std::endl;
      return EXIT_FAILURE;
    }

    /* Target collection lexicon */
    std::unordered_map<std::string, uint32_t> lexicon;
    if (lexicon_filename != nullptr) {
      std::ifstream in_lex(lexicon_filename);
      if (in_lex.is_open()) {
        read_lexicon(in_lex, lexicon);
      }
      else {
        std::cerr << "ERROR: Could not open lexicon file." << std::endl;
      }
    }

    /* Source collection lexicon */
    std::unordered_map<std::string, uint32_t> ext_lexicon;
    if (ext_lexicon_filename != nullptr) {
      std::ifstream ext_in_lex(ext_lexicon_filename);
      if (ext_in_lex.is_open()) {
        read_lexicon(ext_in_lex, ext_lexicon);
      }
      else {
        std::cerr << "ERROR: Could not open lexicon file." << std::endl;
      }
    }

    /* Build the back map */
    std::unordered_map<term_id_type, term_id_type> back_map;

    for (auto it : ext_lexicon) {
        std::unordered_map<std::string, uint32_t>::const_iterator got = lexicon.find (it.first);
        if ( got != lexicon.end() ) {
            back_map.emplace(it.second, got->second);
        }
    }

    term_id_vec q;
    uint32_t qid;
    if(query_filename){
        std::filebuf fb;
        if (fb.open(query_filename, std::ios::in)) {
            std::istream is(&fb);
            if (lexicon_filename != nullptr) {
              while (read_query(q, qid, lexicon, is)) queries.emplace_back(qid, q);
            }
            else {
              while (read_query(q, qid, is)) queries.emplace_back(qid, q);
            }
        }
    } else {
        if (lexicon_filename != nullptr) {
          while (read_query(q, qid, lexicon)) queries.emplace_back(qid, q);
        }
        else {
          while (read_query(q, qid)) queries.emplace_back(qid, q);
        }
    }

    term_id_vec ext_q;
    uint32_t ext_qid;
    if(query_filename){
        std::filebuf ext_fb;
        if (ext_fb.open(query_filename, std::ios::in)) {
            std::istream ext_is(&ext_fb);
            while (read_query(ext_q, ext_qid, ext_lexicon, ext_is)) ext_queries.emplace_back(ext_qid, ext_q);
        }
    }

    /**/
    if (false) {
#define LOOP_BODY(R, DATA, T)                                                       \
        } else if (type == BOOST_PP_STRINGIZE(T)) {                                 \
            if (compressed) {                                                       \
                 rm_three_expansion_external<BOOST_PP_CAT(T, _index), wand_uniform_index>              \
                 (index_filename, ext_index_filename, wand_data_filename, ext_wand_data_filename, forward_filename, ext_forward_filename, queries, ext_queries, type, query_type, map_filename, ext_map_filename, out_filename, m_k, exp_k, exp_t, r_weight, back_map);   \
            } else {                                                                \
                rm_three_expansion_external<BOOST_PP_CAT(T, _index), wand_raw_index>                   \
                (index_filename, ext_index_filename, wand_data_filename, ext_wand_data_filename, forward_filename, ext_forward_filename, queries, ext_queries, type, query_type, map_filename, ext_map_filename, out_filename, m_k, exp_k, exp_t, r_weight, back_map);    \
            }                                                                       \
    /**/

BOOST_PP_SEQ_FOR_EACH(LOOP_BODY, _, DS2I_INDEX_TYPES);
#undef LOOP_BODY

    } else {
        logger() << "ERROR: Unknown type " << type << std::endl;
    }

}

max_stars_repo_name: rmit-ir/RMQV

max_issues_repo_name: JMMackenzie/RMQV

max_forks_repo_name: JMMackenzie/RMQV


text:
// Copyright 2018, Intel Corporation

#include "tile/codegen/partition.h"

#include <boost/optional.hpp>
#include <boost/optional/optional_io.hpp>

#include "base/util/logging.h"
#include "base/util/throw.h"
#include "tile/codegen/alias.h"
#include "tile/codegen/tidy.h"
#include "tile/codegen/tile.h"
#include "tile/math/util.h"
#include "tile/stripe/stripe.h"

namespace vertexai {
namespace tile {
namespace codegen {

using namespace stripe;  // NOLINT

namespace {

struct UseInfo {
  std::string idx_name;
};

using UseMap = std::map<Block*, UseInfo>;

struct BankInfo {
  std::string base_ref;
  size_t dim_pos;
  UseMap uses;
  TensorShape banked_shape;
  size_t num_banks;
};

Index* BiggestIndexForDim(Block* block, const Refinement& ref, int dim_pos) {
  Index* biggest_idx = nullptr;
  size_t biggest_range = 0;
  for (const auto& kvp : ref.access[dim_pos].getMap()) {
    auto idx = block->idx_by_name(kvp.first);
    if (idx && idx->range > biggest_range) {
      biggest_range = idx->range;
      biggest_idx = idx;
    }
  }
  return biggest_idx;
}

// bool IsRefRelatedToIdx(const Refinement& ref, const std::string& idx_name) {
//   for (const auto& aff : ref.access) {
//     if (aff.get(idx_name)) {
//       return true;
//     }
//   }
//   return false;
// }

// struct SplitRefInfo {
//   std::string name;
//   TensorShape scalar_shape;
//   TensorShape banked_shape;
//   std::vector<Affine> scalar_access;
//   std::vector<Affine> banked_access;
// };

// void FixupLoad() {
//   std::map<std::string, SplitRefInfo> related_map;
//   for (auto& ref : block->refs) {
//     if (IsRefRelatedToIdx(ref, idx_name)) {
//       std::string orig_name(ref.into());
//       auto raw_name = block->unique_ref_name(orig_name + "_raw");
//       std::vector<size_t> sizes(ref.shape.dims.size(), 1);
//       TensorShape scalar_shape = SimpleShape(ref.shape.type, sizes);
//       std::vector<Affine> scalar_access(ref.shape.dims.size());
//       TensorShape banked_shape = ref.shape;
//       for (auto& dim : banked_shape.dims) {
//         dim.size = 1;
//       }
//       SplitRefInfo info{
//           raw_name,       // name
//           banked_shape,   // banked_shape
//           scalar_shape,   // scalar_shape
//           scalar_access,  // scalar_access
//           ref.access,     // banked_access
//       };
//       related_map.emplace(orig_name, info);
//       ref.into = raw_name;
//     }
//   }

//   // Replace related loads and stores with switch blocks to handle halo conditions
//   for (auto it = block->stmts.begin(); it != block->stmts.end(); it++) {
//     auto load = Load::Downcast(*it);
//     if (load) {
//       auto it_related = related_map.find(load->from);
//       if (it_related != related_map.end()) {
//         const auto& info = it_related->second;
//         IVLOG(2, "load: ref " << load->from << " is related to " << idx_name << " via " << info.name);

//         auto interior = std::make_shared<Block>();
//         interior->name = load->from + "_load_interior";
//         interior->stmts.emplace_back(std::make_shared<Load>("src", "$X"));
//         interior->stmts.emplace_back(std::make_shared<Store>("$X", "dst"));
//         interior->refs.emplace_back(Refinement{
//             RefDir::In,          // dir
//             info.name,           // from
//             "src",               // into
//             info.banked_access,  // access
//             info.banked_shape,   // shape
//         });
//         interior->refs.emplace_back(Refinement{
//             RefDir::Out,         // dir
//             load->from,          // from
//             "dst",               // into
//             info.scalar_access,  // access
//             info.banked_shape,   // shape
//         });
//         // interior->constraints.emplace_back(info.banked_access - Affine(bank_name, part_size) -
//         Affine(offset_name)); block->stmts.emplace(it, interior);

//         auto low_edge = std::make_shared<Block>();
//         low_edge->name = load->from + "_load_low_edge";
//         block->stmts.emplace(it, low_edge);

//         auto high_edge = std::make_shared<Block>();
//         high_edge->name = load->from + "_load_high_edge";
//         block->stmts.emplace(it, high_edge);
//       }
//     }
//     auto store = Store::Downcast(*it);
//     if (store) {
//       auto ref = block->ref_by_into(store->into);
//       if (IsRefRelatedToIdx(*ref, idx_name)) {
//         LOG(INFO) << "store: ref " << ref->into() << " is related to " << idx_name;
//       }
//     }
//   }
// Finally add new refs
// for (const auto& info : related_map) {
//   block->refs.emplace_back(Refinement{
//       RefDir::None,              // dir
//       "",                        // from
//       info.first,                // into
//       {},                        // access
//       info.second.scalar_shape,  // shape
//   });
// }

// void FixupJumps(Block* block, Refinement* ref, size_t num_banks, const std::string& bank_name, bool is_primary) {
//   for (auto it = block->stmts.begin(); it != block->stmts.end(); it++) {
//     switch ((*it)->kind()) {
//       case StmtKind::Load: {
//         auto load = Load::Downcast(*it);
//         if (load->from == ref->into()) {
//           IVLOG(2, "Load needs fixup: " << *load);
//         }
//       } break;
//       case StmtKind::Store: {
//         auto store = Store::Downcast(*it);
//         if (store->into == ref->into()) {
//           IVLOG(2, "Store needs fixup: " << *store);
//         }
//       } break;
//       case StmtKind::Block: {
//         auto inner = Block::Downcast(*it);
//         auto inner_ref = inner->ref_by_from(ref->into(), false);
//         if (inner_ref != inner->refs.end()) {
//           // add passthru for bank idx
//           auto new_bank_name = inner->unique_idx_name(bank_name);
//           Index bank_idx{new_bank_name, 1, Affine{bank_name}};
//           if (is_primary) {
//             bank_idx.set_tag("bank");
//           }
//           inner->idxs.emplace_back(bank_idx);
//           // extend dim of refinement
//           for (size_t i = 0; i < ref->interior_shape.dims.size() - 1; i++) {
//             inner_ref->interior_shape.dims[i].stride = ref->interior_shape.dims[i].stride;
//           }
//           inner_ref->location.unit = Affine{new_bank_name};
//           inner_ref->bank_dim = ref->bank_dim;
//           FixupJumps(inner.get(), &*inner_ref, num_banks, new_bank_name, is_primary);
//         }
//       } break;
//       default:
//         break;
//     }
//   }
// }

struct BankedRef {
  BankDimension bank_dim;
  Affine cache_unit;
};

void PartitionBuffer(const AliasMap& alias_map,                          //
                     Block* block,                                       //
                     const std::map<std::string, BankInfo>& bank_infos,  //
                     const Tags& set_tags,                               //
                     const std::string& idx_tag) {
  IVLOG(2, "PartitionBuffer> " << block->name);
  std::map<std::string, size_t> tile_by_name;
  std::map<std::string, BankedRef> banked_refs;
  std::set<std::string> primary_idxs;
  // Initialize tile_by_name from the block's index ranges
  for (const auto& idx : block->idxs) {
    tile_by_name[idx.name] = idx.range;
  }
  // Look for refinements that need to be banked
  for (auto& ref : block->refs) {
    const auto& alias_info = alias_map.at(ref.into());
    auto it_bank_info = bank_infos.find(alias_info.base_name);
    if (it_bank_info == bank_infos.end()) {
      continue;
    }
    Index* idx = nullptr;
    const auto& bank_info = it_bank_info->second;
    auto it_usage = bank_info.uses.find(block);
    if (it_usage == bank_info.uses.end()) {
      idx = BiggestIndexForDim(block, ref, bank_info.dim_pos);
      if (!idx) {
        throw_with_trace(
            std::runtime_error(str(boost::format("Could not find valid index to bank on ref %1% in block %2%") %  //
                                   ref.into() % block->name)));
      }
      IVLOG(3, "  secondary> ref: " << ref.into() << ", idx: " << *idx);
    } else {
      idx = block->idx_by_name(it_usage->second.idx_name);
      primary_idxs.insert(idx->name);
      IVLOG(3, "  primary>   ref: " << ref.into() << ", idx: " << *idx);
    }
    // This tag is to prevent idxs from being pruned.
    // A later unroll pass will need these pinned idxs for expansion so that location affine expressions
    // can get fully resolved.
    idx->set_tag("$part");
    // Update the tile size for this index
    tile_by_name[idx->name] = math::RoundUp(idx->range, bank_info.num_banks);
    // Record information used for updating the outer block post-tiling.
    banked_refs[ref.into()] = BankedRef{BankDimension{bank_info.dim_pos}, Affine{idx->name}};
  }
  // Convert the tile based on index name to a tile based on index position
  TileShape tile(block->idxs.size());
  for (size_t i = 0; i < block->idxs.size(); i++) {
    tile[i] = tile_by_name.at(block->idxs[i].name);
  }
  IVLOG(2, "  tile: " << tile_by_name << " " << tile);
  ApplyTile(block, tile, false, true);
  // Update refinements, do this after tiling so that only the outer block gets these updates.
  for (const auto& item : banked_refs) {
    // Use ref_by_into because ApplyTile might cause Refinement pointers to become invalid
    block->ref_by_into(item.first)->mut().cache_unit = item.second.cache_unit;
  }
  block->set_tags(set_tags);
  for (const auto& idx_name : primary_idxs) {
    auto idx = block->idx_by_name(idx_name);
    if (!idx) {
      throw_with_trace(std::runtime_error(str(boost::format("Could not find primary index %1% on block %2%") %  //
                                              idx_name % block->name)));
    }
    idx->set_tag(idx_tag);
  }
  PruneIndexes(block, {"$part"});
}

void CollectBankInfo(std::map<std::string, BankInfo>* bank_infos,  //
                     const AliasMap& alias_map,                    //
                     Block* block,                                 //
                     const proto::PartitionMemoryPass& options) {
  IVLOG(2, "Partition> block: " << block->name);
  if (block->ref_outs().size() != 1) {
    IVLOG(1, boost::format("Partition> skipped '%1%' due to multiple outputs") % block->name);
    return;  // Only work on blocks with a single output
  }
  // Get the (only) output
  const Refinement* out_ref = block->ref_outs()[0];
  // Find the largest input
  size_t biggest = 0;
  AliasInfo big_alias;
  const Refinement* big_ref = nullptr;
  for (const auto& ref : block->ref_ins()) {
    // Get the source buffer size
    const auto& alias_info = alias_map.at(ref->into());
    size_t src_size = alias_info.base_ref->interior_shape.elem_size();
    if (src_size > biggest) {
      biggest = src_size;
      big_ref = ref;
      big_alias = alias_info;
    }
  }
  if (big_ref == nullptr) {
    IVLOG(1, "Partition> skipped due to no inputs");
    return;  // No inputs?  Skip this block
  }
  IVLOG(2, "  big_ref: " << PrintRefinement(*big_ref, block));
  // Find the evenest index that has a non-zero stride both on the large
  // input and on the output refinement
  std::string idx_name;
  size_t best_tile = 0;
  double best_ratio = 0;
  for (size_t i = 0; i < block->idxs.size(); i++) {
    const auto& idx = block->idxs[i];
    if (big_ref->FlatAccess().get(idx.name) == 0 || out_ref->FlatAccess().get(idx.name) == 0 || idx.range == 1) {
      continue;
    }
    size_t tile_size = math::RoundUp(idx.range, options.num_parts());
    size_t rounded_size = tile_size * options.num_parts();
    double ratio = static_cast<double>(idx.range) / static_cast<double>(rounded_size);
    IVLOG(3, "    "
                 << "idx: " << idx                          //
                 << ", num_parts: " << options.num_parts()  //
                 << ", tile_size: " << tile_size            //
                 << ", rounded_size: " << rounded_size      //
                 << ", ratio: " << ratio);
    if (ratio > best_ratio) {
      best_ratio = idx.range;
      best_tile = tile_size;
      idx_name = idx.name;
    }
  }
  if (best_tile == 0) {
    IVLOG(1, "Partition> skipped due to no valid indexes");
    return;  // No valid indexes?  Skip this block
  }
  // Determine the bank_dim for the split memory
  boost::optional<size_t> dim_pos;
  for (size_t i = 0; i < big_ref->access.size(); i++) {
    // Determine the dimension that is associated with the index we want to split on
    if (big_ref->access[i].get(idx_name)) {
      if (dim_pos) {
        IVLOG(1, "Partition> skipped due to complex banking");
        return;
      }
      dim_pos = i;
    }
  }
  IVLOG(2, "    dim_pos: " << dim_pos << ", base_ref: " << *big_alias.base_ref);
  if (!dim_pos) {
    IVLOG(1, "Could not find dimension to bank on for block: " << block->name << ", ref: " << *big_alias.base_ref);
    return;
  }

  auto& bank_info = (*bank_infos)[big_alias.base_name];
  if (bank_info.dim_pos && bank_info.dim_pos != dim_pos) {
    throw_with_trace(
        std::runtime_error(str(boost::format("Multiple uses differ about dim_pos on %1%") % big_alias.base_name)));
  }
  auto base_ref = big_alias.base_ref;
  bank_info.base_ref = base_ref->into();
  bank_info.dim_pos = *dim_pos;
  bank_info.uses[block].idx_name = idx_name;
  bank_info.banked_shape = base_ref->interior_shape;
  auto part_size = math::RoundUp(bank_info.banked_shape.dims[*dim_pos].size, options.num_parts());
  bank_info.banked_shape.resize_dim(bank_info.dim_pos, part_size);
  bank_info.num_banks = options.num_parts();
  base_ref->bank_dim = BankDimension{bank_info.dim_pos};
}

}  // namespace

void PartitionMemoryPass::Apply(CompilerState* state) const {
  std::map<std::string, BankInfo> bank_infos;
  auto reqs = FromProto(options_.reqs());
  RunOnBlocks(state->entry(), reqs, [&](const AliasMap& alias_map, Block* block) {  //
    CollectBankInfo(&bank_infos, alias_map, block, options_);
  });

  auto set_tags = FromProto(options_.set_tags());
  RunOnBlocks(state->entry(), reqs, [&](const AliasMap& alias_map, Block* block) {  //
    PartitionBuffer(alias_map, block, bank_infos, set_tags, options_.idx_tag());
  });
}

namespace {
[[gnu::unused]] char reg = []() -> char {
  CompilePassFactory<PartitionMemoryPass, proto::PartitionMemoryPass>::Register();
  return 0;
}();
}  // namespace
}  // namespace codegen
}  // namespace tile
}  // namespace vertexai

max_stars_repo_name: TolyaTalamanov/plaidml

max_issues_repo_name: HubBucket-Team/plaidml

max_forks_repo_name: HubBucket-Team/plaidml


text:
/*!
@file
Defines `boost::hana::detail::minimal::List`.

@copyright Louis Dionne 2014
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
 */

#ifndef BOOST_HANA_DETAIL_MINIMAL_LIST_HPP
#define BOOST_HANA_DETAIL_MINIMAL_LIST_HPP

#include <boost/hana/bool.hpp>
#include <boost/hana/comparable/comparable.hpp>
#include <boost/hana/core/datatype.hpp>
#include <boost/hana/foldable/foldable.hpp>
#include <boost/hana/iterable/mcd.hpp>
#include <boost/hana/list/mcd.hpp>
#include <boost/hana/monad/monad.hpp>

namespace boost { namespace hana {
namespace detail { namespace minimal {
    template <typename mcd = hana::List::mcd<void>>
    struct List { };

    template <typename Storage, typename mcd,
        typename = operators<hana::Comparable, hana::Monad>
    >
    struct list_type {
        Storage storage;
        using hana_datatype = List<mcd>;
    };

    template <typename mcd>
    struct make_list_impl {
        template <typename ...Xs>
        constexpr auto operator()(Xs ...xs) const {
            auto storage = [=](auto f) { return f(xs...); };
            return list_type<decltype(storage), mcd>{storage};
        }
    };

    template <typename mcd = hana::List::mcd<void>>
    constexpr make_list_impl<mcd> list{};
}} // end namespace detail::minimal

template <typename mcd>
struct Foldable::instance<detail::minimal::List<mcd>>
    : detail::FoldableFromIterable
{ };

template <typename Mcd>
struct Iterable::instance<detail::minimal::List<Mcd>>
    : Iterable::mcd
{
    template <typename Xs>
    static constexpr auto head_impl(Xs xs) {
        return xs.storage([](auto x, auto ...xs) {
            return x;
        });
    }

    template <typename Xs>
    static constexpr auto tail_impl(Xs xs) {
        return xs.storage([](auto x, auto ...xs) {
            return detail::minimal::list<Mcd>(xs...);
        });
    }

    template <typename Xs>
    static constexpr auto is_empty_impl(Xs xs) {
        return xs.storage([](auto ...xs) {
            return bool_<sizeof...(xs) == 0>;
        });
    }
};

template <>
struct List::instance<detail::minimal::List<List::mcd<void>>>
    : List::mcd<detail::minimal::List<List::mcd<void>>>
{
    template <typename X, typename Xs>
    static constexpr auto cons_impl(X x, Xs xs) {
        return xs.storage([=](auto ...xs) {
            return detail::minimal::list<List::mcd<void>>(x, xs...);
        });
    }

    static constexpr auto nil_impl() {
        return detail::minimal::list<List::mcd<void>>();
    }
};
}} // end namespace boost::hana

#endif // !BOOST_HANA_DETAIL_MINIMAL_LIST_HPP

max_stars_repo_name: rbock/hana

max_issues_repo_name: rbock/hana

max_forks_repo_name: rbock/hana


text:
// Boost.Maral library (Molecular Archiving, Retrieval & Algorithm Library)
//
// Copyright (C) 2014 Armin Madadkar Sobhani
//
// Distributed under the Boost Software License Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
// $Id$

#ifndef MARAL_MAKE_HPP
#define MARAL_MAKE_HPP

#include <memory>

#include <boost/any.hpp>

namespace maral {

////////////////////////////////////////////////////////////////////////////////

template<class T>
struct _Unique_if
{
    typedef std::unique_ptr<T> _Single_object;
};

template<class T>
struct _Unique_if<T[]>
{
    typedef std::unique_ptr<T[]> _Unknown_bound;
};

template<class T, size_t N>
struct _Unique_if<T[N]>
{
    typedef void _Known_bound;
};

template<class T, class... Args>
typename _Unique_if<T>::_Single_object
make(Args&&... args)
{
    return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
}

template<class T>
typename _Unique_if<T>::_Unknown_bound
make(size_t n)
{
    typedef typename std::remove_extent<T>::type U;
    return std::unique_ptr<T>(new U[n]());
}

template<class T, class... Args>
typename _Unique_if<T>::_Known_bound
make(Args&&...) = delete;

template <typename T>
using entity = std::unique_ptr<T>;

//using any = boost::any;

}    // namespace maral

#endif    // MARAL_MAKE_HPP

max_stars_repo_name: arminms/maral

max_issues_repo_name: arminms/maral

max_forks_repo_name: arminms/maral


text:
// from: https://stackoverflow.com/a/4548854/12291425

#ifdef UNICODE
#undef UNICODE
#endif
#include <Windows.h>
#include <Tlhelp32.h>
#include <string>
#include <vector>
#include <boost/algorithm/string.hpp>
#include <thread>
#include <chrono>
using namespace std;

// from: https://stackoverflow.com/a/32770084/12291425
BOOL TerminateMyProcess(DWORD dwProcessId, UINT uExitCode)
{
    DWORD dwDesiredAccess = PROCESS_TERMINATE;
    BOOL bInheritHandle = FALSE;
    HANDLE hProcess = OpenProcess(dwDesiredAccess, bInheritHandle, dwProcessId);
    if (hProcess == NULL)
        return FALSE;

    BOOL result = TerminateProcess(hProcess, uExitCode);

    CloseHandle(hProcess);

    return result;
}

void find_process(string sAppPath, string sAppName)
{
    PROCESSENTRY32 pe32 = {0};
    HANDLE hSnap;
    int iDone;
    bool bProcessFound;

    hSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
    pe32.dwSize = sizeof(PROCESSENTRY32);
    Process32First(hSnap, &pe32); // Can throw away, never an actual app

    bProcessFound = false; //init values
    iDone = 1;
    DWORD dwProcessID = 0;
    while (iDone) // go until out of Processes
    {
        iDone = Process32Next(hSnap, &pe32);
        string s{pe32.szExeFile};
        if (boost::starts_with(boost::to_lower_copy(s), "notepad.exe"))
        {
            iDone = 0;
            bProcessFound = true;
            dwProcessID = pe32.th32ProcessID;
        }
    }

    if (!bProcessFound) // if we didn't find it running...
    {
        // additional information
        STARTUPINFO si;
        PROCESS_INFORMATION pi;

        // set the size of the structures
        ZeroMemory(&si, sizeof(si));
        si.cb = sizeof(si);
        ZeroMemory(&pi, sizeof(pi));
        CreateProcess((sAppPath + sAppName).c_str(),
                      NULL,
                      NULL,
                      NULL, FALSE, 0, NULL, NULL, &si, &pi); // start it
        dwProcessID = pi.dwProcessId;

    }

    this_thread::sleep_for(5s);
    TerminateMyProcess(dwProcessID, 1);
}

int main()
{
    string appPath{"C:\\Windows\\"};
    string appName{"notepad.exe"};
    find_process(appPath, appName);

    return 0;
}


text:
#include <dirent.h>
#include <linux_parser.h>
#include <unistd.h>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <cassert>
#include <regex>
#include <iterator>
#include <stdio.h>
#include <math.h>
#include <boost/algorithm/string.hpp>
#include <unistd.h>

using std::stof;
using std::string;
using std::to_string;
using std::vector;

// DONE: An example of how to read data from the filesystem
string LinuxParser::OperatingSystem() {

  // Specify file and regex patterns
  std::string targetfile = "/etc/os-release";
  std::regex rgx("^PRETTY_NAME=\"(.+)\"$");

  // Find regex match in specified file
  std::string os_system = LineByLineRegexGetter(targetfile, rgx, "OS NOT FOUND");

  // Return OS string
  return os_system;

}

// DONE: An example of how to read data from the filesystem
string LinuxParser::Kernel() {

  // Specify file and regex patterns
  std::string targetfile = "/proc/version";
  std::regex rgx("^Linux\\s+version\\s([\\w\\.\\-]+)\\s.*$");

  // Find regex match in specified file
  std::string kernel = LineByLineRegexGetter(targetfile, rgx, "KERNEL NOT FOUND");

  // Return OS string
  return kernel;
}

// BONUS: Update this to use std::filesystem
vector<int> LinuxParser::Pids() {
  vector<int> pids;
  DIR* directory = opendir(kProcDirectory.c_str());
  struct dirent* file;
  while ((file = readdir(directory)) != nullptr) {
    // Is this a directory?
    if (file->d_type == DT_DIR) {
      // Is every character of the name a digit?
      string filename(file->d_name);
      if (std::all_of(filename.begin(), filename.end(), isdigit)) {
        int pid = stoi(filename);
        pids.push_back(pid);
      }
    }
  }
  closedir(directory);
  return pids;
}

float LinuxParser::CalculateMemoryUtilization(const int& memtotal, const int& memfree) {
  float memory_utilization = 1.0 - (float) memfree / (float) memtotal;
  return memory_utilization;
}

// TODO: Read and return the system memory utilization
float LinuxParser::MemoryUtilization() {

  // Vector to store free and total memory
  std::vector<int> memory_utilization = {-1, -1};

  // Regular Expression needed to grab info from line
  std::regex rgx("^\\w+\\:\\s+(\\d+)\\s+.*$");
  std::smatch memory_info;

  // Prepare strings required
  std::string filename = "/proc/meminfo";
  std::string fileline;

  // Open file and ensure it exists
  std::ifstream myfile;
  myfile.open(filename);
  if(myfile){
    for (int linenumber = 0; linenumber < memory_utilization.size(); linenumber++) {
      getline(myfile, fileline);
      if(std::regex_match(fileline, memory_info, rgx)) {
        memory_utilization[linenumber] = std::stof(memory_info[1]);
      }
    }
  } else {
    memory_utilization = {-1, -1};
  }
  myfile.close();

  return LinuxParser::CalculateMemoryUtilization(memory_utilization[0], memory_utilization[1]);
}

// TODO: Read and return the system uptime
long LinuxParser::UpTime() {

  // Specify target file and target rgx
  std::string targetfile = "/proc/uptime";
  std::regex rgx("^([\\d\\.]+)\\s+[\\d+\\.]+\\s*$");

  // find target in file
  std::string uptime = LinuxParser::LineByLineRegexGetter(targetfile, rgx, "-1");

  // return converted string
  return std::stol(uptime);
}

// // TODO: Read and return the number of jiffies for the system
// long LinuxParser::Jiffies() { return 0; }

// // TODO: Read and return the number of active jiffies for a PID
// // REMOVE: [[maybe_unused]] once you define the function
// long LinuxParser::ActiveJiffies(int pid[[maybe_unused]]) { return 0; }

// // TODO: Read and return the number of active jiffies for the system
// long LinuxParser::ActiveJiffies() { return 0; }

// // TODO: Read and return the number of idle jiffies for the system
// long LinuxParser::IdleJiffies() { return 0; }

// TODO: Read and return CPU utilization
vector<int> LinuxParser::CpuUtilization() {
  std::string line;
  std::vector<int> cpu_usages(10, -1);
  std::ifstream myfile;
  myfile.open ("/proc/stat");
  if (myfile) {
    getline(myfile, line);
    char char_array[line.length()];
    strcpy(char_array, line.c_str());
    char *token = strtok(char_array, " ");
    for (int index = 0; index < cpu_usages.size(); index ++) {
      if (index > 0) {
        cpu_usages[index-1] = std::stoi(token);
      };
      token = strtok(NULL, " ");
    }
  }
  myfile.close();
  return cpu_usages;
}

// TODO: Read and return the total number of processes
int LinuxParser::TotalProcesses() {
  std::regex rgx("^processes\\s+(\\d+)");
  std::string filename = "/proc/stat";
  return std::stoi(LinuxParser::LineByLineRegexGetter(filename, rgx, "-1"));
}

// TODO: Read and return the number of running processes
int LinuxParser::RunningProcesses() {
  std::regex rgx("^procs_running\\s+(\\d+)");
  std::string filename = "/proc/stat";
  return std::stoi(LinuxParser::LineByLineRegexGetter(filename, rgx, "-1"));
}

// TODO: Read and return the command associated with a process
// REMOVE: [[maybe_unused]] once you define the function
std::string LinuxParser::Command(int pid) {
  std::string file_location = "/proc/" + std::to_string(pid) + "/cmdline";
  std::string cmdline{""};
  std::ifstream myfile;
  myfile.open(file_location);
  if (myfile) {
    getline(myfile, cmdline);
  } else {
    cmdline = "CMDLINE NOT FOUND";
  }
  myfile.close();
  return cmdline;
}

float LinuxParser::Cpu(int pid, int uptime) {
  std::string file_location = "/proc/" + std::to_string(pid) + "/stat";
  std::vector<int> cpu_times = {-1, -1, -1, -1, -1};

  std::string line;
  std::fstream myfile;
  float cpu_util;
  myfile.open(file_location);
  if (myfile){
    getline(myfile, line);
    std::vector<std::string> line_strs;
    boost::split(line_strs, line, boost::is_any_of("\t "));
    myfile.close();

    int utime   = std::stoi(line_strs[13]);
    int stime   = std::stoi(line_strs[14]);
    int cutime  = std::stoi(line_strs[15]);
    int cstime  = std::stoi(line_strs[17]);
    int startt  = std::stoi(line_strs[21]);
    float HERTZ = sysconf(_SC_CLK_TCK);

    int total_time = utime + stime + cutime + cstime;
    float seconds  = (float)uptime - ((float)startt / HERTZ);
    cpu_util = ((float)total_time / HERTZ ) / seconds;
  } else {
    cpu_util = -1;
  }
  myfile.close();
  return cpu_util;
}

// TODO: Read and return the memory used by a process
// REMOVE: [[maybe_unused]] once you define the function
int LinuxParser::Ram(int pid) {
  std::string file_location = "/proc/" + std::to_string(pid) + "/status";
  std::regex rgx("^VmSize:\\s+(\\d+)\\s+kB\\s?$");
  std:string ram_useage = LinuxParser::LineByLineRegexGetter(file_location, rgx, "-1");
  return std::stol(ram_useage) / 1000;
}

// TODO: Read and return the user ID associated with a process
// REMOVE: [[maybe_unused]] once you define the function
string LinuxParser::Uid(int pid) {
  std::string file_location = "/proc/" + std::to_string(pid) + "/status";
  std::regex rgx("^Uid:\\s+(\\d+)\\s+.+$");
  return LinuxParser::LineByLineRegexGetter(file_location, rgx, "-1");
}

// TODO: Read and return the user associated with a process
// REMOVE: [[maybe_unused]] once you define the function
string LinuxParser::User(int uid) {
  std::string rgx_expr = "^([\\w\\d]+):x+:" + std::to_string(uid) + ":" + std::to_string(uid) + ".+$";
  std::regex rgx(rgx_expr);
  std::string file_location = "/etc/passwd";
  return LinuxParser::LineByLineRegexGetter(file_location, rgx, "USER NOT FOUND");
}

// TODO: Read and return the uptime of a process
// REMOVE: [[maybe_unused]] once you define the function
long int LinuxParser::UpTime(int pid) {

  // Intialize the two file reading variables needed
  std::string ss_key;
  std::string line;

  // Setup filestream and ensure it exists
  std::ifstream filestream("/proc/" + std::to_string(pid) + "/stat") ;
  if(filestream.is_open()) {
    while ( std::getline(filestream,line) ) {
      std::istringstream linestream(line);
      int index = 0;
      while(linestream >> ss_key ) {
        if (index == 21) {
          long int system_uptime =  LinuxParser::UpTime();
          long int process_uptime = system_uptime - std::stol(ss_key) / sysconf(_SC_CLK_TCK);
          return process_uptime;
        }
        index++;
      }
    }
  } else {
    return -1;
  }
}

std::string LinuxParser::LineByLineRegexGetter(std::string file_location, std::regex rgx, std::string std_return){
  //std::cout  << file_location << "\n";

  // reate File object and ensure it exists
  std::ifstream myfile;
  std::string line;
  std::string rgx_return;

  // Find rgx in file (if it exists)
  myfile.open(file_location);
  if(myfile) {
    std::smatch rgx_match;
    while (!std::regex_match(line, rgx_match, rgx)) {
      getline(myfile, line); // Iterate over each line of file until a hit is obtained
    }
    rgx_return = rgx_match[1]; // Return only the Regex Catch
  } else {
    rgx_return = std_return;
  }

  // Close file and return hit (if found)
  myfile.close();
  return rgx_return;

}

max_stars_repo_name: henry-ehlers/CppND-System-Monitor

max_issues_repo_name: henry-ehlers/CppND-System-Monitor

max_forks_repo_name: henry-ehlers/CppND-System-Monitor


text:
#include "games_storage_config.h"
#include <boost/filesystem.hpp>
#include <iostream>

namespace cavoke::server::model {

GamesStorageConfig GamesStorageConfig::load(const std::string &config_file) {
    if (config_file.empty()) {
        // FIXME: rewrite this mess...
        std::cerr << "No games storage config specified. Using a default one. "
                     "Please use `-c` option.\n";
        return GamesStorageConfig{"../../local_server/games", "logic",
                                  "client.zip", "config.json"};
    }
    if (!boost::filesystem::exists(config_file) ||
        !boost::filesystem::is_regular_file(config_file)) {  // FIXME
        throw std::invalid_argument(
            "No such game storage configuration file: " + config_file);
    }

    std::ifstream file(config_file);
    json j = json::parse(file);
    return j.get<GamesStorageConfig>();
}

}  // namespace cavoke::server::model

max_stars_repo_name: cavoke-project/cavoke

max_issues_repo_name: cavoke-project/cavoke

max_forks_repo_name: cavoke-project/cavoke


text:
#include "TranslationFirstPoseController.h"
#include <Eigen/Dense>
#include <iostream>

using namespace Eigen;

namespace DQ_robotics
{

TranslationFirstPoseController::TranslationFirstPoseController(DQ_kinematics robot, MatrixXd translation_feedback_gain, MatrixXd rotation_feedback_gain, double translation_damping, double rotation_damping) : DQ_controller()
{

    //Initialization of argument parameters
    robot_dofs_          = (robot.links() - robot.n_dummy());
    robot_               = robot;
    kp_                  = translation_feedback_gain;
	kr_                  = rotation_feedback_gain;
	translation_damping_ = translation_damping;
	rotation_damping_    = rotation_damping;

    //Initilization of remaining parameters
    thetas_         = MatrixXd(robot_dofs_,1);
    delta_thetas_   = MatrixXd::Zero(robot_dofs_,1);

    analytical_jacobian_  = MatrixXd(8,robot_dofs_);
	rotation_jacobian_    = MatrixXd(4,robot_dofs_);
	translation_jacobian_ = MatrixXd(4,robot_dofs_);

	nullspace_projector_  = MatrixXd(robot_dofs_, robot_dofs_);

    translation_jacobian_pseudoinverse_   = MatrixXd(robot_dofs_,4);
	rotation_jacobian_pseudoinverse_   = MatrixXd(robot_dofs_,4);

    identity4_           = Matrix<double,4,4>::Identity();
	identityDOFS_        = MatrixXd::Identity(robot_dofs_,robot_dofs_);

    error_translation_  = MatrixXd(4,1);
	error_rotation_     = MatrixXd(4,1);

    end_effector_pose_           = DQ(0,0,0,0,0,0,0,0);
	reference_translation_       = DQ(0,0,0,0,0,0,0,0);
	reference_rotation_          = DQ(0,0,0,0,0,0,0,0);

}

VectorXd TranslationFirstPoseController::getNewJointPositions( const DQ reference, const VectorXd thetas)
{

    delta_thetas_ = getNewJointVelocities(reference, thetas);

    // Send updated thetas to simulation
    return (thetas_ + delta_thetas_);

}

VectorXd TranslationFirstPoseController::getNewJointVelocities( const DQ reference, const VectorXd thetas)
{

    ///--Remapping arguments
    thetas_ = thetas;

	//Get translation and rotation individually
	reference_translation_ = reference.translation();
	reference_rotation_    = P(reference);

    ///--Controller Step

    //Calculate jacobian and FKM
    analytical_jacobian_  = robot_.analyticalJacobian(thetas_);
    end_effector_pose_    = robot_.fkm(thetas_);

    //Error
    error_translation_ = vec4( reference_translation_ -  end_effector_pose_.translation() );

	error_rotation_    = vec4( reference_rotation_    - P(end_effector_pose_) );

	//Calculate Jacobians
	translation_jacobian_ = translationJacobian(analytical_jacobian_, vec8(end_effector_pose_));
	rotation_jacobian_    = rotationJacobian(analytical_jacobian_);

	//Pseudoinverses calculation
	translation_jacobian_pseudoinverse_ = (translation_jacobian_.transpose())
										 *(translation_jacobian_*translation_jacobian_.transpose() + translation_damping_*translation_damping_*identity4_).inverse();

	rotation_jacobian_pseudoinverse_ =    (rotation_jacobian_.transpose())
										 *(rotation_jacobian_*rotation_jacobian_.transpose() + rotation_damping_*rotation_damping_*identity4_).inverse();

	//Nullspace projector
	nullspace_projector_ = (identityDOFS_ - (pseudoInverse(translation_jacobian_))*translation_jacobian_ );

    delta_thetas_ = translation_jacobian_pseudoinverse_ * kp_ * error_translation_ +
					nullspace_projector_ * rotation_jacobian_pseudoinverse_ * kr_ * error_rotation_;

    return delta_thetas_;

}

}