text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "util/file_utils.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <dirent.h>

#include <sstream>
#include <algorithm>
#include <iomanip>

#include <boost/filesystem.hpp>
#include <boost/system/error_code.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/trim.hpp>

#include <openssl/md5.h>

#include "olap/file_helper.h"
#include "util/defer_op.h"

namespace doris {

Status FileUtils::create_dir(const std::string& dir_path) {
    try {
        if (boost::filesystem::exists(dir_path.c_str())) {
            // No need to create one
            if (!boost::filesystem::is_directory(dir_path.c_str())) {
                std::stringstream ss;
                ss << "Path(" << dir_path << ") already exists, but not a directory.";
                return Status(ss.str());
            }
        } else {
            if (!boost::filesystem::create_directories(dir_path.c_str())) {
                std::stringstream ss;
                ss << "make directory failed. path=" << dir_path;
                return Status(ss.str());
            }
        }
    } catch (...) {
        std::stringstream ss;
        ss << "make directory failed. path=" << dir_path;
        return Status(ss.str());
    }
    return Status::OK;
}

Status FileUtils::remove_all(const std::string& file_path) {
    try {
        boost::filesystem::path boost_path(file_path);
        boost::system::error_code ec;
        boost::filesystem::remove_all(boost_path, ec);
        if (ec != boost::system::errc::success) {
            std::stringstream ss;
            ss << "remove all(" << file_path << ") failed, because: "
                << ec;
            return Status(ss.str());
        }
    } catch (...) {
        std::stringstream ss;
        ss << "remove all(" << file_path << ") failed, because: exception";
        return Status(ss.str());
    }
    return Status::OK;
}

Status FileUtils::scan_dir(
        const std::string& dir_path, std::vector<std::string>* files,
        int64_t* file_count) {

    DIR* dir = opendir(dir_path.c_str());
    if (dir == nullptr) {
        char buf[64];
        std::stringstream ss;
        ss << "opendir(" << dir_path << ") failed, because: " << strerror_r(errno, buf, 64);
        return Status(ss.str());
    }
    DeferOp close_dir(std::bind<void>(&closedir, dir));

    struct dirent entry;
    struct dirent* result = nullptr;
    int64_t count = 0;
    while (true) {
        int ret = readdir_r(dir, &entry, &result);
        if (ret != 0) {
            char buf[64];
            std::stringstream ss;
            ss << "readdir(" << dir_path << ") failed, because: " << strerror_r(errno, buf, 64);
            return Status(ss.str());
        }
        if (result == nullptr) {
            // Over
            break;
        }
        std::string file_name = result->d_name;
        if (file_name == "." || file_name == "..") {
            continue;
        }

        if (files != nullptr) {
            files->emplace_back(std::move(file_name));
        }
        count++;
    }

    if (file_count != nullptr) {
        *file_count = count;
    }

    return Status::OK;
}

bool FileUtils::is_dir(const std::string& path) {
    struct stat path_stat;
    if (stat(path.c_str(), &path_stat) != 0) {
        return false;
    }

    if (path_stat.st_mode & S_IFDIR) {
        return true;
    }

    return false;
}

// Through proc filesystem
std::string FileUtils::path_of_fd(int fd) {
    const int PATH_SIZE = 256;
    char proc_path[PATH_SIZE];
    snprintf(proc_path, PATH_SIZE, "/proc/self/fd/%d", fd);
    char path[PATH_SIZE];
    if (readlink(proc_path, path, PATH_SIZE) < 0) {
        path[0] = '\0';
    }
    return path;
}

Status FileUtils::split_pathes(const char* path, std::vector<std::string>* path_vec) {
    path_vec->clear();
    try {
        boost::split(*path_vec, path,
                     boost::is_any_of(";"),
                     boost::token_compress_on);
    } catch (...) {
        std::stringstream ss;
        ss << "Boost split path failed.[path=" << path << "]";
        return Status(ss.str());
    }

    for (std::vector<std::string>::iterator it = path_vec->begin(); it != path_vec->end();) {
        boost::trim(*it);

        it->erase(it->find_last_not_of("/") + 1);
        if (it->size() == 0) {
            it = path_vec->erase(it);
        } else {
            ++it;
        }
    }

    // Check if
    std::sort(path_vec->begin(), path_vec->end());
    if (std::unique(path_vec->begin(), path_vec->end()) != path_vec->end()) {
        std::stringstream ss;
        ss << "Same path in path.[path=" << path << "]";
        return Status(ss.str());
    }

    if (path_vec->size() == 0) {
        std::stringstream ss;
        ss << "Size of vector after split is zero.[path=" << path << "]";
        return Status(ss.str());
    }

    return Status::OK;
}

Status FileUtils::copy_file(const std::string& src_path, const std::string& dest_path) {
   // open src file
    FileHandler src_file;
    if (src_file.open(src_path.c_str(), O_RDONLY) != OLAP_SUCCESS) {
        char errmsg[64];
        LOG(ERROR) << "open file failed: " << src_path << strerror_r(errno, errmsg, 64);
        return Status("Internal Error");
    }
    // create dest file and overwrite existing file
    FileHandler dest_file;
    if (dest_file.open_with_mode(dest_path.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU)
        != OLAP_SUCCESS) {
        char errmsg[64];
        LOG(ERROR) << "open file failed: " << dest_path << strerror_r(errno, errmsg, 64);
        return Status("Internal Error");
    }

    const int64_t BUF_SIZE = 8192;
    char *buf = new char[BUF_SIZE];
    DeferOp free_buf(std::bind<void>(std::default_delete<char[]>(), buf));
    int64_t src_length = src_file.length();
    int64_t offset = 0;
    while (src_length > 0) {
        int64_t to_read = BUF_SIZE < src_length ? BUF_SIZE : src_length;
        if (OLAP_SUCCESS != (src_file.pread(buf, to_read, offset))) {
            return Status("Internal Error");
        }
        if (OLAP_SUCCESS != (dest_file.pwrite(buf, to_read, offset))) {
            return Status("Internal Error");
        }

        offset += to_read;
        src_length -= to_read;
    }
    return Status::OK;
}

Status FileUtils::md5sum(const std::string& file, std::string* md5sum) {
    int fd = open(file.c_str(), O_RDONLY);
    if (fd < 0) {
        return Status("failed to open file");
    }

    struct stat statbuf;
    if (fstat(fd, &statbuf) < 0) {
        close(fd);
        return Status("failed to stat file");
    }
    size_t file_len = statbuf.st_size;
    void* buf = mmap(0, file_len, PROT_READ, MAP_SHARED, fd, 0);

    unsigned char result[MD5_DIGEST_LENGTH];
    MD5((unsigned char*) buf, file_len, result);
    munmap(buf, file_len);

    std::stringstream ss;
    for (int32_t i = 0; i < MD5_DIGEST_LENGTH; i++) {
        ss << std::setfill('0') << std::setw(2) << std::hex << (int) result[i];
    }
    ss >> *md5sum;

    close(fd);
    return Status::OK;
}

bool FileUtils::check_exist(const std::string& path) {
    boost::system::error_code errcode;
    bool exist = boost::filesystem::exists(path, errcode);
    if (errcode != boost::system::errc::success && errcode != boost::system::errc::no_such_file_or_directory) {
        LOG(WARNING) << "error when check path:" << path << ", error code:" << errcode;
        return false;
    }
    return exist;
}

}

max_stars_repo_name: txqyou/incubator-doris

max_issues_repo_name: txqyou/incubator-doris

max_forks_repo_name: doris-ci/incubator-doris


text:
#include <math.h>
#include <string>
#include <vector>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdio.h>
#include <stdlib.h>

#include <sys/types.h>
//#include <dirent.h>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>

#include "caffe/blob.hpp"
#include "caffe/filler.hpp"
#include "caffe/layers/DenseBlock_layer.hpp"

namespace caffe {

	bool dirExists(string dirStr) {
		//     const char* dirCStr = dirStr.c_str();
		//     DIR* dir = opendir(dirCStr);
		//     if (ENOENT == errno){
		//       return false;
		//     }
		//     closedir(dir);
		return true;
	}

	void tryCreateDirectory(string fileName) {
		//     vector<string> strVec;
		//     boost::split(strVec,fileName,boost::is_any_of("/"));
		//     string newStr="";
		//     for (int i=0;i<strVec.size()-1;++i){
		//       newStr += strVec[i] + (i==strVec.size()-2?"":"/");
		//     }
		//     boost::filesystem::path dirToCreate(newStr);
		//     if (!dirExists(newStr)){
		//       boost::filesystem::create_directories(dirToCreate);
		//     }
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top)
	{
		this->cpuInited = false;
// #ifndef CPU_ONLY
// 		this->gpuInited = false;
// #endif

		this->N = bottom[0]->shape()[0];
		this->H = bottom[0]->shape()[2];
		this->W = bottom[0]->shape()[3];

		DenseBlockParameter dbParam = this->layer_param_.denseblock_param();
		this->numTransition = dbParam.numtransition();
		//this->initChannel = dbParam.initchannel();
		this->initChannel = bottom[0]->channels();//modified by jxs
		this->growthRate = dbParam.growthrate();
		this->trainCycleIdx = 0; //initially, trainCycleIdx = 0
		this->EMA_decay = dbParam.moving_average_fraction();
#ifndef CPU_ONLY
		this->workspace_size_bytes = dbParam.workspace_mb() * 1024 * 1024;
		this->gpu_idx_ = dbParam.gpuidx();
#endif
		this->useDropout = dbParam.use_dropout();
		this->dropoutAmount = dbParam.dropout_amount();
		this->DB_randomSeed = 124816;
		this->useBC = dbParam.use_bc();
		this->BC_ultra_spaceEfficient = dbParam.bc_ultra_space_efficient();
		//Parameter Blobs
	//for transition i,
	//blobs_[i] is its filter blob
	//blobs_[numTransition + i] is its scaler blob
	//blobs_[2*numTransition + i] is its bias blob
	//blobs_[3*numTransition + i] is its globalMean
	//blobs_[4*numTransition + i] is its globalVar
		if (useBC) {
			this->blobs_.resize(10 * this->numTransition + 1);
		}
		else {
			this->blobs_.resize(5 * this->numTransition + 1);
		}
		for (int transitionIdx = 0; transitionIdx < this->numTransition; ++transitionIdx) {
			//filter
			//No BC case
			if (!useBC) {
				int inChannels = initChannel + transitionIdx * growthRate;
				int filterShape_Arr[] = { growthRate,inChannels,3,3 };
				vector<int> filterShape(filterShape_Arr, filterShape_Arr + 4);
				this->blobs_[transitionIdx].reset(new Blob<Dtype>(filterShape));
				shared_ptr<Filler<Dtype> > filter_Filler(GetFiller<Dtype>(dbParam.filter_filler()));
				filter_Filler->Fill(this->blobs_[transitionIdx].get());
			}
			else {
				//3*3 kernel
				int filter_33_shapeArr[] = { growthRate,4 * growthRate,3,3 };
				vector<int> filter33Shape(filter_33_shapeArr, filter_33_shapeArr + 4);
				this->blobs_[transitionIdx].reset(new Blob<Dtype>(filter33Shape));
				shared_ptr<Filler<Dtype> > filter_Filler3(GetFiller<Dtype>(dbParam.filter_filler()));
				filter_Filler3->Fill(this->blobs_[transitionIdx].get());

				//1*1 kernel
				int inChannels = initChannel + transitionIdx * growthRate;
				int filter_11_shapeArr[] = { 4 * growthRate,inChannels,1,1 };
				vector<int> filter11Shape(filter_11_shapeArr, filter_11_shapeArr + 4);
				this->blobs_[5 * numTransition + transitionIdx].reset(new Blob<Dtype>(filter11Shape));
				shared_ptr<Filler<Dtype> > filter_Filler1(GetFiller<Dtype>(dbParam.filter_filler()));
				filter_Filler1->Fill(this->blobs_[5 * numTransition + transitionIdx].get());
			}
			//scaler & bias
			int inChannels = initChannel + transitionIdx * growthRate;
			int BNparamShape_Arr[] = { 1,inChannels,1,1 };
			vector<int> BNparamShape(BNparamShape_Arr, BNparamShape_Arr + 4);
			//scaler
			this->blobs_[numTransition + transitionIdx].reset(new Blob<Dtype>(BNparamShape));
			shared_ptr<Filler<Dtype> > weight_filler0(GetFiller<Dtype>(dbParam.bn_scaler_filler()));
			weight_filler0->Fill(this->blobs_[numTransition + transitionIdx].get());

			int BN_4G_Shape[] = { 1,4 * growthRate,1,1 };
			vector<int> BN_4Gparam_ShapeVec(BN_4G_Shape, BN_4G_Shape + 4);
			//scaler BC
			if (useBC) {
				this->blobs_[6 * numTransition + transitionIdx].reset(new Blob<Dtype>(BN_4Gparam_ShapeVec));
				shared_ptr<Filler<Dtype> > weight_filler0_4G(GetFiller<Dtype>(dbParam.bn_scaler_filler()));
				weight_filler0_4G->Fill(this->blobs_[6 * numTransition + transitionIdx].get());
			}
			//bias
			this->blobs_[2 * numTransition + transitionIdx].reset(new Blob<Dtype>(BNparamShape));
			shared_ptr<Filler<Dtype> > weight_filler1(GetFiller<Dtype>(dbParam.bn_bias_filler()));
			weight_filler1->Fill(this->blobs_[2 * numTransition + transitionIdx].get());
			//bias BC
			if (useBC) {
				this->blobs_[7 * numTransition + transitionIdx].reset(new Blob<Dtype>(BN_4Gparam_ShapeVec));
				shared_ptr<Filler<Dtype> > weight_filler1_4G(GetFiller<Dtype>(dbParam.bn_bias_filler()));
				weight_filler1_4G->Fill(this->blobs_[7 * numTransition + transitionIdx].get());
			}
			//globalMean
			this->blobs_[3 * numTransition + transitionIdx].reset(new Blob<Dtype>(BNparamShape));
			for (int blobIdx = 0; blobIdx < inChannels; ++blobIdx) {
				shared_ptr<Blob<Dtype> > localB = this->blobs_[3 * numTransition + transitionIdx];
				localB->mutable_cpu_data()[localB->offset(0, blobIdx, 0, 0)] = 0;
			}
			//globalMean BC
			if (useBC) {
				this->blobs_[8 * numTransition + transitionIdx].reset(new Blob<Dtype>(BN_4Gparam_ShapeVec));
				shared_ptr<Blob<Dtype> > localB = this->blobs_[8 * numTransition + transitionIdx];
				for (int blobIdx = 0; blobIdx < 4 * growthRate; ++blobIdx) {
					localB->mutable_cpu_data()[localB->offset(0, blobIdx, 0, 0)] = 0;
				}
			}
			//globalVar
			this->blobs_[4 * numTransition + transitionIdx].reset(new Blob<Dtype>(BNparamShape));
			for (int blobIdx = 0; blobIdx < inChannels; ++blobIdx) {
				shared_ptr<Blob<Dtype> > localB = this->blobs_[4 * numTransition + transitionIdx];
				localB->mutable_cpu_data()[localB->offset(0, blobIdx, 0, 0)] = 1;
			}
			//globalVar BC
			if (useBC) {
				this->blobs_[9 * numTransition + transitionIdx].reset(new Blob<Dtype>(BN_4Gparam_ShapeVec));
				shared_ptr<Blob<Dtype> > localB = this->blobs_[9 * numTransition + transitionIdx];
				for (int blobIdx = 0; blobIdx < 4 * growthRate; ++blobIdx) {
					localB->mutable_cpu_data()[localB->offset(0, blobIdx, 0, 0)] = 1;
				}
			}
		}
		//final parameter for the equivalent of blobs_[2] in Caffe-BN
		vector<int> singletonShapeVec;
		singletonShapeVec.push_back(1);
		int singletonIdx = useBC ? 10 * numTransition : 5 * numTransition;
		this->blobs_[singletonIdx].reset(new Blob<Dtype>(singletonShapeVec));
		this->blobs_[singletonIdx]->mutable_cpu_data()[0] = Dtype(0);
		//parameter specification: globalMean/Var weight decay and lr is 0
		if (!useBC) {
			for (int i = 0; i < this->blobs_.size(); ++i) {
				if (this->layer_param_.param_size() != i) {
					CHECK_EQ(0, 1)
						<< "Nope";
				}
				ParamSpec* fixed_param_spec = this->layer_param_.add_param();
				//global Mean/Var
				if (i >= 3 * this->numTransition) {
					fixed_param_spec->set_lr_mult(0.f);
					fixed_param_spec->set_decay_mult(0.f);
				}
				//BN Scaler and Bias
				else if (i >= this->numTransition) {
					fixed_param_spec->set_lr_mult(1.f);
					fixed_param_spec->set_decay_mult(1.f);
				}
				else {
					fixed_param_spec->set_lr_mult(1.f);
					fixed_param_spec->set_decay_mult(1.f);
				}
			}
		}
		else {
			for (int i = 0; i < this->blobs_.size(); ++i) {
				if (this->layer_param_.param_size() != i) {
					CHECK_EQ(0, 1)
						<< "Nope";
				}
				ParamSpec* fixed_param_spec = this->layer_param_.add_param();
				if ((i >= 3 * numTransition) && (i < 5 * numTransition)) {
					fixed_param_spec->set_lr_mult(0.f);
					fixed_param_spec->set_decay_mult(0.f);
				}
				else if (i >= 8 * numTransition) {
					fixed_param_spec->set_lr_mult(0.f);
					fixed_param_spec->set_decay_mult(0.f);
				}
				else {
					fixed_param_spec->set_lr_mult(1.f);
					fixed_param_spec->set_decay_mult(1.f);
				}
			}
		}

#ifndef CPU_ONLY
		GPU_Initialization();
#endif
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
		int batch_size = bottom[0]->shape()[0];
		int h = bottom[0]->shape()[2];
		int w = bottom[0]->shape()[3];

#ifndef CPU_ONLY
		reshape_gpu_data(this->H, this->W,this->N, h, w,batch_size);
#endif
		this->N = batch_size;
		this->H = h;
		this->W = w;
		int topShapeArr[] = { this->N, this->initChannel + this->numTransition*this->growthRate,this->H,this->W };
		vector<int> topShape(topShapeArr, topShapeArr + 4);
		top[0]->Reshape(topShape);
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::syncBlobs(DenseBlockLayer<Dtype>* originLayer) {
		vector<shared_ptr<Blob<Dtype> > >& originBlobs = originLayer->blobs();
		for (int blobIdx = 0; blobIdx < originBlobs.size(); ++blobIdx) {
			shared_ptr<Blob<Dtype> > localBlob = originBlobs[blobIdx];
			Blob<Dtype> * newBlob = new Blob<Dtype>(localBlob->shape());
			newBlob->CopyFrom(*(localBlob.get()), false);
			shared_ptr<Blob<Dtype> > sharedPtrBlob(newBlob);
			this->blobs_[blobIdx] = sharedPtrBlob;
		}
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::setLogId(int uid) {
		this->logId = uid;
	}

	template <typename Dtype>
	void logBlob(Blob<Dtype>* B, string fileName) {
		string dataNameStr = fileName + "_data";
		string gradNameStr = fileName + "_grad";
		const char* dataName = (dataNameStr).c_str();
		const char* gradName = (gradNameStr).c_str();

		tryCreateDirectory(dataName);
		tryCreateDirectory(gradName);
		std::ofstream outWriter_data(dataName, std::ofstream::out);
		std::ofstream outWriter_grad(gradName, std::ofstream::out);
		for (int n = 0; n < B->shape(0); ++n) {
			for (int c = 0; c < B->shape(1); ++c) {
				for (int h = 0; h < B->shape(2); ++h) {
					for (int w = 0; w < B->shape(3); ++w) {
						outWriter_data << B->data_at(n, c, h, w) << ",";
						outWriter_grad << B->diff_at(n, c, h, w) << ",";
					}
				}
			}
		}
		outWriter_data << std::endl;
		outWriter_grad << std::endl;
	}

	string itos(int i) {
		string output = boost::lexical_cast<string>(i);
		return output;
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::logInternal_cpu(string dir) {
		string localDir = dir + "/cpu_" + itos(this->logId) + "/";
		//batch_Mean
		for (int i = 0; i < this->batch_Mean.size(); ++i) {
			string blobStr = localDir + "batch_Mean_" + itos(i);
			logBlob(this->batch_Mean[i], blobStr);
		}
		//batch_Var
		for (int i = 0; i < this->batch_Var.size(); ++i) {
			string blobStr = localDir + "batch_Var_" + itos(i);
			logBlob(this->batch_Var[i], blobStr);
		}
		if (useBC) {
			//batch_Mean
			for (int i = 0; i < this->batch_Mean4G.size(); ++i) {
				string blobStr = localDir + "batch_Mean_BC_" + itos(i);
				logBlob(this->batch_Mean4G[i], blobStr);
			}
			//batch_Var
			for (int i = 0; i < this->batch_Var4G.size(); ++i) {
				string blobStr = localDir + "batch_Var_BC_" + itos(i);
				logBlob(this->batch_Var4G[i], blobStr);
			}
		}
		//merged_conv
		for (int i = 0; i < this->merged_conv.size(); ++i) {
			string blobStr = localDir + "merged_conv_" + itos(i);
			logBlob(this->merged_conv[i], blobStr);
		}
		//BN_XhatVec
		for (int i = 0; i < this->BN_XhatVec.size(); ++i) {
			string blobStr = localDir + "BN_XhatVec_" + itos(i);
			logBlob(this->BN_XhatVec[i], blobStr);
		}
		//postBN_blobVec
		for (int i = 0; i < this->postBN_blobVec.size(); ++i) {
			string blobStr = localDir + "postBN_blobVec_" + itos(i);
			logBlob(this->postBN_blobVec[i], blobStr);
		}
		//postReLU_blobVec
		for (int i = 0; i < this->postReLU_blobVec.size(); ++i) {
			string blobStr = localDir + "postReLU_blobVec_" + itos(i);
			logBlob(this->postReLU_blobVec[i], blobStr);
		}
		//postConv_blobVec
		for (int i = 0; i < this->postConv_blobVec.size(); ++i) {
			string blobStr = localDir + "postConv_blobVec_" + itos(i);
			logBlob(this->postConv_blobVec[i], blobStr);
		}
		if (useBC) {
			//BC_BN_XhatVec
			for (int i = 0; i < this->BC_BN_XhatVec.size(); ++i) {
				string blobStr = localDir + "BC_BN_XhatVec_" + itos(i);
				logBlob(this->BC_BN_XhatVec[i], blobStr);
			}
			//postBN_BCVec
			for (int i = 0; i < this->postBN_BCVec.size(); ++i) {
				string blobStr = localDir + "postBN_BCVec_" + itos(i);
				logBlob(this->postBN_BCVec[i], blobStr);
			}
			//postReLU_BCVec
			for (int i = 0; i < this->postReLU_BCVec.size(); ++i) {
				string blobStr = localDir + "postReLU_BCVec_" + itos(i);
				logBlob(this->postReLU_BCVec[i], blobStr);
			}
			//postConv_BCVec
			for (int i = 0; i < this->postConv_BCVec.size(); ++i) {
				string blobStr = localDir + "postConv_BCVec_" + itos(i);
				logBlob(this->postConv_BCVec[i], blobStr);
			}
		}
		//filter
		for (int i = 0; i < this->numTransition; ++i) {
			string blobStr = localDir + "filter_" + itos(i);
			logBlob(this->blobs_[i].get(), blobStr);
		}
		//scaler
		for (int i = 0; i < this->numTransition; ++i) {
			string blobStr = localDir + "scaler_" + itos(i);
			logBlob(this->blobs_[this->numTransition + i].get(), blobStr);
		}
		//bias
		for (int i = 0; i < this->numTransition; ++i) {
			string blobStr = localDir + "bias_" + itos(i);
			logBlob(this->blobs_[this->numTransition * 2 + i].get(), blobStr);
		}
		if (useBC) {
			//filter
			for (int i = 0; i < this->numTransition; ++i) {
				string blobStr = localDir + "filter_BC_" + itos(i);
				logBlob(this->blobs_[5 * numTransition + i].get(), blobStr);
			}
			//scaler
			for (int i = 0; i < this->numTransition; ++i) {
				string blobStr = localDir + "scaler_BC_" + itos(i);
				logBlob(this->blobs_[6 * numTransition + i].get(), blobStr);
			}
			//bias
			for (int i = 0; i < this->numTransition; ++i) {
				string blobStr = localDir + "bias_BC_" + itos(i);
				logBlob(this->blobs_[7 * numTransition + i].get(), blobStr);
			}
			//Mean
			for (int i = 0; i < this->numTransition; ++i) {
				string blobStr = localDir + "Mean_BC_" + itos(i);
				logBlob(this->blobs_[8 * numTransition + i].get(), blobStr);
			}
			//Var
			for (int i = 0; i < this->numTransition; ++i) {
				string blobStr = localDir + "Var_BC_" + itos(i);
				logBlob(this->blobs_[9 * numTransition + i].get(), blobStr);
			}
		}
	}

	template <typename Dtype>
	Dtype getZeroPaddedValue(bool isDiff, Blob<Dtype>* inputData, int n, int c, int h, int w) {
		int n_blob = inputData->shape(0);
		int c_blob = inputData->shape(1);
		int h_blob = inputData->shape(2);
		int w_blob = inputData->shape(3);
		if ((n < 0) || (n >= n_blob)) return 0;
		if ((c < 0) || (c >= c_blob)) return 0;
		if ((h < 0) || (h >= h_blob)) return 0;
		if ((w < 0) || (w >= w_blob)) return 0;
		if (isDiff) return inputData->diff_at(n, c, h, w);
		else return inputData->data_at(n, c, h, w);
	}

	//Assumption, h_filter and w_filter must be 3 for now
	//naivest possible implementation of convolution, CPU forward and backward should not be used in production.
	//CPU version of convolution assume img H,W does not change after convolution, which corresponds to denseBlock without BC
	//input of size N*c_input*h_img*w_img
	template <typename Dtype>
	void convolution_Fwd(Blob<Dtype>* input, Blob<Dtype>* output, Blob<Dtype>* filter, int N, int c_output, int c_input, int h_img, int w_img, int h_filter, int w_filter) {
		int outputShape[] = { N,c_output,h_img,w_img };
		vector<int> outputShapeVec(outputShape, outputShape + 4);
		output->Reshape(outputShapeVec);
		Dtype * outputPtr = output->mutable_cpu_data();
		for (int n = 0; n < N; ++n) {
			for (int c_outIdx = 0; c_outIdx < c_output; ++c_outIdx) {
				for (int hIdx = 0; hIdx < h_img; ++hIdx) {
					for (int wIdx = 0; wIdx < w_img; ++wIdx) {
						outputPtr[output->offset(n, c_outIdx, hIdx, wIdx)] = 0;
						for (int c_inIdx = 0; c_inIdx < c_input; ++c_inIdx) {
							for (int filter_x = 0; filter_x < h_filter; ++filter_x) {
								for (int filter_y = 0; filter_y < w_filter; ++filter_y) {
									int localX = hIdx + (h_filter / 2) - filter_x;
									int localY = wIdx + (w_filter / 2) - filter_y;
									outputPtr[output->offset(n, c_outIdx, hIdx, wIdx)] += (filter->data_at(c_outIdx, c_inIdx, filter_x, filter_y) * getZeroPaddedValue(false, input, n, c_inIdx, localX, localY));
								}
							}
						}
					}
				}
			}
		}
	}

	//beta = 1 Convolution for bottomDiff
	template <typename Dtype>
	void convolution_Bwd(Blob<Dtype>* bottom, Blob<Dtype>* top, Blob<Dtype>* filter, int N, int c_output, int c_input, int h_img, int w_img, int h_filter, int w_filter) {
		Dtype * filterDiffPtr = filter->mutable_cpu_diff();
		Dtype * bottomDiffPtr = bottom->mutable_cpu_diff();
		//compute FilterGrad
		for (int coutIdx = 0; coutIdx < c_output; ++coutIdx) {
			for (int cinIdx = 0; cinIdx < c_input; ++cinIdx) {
				for (int filter_x = 0; filter_x < h_filter; ++filter_x) {
					for (int filter_y = 0; filter_y < w_filter; ++filter_y) {
						Dtype localGradSum = 0;
						for (int n = 0; n < N; ++n) {
							for (int i_img = 0; i_img < h_img; ++i_img) {
								for (int j_img = 0; j_img < w_img; ++j_img) {
									int localX = i_img + (h_filter / 2) - filter_x;
									int localY = j_img + (w_filter / 2) - filter_y;
									localGradSum += top->diff_at(n, coutIdx, i_img, j_img) * getZeroPaddedValue(false, bottom, n, cinIdx, localX, localY);
								}
							}
						}
						filterDiffPtr[filter->offset(coutIdx, cinIdx, filter_x, filter_y)] = localGradSum;
					}
				}
			}
		}
		//compute BottomGrad
		for (int n = 0; n < N; ++n) {
			for (int cinIdx = 0; cinIdx < c_input; ++cinIdx) {
				for (int i_img = 0; i_img < h_img; ++i_img) {
					for (int j_img = 0; j_img < w_img; ++j_img) {
						Dtype localGradSum = 0;
						for (int coutIdx = 0; coutIdx < c_output; ++coutIdx) {
							for (int x_img = 0; x_img < h_img; ++x_img) {
								for (int y_img = 0; y_img < w_img; ++y_img) {
									int localX = x_img - i_img + (h_filter / 2);
									int localY = y_img - j_img + (w_filter / 2);
									localGradSum += top->diff_at(n, coutIdx, x_img, y_img) * getZeroPaddedValue(false, filter, coutIdx, cinIdx, localX, localY);
								}
							}
						}
						bottomDiffPtr[bottom->offset(n, cinIdx, i_img, j_img)] = localGradSum;
					}
				}
			}
		}
	}

	template <typename Dtype>
	void ReLU_Fwd(Blob<Dtype>* bottom, Blob<Dtype>* top, int N, int C, int h_img, int w_img) {
		//Reshape top
		int topShapeArr[] = { N,C,h_img,w_img };
		vector<int> topShapeVec(topShapeArr, topShapeArr + 4);
		top->Reshape(topShapeVec);
		//ReLU Fwd
		Dtype* topPtr = top->mutable_cpu_data();
		for (int n = 0; n < N; ++n) {
			for (int cIdx = 0; cIdx < C; ++cIdx) {
				for (int hIdx = 0; hIdx < h_img; ++hIdx) {
					for (int wIdx = 0; wIdx < w_img; ++wIdx) {
						Dtype bottomData = bottom->data_at(n, cIdx, hIdx, wIdx);
						topPtr[top->offset(n, cIdx, hIdx, wIdx)] = bottomData >= 0 ? bottomData : 0;
					}
				}
			}
		}
	}

	template <typename Dtype>
	void ReLU_Bwd(Blob<Dtype>* bottom, Blob<Dtype>* top, int N, int C, int h_img, int w_img) {
		Dtype* bottomDiffPtr = bottom->mutable_cpu_diff();
		for (int n = 0; n < N; ++n) {
			for (int cIdx = 0; cIdx < C; ++cIdx) {
				for (int hIdx = 0; hIdx < h_img; ++hIdx) {
					for (int wIdx = 0; wIdx < w_img; ++wIdx) {
						bottomDiffPtr[bottom->offset(n, cIdx, hIdx, wIdx)] = bottom->data_at(n, cIdx, hIdx, wIdx) >= 0 ? top->diff_at(n, cIdx, hIdx, wIdx) : 0;
					}
				}
			}
		}
	}

	template <typename Dtype>
	Dtype getMean(Blob<Dtype>* A, int channelIdx) {
		int N = A->shape(0);
		int H = A->shape(2);
		int W = A->shape(3);
		int totalCount = N*H*W;

		Dtype sum = 0;
		for (int n = 0; n < N; ++n) {
			for (int h = 0; h < H; ++h) {
				for (int w = 0; w < W; ++w) {
					sum += A->data_at(n, channelIdx, h, w);
				}
			}
		}
		return sum / totalCount;
	}

	template <typename Dtype>
	Dtype getVar(Blob<Dtype>* A, int channelIdx) {
		int N = A->shape(0);
		int H = A->shape(2);
		int W = A->shape(3);
		int totalCount = N*H*W;
		Dtype mean = getMean(A, channelIdx);

		Dtype sum = 0;
		for (int n = 0; n < N; ++n) {
			for (int h = 0; h < H; ++h) {
				for (int w = 0; w < W; ++w) {
					sum += (A->data_at(n, channelIdx, h, w) - mean) * (A->data_at(n, channelIdx, h, w) - mean);
				}
			}
		}
		return sum / totalCount;
	}

	template <typename Dtype>
	void BN_inf_Fwd(Blob<Dtype>* input, Blob<Dtype>* output, int N, int C, int h_img, int w_img, Blob<Dtype>* globalMean, Blob<Dtype>* globalVar, Blob<Dtype>* scaler, Blob<Dtype>* bias, Blob<Dtype>* factor_b) {
		int channelShape[] = { 1,C,1,1 };
		vector<int> channelShapeVec(channelShape, channelShape + 4);
		Blob<Dtype>* localInf_Mean = new Blob<Dtype>(channelShapeVec);
		Blob<Dtype>* localInf_Var = new Blob<Dtype>(channelShapeVec);
		Dtype scale_factor = factor_b->cpu_data()[0] == 0 ? 0 : (1 / factor_b->cpu_data()[0]);
		caffe_cpu_scale(localInf_Mean->count(), scale_factor, globalMean->cpu_data(), localInf_Mean->mutable_cpu_data());
		caffe_cpu_scale(localInf_Var->count(), scale_factor, globalVar->cpu_data(), localInf_Var->mutable_cpu_data());
		//Reshape output
		int outputShape[] = { N,C,h_img,w_img };
		vector<int> outputShapeVec(outputShape, outputShape + 4);
		output->Reshape(outputShapeVec);
		//BN Fwd inf
		double epsilon = 1e-5;
		Dtype* outputPtr = output->mutable_cpu_data();

		for (int n = 0; n < N; ++n) {
			for (int cIdx = 0; cIdx < C; ++cIdx) {
				Dtype denom = 1.0 / sqrt(localInf_Var->data_at(0, cIdx, 0, 0) + epsilon);
				for (int hIdx = 0; hIdx < h_img; ++hIdx) {
					for (int wIdx = 0; wIdx < w_img; ++wIdx) {
						outputPtr[output->offset(n, cIdx, hIdx, wIdx)] = scaler->data_at(0, cIdx, 0, 0) * (denom * (input->data_at(n, cIdx, hIdx, wIdx) - localInf_Mean->data_at(0, cIdx, 0, 0))) + bias->data_at(0, cIdx, 0, 0);
					}
				}
			}
		}
	}

	template <typename Dtype>
	void BN_train_Fwd(Blob<Dtype>* bottom, Blob<Dtype>* top, Blob<Dtype>* output_xhat, Blob<Dtype>* globalMean, Blob<Dtype>* globalVar, Blob<Dtype>* batchMean, Blob<Dtype>* batchVar, Blob<Dtype>* scaler, Blob<Dtype>* bias, int N, int C, int h_img, int w_img, Dtype EMA_decay) {
		//reshape output
		int outputShape[] = { N,C,h_img,w_img };
		vector<int> outputShapeVec(outputShape, outputShape + 4);
		top->Reshape(outputShapeVec);
		output_xhat->Reshape(outputShapeVec);
		//BN Fwd train
		double epsilon = 1e-5;
		//get batch/global Mean/Var
		for (int channelIdx = 0; channelIdx < C; ++channelIdx) {
			int variance_adjust_m = N*h_img*w_img;
			//batch
			Dtype* batchMean_mutable = batchMean->mutable_cpu_data();
			Dtype* batchVar_mutable = batchVar->mutable_cpu_data();
			batchMean_mutable[channelIdx] = getMean(bottom, channelIdx);
			batchVar_mutable[channelIdx] = (variance_adjust_m / (variance_adjust_m - 1.0)) * getVar(bottom, channelIdx);
			//global
			Dtype* globalMean_mutable = globalMean->mutable_cpu_data();
			Dtype* globalVar_mutable = globalVar->mutable_cpu_data();
			globalMean_mutable[channelIdx] = EMA_decay * globalMean->data_at(0, channelIdx, 0, 0) + batchMean->data_at(0, channelIdx, 0, 0);
			globalVar_mutable[channelIdx] = EMA_decay * globalVar->data_at(0, channelIdx, 0, 0) + batchVar->data_at(0, channelIdx, 0, 0);
		}
		//process data
		for (int n = 0; n < N; ++n) {
			for (int c = 0; c < C; ++c) {
				for (int h = 0; h < h_img; ++h) {
					for (int w = 0; w < w_img; ++w) {
						Dtype* xhat_mutable = output_xhat->mutable_cpu_data();
						xhat_mutable[output_xhat->offset(n, c, h, w)] = (bottom->data_at(n, c, h, w) - batchMean->data_at(0, c, 0, 0)) / sqrt(batchVar->data_at(0, c, 0, 0) + epsilon);
						Dtype* output_mutable = top->mutable_cpu_data();
						output_mutable[top->offset(n, c, h, w)] = (scaler->data_at(0, c, 0, 0)) * (output_xhat->data_at(n, c, h, w)) + bias->data_at(0, c, 0, 0);
					}
				}
			}
		}
	}

	template <typename Dtype>
	bool decide_channelDiffAllZero(Blob<Dtype>* B, int channelIdx, int N, int C, int H, int W) {
		bool output = true;
		for (int n = 0; n < N; ++n) {
			for (int h = 0; h < H; ++h) {
				for (int w = 0; w < W; ++w) {
					output = output && (B->diff_at(n, channelIdx, h, w) < 0.001) && (B->diff_at(n, channelIdx, h, w) > -0.001);
				}
			}
		}
		return output;
	}

	template <typename Dtype>
	void BN_train_Bwd(Blob<Dtype>* bottom, Blob<Dtype>* bottom_xhat, Blob<Dtype>* top, Blob<Dtype>* batchMean, Blob<Dtype>* batchVar, Blob<Dtype>* scaler, Blob<Dtype>* bias, int N, int C, int h_img, int w_img, bool betaOneData) {
		double epsilon = 1e-5;
		//bias and scaler grad
		Dtype* biasGrad = bias->mutable_cpu_diff();
		Dtype* scalerGrad = scaler->mutable_cpu_diff();
		for (int channelIdx = 0; channelIdx < C; ++channelIdx) {
			biasGrad[channelIdx] = 0;
			scalerGrad[channelIdx] = 0;
			for (int n = 0; n < N; ++n) {
				for (int hIdx = 0; hIdx < h_img; ++hIdx) {
					for (int wIdx = 0; wIdx < w_img; ++wIdx) {
						biasGrad[channelIdx] += top->diff_at(n, channelIdx, hIdx, wIdx);
						scalerGrad[channelIdx] += top->diff_at(n, channelIdx, hIdx, wIdx) * bottom_xhat->data_at(n, channelIdx, hIdx, wIdx);
					}
				}
			}
		}
		//bottom data grad
		//helper 1:
		Dtype* XhatGrad = bottom_xhat->mutable_cpu_diff();
		for (int n = 0; n < N; ++n) {
			for (int c = 0; c < C; ++c) {
				for (int h = 0; h < h_img; ++h) {
					for (int w = 0; w < w_img; ++w) {
						XhatGrad[bottom_xhat->offset(n, c, h, w)] = top->diff_at(n, c, h, w) * scaler->data_at(0, c, 0, 0);
					}
				}
			}
		}
		//helper 2:
		Dtype* varGrad = batchVar->mutable_cpu_diff();
		for (int c = 0; c < C; ++c) {
			for (int n = 0; n < N; ++n) {
				for (int h = 0; h < h_img; ++h) {
					for (int w = 0; w < w_img; ++w) {
						//varGrad[c] += bottom_xhat->diff_at(n,c,h,w) * (bottom->data_at(n,c,h,w)-batchMean->data_at(0,c,0,0)) * (-0.5) * pow(batchVar->data_at(0,c,0,0) + epsilon,-1.5);
						varGrad[c] += bottom_xhat->diff_at(n, c, h, w) * (bottom->data_at(n, c, h, w) - batchMean->data_at(0, c, 0, 0)) * (-0.5) * (1.0 / ((batchVar->data_at(0, c, 0, 0) + epsilon) * sqrt(batchVar->data_at(0, c, 0, 0) + epsilon)));
						//flag
						//if (decide_channelDiffAllZero<Dtype>(top,c,N,C,h_img,w_img)){
						//  std::cout<<varGrad[c]<<std::endl;
						//}

					}
				}
			}
		}

		//helper 3:
		double m = N * h_img * w_img;
		Dtype* meanGrad = batchMean->mutable_cpu_diff();
		for (int c = 0; c < C; ++c) {
			for (int n = 0; n < N; ++n) {
				for (int h = 0; h < h_img; ++h) {
					for (int w = 0; w < w_img; ++w) {
						meanGrad[c] += bottom_xhat->diff_at(n, c, h, w) * (-1.0 / sqrt(batchVar->data_at(0, c, 0, 0) + epsilon)) + batchVar->diff_at(0, c, 0, 0) * (-2.0) * (bottom->data_at(n, c, h, w) - batchMean->data_at(0, c, 0, 0)) / m;
						//if (decide_channelDiffAllZero<Dtype>(top,c,N,C,h_img,w_img)){
					//  std::cout<<varGrad[c]<<std::endl;
					//}

					}
				}
			}
		}

		//combine helpers
		Dtype* bottomDataGrad = bottom->mutable_cpu_diff();
		for (int n = 0; n < N; ++n) {
			for (int c = 0; c < C; ++c) {
				for (int h = 0; h < h_img; ++h) {
					for (int w = 0; w < w_img; ++w) {
						//Dtype term1=bottom_xhat->diff_at(n,c,h,w)*pow(batchVar->data_at(0,c,0,0)+epsilon,-0.5);
						Dtype term1 = bottom_xhat->diff_at(n, c, h, w) / (sqrt(batchVar->data_at(0, c, 0, 0) + epsilon));
						Dtype term2 = batchVar->diff_at(0, c, 0, 0)*2.0*(bottom->data_at(n, c, h, w) - batchMean->data_at(0, c, 0, 0)) / m;
						Dtype term3 = batchMean->diff_at(0, c, 0, 0) / m;
						if (betaOneData) {
							bottomDataGrad[bottom->offset(n, c, h, w)] += term1 + term2 + term3;
						}
						else {
							bottomDataGrad[bottom->offset(n, c, h, w)] = term1 + term2 + term3;
						}
						//std::cout<<term1<<","<<term2<<","<<term3<<std::endl;
					}
				}
			}
		}

	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::CPU_Initialization() {
		this->batch_Mean.resize(this->numTransition);
		this->batch_Var.resize(this->numTransition);

		this->merged_conv.resize(this->numTransition + 1);
		this->BN_XhatVec.resize(this->numTransition);
		this->postBN_blobVec.resize(this->numTransition);
		this->postReLU_blobVec.resize(this->numTransition);
		this->postConv_blobVec.resize(this->numTransition);
		if (useBC) {
			BC_BN_XhatVec.resize(this->numTransition);
			postBN_BCVec.resize(this->numTransition);
			postReLU_BCVec.resize(this->numTransition);
			postConv_BCVec.resize(this->numTransition);
			batch_Mean4G.resize(numTransition);
			batch_Var4G.resize(numTransition);
		}
		for (int transitionIdx = 0; transitionIdx < this->numTransition; ++transitionIdx) {
			int conv_y_Channels = this->growthRate;
			int mergeChannels = this->initChannel + this->growthRate * transitionIdx;
			int channelShapeArr[] = { 1,mergeChannels,1,1 };
			int conv_y_ShapeArr[] = { this->N,conv_y_Channels,this->H,this->W };
			int mergeShapeArr[] = { this->N,mergeChannels,this->H,this->W };
			vector<int> channelShape(channelShapeArr, channelShapeArr + 4);
			vector<int> conv_y_Shape(conv_y_ShapeArr, conv_y_ShapeArr + 4);
			vector<int> mergeShape(mergeShapeArr, mergeShapeArr + 4);

			this->batch_Mean[transitionIdx] = new Blob<Dtype>(channelShape);
			this->batch_Var[transitionIdx] = new Blob<Dtype>(channelShape);

			this->merged_conv[transitionIdx] = new Blob<Dtype>(mergeShape);
			this->BN_XhatVec[transitionIdx] = new Blob<Dtype>(mergeShape);
			this->postBN_blobVec[transitionIdx] = new Blob<Dtype>(mergeShape);
			this->postReLU_blobVec[transitionIdx] = new Blob<Dtype>(mergeShape);
			this->postConv_blobVec[transitionIdx] = new Blob<Dtype>(conv_y_Shape);
			if (useBC) {
				int quadGShapeArr[] = { N,4 * growthRate,H,W };
				int quadChannelArr[] = { 1,4 * growthRate,1,1 };
				vector<int> quadGShape(quadGShapeArr, quadGShapeArr + 4);
				vector<int> quadChannelShape(quadChannelArr, quadChannelArr + 4);
				this->BC_BN_XhatVec[transitionIdx] = new Blob<Dtype>(quadGShape);
				this->postBN_BCVec[transitionIdx] = new Blob<Dtype>(quadGShape);
				this->postReLU_BCVec[transitionIdx] = new Blob<Dtype>(quadGShape);
				this->postConv_BCVec[transitionIdx] = new Blob<Dtype>(quadGShape);
				batch_Mean4G[transitionIdx] = new Blob<Dtype>(quadChannelShape);
				batch_Var4G[transitionIdx] = new Blob<Dtype>(quadChannelShape);
			}
		}
		//the last element of merged_conv serve as output of forward
		int extraMergeOutputShapeArr[] = { this->N,this->initChannel + this->growthRate*this->numTransition,this->H,this->W };
		vector<int> extraMergeOutputShapeVector(extraMergeOutputShapeArr, extraMergeOutputShapeArr + 4);
		this->merged_conv[this->numTransition] = new Blob<Dtype>(extraMergeOutputShapeVector);
	}

	template <typename Dtype>
	void mergeChannelData(Blob<Dtype>* outputBlob, Blob<Dtype>* blobA, Blob<Dtype>* blobB) {
		int N = blobA->shape(0);
		int frontC = blobA->shape(1); int backC = blobB->shape(1);
		int H = blobA->shape(2);
		int W = blobA->shape(3);

		for (int n = 0; n < N; ++n) {
			for (int c = 0; c < frontC + backC; ++c) {
				for (int h = 0; h < H; ++h) {
					for (int w = 0; w < W; ++w) {
						Dtype inData;
						if (c < frontC) {
							inData = blobA->cpu_data()[blobA->offset(n, c, h, w)];
						}
						else {
							int readC = c - frontC;
							inData = blobB->cpu_data()[blobB->offset(n, readC, h, w)];
						}
						outputBlob->mutable_cpu_data()[outputBlob->offset(n, c, h, w)] = inData;
					}
				}
			}
		}
	}

	template <typename Dtype>
	void distributeChannelDiff(Blob<Dtype>* inputBlob, Blob<Dtype>* blobA, Blob<Dtype>* blobB) {
		int N = blobA->shape(0);
		int frontC = blobA->shape(1); int backC = blobB->shape(1);
		int H = blobA->shape(2);
		int W = blobA->shape(3);

		for (int n = 0; n < N; ++n) {
			for (int c = 0; c < frontC + backC; ++c) {
				for (int h = 0; h < H; ++h) {
					for (int w = 0; w < W; ++w) {
						Dtype readData = inputBlob->cpu_diff()[inputBlob->offset(n, c, h, w)];
						if (c < frontC) {
							blobA->mutable_cpu_diff()[blobA->offset(n, c, h, w)] = readData;
						}
						else {
							int writeC = c - frontC;
							blobB->mutable_cpu_diff()[blobB->offset(n, writeC, h, w)] = readData;
						}
					}
				}
			}
		}
	}

	template <typename Dtype>
	void BlobSetZero(Blob<Dtype>* B, int count) {
		Dtype* B_mutable_data = B->mutable_cpu_data();
		Dtype* B_mutable_diff = B->mutable_cpu_diff();
		for (int i = 0; i < count; ++i) {
			B_mutable_data[i] = 0;
			B_mutable_diff[i] = 0;
		}
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::LoopEndCleanup_cpu() {
		for (int transitionIdx = 0; transitionIdx < this->numTransition; ++transitionIdx) {
			int tensorCount = this->N * growthRate * this->H * this->W;
			int tensorMergeCount = this->N * (this->initChannel + this->growthRate * transitionIdx) * this->H * this->W;
			BlobSetZero<Dtype>(this->merged_conv[transitionIdx], tensorMergeCount);
			BlobSetZero<Dtype>(this->BN_XhatVec[transitionIdx], tensorMergeCount);
			BlobSetZero<Dtype>(this->postBN_blobVec[transitionIdx], tensorMergeCount);
			BlobSetZero<Dtype>(this->postReLU_blobVec[transitionIdx], tensorMergeCount);
			BlobSetZero<Dtype>(this->postConv_blobVec[transitionIdx], tensorCount);
		}
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
		const vector<Blob<Dtype>*>& top)
	{
		//init CPU
		if (!this->cpuInited) {
			//std::cout<<"fwd cpu init"<<std::endl;
			this->CPU_Initialization();
			this->cpuInited = true;
			//std::cout<<"fwd cpu init done"<<std::endl;
		}
		int bnTimerIdx = useBC ? 10 * numTransition : 5 * numTransition;
		//deploy init data
		this->merged_conv[0]->CopyFrom(*(bottom[0]));
		//init CPU finish
		for (int transitionIdx = 0; transitionIdx < this->numTransition; ++transitionIdx) {
			//BN
			Blob<Dtype>* BN_bottom = this->merged_conv[transitionIdx];
			Blob<Dtype>* BN_top = this->postBN_blobVec[transitionIdx];
			Blob<Dtype>* Scaler = this->blobs_[numTransition + transitionIdx].get();
			Blob<Dtype>* Bias = this->blobs_[2 * numTransition + transitionIdx].get();
			int localChannels = this->initChannel + transitionIdx*this->growthRate;
			if (this->phase_ == TEST) {
				//std::cout<<"cpu BN test forward"<<std::endl;
				BN_inf_Fwd<Dtype>(BN_bottom, BN_top, this->N, localChannels, this->H, this->W, this->blobs_[3 * this->numTransition + transitionIdx].get(), this->blobs_[4 * this->numTransition + transitionIdx].get(), Scaler, Bias, this->blobs_[bnTimerIdx].get());
			}
			else {
				//std::cout<<"cpu BN train forward"<<std::endl;
				BN_train_Fwd<Dtype>(BN_bottom, BN_top, this->BN_XhatVec[transitionIdx], this->blobs_[3 * this->numTransition + transitionIdx].get(), this->blobs_[4 * this->numTransition + transitionIdx].get(), this->batch_Mean[transitionIdx], this->batch_Var[transitionIdx], Scaler, Bias, this->N, localChannels, this->H, this->W, this->EMA_decay);
			}
			//ReLU
			Blob<Dtype>* ReLU_top = this->postReLU_blobVec[transitionIdx];
			ReLU_Fwd<Dtype>(BN_top, ReLU_top, this->N, localChannels, this->H, this->W);
			//if useBC, Conv1*1-BN(BC)-ReLU(BC)
			if (useBC) {
				//BC Conv 1*1
				Blob<Dtype>* BC_filterBlob = this->blobs_[5 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_conv_x = postReLU_blobVec[transitionIdx];
				Blob<Dtype>* BC_conv_y = postConv_BCVec[transitionIdx];
				int BC_conv_inChannel = initChannel + growthRate*transitionIdx;
				int BC_conv_outChannel = 4 * growthRate;
				convolution_Fwd<Dtype>(BC_conv_x, BC_conv_y, BC_filterBlob, N, BC_conv_outChannel, BC_conv_inChannel, H, W, 1, 1);
				//BC BN
				Blob<Dtype>* BC_BN_x = postConv_BCVec[transitionIdx];
				Blob<Dtype>* BC_BN_y = postBN_BCVec[transitionIdx];
				Blob<Dtype>* BC_Scaler = this->blobs_[6 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_Bias = this->blobs_[7 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_Mean = this->blobs_[8 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_Var = this->blobs_[9 * numTransition + transitionIdx].get();
				if (this->phase_ == TEST) {
					BN_inf_Fwd<Dtype>(BC_BN_x, BC_BN_y, N, 4 * growthRate, H, W, BC_Mean, BC_Var, BC_Scaler, BC_Bias, this->blobs_[bnTimerIdx].get());
				}
				else {
					Blob<Dtype>* BC_xhat = BC_BN_XhatVec[transitionIdx];
					Blob<Dtype>* BC_batchMean = batch_Mean4G[transitionIdx];
					Blob<Dtype>* BC_batchVar = batch_Var4G[transitionIdx];
					BN_train_Fwd<Dtype>(BC_BN_x, BC_BN_y, BC_xhat, BC_Mean, BC_Var, BC_batchMean, BC_batchVar, BC_Scaler, BC_Bias, N, 4 * growthRate, H, W, EMA_decay);
				}
				//BC ReLU
				Blob<Dtype>* ReLU_x = postBN_BCVec[transitionIdx];
				Blob<Dtype>* ReLU_y = postReLU_BCVec[transitionIdx];
				ReLU_Fwd<Dtype>(ReLU_x, ReLU_y, N, 4 * growthRate, H, W);
			}
			//Conv
			Blob<Dtype>* filterBlob = this->blobs_[transitionIdx].get();
			Blob<Dtype>* conv_x = useBC ? postReLU_BCVec[transitionIdx] : postReLU_blobVec[transitionIdx];
			Blob<Dtype>* conv_y = this->postConv_blobVec[transitionIdx];
			int inConvChannel = useBC ? 4 * growthRate : initChannel + growthRate*transitionIdx;
			convolution_Fwd<Dtype>(conv_x, conv_y, filterBlob, N, growthRate, inConvChannel, H, W, 3, 3);
			//post Conv merge
			Blob<Dtype>* mergeOutput = merged_conv[transitionIdx + 1];
			Blob<Dtype>* mergeInputA = merged_conv[transitionIdx];
			Blob<Dtype>* mergeInputB = postConv_blobVec[transitionIdx];
			mergeChannelData(mergeOutput, mergeInputA, mergeInputB);
		}
		//deploy output data
		top[0]->CopyFrom(*(this->merged_conv[this->numTransition]));
		if (this->phase_ == TRAIN) {
			this->blobs_[bnTimerIdx]->mutable_cpu_data()[0] *= this->EMA_decay;
			this->blobs_[bnTimerIdx]->mutable_cpu_data()[0] += 1;
			this->trainCycleIdx += 1;
		}
		//logInternal_cpu("TC_TrueFwdlog");
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
		const vector<bool>& propagate_down,
		const vector<Blob<Dtype>*>& bottom)
	{
		if (!this->cpuInited) {
			this->CPU_Initialization();
			this->cpuInited = true;
		}
		//deploy top diff
		this->merged_conv[this->numTransition]->CopyFrom(*(top[0]), true);
		for (int transitionIdx = this->numTransition - 1; transitionIdx >= 0; --transitionIdx) {
			//distribute diff
			distributeChannelDiff(this->merged_conv[transitionIdx + 1], this->merged_conv[transitionIdx], this->postConv_blobVec[transitionIdx]);
			//Conv Bwd
			Blob<Dtype>* conv_top = this->postConv_blobVec[transitionIdx];
			Blob<Dtype>* conv_bottom = useBC ? postReLU_BCVec[transitionIdx] : postReLU_blobVec[transitionIdx];
			Blob<Dtype>* filter = this->blobs_[transitionIdx].get();
			int c_input = useBC ? 4 * growthRate : initChannel + growthRate*transitionIdx;
			convolution_Bwd<Dtype>(conv_bottom, conv_top, filter, this->N, this->growthRate, c_input, this->H, this->W, 3, 3);
			//BC ReLU_BC_Bwd - BN_BC_Bwd - Conv1*1_BC_Bwd
			if (useBC) {
				//ReLU BC Bwd
				Blob<Dtype>* BC_ReLU_y = postReLU_BCVec[transitionIdx];
				Blob<Dtype>* BC_ReLU_x = postBN_BCVec[transitionIdx];
				ReLU_Bwd<Dtype>(BC_ReLU_x, BC_ReLU_y, N, 4 * growthRate, H, W);
				//BN BC Bwd
				Blob<Dtype>* BC_BN_y = postBN_BCVec[transitionIdx];
				Blob<Dtype>* BC_BN_x = postConv_BCVec[transitionIdx];
				Blob<Dtype>* BC_BN_xhat = BC_BN_XhatVec[transitionIdx];
				Blob<Dtype>* BC_Scaler = this->blobs_[6 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_Bias = this->blobs_[7 * numTransition + transitionIdx].get();
				Blob<Dtype>* BC_batchMean = batch_Mean4G[transitionIdx];
				Blob<Dtype>* BC_batchVar = batch_Var4G[transitionIdx];
				BN_train_Bwd<Dtype>(BC_BN_x, BC_BN_xhat, BC_BN_y, BC_batchMean, BC_batchVar, BC_Scaler, BC_Bias, N, 4 * growthRate, H, W, false);
				//Conv1*1 BC Bwd
				Blob<Dtype>* BC_conv_x = postReLU_blobVec[transitionIdx];
				Blob<Dtype>* BC_conv_y = postConv_BCVec[transitionIdx];
				Blob<Dtype>* BC_filter = this->blobs_[5 * numTransition + transitionIdx].get();
				int BC_c_input = initChannel + growthRate*transitionIdx;
				int BC_c_output = 4 * growthRate;
				convolution_Bwd<Dtype>(BC_conv_x, BC_conv_y, BC_filter, N, BC_c_output, BC_c_input, H, W, 1, 1);
			}
			//ReLU Bwd
			int localChannel = this->initChannel + this->growthRate*transitionIdx;
			ReLU_Bwd<Dtype>(postBN_blobVec[transitionIdx], postReLU_blobVec[transitionIdx], this->N, localChannel, this->H, this->W);
			//BN Bwd
			Blob<Dtype>* BN_bottom = this->merged_conv[transitionIdx];
			Blob<Dtype>* scaler = this->blobs_[this->numTransition + transitionIdx].get();
			Blob<Dtype>* bias = this->blobs_[2 * this->numTransition + transitionIdx].get();
			BN_train_Bwd<Dtype>(BN_bottom, this->BN_XhatVec[transitionIdx], this->postBN_blobVec[transitionIdx], this->batch_Mean[transitionIdx], this->batch_Var[transitionIdx], scaler, bias, this->N, localChannel, this->H, this->W, true);
		}
		bottom[0]->CopyFrom(*(this->merged_conv[0]), true);
		//logInternal_cpu("TC_TrueBwdlog");
		this->LoopEndCleanup_cpu();
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::Forward_cpu_public(const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
		this->Forward_cpu(bottom, top);
	}

	template <typename Dtype>
	void DenseBlockLayer<Dtype>::Backward_cpu_public(const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
		this->Backward_cpu(top, propagate_down, bottom);
	}

#ifdef CPU_ONLY
	STUB_GPU(DenseBlockLayer);
#endif

	INSTANTIATE_CLASS(DenseBlockLayer);
	REGISTER_LAYER_CLASS(DenseBlock);

}  // namespace caffe

ext: cpp


text:
#include <CGAL/Exact_predicates_inexact_constructions_kernel.h>

#include <CGAL/Mesh_triangulation_3.h>
#include <CGAL/Mesh_complex_3_in_triangulation_3.h>
#include <CGAL/Mesh_criteria_3.h>

#include <CGAL/Labeled_mesh_domain_3.h>
#include <CGAL/make_mesh_3.h>
#include <CGAL/Image_3.h>
#include <cstdlib>
#include <iostream>

#include <boost/container/flat_set.hpp>

typedef float Image_word_type;

// Domain
typedef CGAL::Exact_predicates_inexact_constructions_kernel K;
typedef CGAL::Labeled_mesh_domain_3<K> Mesh_domain;

// Triangulation
typedef CGAL::Mesh_triangulation_3<Mesh_domain>::type Tr;
typedef CGAL::Mesh_complex_3_in_triangulation_3<Tr> C3t3;

// Criteria
typedef CGAL::Mesh_criteria_3<Tr> Mesh_criteria;

// To avoid verbose function and named parameters call
using namespace CGAL::parameters;

template <typename Set>
struct Image_to_multiple_iso_level_sets {
  const Set& set;
  Image_to_multiple_iso_level_sets(const Set& set) : set(set) {}

  int operator()(double v) const {
       return int(std::distance(set.begin(),
                                set.lower_bound(float(v))));
  }
};

int main(int argc, char*argv[])
{
  const std::string fname = (argc>1)?argv[1]:CGAL::data_file_path("images/skull_2.9.inr");
  // Load image
  CGAL::Image_3 image;
  if(!image.read(fname)){
    std::cerr << "Error: Cannot read file " <<  fname << std::endl;
    return EXIT_FAILURE;
  }

  typedef boost::container::flat_set<float> Flat_set;
  Flat_set iso_values;
  if(argc < 2) {
    iso_values.insert(1.5f);
    iso_values.insert(2.9f);
    iso_values.insert(3.5f);
  } else {
    for(int i = 2; i < argc; ++i) {
      iso_values.insert(static_cast<float>(std::atof(argv[i])));
    }
  }

  // Domain
  namespace p = CGAL::parameters;
  Mesh_domain domain =
    Mesh_domain::create_gray_image_mesh_domain
    (p::image = image,
     p::image_values_to_subdomain_indices =
       Image_to_multiple_iso_level_sets<Flat_set>(iso_values),
     p::value_outside = 0.f
     );

  // Mesh criteria
  Mesh_criteria criteria(facet_angle=30, facet_size=6, facet_distance=2,
                         cell_radius_edge_ratio=3, cell_size=8);

  // Meshing
  C3t3 c3t3 = CGAL::make_mesh_3<C3t3>(domain, criteria);

  // Output
  std::ofstream medit_file("out.mesh");
  c3t3.output_to_medit(medit_file);

  return 0;
}

max_stars_repo_name: ffteja/cgal

max_issues_repo_name: ffteja/cgal

max_forks_repo_name: ffteja/cgal


text:
/*------------------------------------------------------------------------------
                Copyright Butterfly Energy Systems 2014-2015.
           Distributed under the Boost Software License, Version 1.0.
              (See accompanying file LICENSE_1_0.txt or copy at
                    http://www.boost.org/LICENSE_1_0.txt)
------------------------------------------------------------------------------*/

#include <boost/endian/conversion.hpp>

namespace wamp
{

namespace internal
{

namespace endian
{

inline uint32_t nativeToBig32(uint32_t native)
{
    return boost::endian::native_to_big(native);
}

inline uint32_t bigToNative32(uint32_t big)
{
    return boost::endian::big_to_native(big);
}

} // namespace endian

} // namespace internal

} // namespace wamp

ext: ipp

max_stars_repo_name: katreniak/cppwamp

max_issues_repo_name: katreniak/cppwamp

max_forks_repo_name: katreniak/cppwamp


text:
/****************************************************************************
*
*    Copyright (c) 2019 Vivante Corporation
*
*    Permission is hereby granted, free of charge, to any person obtaining a
*    copy of this software and associated documentation files (the "Software"),
*    to deal in the Software without restriction, including without limitation
*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
*    and/or sell copies of the Software, and to permit persons to whom the
*    Software is furnished to do so, subject to the following conditions:
*
*    The above copyright notice and this permission notice shall be included in
*    all copies or substantial portions of the Software.
*
*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
*    DEALINGS IN THE SOFTWARE.
*
*****************************************************************************/

#pragma once

#include <backendsCommon/CpuTensorHandle.hpp>
#include <backendsCommon/Workload.hpp>
#include <backendsCommon/WorkloadData.hpp>
#include <boost/log/trivial.hpp>
#include "TNpuWorkloads.hpp"

namespace armnn {
template <typename armnn::DataType... DataTypes>
class NpuSpaceToBatchNdWorkload : public TNpuWorkload<SpaceToBatchNdQueueDescriptor, DataTypes...> {
   public:
    using base_type = TNpuWorkload<SpaceToBatchNdQueueDescriptor, DataTypes...>;
    explicit NpuSpaceToBatchNdWorkload(const SpaceToBatchNdQueueDescriptor& descriptor,
                                       const WorkloadInfo& info)
        : TNpuWorkload<SpaceToBatchNdQueueDescriptor, DataTypes...>(descriptor, info),
          m_BlockShape(descriptor.m_Parameters.m_BlockShape),
          m_PadList(descriptor.m_Parameters.m_PadList),
          m_DataLayout(descriptor.m_Parameters.m_DataLayout) {
        // Add input operand
        // Only 1 input
        std::vector<uint32_t> inOperandIds;
        NpuTensorHandler* inputTensorHandle =
            dynamic_cast<NpuTensorHandler*>(descriptor.m_Inputs[0]);
        uint32_t inputOperandId = this->AddOperandAndSetValue(
            inputTensorHandle->GetTensorInfo(), inputTensorHandle->GetShape(), nullptr);
        inOperandIds.push_back(inputOperandId);

        // Add block shape operand
        TensorShape blockShape({(uint32_t)m_BlockShape.size()});
        TensorInfo blockInfo(blockShape, DataType::Signed32);
        inOperandIds.push_back(
            this->AddOperandAndSetValue(blockInfo, blockShape, m_BlockShape.data()));

        // Add pad list operand
        TensorShape padListShape({2, 2});
        TensorInfo padListInfo(padListShape, DataType::Signed32);

        m_Pad.push_back(m_PadList[0].first);
        m_Pad.push_back(m_PadList[0].second);

        m_Pad.push_back(m_PadList[1].first);
        m_Pad.push_back(m_PadList[1].second);

        inOperandIds.push_back(
            this->AddOperandAndSetValue(padListInfo, padListShape, m_Pad.data()));

        // Add layout operand
        int32_t layoutCode = m_DataLayout == armnn::DataLayout::NCHW
                                 ? int32_t(nnrt::DataLayout::NCHW)
                                 : int32_t(nnrt::DataLayout::NHWC);
        inOperandIds.push_back(this->AddOperandAndSetValue(layoutCode));

        // Add output operand
        std::vector<uint32_t> outOperandIds;
        NpuTensorHandler* outputTensorHandle =
            dynamic_cast<NpuTensorHandler*>(descriptor.m_Outputs[0]);
        uint32_t outputTensorId = this->AddOperandAndSetValue(
            outputTensorHandle->GetTensorInfo(), outputTensorHandle->GetShape(), nullptr);
        outOperandIds.push_back(outputTensorId);

        this->AddOperation(nnrt::OperationType::SPACE_TO_BATCH_ND,
                           inOperandIds.size(),
                           inOperandIds.data(),
                           outOperandIds.size(),
                           outOperandIds.data());
    }

   private:
    // Block shape value.
    std::vector<unsigned int> m_BlockShape;
    // Specifies the padding values for the input dimension:
    /// heightPad{top, bottom} widthPad{left, right}.
    std::vector<std::pair<unsigned int, unsigned int>> m_PadList;
    std::vector<unsigned int> m_Pad;
    // The data layout to be used (NCHW, NHWC).
    DataLayout m_DataLayout;
};
using NpuSpaceToBatchNDFloat32Workload = NpuSpaceToBatchNdWorkload<armnn::DataType::Float32>;
using NpuSpaceToBatchNDFloat16Workload = NpuSpaceToBatchNdWorkload<armnn::DataType::Float16>;
using NpuSpaceToBatchNDUint8Workload = NpuSpaceToBatchNdWorkload<armnn::DataType::QuantisedAsymm8>;
}  // namespace armnn

ext: hpp

max_stars_repo_name: phytec-mirrors/nn-imx

max_issues_repo_name: phytec-mirrors/nn-imx

max_forks_repo_name: phytec-mirrors/nn-imx


text: #include <boost/spirit/home/karma/what.hpp>

max_stars_repo_name: miathedev/BoostForArduino

max_issues_repo_name: miathedev/BoostForArduino

max_forks_repo_name: miathedev/BoostForArduino


text:
#include <vban/lib/config.hpp>
#include <vban/lib/jsonconfig.hpp>
#include <vban/lib/tomlconfig.hpp>
#include <vban/node/node_rpc_config.hpp>

#include <boost/property_tree/ptree.hpp>

vban::error vban::node_rpc_config::serialize_json (vban::jsonconfig & json) const
{
	json.put ("version", json_version ());
	json.put ("enable_sign_hash", enable_sign_hash);

	vban::jsonconfig child_process_l;
	child_process_l.put ("enable", child_process.enable);
	child_process_l.put ("rpc_path", child_process.rpc_path);
	json.put_child ("child_process", child_process_l);
	return json.get_error ();
}

vban::error vban::node_rpc_config::serialize_toml (vban::tomlconfig & toml) const
{
	toml.put ("enable_sign_hash", enable_sign_hash, "Allow or disallow signing of hashes.\ntype:bool");

	vban::tomlconfig child_process_l;
	child_process_l.put ("enable", child_process.enable, "Enable or disable RPC child process. If false, an in-process RPC server is used.\ntype:bool");
	child_process_l.put ("rpc_path", child_process.rpc_path, "Path to the vban_rpc executable. Must be set if child process is enabled.\ntype:string,path");
	toml.put_child ("child_process", child_process_l);
	return toml.get_error ();
}

vban::error vban::node_rpc_config::deserialize_toml (vban::tomlconfig & toml)
{
	toml.get_optional ("enable_sign_hash", enable_sign_hash);
	toml.get_optional<bool> ("enable_sign_hash", enable_sign_hash);

	auto child_process_l (toml.get_optional_child ("child_process"));
	if (child_process_l)
	{
		child_process_l->get_optional<bool> ("enable", child_process.enable);
		child_process_l->get_optional<std::string> ("rpc_path", child_process.rpc_path);
	}

	return toml.get_error ();
}

vban::error vban::node_rpc_config::deserialize_json (bool & upgraded_a, vban::jsonconfig & json, boost::filesystem::path const & data_path)
{
	json.get_optional<bool> ("enable_sign_hash", enable_sign_hash);

	auto child_process_l (json.get_optional_child ("child_process"));
	if (child_process_l)
	{
		child_process_l->get_optional<bool> ("enable", child_process.enable);
		child_process_l->get_optional<std::string> ("rpc_path", child_process.rpc_path);
	}

	return json.get_error ();
}

void vban::node_rpc_config::set_request_callback (std::function<void (boost::property_tree::ptree const &)> callback_a)
{
	debug_assert (vban::network_constants ().is_dev_network ());
	request_callback = std::move (callback_a);
}

max_stars_repo_name: EduardAl/vban-node

max_issues_repo_name: EduardAl/vban-node

max_forks_repo_name: EduardAl/vban-node


text:
//===------------------------------------------------------------*- C++ -*-===//
///
/// \file
/// \brief Asio
/// \details 可移植的网络和其他底层 I/O，包括套接字、计时器、主机名解析、套接字 I/O 流、串行端口、文件描述符和 Windows 句柄。
///
/// \sa [Boost.Asio](https://boost.org/doc/libs/release/libs/asio/)
/// \sa [Boost.Asio C++ 网络编程](https://mmoaay.gitbooks.io/boost-asio-cpp-network-programming-chinese/)
///
/// \version 2021-09-29
/// \since 2019-01-08
/// \authors zhengrr
/// \copyright Unlicense
///
//===----------------------------------------------------------------------===//

#include <iostream>

#include <boost/array.hpp>
#include <boost/asio.hpp>
#include <boost/bind.hpp>
#include <gtest/gtest.h>

using namespace std;

/// \brief 标记当前时间。
template <typename T>
void tag(const T &label) noexcept
{
    const auto now = chrono::steady_clock::now();
    const auto since_epoch_mss = chrono::duration_cast<chrono::milliseconds>(now.time_since_epoch());

    cout << '[' << since_epoch_mss.count() << "ms] " << label << endl;
}

namespace rrboost {

/// \brief 同步计时器示例。
TEST(asio, wait)
{
    using namespace chrono_literals;

    boost::asio::io_context io;

    boost::asio::steady_timer timer {io, 10ms};
    tag("waiting...");

#if 1  // exception style
    try {
        timer.wait();
    } catch (const boost::system::system_error &e) {
        cerr << e.what() << endl;
    }
#else  // error code style
    boost::system::error_code ec;
    timer.wait(ec);
    if (ec)
        cerr << system_error(ec).what() << endl;
#endif
    tag("timeout.");
}

/// \brief 异步计时器示例。
TEST(asio, async_wait)
{
    using namespace chrono_literals;

    boost::asio::io_context io;

    const auto now {chrono::steady_clock::now()};
    boost::asio::steady_timer timer1 {io, now + 10ms};
    timer1.async_wait([]([[maybe_unused]] const boost::system::error_code &e) {
        tag("timer1 timeout and callback.");
    });
    boost::asio::steady_timer timer2 {io, now + 20ms};
    timer2.async_wait([]([[maybe_unused]] const boost::system::error_code &e) {
        tag("timer2 timeout and callback.");
    });
    tag("waiting...");

    io.run();
    tag("io run return.");
}

/// \brief 异步计时器示例，回调带参数。
TEST(asio, async_wait_arg)
{
    using namespace chrono_literals;

    constexpr auto callback = [](const boost::system::error_code &/*e*/, const string &arg) -> void {
        tag(arg);
    };

    boost::asio::io_context io;

    boost::asio::steady_timer timer(io, 10ms);
    timer.async_wait(boost::bind<void>(callback, boost::asio::placeholders::error, "Aoi Shigure"));

    io.run();
}

/// \brief TCP 示例。
TEST(asio, tcp)
{
    namespace asio = boost::asio;
    using asio::ip::address;
    using asio::ip::tcp;

    thread server([] {
        try {
            asio::io_context io;

            tcp::endpoint listening_ep(tcp::v4(), 49152);
            tcp::acceptor acceptor(io, listening_ep);

            while (true) {
                tcp::socket socket(io);
                acceptor.accept(socket);

                // read from client
                array<char, 1024> buf;
                const auto len = socket.read_some(asio::buffer(buf));
                cout.write(buf.data(), len) << "\n";

                // write to client
                socket.write_some(asio::buffer("TCP server: Hello, TCP client."));

                // disconnect
                break;
            }

        } catch (const exception &e) {
            cerr << "TCP server: " << e.what() << endl;

        }
    });

    thread client([] {
        try {
            asio::io_context io;

#if 1  // ip address
            const tcp::endpoint server_ep(address::from_string("127.0.0.1"), 49152);

            tcp::socket socket(io);
            socket.connect(server_ep);
#else  // domain name
            tcp::resolver resolver(io);
            const auto server_eps {resolver.resolve("localhost", "49152")};

            tcp::socket socket(io);
            asio::connect(socket, server_eps);
#endif
            // write to server
            socket.write_some(asio::buffer("TCP client: Hello, TCP server."));

            // read from server
            array<char, 1024> buf;
            while (true) {
                try {
                    const auto len = socket.read_some(asio::buffer(buf));
                    cout.write(buf.data(), len);
                } catch (const boost::system::system_error &e) {
                    if (e.code() == asio::error::eof)
                        break;
                    throw;
                }
            }
            cout << "\n";

        } catch (const exception &e) {
            cerr << "TCP client: " << e.what() << endl;

        }
    });

    server.join();
    client.join();
}

/// \brief UDP 示例。
TEST(asio, udp)
{
    namespace asio = boost::asio;
    using asio::ip::address;
    using asio::ip::udp;

    thread server([] {
        try {
            asio::io_context io;

            udp::endpoint listening_ep(udp::v4(), 49152);
            udp::socket socket(io, listening_ep);

            array<char, 65535> buf;
            while (true) {
                udp::endpoint remote_ep;
                const auto len = socket.receive_from(asio::buffer(buf), remote_ep);
                cout.write(buf.data(), len) << "\n";

                socket.send_to(asio::buffer("UDP server: Hello, UDP client."), remote_ep);
                break;
            }

        } catch (const exception &e) {
            cerr << "UDP server: " << e.what() << endl;

        }
    });

    thread client([] {
        try {
            asio::io_context io;

            udp::socket socket(io);
            socket.open(udp::v4());

            const udp::endpoint server_receiver_ep(address::from_string("127.0.0.1"), 49152);
            socket.send_to(asio::buffer("UDP client: Hello, UDP server."), server_receiver_ep);

            array<char, 65535> buf;
            udp::endpoint server_sender_ep;
            const auto len = socket.receive_from(asio::buffer(buf), server_sender_ep);
            cout.write(buf.data(), len) << "\n";

        } catch (const exception &e) {
            cerr << "UDP client: " << e.what() << endl;

        }
    });

    server.join();
    client.join();
}

}

max_stars_repo_name: afoolsbag/rrCnCxx

max_issues_repo_name: afoolsbag/rrCnCxx

max_forks_repo_name: afoolsbag/rrCnCxx


text:
/*
 Copyright (C) 2016 Quaternion Risk Management Ltd
 All rights reserved.

 This file is part of ORE, a free-software/open-source library
 for transparent pricing and risk analysis - http://opensourcerisk.org

 ORE is free software: you can redistribute it and/or modify it
 under the terms of the Modified BSD License.  You should have received a
 copy of the license along with this program.
 The license is also available online at <http://opensourcerisk.org>

 This program is distributed on the basis that it will form a useful
 contribution to risk analytics and model standardisation, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

#include <boost/test/unit_test.hpp>
#include <oret/datapaths.hpp>
#include <oret/fileutilities.hpp>
#include <oret/toplevelfixture.hpp>

#include <ored/model/calibrationinstruments/cpicapfloor.hpp>
#include <ored/model/crossassetmodeldata.hpp>
#include <ored/model/inflation/infdkdata.hpp>
#include <ored/utilities/correlationmatrix.hpp>

using namespace QuantLib;
using namespace QuantExt;
using namespace boost::unit_test_framework;
using namespace std;
using namespace ore;
using namespace ore::data;

using ore::test::TopLevelFixture;

namespace {

boost::shared_ptr<vector<boost::shared_ptr<IrLgmData>>> irConfigsData() {

    // Create three instances
    boost::shared_ptr<IrLgmData> lgmData1(new data::IrLgmData());
    boost::shared_ptr<IrLgmData> lgmData2(new data::IrLgmData());
    boost::shared_ptr<IrLgmData> lgmData3(new data::IrLgmData());

    vector<std::string> expiries = {"1Y", "2Y", "36M"};
    vector<std::string> terms = {"5Y", "2Y", "6M"};
    vector<std::string> strikes = {"ATM", "ATM", "ATM"};

    // First instance
    lgmData1->ccy() = "EUR";

    lgmData1->calibrationType() = parseCalibrationType("BOOTSTRAP");
    lgmData1->reversionType() = parseReversionType("HULLWHITE");
    lgmData1->volatilityType() = parseVolatilityType("HAGAN");
    std::vector<Time> hTimes = {1.0, 2.0, 3.0, 4.0};
    std::vector<Real> hValues = {1.0, 2.0, 3.0, 4.0};
    std::vector<Time> aTimes = {1.0, 2.0, 3.0, 4.0};
    std::vector<Real> aValues = {1.0, 2.0, 3.0, 4.0};

    lgmData1->calibrateH() = false;
    lgmData1->hParamType() = parseParamType("PIECEWISE");

    lgmData1->hTimes() = hTimes;

    lgmData1->hValues() = hValues;

    lgmData1->calibrateA() = false;
    lgmData1->aParamType() = parseParamType("PIECEWISE");
    lgmData1->aTimes() = aTimes;
    lgmData1->aValues() = aValues;
    lgmData1->shiftHorizon() = 1.0;

    lgmData1->optionExpiries() = expiries;
    lgmData1->optionTerms() = terms;
    lgmData1->optionStrikes() = strikes;

    lgmData1->scaling() = 1.0;

    // Second instance
    lgmData2->ccy() = "USD";

    lgmData2->calibrationType() = parseCalibrationType("BOOTSTRAP");
    lgmData2->reversionType() = parseReversionType("HULLWHITE");
    lgmData2->volatilityType() = parseVolatilityType("HAGAN");

    lgmData2->calibrateH() = false;
    lgmData2->hParamType() = parseParamType("PIECEWISE");
    lgmData2->hTimes() = hTimes;
    lgmData2->hValues() = hValues;

    lgmData2->calibrateA() = false;
    lgmData2->aParamType() = parseParamType("PIECEWISE");

    lgmData2->aTimes() = aTimes;
    lgmData2->aValues() = aValues;
    lgmData2->shiftHorizon() = 1.0;

    lgmData2->optionExpiries() = expiries;
    lgmData2->optionTerms() = terms;
    lgmData2->optionStrikes() = strikes;

    lgmData2->scaling() = 1.0;

    // Third instance
    lgmData3->ccy() = "JPY";

    lgmData3->calibrationType() = parseCalibrationType("BOOTSTRAP");
    lgmData3->reversionType() = parseReversionType("HULLWHITE");
    lgmData3->volatilityType() = parseVolatilityType("HAGAN");

    lgmData3->calibrateH() = false;
    lgmData3->hParamType() = parseParamType("PIECEWISE");
    lgmData3->hTimes() = hTimes;
    lgmData3->hValues() = hValues;

    lgmData3->calibrateA() = false;
    lgmData3->aParamType() = parseParamType("PIECEWISE");

    lgmData3->aTimes() = aTimes;
    lgmData3->aValues() = aValues;
    lgmData3->shiftHorizon() = 1.0;

    lgmData3->optionExpiries() = expiries;
    lgmData3->optionTerms() = terms;
    lgmData3->optionStrikes() = strikes;

    lgmData3->scaling() = 1.0;

    boost::shared_ptr<vector<boost::shared_ptr<IrLgmData>>> lgmDataVector(new vector<boost::shared_ptr<IrLgmData>>);
    *lgmDataVector = {lgmData1, lgmData2, lgmData3};
    return lgmDataVector;
}

vector<boost::shared_ptr<InflationModelData>> infConfigsData() {

    // TODO: Replacing the data that was here for now. It doesn't make sense.

    vector<boost::shared_ptr<CalibrationInstrument>> instruments;
    vector<Period> expiries = { 1 * Years, 2 * Years, 36 * Months };
    auto strike = boost::make_shared<AbsoluteStrike>(0.03);
    for (const Period& expiry : expiries) {
        instruments.push_back(boost::make_shared<CpiCapFloor>(CapFloor::Floor, expiry, strike));
    }
    CalibrationBasket cb(instruments);
    vector<CalibrationBasket> calibrationBaskets = { cb };

    ReversionParameter reversion(LgmData::ReversionType::HullWhite, false, ParamType::Piecewise,
        { 1.0, 2.0, 3.0, 4.0 }, { 1.0, 2.0, 3.0, 4.0, 4.0 });

    VolatilityParameter volatility(LgmData::VolatilityType::Hagan, false, ParamType::Piecewise,
        { 1.0, 2.0, 3.0, 4.0 }, { 1.0, 2.0, 3.0, 4.0, 4.0 });

    LgmReversionTransformation rt(1.0, 1.0);

    boost::shared_ptr<InfDkData> data = boost::make_shared<InfDkData>(CalibrationType::Bootstrap, calibrationBaskets,
        "EUR", "EUHICPXT", reversion, volatility, rt);

    return {data};
}

boost::shared_ptr<vector<boost::shared_ptr<FxBsData>>> fxConfigsData() {

    // Create two instances
    boost::shared_ptr<FxBsData> fxBsData1(new data::FxBsData());
    boost::shared_ptr<FxBsData> fxBsData2(new data::FxBsData());

    vector<std::string> expiries = {"1Y", "2Y", "36M"};
    vector<std::string> strikes = {"ATMF", "ATMF", "ATMF"};
    std::vector<Time> times = {1.0, 2.0, 3.0, 4.0};

    // First instance
    fxBsData1->foreignCcy() = "USD";
    fxBsData1->domesticCcy() = "EUR";
    fxBsData1->calibrationType() = parseCalibrationType("BOOTSTRAP");
    fxBsData1->calibrateSigma() = true;
    fxBsData1->sigmaParamType() = parseParamType("CONSTANT");
    fxBsData1->sigmaTimes() = times;
    fxBsData1->optionExpiries() = expiries;
    fxBsData1->optionStrikes() = strikes;

    // Second instance
    fxBsData2->foreignCcy() = "JPY";
    fxBsData2->domesticCcy() = "EUR";
    fxBsData2->calibrationType() = parseCalibrationType("BOOTSTRAP");
    fxBsData2->calibrateSigma() = true;
    fxBsData2->sigmaParamType() = parseParamType("CONSTANT");
    fxBsData2->sigmaTimes() = times;
    fxBsData2->optionExpiries() = expiries;
    fxBsData2->optionStrikes() = strikes;

    boost::shared_ptr<vector<boost::shared_ptr<FxBsData>>> fxBsDataVector(new vector<boost::shared_ptr<FxBsData>>);
    *fxBsDataVector = {fxBsData1, fxBsData2};
    return fxBsDataVector;
}

boost::shared_ptr<vector<boost::shared_ptr<EqBsData>>> eqConfigsData() {

    // Create two instances
    boost::shared_ptr<EqBsData> eqBsData1(new data::EqBsData());

    vector<std::string> expiries = {"1Y", "2Y", "36M"};
    vector<std::string> strikes = {"ATMF", "ATMF", "ATMF"};
    std::vector<Time> times = {1.0, 2.0, 3.0, 4.0};

    // First instance
    eqBsData1->eqName() = "SP5";
    eqBsData1->currency() = "EUR";
    eqBsData1->calibrationType() = parseCalibrationType("BOOTSTRAP");
    eqBsData1->calibrateSigma() = true;
    eqBsData1->sigmaParamType() = parseParamType("CONSTANT");
    eqBsData1->sigmaTimes() = times;
    eqBsData1->optionExpiries() = expiries;
    eqBsData1->optionStrikes() = strikes;

    boost::shared_ptr<vector<boost::shared_ptr<EqBsData>>> eqBsDataVector(new vector<boost::shared_ptr<EqBsData>>);
    *eqBsDataVector = {eqBsData1};
    return eqBsDataVector;
}

boost::shared_ptr<vector<boost::shared_ptr<CrLgmData>>> crLgmConfigsData() {

    // Create three instances
    boost::shared_ptr<CrLgmData> lgmData(new data::CrLgmData());

    lgmData->name() = "ItraxxEuropeS9V1";

    lgmData->calibrationType() = parseCalibrationType("BOOTSTRAP");
    lgmData->reversionType() = parseReversionType("HULLWHITE");
    lgmData->volatilityType() = parseVolatilityType("HAGAN");
    std::vector<Time> hTimes = {1.0, 2.0, 3.0, 4.0};
    std::vector<Real> hValues = {1.0, 2.0, 3.0, 4.0};
    std::vector<Time> aTimes = {1.0, 2.0, 3.0, 4.0};
    std::vector<Real> aValues = {1.0, 2.0, 3.0, 4.0};

    lgmData->calibrateH() = false;
    lgmData->hParamType() = parseParamType("CONSTANT");

    lgmData->hTimes() = hTimes;

    lgmData->hValues() = hValues;

    lgmData->calibrateA() = false;
    lgmData->aParamType() = parseParamType("CONSTANT");
    lgmData->aTimes() = aTimes;
    lgmData->aValues() = aValues;
    lgmData->shiftHorizon() = 1.0;

    lgmData->scaling() = 1.0;

    boost::shared_ptr<vector<boost::shared_ptr<CrLgmData>>> lgmDataVector(new vector<boost::shared_ptr<CrLgmData>>);
    *lgmDataVector = {lgmData};
    return lgmDataVector;
}

boost::shared_ptr<vector<boost::shared_ptr<CrCirData>>> crCirConfigsData() {

    // Create three instances
    boost::shared_ptr<CrCirData> cirData(new data::CrCirData());

    cirData->name() = "CDX.NA.S33v1";

    cirData->currency() = "USD";
    cirData->calibrationType() = parseCalibrationType("None");
    cirData->calibrationStrategy() = parseCirCalibrationStrategy("None");
    cirData->startValue() = 0.1;
    cirData->reversionValue() = 0.1;
    cirData->longTermValue() = 0.1;
    cirData->volatility() = 0.1;
    cirData->relaxedFeller() = true;
    cirData->fellerFactor() = 1.1;
    cirData->tolerance() = 1e-8;

    boost::shared_ptr<vector<boost::shared_ptr<CrCirData>>> cirDataVector(new vector<boost::shared_ptr<CrCirData>>);
    *cirDataVector = {cirData};
    return cirDataVector;
}

boost::shared_ptr<data::CrossAssetModelData> crossAssetData() {

    boost::shared_ptr<data::CrossAssetModelData> crossAssetData(new data::CrossAssetModelData());

    crossAssetData->domesticCurrency() = "EUR";
    crossAssetData->currencies() = {"EUR", "USD", "JPY"}; // need to check how to set this up
    crossAssetData->equities() = {"SP5"};
    crossAssetData->infIndices() = {"EUHICPXT"};
    crossAssetData->creditNames() = {"ItraxxEuropeS9V1", "CDX.NA.S33v1"};
    crossAssetData->irConfigs() = *irConfigsData();
    crossAssetData->fxConfigs() = *fxConfigsData();
    crossAssetData->eqConfigs() = *eqConfigsData();
    crossAssetData->infConfigs() = infConfigsData();
    crossAssetData->crLgmConfigs() = *crLgmConfigsData();
    crossAssetData->crCirConfigs() = *crCirConfigsData();

    CorrelationMatrixBuilder cmb;
    cmb.addCorrelation("IR:EUR", "IR:USD", 1.0);
    cmb.addCorrelation("IR:EUR", "IR:JPY", 1.0);
    cmb.addCorrelation("IR:USD", "IR:JPY", 1.0);
    cmb.addCorrelation("INF:EUHICPXT", "IR:EUR", 1.0);
    cmb.addCorrelation("IR:EUR", "CR:ItraxxEuropeS9V1", 1.0);
    cmb.addCorrelation("IR:USD", "CR:CDX.NA.S33v1", 1.0);
    cmb.addCorrelation("CR:ItraxxEuropeS9V1", "CR:CDX.NA.S33v1", 1.0);

    crossAssetData->setCorrelations(cmb.correlations());

    crossAssetData->bootstrapTolerance() = 0.001;

    return crossAssetData;
}

// Fixture to remove output files
class F : public TopLevelFixture {
public:
    F() {}
    ~F() { clearOutput(TEST_OUTPUT_PATH); }
};
} // namespace

BOOST_FIXTURE_TEST_SUITE(OREDataTestSuite, TopLevelFixture)

BOOST_FIXTURE_TEST_SUITE(CrossAssetModelDataTests, F)

BOOST_AUTO_TEST_CASE(testToXMLFromXML) {

    BOOST_TEST_MESSAGE("Testing toXML/fromXML...");

    data::CrossAssetModelData data = *crossAssetData();
    XMLDocument OutDoc;

    XMLNode* simulationNode = OutDoc.allocNode("Simulation");
    OutDoc.appendNode(simulationNode);

    XMLNode* crossAssetModelNode = data.toXML(OutDoc);
    XMLUtils::appendNode(simulationNode, crossAssetModelNode);

    std::string filename = TEST_OUTPUT_FILE("simulationtest.xml");
    OutDoc.toFile(filename);

    data::CrossAssetModelData newData;
    newData.fromFile(filename);

    BOOST_CHECK(data == newData);

    newData.irConfigs() = {};
    BOOST_CHECK(data != newData);
}

BOOST_AUTO_TEST_SUITE_END()

BOOST_AUTO_TEST_SUITE_END()

max_stars_repo_name: mrslezak/Engine

max_issues_repo_name: mrslezak/Engine

max_forks_repo_name: mrslezak/Engine