text:
// -------------------------------------------------------------------------
// This file is part of ZeroBugs, Copyright (c) 2010 Cristian L. Vlasceanu
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// -------------------------------------------------------------------------
//
// $Id$
//
#include <boost/python.hpp>
#include "zdk/get_pointer.h"
#include "zdk/properties.h"
#include "locked.h"
#include "properties.h"

using namespace boost;
using namespace boost::python;

void export_properties()
{
    class_<Properties, bases<>, noncopyable>("Properties", no_init)
        .def("get_word", &Properties::get_word, locked<>())
        .def("set_word", &Properties::set_word, locked<>())
        .def("get_double", &Properties::get_double, locked<>())
        .def("set_double", &Properties::set_double, locked<>())
        .def("get_string", &Properties::get_string, locked<>())
        .def("set_string", &Properties::set_string, locked<>())
        ;

    register_ptr_to_python<RefPtr<Properties> >();
}

// vim: tabstop=4:softtabstop=4:expandtab:shiftwidth=4

max_stars_repo_name: cristivlas/zerobugs

max_issues_repo_name: cristivlas/zerobugs

max_forks_repo_name: cristivlas/zerobugs


text:
#include <cstdlib>
#include <iostream>
#include <memory>
#include <utility>
#include <boost/asio.hpp>
#include <boost/algorithm/string.hpp>
#include <sstream>

#include "buffer.h"
#include "server.h"

using boost::asio::ip::tcp;

enum { max_length = 1024 };

void print(uint8_t *buf,int len)
{
	for(int i=0; i < len; i++) {
		printf("%02x ",buf[i]);
	}
	printf("\n");
}

void process(uint8_t *buf,int length,uint8_t *resp,int *resp_len)
{
/*
	Buffer args(buf,length);
	Buffer res(resp,max_length);
	invoke(args,res);
	*resp_len = res.length();
*/
	print(buf,length);
	memcpy(resp,buf,length);
	*resp_len = length;
}

class session : public std::enable_shared_from_this<session>
{
public:
  session(tcp::socket socket) : socket_(std::move(socket)), length_(0),response_length_(0)
  {
  }

  void start()
  {
    do_read();
  }

private:
  void do_read()
  {
    auto self(shared_from_this());

    socket_.async_read_some(boost::asio::buffer(data_, max_length),
        [this, self](boost::system::error_code ec, std::size_t length)
        {
          if (!ec)
          {
            length_ = data_[0] + (data_[1] << 8);
            process(data_+2,length_,response_+2,&response_length_);
            response_[0] = response_length_ & 0xff;
            response_[1] = (response_length_ >> 8) & 0xff;
            do_write();
          }
        });
  }

  void do_write()
  {
    auto self(shared_from_this());

    boost::asio::async_write(socket_, boost::asio::buffer(response_, response_length_+2),
        [this, self](boost::system::error_code ec, std::size_t /*length*/)
        {
          if (!ec)
          {
            do_read();
          }
        });
  }

  tcp::socket socket_;

  uint8_t data_[max_length];
  int length_;
  uint8_t response_[max_length];
  int response_length_;
};

class server
{
public:
  server(boost::asio::io_service& io_service, short port)
		: acceptor_(io_service, tcp::endpoint(tcp::v4(), port)),  socket_(io_service)
  {
    do_accept();
  }

private:
  void do_accept()
  {
    acceptor_.async_accept(socket_,
        [this](boost::system::error_code ec)
        {
          if (!ec)
          {
            std::make_shared<session>(std::move(socket_))->start();
          }

          do_accept();
        });
  }

  tcp::acceptor acceptor_;
  tcp::socket socket_;
};

int main(int argc, char* argv[])
{
  try {
    boost::asio::io_service io_service;
    server s(io_service, 8000);
    io_service.run();
  } catch (std::exception& e) {
    std::cerr << "Exception: " << e.what() << "\n";
  }

  return 0;
}

ext: cpp


text:
#include <wordring/whatwg/url/host.hpp>

#include <wordring/whatwg/infra/infra.hpp>

#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>

#include <array>
#include <charconv>
#include <cmath>
#include <string>
#include <utility>
#include <vector>

#include <unicode/idna.h>

std::u32string wordring::whatwg::domain_to_ascii(std::u32string const& s, bool beStrict, std::error_code& ec)
{
	std::int32_t opt = UIDNA_USE_STD3_RULES | UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ | UIDNA_CHECK_CONTEXTO;
	UErrorCode u_ec = U_ZERO_ERROR;
	std::unique_ptr<icu::IDNA> idna(icu::IDNA::createUTS46Instance(opt, u_ec));

	icu::IDNAInfo info;
	icu::UnicodeString s1, s2;
	for (std::uint32_t cp : s) s1.append(static_cast<UChar32>(cp));

	idna->nameToASCII(s1, s2, info, u_ec);

	if (U_FAILURE(u_ec) || info.getErrors()) ec = url_error_name::IdnaTranslationFailed;

	std::u32string tmp;
	encoding_cast(s2.getBuffer(), s2.getBuffer() + s2.length(), std::back_inserter(tmp));

	return tmp;
}

std::u32string wordring::whatwg::domain_to_unicode(std::u32string const& s, std::error_code& ec)
{
	std::int32_t opt = UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ | UIDNA_CHECK_CONTEXTO;
	UErrorCode u_ec = U_ZERO_ERROR;
	std::unique_ptr<icu::IDNA> idna(icu::IDNA::createUTS46Instance(opt, u_ec));

	icu::IDNAInfo info;
	icu::UnicodeString s1, s2;
	for (std::uint32_t cp : s) s1.append(static_cast<UChar32>(cp));

	idna->nameToUnicode(s1, s2, info, u_ec);

	if (U_FAILURE(u_ec) || info.getErrors()) ec = url_error_name::IdnaTranslationFailed;

	std::u32string tmp;
	encoding_cast(s2.getBuffer(), s2.getBuffer() + s2.length(), std::back_inserter(tmp));

	return tmp;
}

bool wordring::whatwg::is_valid_domain_string(std::u32string const& s)
{
	// 1.
	std::error_code ec;
	std::u32string result = domain_to_ascii(s, true, ec);
	// 2.
	if (ec) return false;
	// 3.
	std::int32_t opt = UIDNA_USE_STD3_RULES | UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ | UIDNA_CHECK_CONTEXTO;
	UErrorCode u_ec = U_ZERO_ERROR;
	std::unique_ptr<icu::IDNA> idna(icu::IDNA::createUTS46Instance(opt, u_ec));

	icu::IDNAInfo info;
	icu::UnicodeString s1, s2;
	for (std::uint32_t cp : s) s1.append(static_cast<UChar32>(cp));

	idna->nameToUnicode(s1, s2, info, u_ec);

	// 4.
	if (U_FAILURE(u_ec) || info.getErrors()) return false;
	// 5.
	return true;
}

bool wordring::whatwg::is_valid_ipv4_address_string(std::u32string const& s)
{
	std::vector<std::u32string> v;
	boost::algorithm::split(v, s, boost::is_any_of(U"."));

	if (v.size() != 4) return false;

	std::string tmp;
	for (std::u32string const& c : v)
	{
		if (3 < c.size()) return false;
		tmp.clear();
		for (char32_t ch : c) tmp.push_back(static_cast<std::uint8_t>(ch));
		std::uint32_t i;
		std::from_chars(tmp.data(), tmp.data() + tmp.length(), i);
		if (255 < i) return false;
	}
	return true;
}

wordring::whatwg::host_base wordring::whatwg::parse_host(
	std::u32string const& in, bool isNotSpecial, std::error_code& ec)
{
	// 1.
	if (!in.empty() && in.front() == U'[')
	{
		if (in.back() != U']')
		{
			ec = url_error_name::ParseFailed;
			return host_base();
		}
		return parse_ipv6(std::u32string(++in.begin(), --in.end()), ec);
	}
	// 2.
	if (isNotSpecial) return parse_opaque_host(in, ec);
	// 3.
	assert(!in.empty());
	// 4.
	std::string tmp;
	string_percent_decode(in.begin(), in.end(), std::back_inserter(tmp));
	encoding::io_queue<char> in_q = encoding::to_io_queue_convert(tmp.begin(), tmp.end());
	encoding::io_queue<char32_t> out_q;
	encoding::utf8_decode_without_bom(in_q, out_q);
	std::u32string domain;
	encoding::from_io_queue_convert(out_q, std::back_inserter(domain));
	// 5.
	std::u32string asciiDomain = domain_to_ascii(domain, ec);
	// 6.
	if (ec) return host_base();
	// 7.
	if (std::find_if(asciiDomain.begin(), asciiDomain.end(), is_forbidden_host_code_point) != asciiDomain.end())
	{
		ec = url_error_name::ParseFailed;
		return host_base();
	}
	// 8.
	host_base ipv4 = parse_ipv4(asciiDomain, ec);
	// 9.
	if (!ipv4 || ipv4.type() == host_type_name::Ipv4Address) return ipv4;
	// 10.
	return domain_base{ asciiDomain };
}

// https://triple-underscore.github.io/URL-ja.html#concept-ipv4-parser

wordring::whatwg::host_base wordring::whatwg::parse_ipv4(std::u32string const& in, std::error_code& ec)
{
	// 1.
	bool validationError = false;
	// 2.
	std::vector<std::u32string> parts;
	strictly_split_on_particular_delimiter(in.begin(), in.end(), std::back_inserter(parts), U'.');
	// 3.
	if (!parts.empty() && parts.back().empty())
	{
		validationError = true;
		if (1 < parts.size()) parts.erase(--parts.end());
	}
	// 4.
	if (4 < parts.size()) return opaque_host_base{ in };
	// 5.
	std::vector<std::uint32_t> numbers;
	// 6.
	for (std::u32string const& part : parts)
	{
		// 6.1.
		if (part.empty()) return opaque_host_base{ in };
		// 6.2.
		std::pair<std::optional<std::uint32_t>, bool> result = parse_ipv4_number(part);
		// 6.3.
		if (!result.first) return opaque_host_base{ in };
		// 6.4.
		if (result.second) validationError = true;
		// 6.5.
		numbers.push_back(*result.first);
	}
	// 7.
	if (validationError) ec = url_error_name::ValidationError;
	// 8.
	for (std::uint32_t n : numbers) if (255 < n) ec = url_error_name::ValidationError;
	// 9.
	for (std::size_t i = 0; i < numbers.size() - 1; ++i) if (255 < numbers.at(i)) return host_base();
	// 10.
	assert(!numbers.empty());
	if (std::pow(256, (5 - numbers.size())) <= numbers.back())
	{
		ec = url_error_name::ValidationError;
		return host_base();
	}
	// 11.
	std::uint32_t ipv4 = numbers.back();
	// 12.
	numbers.pop_back();
	// 13.
	std::uint32_t counter = 0;
	// 14.
	for (std::uint32_t n : numbers)
	{
		ipv4 += static_cast<std::uint32_t>(n * std::pow(256, 3 - counter));
		++counter;
	}
	// 15.
	return host_base(ipv4_address_base{ ipv4 });
}

std::pair<std::optional<std::uint32_t>, bool>
wordring::whatwg::parse_ipv4_number(std::u32string s)
{
	// 1.
	bool validationError = false;
	// 2.
	std::uint32_t R = 10;
	// 3.
	if (2 <= s.size() && s.front() == U'0' && (s.at(1) == U'x' || s.at(1) == U'X'))
	{
		validationError = true;
		s.erase(0, 2);
		R = 16;
	}
	// 4.
	else if (2 <= s.size() && s.front() == U'0')
	{
		validationError = true;
		s.erase(0, 1);
		R = 8;
	}
	// 5.
	if (s.empty()) return std::make_pair(std::optional<std::uint32_t>(), validationError);
	// 6.
	std::u32string_view sv = R == 8 ? U"012345678" : (R == 10 ? U"0123456789" : U"0123456789abcdefABCDEF");
	for (char32_t cp : s) if (std::find(sv.begin(), sv.end(), cp) == sv.end())
	{
		return std::make_pair(std::optional<std::uint32_t>(), validationError);
	}
	// 7.
	std::string tmp;
	for (char32_t ch : s) tmp.push_back(static_cast<std::uint8_t>(ch));
	std::uint32_t i;
	std::from_chars(tmp.data(), tmp.data() + tmp.length(), i, R);
	// 8.
	return std::make_pair(std::optional<std::uint32_t>(i), validationError);
}

wordring::whatwg::host_base wordring::whatwg::parse_ipv6(std::u32string const& in, std::error_code& ec)
{
	// 1.
	ipv6_address_base address{};
	// 2.
	std::uint32_t pieceIndex = 0;
	// 3.
	std::optional<std::uint32_t> compress;
	// 4.
	std::int32_t pointer = 0;
	// 5.
	if (!in.empty() && in[pointer] == U':')
	{
		if (in.size() < 2 || in[pointer + 1] != U':')
		{
			ec = url_error_name::ValidationError;
			return host_base();
		}
		pointer += 2;
		++pieceIndex;
		compress = pieceIndex;
	}
	// 6.
	while (static_cast<std::uint32_t>(pointer) < in.size())
	{
		// 6.1.
		if (pieceIndex == 8) return host_base();
		// 6.2.
		if (static_cast<std::uint32_t>(pointer) < in.size() && in[pointer] == U':')
		{
			if (compress)
			{
				ec = url_error_name::ValidationError;
				return host_base();
			}
			++pointer;
			++pieceIndex;
			compress = pieceIndex;
			continue;
		}
		// 6.3.
		std::uint32_t value = 0;
		std::uint32_t length = 0;
		// 6.4.
		while (length < 4 && static_cast<std::uint32_t>(pointer) < in.size() && is_ascii_hex_digit(in[pointer]))
		{
			char32_t cp = in[pointer];
			std::uint32_t i = (U'0' <= cp && cp <= U'9') ? cp - U'0' : (U'a' <= cp && cp <= U'f' ? cp - U'a' + 10 : cp - U'A' + 10);
			value = value * 0x10 + i;
			++pointer;
			++length;
		}
		// 6.5.
		if (static_cast<std::uint32_t>(pointer) < in.size() && in[pointer] == U'.')
		{
			// 6.5.1.
			if (length == 0)
			{
				ec = url_error_name::ValidationError;
				return host_base();
			}
			// 6.5.2.
			pointer -= length;
			// 6.5.3.
			if (6 < pieceIndex)
			{
				ec = url_error_name::ValidationError;
				return host_base();
			}
			// 6.5.4.
			std::uint32_t numbersSeen = 0;
			// 6.5.5.
			while (0 <= static_cast<std::uint32_t>(pointer) && static_cast<std::uint32_t>(pointer) < in.size())
			{
				// 6.5.5.1.
				std::optional<std::uint32_t> ipv4Piece;
				// 6.5.5.2.
				if (0 < numbersSeen)
				{
					if (in[pointer] == U'.' && numbersSeen < 4) ++pointer;
					else
					{
						ec = url_error_name::ValidationError;
						return host_base();
					}
				}
				// 6.5.5.3.
				if (in.size() <= static_cast<std::uint32_t>(pointer) || !is_ascii_digit(in[pointer]))
				{
					ec = url_error_name::ValidationError;
					return host_base();
				}
				// 6.5.5.4.
				while (static_cast<std::uint32_t>(pointer) < in.size() && is_ascii_digit(in[pointer]))
				{
					// 1.
					std::uint32_t number = in[pointer] - U'0';
					// 2.
					if (!ipv4Piece) *ipv4Piece = number;
					else if (*ipv4Piece == 0)
					{
						ec = url_error_name::ValidationError;
						return host_base();
					}
					else *ipv4Piece = *ipv4Piece * 10 + number;
					// 3.
					if(255 < *ipv4Piece)
					{
						ec = url_error_name::ValidationError;
						return host_base();
					}
					// 4.
					++pointer;
				}
				// 6.5.5.5.
				address[pieceIndex] = address[pieceIndex] * 0x100 + *ipv4Piece;
				// 6.5.5.6.
				++numbersSeen;
				// 6.5.5.7.
				if (numbersSeen == 2 || numbersSeen == 4) ++pieceIndex;
			}
			// 6.5.6.
			if(numbersSeen != 4)
			{
				ec = url_error_name::ValidationError;
				return host_base();
			}
			// 6.5.7.
			break;
		}
		// 6.6.
		else if(static_cast<std::uint32_t>(pointer) < in.size() && in[pointer] == U':')
		{
			// 6.6.1.
			++pointer;
			// 6.6.2.
			if(in.size() <= static_cast<std::uint32_t>(pointer))
			{
				ec = url_error_name::ValidationError;
				return host_base();
			}
		}
		// 6.7.
		else if(static_cast<std::uint32_t>(pointer) < in.size())
		{
			ec = url_error_name::ValidationError;
			return host_base();
		}
		// 6.8.
		address[pieceIndex] = value;
		// 6.9.
		++pieceIndex;
	}
	// 7.
	if (compress)
	{
		std::uint32_t swaps = pieceIndex - *compress;
		pieceIndex = 7;
		while (pieceIndex != 0 && 0 < swaps)
		{
			std::swap(address[pieceIndex], address[*compress + swaps - 1]);
			--pieceIndex;
			--swaps;
		}
	}
	// 8.
	else if(!compress && pieceIndex != 8)
	{
		ec = url_error_name::ValidationError;
		return host_base();
	}
	// 9.
	return address;
}

namespace
{
	// parse_opaque_host() 内で使う。
	// 4.3. URL writing 内に同じ定義があることに注意

	inline bool is_url_code_points_(char32_t cp)
	{
		using namespace wordring::whatwg;
		if (is_ascii_alphanumeric(cp)) return true;
		switch (cp)
		{
		case U'\x21': // (!)
		case U'\x24': // ($)
		case U'\x26': // (&)
		case U'\x27': // (')
		case U'\x28': // LEFT PARENTHESIS
		case U'\x29': // RIGHT PARENTHESIS
		case U'\x2A': // (*)
		case U'\x2B': // (+)
		case U'\x2C': // (,)
		case U'\x2D': // (-)
		case U'\x2E': // (.)
		case U'\x2F': // (/)
		case U'\x3A': // (:)
		case U'\x3B': // (;)
		case U'\x3D': // (=)
		case U'\x3F': // (?)
		case U'\x40': // (@)
		case U'\x5F': // (_)
		case U'\x7E': // (~)
			return true;
		}
		if (U'\xA0' <= cp && cp <= U'\U0010FFFD' && !is_surrogate(cp) && !is_noncharacter(cp)) return true;
		return false;
	}
}

wordring::whatwg::host_base wordring::whatwg::parse_opaque_host(
	std::u32string const& in, std::error_code& ec)
{
	// 1.
	for (char32_t cp : in)
	{
		if (is_forbidden_host_code_point(cp) && cp != U'%')
		{
			ec = url_error_name::ValidationError;
			return host_base();
		}
	}
	// 2.
	for (char32_t cp : in) { if (!is_url_code_points_(cp) && cp != U'%') ec = url_error_name::ValidationError; }
	// 3.
	std::uint32_t state = 0;
	for (std::uint32_t i = 0; i < in.size(); ++i)
	{
		char32_t cp = in[i];
		switch (state)
		{
		case 0:
			if (cp == U'%')
			{
				if (in.size() - i < 3)
				{
					ec = url_error_name::ValidationError;
					break;
				}
				else state = 1;
			}
			break;
		case 1:
			if (!is_ascii_hex_digit(cp))
			{
				ec = url_error_name::ValidationError;
				break;
			}
			else state = 2;
			break;
		case 2:
			if (!is_ascii_hex_digit(cp))
			{
				ec = url_error_name::ValidationError;
				break;
			}
			else state = 0;
			break;
		}
	}
	// 4.
	return opaque_host_base{ utf8_percent_encode(in, is_c0_control_percent_encode_set) };
}

std::u32string wordring::whatwg::serialize_host(wordring::whatwg::host_base const& host)
{
	// 1.
	if (host.type() == host_type_name::Ipv4Address) return serialize_ipv4(host);
	// 2.
	if (host.type() == host_type_name::Ipv6Address)
	{
		std::u32string s;
		s.push_back(U'[');
		s += serialize_ipv6(host);
		s.push_back(U']');
		return s;
	}
	// 3.
	if (host.type() == host_type_name::Domain) return host.get<domain_base>().m_value;
	if (host.type() == host_type_name::OpaqueHost) return host.get<opaque_host_base>().m_value;
	if (host.type() == host_type_name::EmptyHost) return U"";

	assert(false);
	return U"";
}

std::u32string wordring::whatwg::serialize_ipv4(wordring::whatwg::host_base const& ipv4)
{
	// 1.
	std::u32string output;
	// 2.
	std::uint32_t n = ipv4.get<ipv4_address_base>().m_value;
	// 3.
	for (std::uint32_t i = 1; i <= 4; ++i)
	{
		// 3.1.
		std::array<char, 10> a;
		auto [p, ec] = std::to_chars(a.data(), a.data() + a.size(), n % 256);
		output.insert(output.begin(), a.data(), p);
		// 3.2.
		if (i != 4) output.insert(output.begin(), U'.');
		// 3.3.
		n = static_cast<std::uint32_t>(std::floor(n / 256));
	}
	// 4.
	return output;
}

std::u32string wordring::whatwg::serialize_ipv6(wordring::whatwg::host_base const& ipv6)
{
	// 1.
	std::u32string output;
	// 2-3.
	std::optional<std::uint32_t> compress;
	ipv6_address_base const& address = ipv6.get<ipv6_address_base>();
	std::uint32_t longer = 0;
	std::uint32_t length = 0;
	std::uint32_t tail = 0;
	for (std::uint32_t i = 0; i < 8; ++i)
	{
		if (address[i] == 0)
		{
			++length;
			if (i == 7) i = 8;
			else continue;
		}

		if (longer < length)
		{
			tail = i;
			longer = length;
			length = 0;
		}
	}
	if (2 <= longer) compress = std::make_optional<std::uint32_t>(tail - longer);
	// 4.
	bool ignore0 = false;
	// 5.
	for (std::uint32_t pieceIndex = 0; pieceIndex < 8; ++pieceIndex)
	{
		// 5.1.
		if (ignore0 && address[pieceIndex] == 0) continue;
		// 5.2.
		ignore0 = false;
		// 5.3.
		if (compress && *compress == pieceIndex)
		{
			std::u32string separator = pieceIndex == 0 ? U"::" : U":";
			output += separator;
			ignore0 = true;
			continue;
		}
		// 5.4.
		std::array<char, 10> a;
		auto [p, ec] = std::to_chars(a.data(), a.data() + a.size(), address[pieceIndex], 0x10);
		output.insert(output.end(), a.data(), p);
		// 5.5.
		if (pieceIndex != 7) output.push_back(U':');
	}
	// 6.
	return output;
}


text:
#pragma once
#include <vector>
#include <set>
#include <Eigen/Dense>
#include <utility>
#include <ostream>
#include <map>
#include <atomic>
#include <mutex>
#include <span>

namespace ml
{

using namespace Eigen;

struct topology
{
    std::vector<int> layers;
};

std::ostream& operator<<(std::ostream& stream, const topology& topo);

class neural_network
{
private:
    std::vector<MatrixXd>                 m_weights;
    std::vector<VectorXd>                 m_biases;
    topology                              m_topo;
    const std::map<std::string, VectorXd> m_labels;
    std::mutex                            mtx;
    // Ideally m_latest_weight_gradients and m_latest_bias_gradients would not be properties of the class,
    // however back_propagate's function signature was getting too large
    std::vector<MatrixXd>                 m_latest_weight_gradients;
    std::vector<VectorXd>                 m_latest_bias_gradients;

    static void hadamard_product(VectorXd& lhs, const VectorXd& rhs);

    static void logistic(VectorXd& vector);
    static void logistic_derivative(VectorXd& vector);

    [[nodiscard]] static std::map<std::string, VectorXd> generate_class_vector_map(std::set<std::string>);

public:
    neural_network(topology topo, std::set<std::string> labels);
    neural_network(std::vector<MatrixXd> weights, std::vector<VectorXd> biases, std::set<std::string> labels);

    void randomize();

    // returns the largest activation of the final layer and it's coresponding class
    [[nodiscard]] std::pair<std::string, double>                classify(const VectorXd& input_activations) const;
    // Generates a confusion matrix with the columns representing predicted labels and the rows representing actual labels, vector component
    // of pair is to indicate ordering of labels in rows and columns
    [[nodiscard]] std::pair<std::vector<std::string>, MatrixXi> generate_confusion_matrix(const std::vector<std::pair<VectorXd, std::string>>& test_data) const;
    // returns a vector of all calculated activations and weighted sums
    [[nodiscard]] std::vector<std::pair<VectorXd, VectorXd>>    forward_propagate_return_all(const VectorXd& input_activations, const std::vector<MatrixXd>& local_weights, const std::vector<VectorXd>& local_biases) const;

    // performes back propagation. training_data is a vector of pairs of training input data and desired output classification
    double back_propagate(const std::vector<std::pair<VectorXd, VectorXd>>& training_data, const std::span<int>& mini_batch, double learning_rate);
    // trains network
    void   train(const std::vector<std::pair<VectorXd, std::string>>& training_data, bool print_cost, std::atomic_bool& stopped, double learning_rate = 0.1, std::size_t mini_batch_size = 0);

    friend std::ostream& operator<<(std::ostream&, const neural_network&);
};

std::ostream& operator<<(std::ostream& stream, const neural_network& network);

};

ext: hpp


text:
/*******************************************************************************
*
*  Filename    : FillHistogram.cc
*  Description : Command for prefill the Histogram files, call each per group
*  Author      : Yi-Mu "Enoch" Chen [ ensc@hep1.phys.ntu.edu.tw ]
*
*******************************************************************************/
#include "TstarAnalysis/Common/interface/InitSample.hpp"
#include "TstarAnalysis/CompareDataMC/interface/Compare_Common.hpp"
#include "TstarAnalysis/CompareDataMC/interface/SampleErrHistMgr.hpp"
#include "TstarAnalysis/CompareDataMC/interface/SampleHistMgr.hpp"
#include "TstarAnalysis/CompareDataMC/interface/SampleTableMgr.hpp"

#include <boost/program_options.hpp>
#include <iostream>
using namespace std;
namespace opt = boost::program_options;

/*******************************************************************************
*   Main control flow
*******************************************************************************/
int
main( int argc, char* argv[] )
{
  opt::options_description desc( "Options for FillHistogram" );
  desc.add_options()
    ( "group,g",   opt::value<string>()->required(), "What group to fill" )
    ( "type,t",    opt::value<string>()->required(), "What type of histrogram to fill" )
  ;

  compnamer.AddOptions( CompareOptions() ); // Defined in src/Common.cc
  compnamer.AddOptions( desc );
  const int parse = compnamer.ParseOptions( argc, argv ); // defined in Compare_Common.hpp
  if( parse == mgr::OptNamer::PARSE_ERROR ){ return 1; }
  if( parse == mgr::OptNamer::PARSE_HELP ){ return 0; }

  /*******************************************************************************
  *   Additional parsing for this program
  *******************************************************************************/
  if( compnamer.GetInput<string>("group") == "Data" && !compnamer.CheckInput("era") ){
    cerr << "Error! filling of group [Data] must have [era] specified!" << endl;
    cerr << compnamer.GetDescription() << endl;
    return 0;
  }

  /*******************************************************************************
  *   Initializing
  *******************************************************************************/
  InitSampleStatic( compnamer );

  const mgr::ConfigReader& master = compnamer.MasterConfig();

  const string tag = ( compnamer.GetInput<string>( "group" ) == "Data" ) ?
                     compnamer.GetChannelEXT( "Data Prefix" )+ compnamer.GetExt<string>( "era", "Data Postfix" ) :
                     compnamer.GetInput<string>( "group" );

  if( compnamer.GetInput<string>( "type" ) == "Err" ){
    SampleErrHistMgr errmgr( tag, master );
    errmgr.FillFromSample();
  } else if( compnamer.GetInput<string>( "type" ) == "Qk" ){
    SampleHistMgr mgr( tag, master );
    mgr.FillFromSample();
  } else if( compnamer.GetInput<string>( "type" ) == "Tb" ){
    SampleTableMgr mgr( tag, master );
    mgr.LoadFromEDM();
  } else {
    cerr << "Unknown histogram type!" << endl;
    return 1;
  }

  return 0;
}

ext: cc

max_stars_repo_name: NTUHEP-Tstar/TstarAnalysis

max_issues_repo_name: NTUHEP-Tstar/TstarAnalysis

max_forks_repo_name: NTUHEP-Tstar/TstarAnalysis


text:
// Copyright 2020 The Defold Foundation
// Licensed under the Defold License version 1.0 (the "License"); you may not use
// this file except in compliance with the License.
//
// You may obtain a copy of the License, together with FAQs at
// https://www.defold.com/license
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "math.h"
#include "http_client.h"
#include "http_client_private.h"
#include "log.h"
#include "sys.h"
#include "dstrings.h"
#include "uri.h"
#include "path.h"
#include "time.h"
#include "connection_pool.h"
#include <dlib/dns.h>
#include <dlib/mutex.h>
#include <dlib/socket.h>
#include <dlib/sslsocket.h>

namespace dmHttpClient
{
    // Be careful with this buffer. See comment in Receive about ssl_read.
    const int BUFFER_SIZE = 64 * 1024;

    const unsigned int HTTP_CLIENT_MAXIMUM_CACHE_AGE = 30U * 24U * 60U * 60U; // 30 days

    const uint32_t MAX_POOL_CONNECTIONS = 32;

    const int SOCKET_TIMEOUT = 500 * 1000;

    // Since https post requests have an upper limit of 2^14 bytes
    // we need to be able to handle chunked uploads.
    // See https://tools.ietf.org/html/rfc8446, chapter 5.1
    const uint32_t MAX_HTTPS_POST_CHUNK_SIZE = 16384;

    // TODO: This is not good. Singleton like stuff
    // that requires a lock for initialization
    // See comment in GetPool()
    struct PoolCreator
    {
        PoolCreator()
        {
            m_Pool = 0;
            m_Mutex = dmMutex::New();
        }

        ~PoolCreator()
        {
            dmMutex::Lock(m_Mutex);
            if (m_Pool) {
                dmConnectionPool::Delete(m_Pool);
            }
            dmMutex::Unlock(m_Mutex);
            dmMutex::Delete(m_Mutex);
        }

        // Create the pool lazily for two reasons
        // 1. If dlib is imported from loaded from python it will crash and burn
        //    in OpenSSL as the we several symbol conflics, e.g. RSA_free, and
        //    the version from OpenSSL will be used instead of axTLS!
        // 2. Startup performance. Minor in this context
        dmConnectionPool::HPool GetPool()
        {
            DM_MUTEX_SCOPED_LOCK(m_Mutex);
            if (m_Pool == 0) {
                dmConnectionPool::Params params;
                params.m_MaxConnections = MAX_POOL_CONNECTIONS;
                dmConnectionPool::Result r = dmConnectionPool::New(&params, &m_Pool);
                assert(r == dmConnectionPool::RESULT_OK);
            }
            return m_Pool;
        }

        dmConnectionPool::HPool GetPoolNoCreate()
        {
            DM_MUTEX_SCOPED_LOCK(m_Mutex);
            return m_Pool;
        }

    private:
        dmConnectionPool::HPool m_Pool;
        dmMutex::HMutex         m_Mutex;
    };

    PoolCreator g_PoolCreator;

    struct Response
    {
        HClient m_Client;
        int m_Major;
        int m_Minor;
        int m_Status;

        // Offset to actual content in Client.m_Buffer,
        // ie after meta-data such as http-headers or chunk-size for transferring data with chunked encoding.
        int m_ContentOffset;
        // Total amount of data received in Client.m_Buffer
        int m_TotalReceived;

        // Headers
        int      m_ContentLength;
        char     m_ETag[64];
        uint32_t m_Chunked : 1;
        uint32_t m_CloseConnection : 1;
        uint32_t m_MaxAge;

        // Cache
        dmHttpCache::HCacheCreator m_CacheCreator;

        // Connection
        dmConnectionPool::HPool         m_Pool;
        dmConnectionPool::HConnection   m_Connection;
        dmSocket::Socket                m_Socket;
        dmSSLSocket::Socket             m_SSLSocket;

        Response(HClient client)
        {
            m_Client = client;
            m_Major = 0;
            m_Minor = 0;
            m_Status = 0;
            m_ContentLength = -1;
            m_ETag[0] = '\0';
            m_ContentOffset = -1;
            m_TotalReceived = 0;
            m_Chunked = 0;
            m_CloseConnection = 0;
            m_MaxAge = 0;
            m_CacheCreator = 0;
            m_Pool = 0;
            m_Connection = 0;
            m_Socket = 0;
            m_SSLSocket = 0;
        }
        Result Connect(const char* host, uint16_t port, bool secure, int timeout);
        ~Response();
    };

    /*
     Client.m_Buffer layout

     |-------------------------------------------------------------------------|
     |                    |                               |                    |
     |      Meta-data     |        Content                |                    |X (extra byte for null termination)
     |                    |                               |                    |
     |-------------------------------------------------------------------------|
         m_ContentOffset ->             m_TotalReceived ->          BUFFER_SIZE->
     */

    struct Client
    {
        char*               m_Hostname;
        char                m_URI[dmURI::MAX_URI_LEN];
        dmSocket::Result    m_SocketResult;

        void*               m_Userdata;
        HttpContent         m_HttpContent;
        HttpHeader          m_HttpHeader;
        HttpSendContentLength m_HttpSendContentLength;
        HttpWrite           m_HttpWrite;
        HttpWriteHeaders    m_HttpWriteHeaders;
        int                 m_MaxGetRetries;
        int                 m_RequestTimeout;
        uint64_t            m_RequestStart;
        Statistics          m_Statistics;

        dmHttpCache::HCache m_HttpCache;
        dmDNS::HChannel     m_DNSChannel;

        bool                m_Secure;
        uint16_t            m_Port;

        // Used both for reading header and content. NOTE: Extra byte for null-termination
        char                m_Buffer[BUFFER_SIZE + 1];
    };

    Result Response::Connect(const char* host, uint16_t port, bool secure, int timeout)
    {
        m_Pool = g_PoolCreator.GetPool();
        dmConnectionPool::Result r = dmConnectionPool::Dial(m_Pool, host, port, m_Client->m_DNSChannel, secure, timeout, &m_Connection, &m_Client->m_SocketResult);

        if (r == dmConnectionPool::RESULT_OK) {

            m_Socket = dmConnectionPool::GetSocket(m_Pool, m_Connection);
            m_SSLSocket = dmConnectionPool::GetSSLSocket(m_Pool, m_Connection);

            dmSocket::SetSendTimeout(m_Socket, SOCKET_TIMEOUT);
            dmSocket::SetReceiveTimeout(m_Socket, SOCKET_TIMEOUT);

            return RESULT_OK;
        } else {
            return RESULT_SOCKET_ERROR;
        }
    }

    Response::~Response()
    {
        if (m_Connection) {
            if (m_CloseConnection || m_Client->m_SocketResult != dmSocket::RESULT_OK) {
                dmConnectionPool::Close(m_Pool, m_Connection);
            } else {
                dmConnectionPool::Return(m_Pool, m_Connection);
            }
        }
    }

    static void DefaultHttpContentData(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
    {
        (void) response;
        (void) user_data;
        (void) status_code;
        (void) content_data;
        (void) content_data_size;
    }

    void SetDefaultParams(NewParams* params)
    {
        params->m_HttpContent = &DefaultHttpContentData;
        params->m_HttpHeader = 0;
        params->m_HttpSendContentLength = 0;
        params->m_HttpWrite = 0;
        params->m_HttpWriteHeaders = 0;
        params->m_HttpCache = 0;
        params->m_MaxGetRetries = 1;
        params->m_RequestTimeout = 0;
    }

    HClient New(const NewParams* params, const char* hostname, uint16_t port, bool secure)
    {
        dmSocket::Address address;

        if (params->m_DNSChannel)
        {
            if (dmDNS::GetHostByName(hostname, &address, params->m_DNSChannel, params->m_RequestTimeout) != dmDNS::RESULT_OK)
            {
                // If the DNS request failed, we refresh the DNS configuration and try again.
                // This is needed if we first perform an HTTP request when there's no network available,
                // and then later on do more requests once we have gained connectivity.
                dmDNS::RefreshChannel(params->m_DNSChannel);
                if (dmDNS::GetHostByName(hostname, &address, params->m_DNSChannel, params->m_RequestTimeout) != dmDNS::RESULT_OK)
                {
                    return 0;
                }
            }
        }
        else
        {
            // Fallback to socket implementation of GetHostByNamea
            // in case we couldn't create a proper channel
            if (dmSocket::GetHostByName(hostname, &address) != dmSocket::RESULT_OK)
            {
                return 0;
            }
        }

        Client* client = new Client();

        client->m_Hostname = strdup(hostname);
        client->m_SocketResult = dmSocket::RESULT_OK;

        client->m_Userdata = params->m_Userdata;
        client->m_HttpContent = params->m_HttpContent;
        client->m_HttpHeader = params->m_HttpHeader;
        client->m_HttpSendContentLength = params->m_HttpSendContentLength;
        client->m_HttpWrite = params->m_HttpWrite;
        client->m_HttpWriteHeaders = params->m_HttpWriteHeaders;
        client->m_MaxGetRetries = params->m_MaxGetRetries;
        client->m_RequestTimeout = params->m_RequestTimeout;
        client->m_RequestStart = 0;
        memset(&client->m_Statistics, 0, sizeof(client->m_Statistics));
        client->m_HttpCache = params->m_HttpCache;
        client->m_Secure = secure;
        client->m_Port = port;
        client->m_DNSChannel = params->m_DNSChannel;

        return client;
    }

    HClient New(const NewParams* params, const char* hostname, uint16_t port)
    {
        return New(params, hostname, port, false);
    }

    Result SetOptionInt(HClient client, Option option, int64_t value)
    {
        switch (option) {
            case OPTION_MAX_GET_RETRIES:
                if (value < 1)
                    return RESULT_INVAL_ERROR;
                client->m_MaxGetRetries = (int) value;
                break;
            case OPTION_REQUEST_TIMEOUT:
                client->m_RequestTimeout = (int) value;
                break;
            default:
                return RESULT_INVAL_ERROR;
        }
        return RESULT_OK;
    }

    void Delete(HClient client)
    {
        free(client->m_Hostname);
        delete client;
    }

    dmSocket::Result GetLastSocketResult(HClient client)
    {
        return client->m_SocketResult;
    }

    static bool HasRequestTimedOut(HClient client)
    {
        if( client->m_RequestTimeout == 0 )
            return false;
        uint64_t currenttime = dmTime::GetTime();
        return int(currenttime - client->m_RequestStart) >= client->m_RequestTimeout;
    }

    static dmSocket::Result SendAll(Response* response, const char* buffer, int length)
    {
        int total_sent_bytes = 0;
        int sent_bytes = 0;

        while (total_sent_bytes < length) {
            dmSocket::Result r;
            if (response->m_SSLSocket)
                r = dmSSLSocket::Send(response->m_SSLSocket, buffer + total_sent_bytes, length - total_sent_bytes, &sent_bytes);
            else
                r = dmSocket::Send(response->m_Socket, buffer + total_sent_bytes, length - total_sent_bytes, &sent_bytes);

            if( r == dmSocket::RESULT_WOULDBLOCK )
            {
                r = dmSocket::RESULT_TRY_AGAIN;
            }
            if( (r == dmSocket::RESULT_OK || r == dmSocket::RESULT_TRY_AGAIN) && HasRequestTimedOut(response->m_Client) )
            {
                r = dmSocket::RESULT_WOULDBLOCK;
            }

            if (r == dmSocket::RESULT_TRY_AGAIN)
                continue;

            if (r != dmSocket::RESULT_OK) {
                return r;
            }

            total_sent_bytes += sent_bytes;
        }
        return dmSocket::RESULT_OK;
    }

    static dmSocket::Result Receive(Response* response, void* buffer, int length, int* received_bytes)
    {
        if (response->m_SSLSocket != 0) {
            return dmSSLSocket::Receive(response->m_SSLSocket, buffer, length, received_bytes);
        } else {
            return dmSocket::Receive(response->m_Socket, buffer, length, received_bytes);
        }
    }

    static void HandleVersion(void* user_data, int major, int minor, int status, const char* status_str)
    {
        Response* resp = (Response*) user_data;
        resp->m_Major = major;
        resp->m_Minor = minor;
        resp->m_Status = status;

        if ((major << 16 | minor) < (1 << 16 | 1))
        {
            // Close connection for HTTP protocol version < 1.1
            resp->m_CloseConnection = 1;
        }
    }

    static void HandleHeader(void* user_data, const char* key, const char* value)
    {
        Response* resp = (Response*) user_data;

        if (dmStrCaseCmp(key, "Content-Length") == 0)
        {
            resp->m_ContentLength = strtol(value, 0, 10);
        }
        else if (dmStrCaseCmp(key, "Transfer-Encoding") == 0 && dmStrCaseCmp(value, "chunked") == 0)
        {
            resp->m_Chunked = 1;
        }
        else if (dmStrCaseCmp(key, "Connection") == 0 && dmStrCaseCmp(value, "close") == 0)
        {
            resp->m_CloseConnection = 1;
        }
        else if (dmStrCaseCmp(key, "ETag") == 0)
        {
            dmStrlCpy(resp->m_ETag, value, sizeof(resp->m_ETag));
        }
        else if (dmStrCaseCmp(key, "Cache-Control") == 0)
        {
            const char* substr = "max-age=";
            const char* max_age = strstr(value, "max-age=");
            if (max_age) {
                max_age += strlen(substr);
                resp->m_MaxAge = dmMath::Max(0, atoi(max_age));
                if (resp->m_MaxAge > HTTP_CLIENT_MAXIMUM_CACHE_AGE)
                {
                    resp->m_MaxAge = HTTP_CLIENT_MAXIMUM_CACHE_AGE;
                }
            }
        }

        HClient c = resp->m_Client;
        if (c->m_HttpHeader)
        {
            resp->m_Client->m_HttpHeader(resp, c->m_Userdata, resp->m_Status, key, value);
        }
    }

    static void HandleContent(void* user_data, int offset)
    {
        Response* resp = (Response*) user_data;
        resp->m_ContentOffset = offset;
    }

    static Result RecvAndParseHeaders(HClient client, Response* response)
    {
        response->m_TotalReceived = 0;

        while (1)
        {
            int max_to_recv = BUFFER_SIZE - response->m_TotalReceived;

            if (max_to_recv <= 0)
            {
                return RESULT_HTTP_HEADERS_ERROR;
            }

            int recv_bytes = 0;
            dmSocket::Result r = Receive(response, client->m_Buffer + response->m_TotalReceived, max_to_recv, &recv_bytes);

            if( r == dmSocket::RESULT_WOULDBLOCK )
            {
                r = dmSocket::RESULT_TRY_AGAIN;
            }
            if( (r == dmSocket::RESULT_OK || r == dmSocket::RESULT_TRY_AGAIN) && HasRequestTimedOut(client) )
            {
                r = dmSocket::RESULT_WOULDBLOCK;
            }

            if (r == dmSocket::RESULT_TRY_AGAIN)
                continue;

            if (r != dmSocket::RESULT_OK)
            {
                client->m_SocketResult = r;
                return RESULT_SOCKET_ERROR;
            }

            response->m_TotalReceived += recv_bytes;

            // NOTE: We have an extra byte for null-termination so no buffer overrun here.
            client->m_Buffer[response->m_TotalReceived] = '\0';

            dmHttpClientPrivate::ParseResult parse_res;
            parse_res = dmHttpClientPrivate::ParseHeader(client->m_Buffer, response, recv_bytes == 0, &HandleVersion, &HandleHeader, &HandleContent);
            if (parse_res == dmHttpClientPrivate::PARSE_RESULT_NEED_MORE_DATA)
            {
                if (recv_bytes == 0)
                {
                    dmLogWarning("Unexpected eof for socket connection.");
                    return RESULT_UNEXPECTED_EOF;
                }
                continue;
            }
            else if (parse_res == dmHttpClientPrivate::PARSE_RESULT_SYNTAX_ERROR)
            {
                return RESULT_HTTP_HEADERS_ERROR;
            }
            else if (parse_res == dmHttpClientPrivate::PARSE_RESULT_OK)
            {
                break;
            }
            else
            {
                assert(0);
            }
        }

        return RESULT_OK;
    }

    Result Write(HResponse response, const void* buffer, uint32_t buffer_size)
    {
        HClient client = response->m_Client;
        if (client->m_SocketResult != dmSocket::RESULT_OK) {
            return RESULT_SOCKET_ERROR;
        }
        dmSocket::Result sock_res = SendAll(response, (const char*) buffer, buffer_size);
        if (sock_res != dmSocket::RESULT_OK)
        {
            client->m_SocketResult = sock_res;
            return RESULT_SOCKET_ERROR;
        }

        return RESULT_OK;
    }

    Result WriteHeader(HResponse response, const char* name, const char* value)
    {
        HClient client = response->m_Client;
        if (client->m_SocketResult != dmSocket::RESULT_OK) {
            return RESULT_SOCKET_ERROR;
        }
        dmSocket::Result sock_res;

        // DEF-2889 most webservers have a header length limit of 8096 bytes
        char buf[8096];
        const int bufsize = sizeof(buf);
        if(dmSnPrintf(buf, bufsize, "%s: %s\r\n", name, value) > bufsize) {
            dmLogWarning("Truncated HTTP request header %s since it was larger than %d", name, bufsize);
        }

        sock_res = SendAll(response, buf, strlen(buf));
        if (sock_res != dmSocket::RESULT_OK) {
            client->m_SocketResult = sock_res;
            return RESULT_SOCKET_ERROR;
        }
        return RESULT_OK;
    }

#define HTTP_CLIENT_SENDALL_AND_BAIL(s) \
sock_res = SendAll(response, s, strlen(s));\
if (sock_res != dmSocket::RESULT_OK)\
{\
    client->m_SocketResult = sock_res;\
    goto bail;\
}\

    static dmSocket::Result SendRequest(HClient client, HResponse response, const char* encoded_path, const char* method)
    {
        dmSocket::Result sock_res;
        uint32_t send_content_length = 0;
        int chunked = 0;

        HTTP_CLIENT_SENDALL_AND_BAIL(method);
        HTTP_CLIENT_SENDALL_AND_BAIL(" ")
        HTTP_CLIENT_SENDALL_AND_BAIL(encoded_path)
        HTTP_CLIENT_SENDALL_AND_BAIL(" HTTP/1.1\r\n")
        HTTP_CLIENT_SENDALL_AND_BAIL("Host: ");
        HTTP_CLIENT_SENDALL_AND_BAIL(client->m_Hostname);
        HTTP_CLIENT_SENDALL_AND_BAIL("\r\n");
        if (client->m_HttpWriteHeaders) {
            Result header_result = client->m_HttpWriteHeaders(response, client->m_Userdata);
            if (header_result != RESULT_OK) {
                goto bail;
            }
        }
        if (client->m_HttpCache)
        {
            char etag[64];
            dmHttpCache::Result cache_result = dmHttpCache::GetETag(client->m_HttpCache, client->m_URI, etag, sizeof(etag));
            if (cache_result == dmHttpCache::RESULT_OK)
            {
                HTTP_CLIENT_SENDALL_AND_BAIL("If-None-Match: ");
                HTTP_CLIENT_SENDALL_AND_BAIL(etag);
                HTTP_CLIENT_SENDALL_AND_BAIL("\r\n");
            }
        }

        if (strcmp(method, "POST") == 0 || strcmp(method, "PUT") == 0 || strcmp(method, "PATCH") == 0) {
            send_content_length = client->m_HttpSendContentLength(response, client->m_Userdata);

            if (client->m_Secure && send_content_length > MAX_HTTPS_POST_CHUNK_SIZE)
            {
                chunked = 1;
            }

            // If we want to send more that this, we need to send it chunked
            if (chunked) {
                HTTP_CLIENT_SENDALL_AND_BAIL("Transfer-Encoding: chunked\r\n");
            } else {
                char buf[64];
                dmSnPrintf(buf, sizeof(buf), "Content-Length: %d\r\n", send_content_length);
                HTTP_CLIENT_SENDALL_AND_BAIL(buf);
            }
        }

        HTTP_CLIENT_SENDALL_AND_BAIL("\r\n")

        if (strcmp(method, "POST") == 0 || strcmp(method, "PUT") == 0 || strcmp(method, "PATCH") == 0)
        {
            if (!chunked)
            {
                Result post_result = client->m_HttpWrite(response, 0, send_content_length, client->m_Userdata);
                if (post_result != RESULT_OK) {
                    goto bail;
                }
            }
            else
            {
                // https://en.wikipedia.org/wiki/Chunked_transfer_encoding
                uint32_t offset = 0;
                while (offset < send_content_length)
                {
                    uint32_t length = dmMath::Min(send_content_length - offset, MAX_HTTPS_POST_CHUNK_SIZE);

                    // Prefix each chunk with the length in hexadecimal
                    char buf[64];
                    dmSnPrintf(buf, sizeof(buf), "%x\r\n", length);
                    HTTP_CLIENT_SENDALL_AND_BAIL(buf);

                    // Write the chunk payload
                    Result post_result = client->m_HttpWrite(response, offset, length, client->m_Userdata);
                    if (post_result != RESULT_OK) {
                        goto bail;
                    }
                    offset += length;

                    // Finish the chunk
                    HTTP_CLIENT_SENDALL_AND_BAIL("\r\n");
                }

                // The terminating chunk + trailing blank line (currently no trailer properties)
                HTTP_CLIENT_SENDALL_AND_BAIL("0\r\n\r\n");
            }
        }

        return client->m_SocketResult;
bail:
        return client->m_SocketResult;
    }

    static Result DoTransfer(HClient client, Response* response, int to_transfer, HttpContent http_content, bool add_to_cache)
    {
        // to_transfer can be set to -1 when the "Content-Length" is unknown
        int total_transferred = 0;

        while (true)
        {
            int n;
            if (to_transfer == -1) {
                // Unknown "Content-Length". Read as much as we can.
                n = response->m_TotalReceived - response->m_ContentOffset;
            } else {
                n = dmMath::Min(to_transfer - total_transferred, response->m_TotalReceived - response->m_ContentOffset);
            }
            http_content(response, client->m_Userdata, response->m_Status, client->m_Buffer + response->m_ContentOffset, n);

            if (response->m_CacheCreator && add_to_cache)
            {
                dmHttpCache::Add(client->m_HttpCache, response->m_CacheCreator, client->m_Buffer + response->m_ContentOffset, n);
            }

            total_transferred += n;
            assert(total_transferred <= to_transfer || to_transfer == -1);
            response->m_ContentOffset += n;

            if (total_transferred == to_transfer)
            {
                // Move "extra" bytes to buffer start
                memmove(client->m_Buffer, client->m_Buffer + response->m_ContentOffset, response->m_TotalReceived - response->m_ContentOffset);
                response->m_TotalReceived = response->m_TotalReceived - response->m_ContentOffset;
                response->m_ContentOffset = 0;
                break;
            }

            assert(response->m_TotalReceived - response->m_ContentOffset == 0);
            response->m_ContentOffset = 0;
            response->m_TotalReceived = 0;

            int recv_bytes;
            dmSocket::Result sock_res = Receive(response, client->m_Buffer, BUFFER_SIZE, &recv_bytes);

            if( sock_res == dmSocket::RESULT_WOULDBLOCK )
            {
                sock_res = dmSocket::RESULT_TRY_AGAIN;
            }
            if( (sock_res == dmSocket::RESULT_OK || sock_res == dmSocket::RESULT_TRY_AGAIN) && HasRequestTimedOut(client) )
            {
                sock_res = dmSocket::RESULT_WOULDBLOCK;
            }

            if (sock_res == dmSocket::RESULT_OK)
            {
                if (recv_bytes == 0)
                {
                    break;
                }
                else
                {
                    response->m_TotalReceived = recv_bytes;
                }
            }
            else if (sock_res == dmSocket::RESULT_TRY_AGAIN)
            {
                // Continue
            }
            else if (sock_res == dmSocket::RESULT_CONNRESET)
            {
                // Break out of loop and handle below
                break;
            }
            else
            {
                return RESULT_SOCKET_ERROR;
            }
        }
        assert(total_transferred <= to_transfer || to_transfer == -1);

        if (total_transferred != to_transfer && to_transfer != -1)
        {
            return RESULT_PARTIAL_CONTENT;
        }
        else
        {
            return RESULT_OK;
        }
    }

    static void HttpContentConsume(HResponse response, void* user_data, int status_code, const void* content_data, uint32_t content_data_size)
    {
        (void) response;
        (void) user_data;
        (void) status_code;
        (void) content_data;
        (void) content_data_size;
    }

    static Result HandleCached(HClient client, const char* path, Response* response)
    {
        client->m_Statistics.m_CachedResponses++;

        if (client->m_HttpCache == 0)
        {
            dmLogWarning("Got HTTP response NOT MODIFIED (304) but no cache present");
            return RESULT_OK;
        }
        dmHttpCache::Result cache_result;

        char cache_etag[64];
        cache_etag[0] = '\0';
        cache_result = dmHttpCache::GetETag(client->m_HttpCache, client->m_URI, cache_etag, sizeof(cache_etag));

        if (cache_result != dmHttpCache::RESULT_OK)
        {
            dmLogWarning("Got HTTP response NOT MODIFIED (304) but no ETag present. Returning no cached data.");
            return RESULT_OK;
        }

        if (response->m_ETag[0] != '\0')
        {
            // The Entity Tag (ETag) is optional in HTTP 1.1.
            // It is the servers responsibility to verify the ETag in case it is included. The
            // ETag is a mechanism to reduce the bandwidth required by the server, not
            // by the client. There is no guarantee that an ETag will be included in a
            // 403 response even though it's been sent with a previous 200 response.
            // Even though it might be possible at times, the client has no formal responsibility
            // to perform this verification.
            if (strcmp(cache_etag, response->m_ETag) != 0)
            {
                dmLogFatal("ETag mismatch (%s vs %s)", cache_etag, response->m_ETag);
                return RESULT_IO_ERROR;
            }
        }

        FILE* file = 0;
        uint64_t checksum;
        cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, cache_etag, &file, &checksum);
        if (cache_result == dmHttpCache::RESULT_OK)
        {
            // NOTE: We have an extra byte for null-termination so no buffer overrun here.
            size_t nread;
            do
            {
                nread = fread(client->m_Buffer, 1, BUFFER_SIZE, file);
                client->m_Buffer[nread] = '\0';
                client->m_HttpContent(response, client->m_Userdata, response->m_Status, client->m_Buffer, nread);
            }
            while (nread > 0);
            dmHttpCache::Release(client->m_HttpCache, client->m_URI, cache_etag, file);
        }
        else
        {
            return RESULT_IO_ERROR;
        }

        dmHttpCache::SetVerified(client->m_HttpCache, client->m_URI, true);

        return RESULT_OK;
    }

    static Result HandleResponse(HClient client, const char* path, const char* method, Response* response)
    {
        Result r = RESULT_OK;

        client->m_HttpContent(response, client->m_Userdata, response->m_Status, 0, 0);

        if (strcmp(method, "HEAD") == 0) {
            // A response from a HEAD request should not attempt to read any body despite
            // content length being non-zero, but we still call DoTransfer (with a
            // content length of 0) to ensure that the response is setup properly
            r = DoTransfer(client, response, 0, client->m_HttpContent, true);
        }
        else if (response->m_Chunked)
        {
            // Chunked encoding
            // Move actual data to the beginning of the buffer
            memmove(client->m_Buffer, client->m_Buffer + response->m_ContentOffset, response->m_TotalReceived - response->m_ContentOffset);

            response->m_TotalReceived = response->m_TotalReceived - response->m_ContentOffset;
            response->m_ContentOffset = 0;

            int chunk_size;
            int chunk_number = 0;
            while(true)
            {
                chunk_size = 0;
                // NOTE: We have an extra byte for null-termination so no buffer overrun here.
                client->m_Buffer[response->m_TotalReceived] = '\0';

                char* chunk_size_end = strstr(client->m_Buffer, "\r\n");
                if (chunk_size_end)
                {
                    // We found a chunk
                    sscanf(client->m_Buffer, "%x", &chunk_size);
                    chunk_size_end += 2; // "\r\n"

                    // Move content-offset after chunk termination, ie after "\r\n"
                    response->m_ContentOffset = chunk_size_end - client->m_Buffer;
                    r = DoTransfer(client, response, chunk_size, client->m_HttpContent, true);
                    if (r != RESULT_OK)
                        break;

                    // Consume \r\n"
                    // NOTE: *not* added to cache
                    r = DoTransfer(client, response, 2, &HttpContentConsume, false);
                    if (r != RESULT_OK)
                        break;

                    if (chunk_size == 0)
                    {
                        r = RESULT_OK;
                        break;
                    }

                    ++chunk_number;
                }
                else
                {
                    // We need more data
                    int max_to_recv = BUFFER_SIZE - response->m_TotalReceived;

                    if (max_to_recv <= 0)
                    {
                        return RESULT_HTTP_HEADERS_ERROR;
                    }

                    int recv_bytes;
                    dmSocket::Result sock_r = Receive(response, client->m_Buffer + response->m_TotalReceived, max_to_recv, &recv_bytes);

                    if( sock_r == dmSocket::RESULT_WOULDBLOCK )
                    {
                        sock_r = dmSocket::RESULT_TRY_AGAIN;
                    }
                    if( (sock_r == dmSocket::RESULT_OK || sock_r == dmSocket::RESULT_TRY_AGAIN) && HasRequestTimedOut(client) )
                    {
                        sock_r = dmSocket::RESULT_WOULDBLOCK;
                    }

                    if (sock_r == dmSocket::RESULT_TRY_AGAIN)
                        continue;

                    if (sock_r != dmSocket::RESULT_OK)
                    {
                        return RESULT_SOCKET_ERROR;
                    }
                    response->m_TotalReceived += recv_bytes;
                }
            }
        }
        else
        {
            // "Regular" transfer, single chunk
            assert(response->m_ContentOffset != -1);
            r = DoTransfer(client, response, response->m_ContentLength, client->m_HttpContent, true);
        }

        return r;
    }

    static Result DoDoRequest(HClient client, Response& response, const char* path, const char* method)
    {
        dmSocket::Result sock_res;

        sock_res = SendRequest(client, &response, path, method);

        if (sock_res != dmSocket::RESULT_OK)
        {
            return RESULT_SOCKET_ERROR;
        }

        Result r = RecvAndParseHeaders(client, &response);
        if (r != RESULT_OK)
        {
            response.m_CloseConnection = 1;
            return r;
        }

        if (response.m_Status == 204 /* No Content*/)
        {
            assert(response.m_ContentLength == -1);
            response.m_ContentLength = 0;
        }

        if (response.m_Chunked)
        {
            // Ok
        }
        else if (response.m_ContentLength == -1 && response.m_Status != 304)
        {
            // When not chunked (and not 304) we used to require Content-Length attribute to be set
            // but changed to support this behavior in order to be compatible with old/exotic web-servers.
            // The connection is however closed as keep-alive isn't possible for this case.
            response.m_CloseConnection = 1;
        }

        if (response.m_Status == 304 /* NOT MODIFIED */)
        {
            // Use cached version
            if (response.m_ContentLength == 0 || response.m_ContentLength == -1)
            {
                r = HandleCached(client, path, &response);
                response.m_TotalReceived = 0;
            }
            else
            {
                // Cached version can't have payload
                if (response.m_ContentLength != -1)
                    dmLogWarning("Unexpected Content-Length: %d for NOT MODIFIED response (304)", response.m_ContentLength);
                r = RESULT_INVALID_RESPONSE;
            }
        }
        else
        {
            // Non-cached response
            if (client->m_HttpCache && response.m_Status == 200 /* OK */)
            {
                if (response.m_ETag[0] != '\0') {
                    dmHttpCache::Begin(client->m_HttpCache, client->m_URI, response.m_ETag, &response.m_CacheCreator);
                } else if (response.m_MaxAge > 0) {
                    dmHttpCache::Begin(client->m_HttpCache, client->m_URI, response.m_MaxAge, &response.m_CacheCreator);
                }
            }

            r = HandleResponse(client, path, method, &response);

            if (response.m_CacheCreator)
            {
                dmHttpCache::End(client->m_HttpCache, response.m_CacheCreator);
                response.m_CacheCreator = 0;
            }
        }

        // Removed an assert here, in favor of returning an error instead
        // which should allow the user to detect this and act accordingly
        if (response.m_TotalReceived != 0)
        {
            dmLogError("Not all bytes were handled during the response (%d bytes left). Method: %s Status: %d", response.m_TotalReceived, method, response.m_Status);
            r = RESULT_INVALID_RESPONSE;
        }

        if (r == RESULT_OK)
        {
            if (response.m_Status == 200)
                return RESULT_OK;
            else
                return RESULT_NOT_200_OK;
        }
        return r;
    }

    static Result DoRequest(HClient client, const char* path, const char* method)
    {
        // Theoretically we can be in a state where every
        // connections in the pool is closed by the remote peer
        // but not yet closed on the client side (max-keep-alive)
        // Therefore we must loop MAX_POOL_CONNECTIONS + 1 times.
        // The assumption holds only for a single-threaded environment though
        // but as network errors will occur in practice the heuristic is probably
        // "good enough".
        for (uint32_t i = 0; i < MAX_POOL_CONNECTIONS + 1; ++i) {
            Response response(client);
            client->m_Statistics.m_Responses++;

            client->m_SocketResult = dmSocket::RESULT_OK;
            Result r = response.Connect(client->m_Hostname, client->m_Port, client->m_Secure, client->m_RequestTimeout);
            if (r != RESULT_OK) {
                return r;
            }

            if( HasRequestTimedOut(client) )
            {
                return r;
            }

            r = DoDoRequest(client, response, path, method);
            if (r != RESULT_OK && r != RESULT_NOT_200_OK) {

                if( HasRequestTimedOut(client) )
                {
                    return r;
                }

                response.m_CloseConnection = 1;
                uint32_t count = dmConnectionPool::GetReuseCount(response.m_Pool, response.m_Connection);

                if (count > 0 && response.m_TotalReceived == 0) {
                    client->m_Statistics.m_Reconnections++;
                    // We assume that the connection was closed by remote peer
                    // as the total received data is zero bytes.
                    // Moreover, the case when the connection was closed by the remote
                    // peer is only interesting if the connection is reused, i.e. count > 0
                    // Otherwise, we have to regard this type of connection shutdown as
                    // an error.

                    // implicit continue here
                } else {
                    return r;
                }
            } else {
                return r;
            }
        }
        dmLogWarning("All connection attempts to remote host are prematurely closed. This error is very unlikely.");
        return RESULT_UNKNOWN;
    }

    static Result HandleCachedVerified(HClient client, const dmHttpCache::EntryInfo* info)
    {
        Response response(client);
        client->m_Statistics.m_DirectFromCache++;

        FILE* file = 0;
        uint64_t checksum;

        dmHttpCache::Result cache_result = dmHttpCache::Get(client->m_HttpCache, client->m_URI, info->m_ETag, &file, &checksum);
        if (cache_result == dmHttpCache::RESULT_OK)
        {
            // NOTE: We have an extra byte for null-termination so no buffer overrun here.
            size_t nread;
            do
            {
                nread = fread(client->m_Buffer, 1, BUFFER_SIZE, file);
                client->m_Buffer[nread] = '\0';
                client->m_HttpContent(&response, client->m_Userdata, 304, client->m_Buffer, nread);
            }
            while (nread > 0);
            dmHttpCache::Release(client->m_HttpCache, client->m_URI, info->m_ETag, file);
            return RESULT_NOT_200_OK;
        }
        else
        {
            return RESULT_IO_ERROR;
        }
    }

    Result Get(HClient client, const char* path)
    {
        dmSnPrintf(client->m_URI, sizeof(client->m_URI), "%s://%s:%d/%s", client->m_Secure ? "https" : "http", client->m_Hostname, (int) client->m_Port, path);
        client->m_RequestStart = dmTime::GetTime();

        Result r;

        if (client->m_HttpCache)
        {
            dmHttpCache::ConsistencyPolicy policy = dmHttpCache::GetConsistencyPolicy(client->m_HttpCache);
            dmHttpCache::EntryInfo info;
            dmHttpCache::Result cache_r = dmHttpCache::GetInfo(client->m_HttpCache, client->m_URI, &info);
            if (cache_r == dmHttpCache::RESULT_OK) {
                bool ok_etag = info.m_Verified && policy == dmHttpCache::CONSISTENCY_POLICY_TRUST_CACHE;
                if ((ok_etag || info.m_Valid)) {
                    // We have a cache and trust the content of the cache
                    // OR
                    // the entry is valid in terms of max-age
                    r = HandleCachedVerified(client, &info);
                    if (r == RESULT_NOT_200_OK) {
                        return r;
                    }
                }
            }
        }

        for (int i = 0; i < client->m_MaxGetRetries; ++i)
        {
            r = DoRequest(client, path, "GET");
            if (r == RESULT_UNEXPECTED_EOF ||
               (r == RESULT_SOCKET_ERROR && (client->m_SocketResult == dmSocket::RESULT_CONNRESET
                                          || client->m_SocketResult == dmSocket::RESULT_WOULDBLOCK
                                          || client->m_SocketResult == dmSocket::RESULT_PIPE)))
            {
                // Try again
                if (i < client->m_MaxGetRetries - 1) {
                    client->m_Statistics.m_Reconnections++;
                    client->m_RequestStart = dmTime::GetTime();
                    dmLogInfo("HTTPCLIENT: Connection lost, reconnecting. (%d/%d)", i + 1, client->m_MaxGetRetries - 1);
                }
            }
            else
            {
                return r;
            }
        }
        return r;
    }

    Result Post(HClient client, const char* path)
    {
        return Request(client, "POST", path);
    }

    Result Request(HClient client, const char* method, const char* path)
    {
        if (strcmp(method, "GET") == 0) {
            return Get(client, path);
        } else {
            dmSnPrintf(client->m_URI, sizeof(client->m_URI), "%s://%s:%d/%s", client->m_Secure ? "https" : "http", client->m_Hostname, (int) client->m_Port, path);
            client->m_RequestStart = dmTime::GetTime();
            Result r = DoRequest(client, path, method);
            return r;
        }
    }

    void GetStatistics(HClient client, Statistics* statistics)
    {
        *statistics = client->m_Statistics;
    }

    dmHttpCache::HCache GetHttpCache(HClient client)
    {
        return client->m_HttpCache;
    }

    dmDNS::HChannel GetDNSChannel(HClient client)
    {
        return client->m_DNSChannel;
    }

    uint32_t ShutdownConnectionPool()
    {
        dmConnectionPool::HPool pool = g_PoolCreator.GetPoolNoCreate();
        if (pool) {
            return dmConnectionPool::Shutdown(pool, dmSocket::SHUTDOWNTYPE_READWRITE);
        }
        return 0;
    }

    void ReopenConnectionPool()
    {
        dmConnectionPool::HPool pool = g_PoolCreator.GetPool();
        dmConnectionPool::Reopen(pool);
    }

#undef HTTP_CLIENT_SENDALL_AND_BAIL

    #define DM_HTTPCLIENT_RESULT_TO_STRING_CASE(x) case RESULT_##x: return #x;
    const char* ResultToString(Result r)
    {
        switch (r)
        {
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(NOT_200_OK);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(OK);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(SOCKET_ERROR);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(HTTP_HEADERS_ERROR);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(INVALID_RESPONSE);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(PARTIAL_CONTENT);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(UNSUPPORTED_TRANSFER_ENCODING);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(INVAL_ERROR);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(UNEXPECTED_EOF);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(IO_ERROR);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(HANDSHAKE_FAILED);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(INVAL);
            DM_HTTPCLIENT_RESULT_TO_STRING_CASE(UNKNOWN);
            default:
                break;
        }
        dmLogError("Unable to convert result %d to string", r);

        return "RESULT_UNDEFINED";
    }
    #undef DM_HTTPCLIENT_RESULT_TO_STRING_CASE

} // namespace dmHttpClient

max_stars_repo_name: Krzlfx/defold

max_issues_repo_name: Krzlfx/defold

max_forks_repo_name: Krzlfx/defold