yuzu/cpp-httplib
1
0
Fork 0
mirror of https://github.com/yhirose/cpp-httplib synced 2024-11-21 06:26:02 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Brotli support on client

Browse source
This commit is contained in:
yhirose 2020-07-25 20:44:02 -04:00
parent 29a06f852a
commit 12540fe8d3
4 changed files with 294 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
example/Makefile
View file

@ -1,41 +1,46 @@
#CXX = clang++
CXXFLAGS = -std=c++14 -I.. -Wall -Wextra -pthread
OPENSSL_DIR = /usr/local/opt/openssl
OPENSSL_SUPPORT = -DCPPHTTPLIB_OPENSSL_SUPPORT -I$(OPENSSL_DIR)/include -L$(OPENSSL_DIR)/lib -lssl -lcrypto
ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz
BROTLI_DIR = /usr/local/opt/brotli
# BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon-static -lbrotlienc-static -lbrotlidec-static
all: server client hello simplecli simplesvr upload redirect ssesvr ssecli benchmark
server : server.cc ../httplib.h Makefile
$(CXX) -o server $(CXXFLAGS) server.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o server $(CXXFLAGS) server.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
client : client.cc ../httplib.h Makefile
$(CXX) -o client $(CXXFLAGS) client.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o client $(CXXFLAGS) client.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
hello : hello.cc ../httplib.h Makefile
$(CXX) -o hello $(CXXFLAGS) hello.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o hello $(CXXFLAGS) hello.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
simplecli : simplecli.cc ../httplib.h Makefile
$(CXX) -o simplecli $(CXXFLAGS) simplecli.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o simplecli $(CXXFLAGS) simplecli.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
simplesvr : simplesvr.cc ../httplib.h Makefile
$(CXX) -o simplesvr $(CXXFLAGS) simplesvr.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o simplesvr $(CXXFLAGS) simplesvr.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
upload : upload.cc ../httplib.h Makefile
$(CXX) -o upload $(CXXFLAGS) upload.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o upload $(CXXFLAGS) upload.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
redirect : redirect.cc ../httplib.h Makefile
$(CXX) -o redirect $(CXXFLAGS) redirect.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o redirect $(CXXFLAGS) redirect.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
ssesvr : ssesvr.cc ../httplib.h Makefile
$(CXX) -o ssesvr $(CXXFLAGS) ssesvr.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o ssesvr $(CXXFLAGS) ssesvr.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
ssecli : ssecli.cc ../httplib.h Makefile
$(CXX) -o ssecli $(CXXFLAGS) ssecli.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o ssecli $(CXXFLAGS) ssecli.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
benchmark : benchmark.cc ../httplib.h Makefile
$(CXX) -o benchmark $(CXXFLAGS) benchmark.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT)
$(CXX) -o benchmark $(CXXFLAGS) benchmark.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT)
pem:
openssl genrsa 2048 > key.pem

292
httplib.h
View file

@ -215,6 +215,11 @@ inline const unsigned char *ASN1_STRING_get0_data(const ASN1_STRING *asn1) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
#include <zlib.h>
#endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
#include <brotli/decode.h>
#endif
/*
* Declaration
*/
@ -1668,19 +1673,34 @@ inline std::string file_extension(const std::string &path) {
return std::string();
}
inline std::pair<int, int> trim(const char *b, const char *e, int left,
int right) {
while (b + left < e && b[left] == ' ') {
left++;
}
while (right - 1 >= 0 && b[right - 1] == ' ') {
right--;
}
return std::make_pair(left, right);
}
template <class Fn> void split(const char *b, const char *e, char d, Fn fn) {
int i = 0;
int beg = 0;
while (e ? (b + i != e) : (b[i] != '\0')) {
if (b[i] == d) {
fn(&b[beg], &b[i]);
auto r = trim(b, e, beg, i);
fn(&b[r.first], &b[r.second]);
beg = i + 1;
}
i++;
}
if (i) { fn(&b[beg], &b[i]); }
if (i) {
auto r = trim(b, e, beg, i);
fn(&b[r.first], &b[r.second]);
}
}
// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`
@ -2324,21 +2344,34 @@ inline bool can_compress_content_type(const std::string &content_type) {
content_type == "application/xhtml+xml";
}
inline bool can_compress_content(const Request &req, const Response &res) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
const auto &encodings = req.get_header_value("Accept-Encoding");
return encodings.find("gzip") != std::string::npos &&
detail::can_compress_content_type(
res.get_header_value("Content-Type"));
#else
return false;
enum class EncodingType { None = 0, Gzip, Brotli };
inline EncodingType encoding_type(const Request &req, const Response &res) {
auto ret =
detail::can_compress_content_type(res.get_header_value("Content-Type"));
if (!ret) { return EncodingType::None; }
const auto &s = req.get_header_value("Accept-Encoding");
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
// TODO: 'Accept-Encoding' has br, not br;q=0
ret = s.find("br") != std::string::npos;
if (ret) { return EncodingType::Brotli; }
#endif
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
// TODO: 'Accept-Encoding' has gzip, not gzip;q=0
ret = s.find("gzip") != std::string::npos;
if (ret) { return EncodingType::Gzip; }
#endif
return EncodingType::None;
}
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
class compressor {
class gzip_compressor {
public:
compressor() {
gzip_compressor() {
std::memset(&strm_, 0, sizeof(strm_));
strm_.zalloc = Z_NULL;
strm_.zfree = Z_NULL;
@ -2348,7 +2381,7 @@ public:
Z_DEFAULT_STRATEGY) == Z_OK;
}
~compressor() { deflateEnd(&strm_); }
~gzip_compressor() { deflateEnd(&strm_); }
template <typename T>
bool compress(const char *data, size_t data_length, bool last, T callback) {
@ -2384,9 +2417,9 @@ private:
z_stream strm_;
};
class decompressor {
class gzip_decompressor {
public:
decompressor() {
gzip_decompressor() {
std::memset(&strm_, 0, sizeof(strm_));
strm_.zalloc = Z_NULL;
strm_.zfree = Z_NULL;
@ -2399,7 +2432,7 @@ public:
is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK;
}
~decompressor() { inflateEnd(&strm_); }
~gzip_decompressor() { inflateEnd(&strm_); }
bool is_valid() const { return is_valid_; }
@ -2439,6 +2472,59 @@ private:
};
#endif
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
class brotli_decompressor {
public:
brotli_decompressor() {
decoder_s = BrotliDecoderCreateInstance(0, 0, 0);
decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
: BROTLI_DECODER_RESULT_ERROR;
}
~brotli_decompressor() {
if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); }
}
bool is_valid() const { return decoder_s; }
template <typename T>
bool decompress(const char *data, size_t data_length, T callback) {
if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
decoder_r == BROTLI_DECODER_RESULT_ERROR)
return 0;
const uint8_t *next_in = (const uint8_t *)data;
size_t avail_in = data_length;
size_t total_out;
decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
char output[1024];
char *next_out = output;
size_t avail_out = sizeof(output);
decoder_r = BrotliDecoderDecompressStream(
decoder_s, &avail_in, &next_in, &avail_out,
reinterpret_cast<unsigned char **>(&next_out), &total_out);
if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; }
if (!callback((const char *)output, sizeof(output) - avail_out)) {
return false;
}
}
return decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||
decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
}
private:
BrotliDecoderResult decoder_r;
BrotliDecoderState *decoder_s = nullptr;
};
#endif
inline bool has_header(const Headers &headers, const char *key) {
return headers.find(key) != headers.end();
}
@ -2611,63 +2697,86 @@ inline bool is_chunked_transfer_encoding(const Headers &headers) {
"chunked");
}
template <typename T>
bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
Progress progress, ContentReceiver receiver,
bool decompress) {
template <typename T, typename U>
bool prepare_content_receiver(T &x, int &status, ContentReceiver receiver,
bool decompress, U callback) {
if (decompress) {
std::string encoding = x.get_header_value("Content-Encoding");
if (encoding.find("gzip") != std::string::npos ||
encoding.find("deflate") != std::string::npos) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
gzip_decompressor decompressor;
if (decompressor.is_valid()) {
ContentReceiver out = [&](const char *buf, size_t n) {
return decompressor.decompress(
buf, n,
[&](const char *buf, size_t n) { return receiver(buf, n); });
};
return callback(out);
} else {
status = 500;
return false;
}
#else
status = 415;
return false;
#endif
} else if (encoding.find("br") != std::string::npos) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
brotli_decompressor decompressor;
if (decompressor.is_valid()) {
ContentReceiver out = [&](const char *buf, size_t n) {
return decompressor.decompress(
buf, n,
[&](const char *buf, size_t n) { return receiver(buf, n); });
};
return callback(out);
} else {
status = 500;
return false;
}
#else
status = 415;
return false;
#endif
}
}
ContentReceiver out = [&](const char *buf, size_t n) {
return receiver(buf, n);
};
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
decompressor decompressor;
#endif
return callback(out);
}
if (decompress) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
std::string content_encoding = x.get_header_value("Content-Encoding");
if (content_encoding.find("gzip") != std::string::npos ||
content_encoding.find("deflate") != std::string::npos) {
if (!decompressor.is_valid()) {
status = 500;
return false;
}
template <typename T>
bool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,
Progress progress, ContentReceiver receiver,
bool decompress) {
return prepare_content_receiver(
x, status, receiver, decompress, [&](ContentReceiver &out) {
auto ret = true;
auto exceed_payload_max_length = false;
out = [&](const char *buf, size_t n) {
return decompressor.decompress(buf, n, [&](const char *buf, size_t n) {
return receiver(buf, n);
});
};
}
#else
if (x.get_header_value("Content-Encoding") == "gzip") {
status = 415;
return false;
}
#endif
}
if (is_chunked_transfer_encoding(x.headers)) {
ret = read_content_chunked(strm, out);
} else if (!has_header(x.headers, "Content-Length")) {
ret = read_content_without_length(strm, out);
} else {
auto len = get_header_value<uint64_t>(x.headers, "Content-Length");
if (len > payload_max_length) {
exceed_payload_max_length = true;
skip_content_with_length(strm, len);
ret = false;
} else if (len > 0) {
ret = read_content_with_length(strm, len, progress, out);
}
}
auto ret = true;
auto exceed_payload_max_length = false;
if (is_chunked_transfer_encoding(x.headers)) {
ret = read_content_chunked(strm, out);
} else if (!has_header(x.headers, "Content-Length")) {
ret = read_content_without_length(strm, out);
} else {
auto len = get_header_value<uint64_t>(x.headers, "Content-Length");
if (len > payload_max_length) {
exceed_payload_max_length = true;
skip_content_with_length(strm, len);
ret = false;
} else if (len > 0) {
ret = read_content_with_length(strm, len, progress, out);
}
}
if (!ret) { status = exceed_payload_max_length ? 413 : 400; }
return ret;
if (!ret) { status = exceed_payload_max_length ? 413 : 400; }
return ret;
});
}
template <typename T>
@ -2733,7 +2842,7 @@ inline ssize_t write_content(Stream &strm, ContentProvider content_provider,
template <typename T>
inline ssize_t write_content_chunked(Stream &strm,
ContentProvider content_provider,
T is_shutting_down, bool compress) {
T is_shutting_down, EncodingType type) {
size_t offset = 0;
auto data_available = true;
ssize_t total_written_length = 0;
@ -2742,7 +2851,7 @@ inline ssize_t write_content_chunked(Stream &strm,
DataSink data_sink;
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
detail::compressor compressor;
detail::gzip_compressor compressor;
#endif
data_sink.write = [&](const char *d, size_t l) {
@ -2752,7 +2861,7 @@ inline ssize_t write_content_chunked(Stream &strm,
offset += l;
std::string payload;
if (compress) {
if (type == EncodingType::Gzip) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
if (!compressor.compress(d, l, false,
[&](const char *data, size_t data_len) {
@ -2762,6 +2871,9 @@ inline ssize_t write_content_chunked(Stream &strm,
ok = false;
return;
}
#endif
} else if (type == EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
#endif
} else {
payload = std::string(d, l);
@ -2784,7 +2896,7 @@ inline ssize_t write_content_chunked(Stream &strm,
data_available = false;
if (compress) {
if (type == EncodingType::Gzip) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
std::string payload;
if (!compressor.compress(nullptr, 0, true,
@ -2806,6 +2918,9 @@ inline ssize_t write_content_chunked(Stream &strm,
return;
}
}
#endif
} else if (type == EncodingType::Brotli) {
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
#endif
}
@ -3964,7 +4079,7 @@ inline bool Server::write_response(Stream &strm, bool close_connection,
"multipart/byteranges; boundary=" + boundary);
}
bool compress = detail::can_compress_content(req, res);
auto type = detail::encoding_type(req, res);
if (res.body.empty()) {
if (res.content_length_ > 0) {
@ -3987,7 +4102,11 @@ inline bool Server::write_response(Stream &strm, bool close_connection,
} else {
if (res.content_provider_) {
res.set_header("Transfer-Encoding", "chunked");
if (compress) { res.set_header("Content-Encoding", "gzip"); }
if (type == detail::EncodingType::Gzip) {
res.set_header("Content-Encoding", "gzip");
} else if (type == detail::EncodingType::Brotli) {
res.set_header("Content-Encoding", "br");
}
} else {
res.set_header("Content-Length", "0");
}
@ -4009,20 +4128,25 @@ inline bool Server::write_response(Stream &strm, bool close_connection,
detail::make_multipart_ranges_data(req, res, boundary, content_type);
}
// TODO: 'Accept-Encoding' has gzip, not gzip;q=0
if (compress) {
if (type != detail::EncodingType::None) {
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
std::string compressed;
detail::compressor compressor;
if (!compressor.compress(res.body.data(), res.body.size(), true,
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
return false;
if (type == detail::EncodingType::Gzip) {
detail::gzip_compressor compressor;
if (!compressor.compress(res.body.data(), res.body.size(), true,
[&](const char *data, size_t data_len) {
compressed.append(data, data_len);
return true;
})) {
return false;
}
res.set_header("Content-Encoding", "gzip");
} else if (type == detail::EncodingType::Brotli) {
// TODO:
}
res.body.swap(compressed);
res.set_header("Content-Encoding", "gzip");
#endif
}
@ -4085,9 +4209,9 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
}
}
} else {
auto compress = detail::can_compress_content(req, res);
auto type = detail::encoding_type(req, res);
if (detail::write_content_chunked(strm, res.content_provider_,
is_shutting_down, compress) < 0) {
is_shutting_down, type) < 0) {
return false;
}
}
@ -4827,7 +4951,7 @@ inline std::shared_ptr<Response> Client::send_with_content_provider(
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
if (compress_) {
detail::compressor compressor;
detail::gzip_compressor compressor;
if (content_provider) {
auto ok = true;

9
test/Makefile
View file

@ -1,10 +1,15 @@
#CXX = clang++
CXXFLAGS = -ggdb -O0 -std=c++11 -DGTEST_USE_OWN_TR1_TUPLE -I.. -I. -Wall -Wextra -Wtype-limits -Wconversion
OPENSSL_DIR = /usr/local/opt/openssl
OPENSSL_SUPPORT = -DCPPHTTPLIB_OPENSSL_SUPPORT -I$(OPENSSL_DIR)/include -L$(OPENSSL_DIR)/lib -lssl -lcrypto
ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz
BROTLI_DIR = /usr/local/opt/brotli
# BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon-static -lbrotlienc-static -lbrotlidec-static
all : test
./test
@ -12,10 +17,10 @@ proxy : test_proxy
./test_proxy
test : test.cc ../httplib.h Makefile cert.pem
$(CXX) -o test $(CXXFLAGS) test.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) -pthread
$(CXX) -o test $(CXXFLAGS) test.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread
test_proxy : test_proxy.cc ../httplib.h Makefile cert.pem
$(CXX) -o test_proxy $(CXXFLAGS) test_proxy.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) -pthread
$(CXX) -o test_proxy $(CXXFLAGS) test_proxy.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread
cert.pem:
openssl genrsa 2048 > key.pem

64
test/test.cc
View file

@ -216,6 +216,58 @@ TEST(ParseHeaderValueTest, Range) {
}
}
TEST(ParseAcceptEncoding1, AcceptEncoding) {
Request req;
req.set_header("Accept-Encoding", "gzip");
Response res;
res.set_header("Content-Type", "text/plain");
auto ret = detail::encoding_type(req, res);
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
EXPECT_TRUE(ret == detail::EncodingType::Gzip);
#else
EXPECT_TRUE(ret == detail::EncodingType::None);
#endif
}
TEST(ParseAcceptEncoding2, AcceptEncoding) {
Request req;
req.set_header("Accept-Encoding", "gzip, deflate, br");
Response res;
res.set_header("Content-Type", "text/plain");
auto ret = detail::encoding_type(req, res);
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
EXPECT_TRUE(ret == detail::EncodingType::Brotli);
#elif CPPHTTPLIB_ZLIB_SUPPORT
EXPECT_TRUE(ret == detail::EncodingType::Gzip);
#else
EXPECT_TRUE(ret == detail::EncodingType::None);
#endif
}
TEST(ParseAcceptEncoding3, AcceptEncoding) {
Request req;
req.set_header("Accept-Encoding", "br;q=1.0, gzip;q=0.8, *;q=0.1");
Response res;
res.set_header("Content-Type", "text/plain");
auto ret = detail::encoding_type(req, res);
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
EXPECT_TRUE(ret == detail::EncodingType::Brotli);
#elif CPPHTTPLIB_ZLIB_SUPPORT
EXPECT_TRUE(ret == detail::EncodingType::Gzip);
#else
EXPECT_TRUE(ret == detail::EncodingType::None);
#endif
}
TEST(BufferStreamTest, read) {
detail::BufferStream strm1;
Stream &strm = strm1;
@ -3050,6 +3102,18 @@ TEST(YahooRedirectTest3, SimpleInterface) {
EXPECT_EQ(200, res->status);
}
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
TEST(DecodeWithChunkedEncoding, BrotliEncoding) {
httplib::Client2 cli("https://cdnjs.cloudflare.com");
auto res = cli.Get("/ajax/libs/jquery/3.5.1/jquery.js", {{"Accept-Encoding", "brotli"}});
ASSERT_TRUE(res != nullptr);
EXPECT_EQ(200, res->status);
EXPECT_EQ(287630, res->body.size());
EXPECT_EQ("application/javascript; charset=utf-8", res->get_header_value("Content-Type"));
}
#endif
#if 0
TEST(HttpsToHttpRedirectTest2, SimpleInterface) {
auto res =