From 7e420aeed361f89d041af997c9a89895043846d0 Mon Sep 17 00:00:00 2001 From: Ray Beck <115stingray@gmail.com> Date: Sun, 8 Jan 2023 18:38:14 -0500 Subject: [PATCH] add support for requests with both MultipartFormDataItems and Content Providers (#1454) * add support for requests with both MultipartFormDataItems and ContentProviders * rework implementation * use const auto & and fix offset calculation * fix zero items * snake case variables * clang-format * commonize get_multipart_content_provider, add Put() with MultipartFormDataProviderItems * fix linker multiple definition error * add test MultipartFormDataTest.DataProviderItems --- httplib.h | 189 +++++++++++++++++++++++++++++++++++++++--------- test/test.cc | 198 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 355 insertions(+), 32 deletions(-) diff --git a/httplib.h b/httplib.h index 751aa92..1d0e628 100644 --- a/httplib.h +++ b/httplib.h @@ -369,6 +369,14 @@ using ContentProviderWithoutLength = using ContentProviderResourceReleaser = std::function; +struct MultipartFormDataProvider { + std::string name; + ContentProviderWithoutLength provider; + std::string filename; + std::string content_type; +}; +using MultipartFormDataProviderItems = std::vector; + using ContentReceiverWithProgress = std::function; @@ -934,6 +942,9 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, @@ -963,6 +974,9 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, @@ -1201,6 +1215,9 @@ private: ContentProvider content_provider, ContentProviderWithoutLength content_provider_without_length, const std::string &content_type); + ContentProviderWithoutLength get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); std::string adjust_host_string(const std::string &host) const; @@ -1296,6 +1313,10 @@ public: const MultipartFormDataItems &items); Result Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Put(const std::string &path); Result Put(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -1324,6 +1345,10 @@ public: const MultipartFormDataItems &items); Result Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items, const std::string &boundary); + Result Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items); + Result Patch(const std::string &path); Result Patch(const std::string &path, const char *body, size_t content_length, const std::string &content_type); @@ -2854,8 +2879,7 @@ inline socket_t create_client_socket( } inline bool get_ip_and_port(const struct sockaddr_storage &addr, - socklen_t addr_len, std::string &ip, - int &port) { + socklen_t addr_len, std::string &ip, int &port) { if (addr.ss_family == AF_INET) { port = ntohs(reinterpret_cast(&addr)->sin_port); } else if (addr.ss_family == AF_INET6) { @@ -4129,29 +4153,48 @@ inline bool is_multipart_boundary_chars_valid(const std::string &boundary) { return valid; } +template +inline std::string +serialize_multipart_formdata_item_begin(const T &item, + const std::string &boundary) { + std::string body = "--" + boundary + "\r\n"; + body += "Content-Disposition: form-data; name=\"" + item.name + "\""; + if (!item.filename.empty()) { + body += "; filename=\"" + item.filename + "\""; + } + body += "\r\n"; + if (!item.content_type.empty()) { + body += "Content-Type: " + item.content_type + "\r\n"; + } + body += "\r\n"; + + return body; +} + +inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; } + +inline std::string +serialize_multipart_formdata_finish(const std::string &boundary) { + return "--" + boundary + "--\r\n"; +} + +inline std::string +serialize_multipart_formdata_get_content_type(const std::string &boundary) { + return "multipart/form-data; boundary=" + boundary; +} + inline std::string serialize_multipart_formdata(const MultipartFormDataItems &items, - const std::string &boundary, - std::string &content_type) { + const std::string &boundary, bool finish = true) { std::string body; for (const auto &item : items) { - body += "--" + boundary + "\r\n"; - body += "Content-Disposition: form-data; name=\"" + item.name + "\""; - if (!item.filename.empty()) { - body += "; filename=\"" + item.filename + "\""; - } - body += "\r\n"; - if (!item.content_type.empty()) { - body += "Content-Type: " + item.content_type + "\r\n"; - } - body += "\r\n"; - body += item.content + "\r\n"; + body += serialize_multipart_formdata_item_begin(item, boundary); + body += item.content + serialize_multipart_formdata_item_end(); } - body += "--" + boundary + "--\r\n"; + if (finish) body += serialize_multipart_formdata_finish(boundary); - content_type = "multipart/form-data; boundary=" + boundary; return body; } @@ -4536,8 +4579,8 @@ inline void hosted_at(const std::string &hostname, *reinterpret_cast(rp->ai_addr); std::string ip; int dummy = -1; - if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), - ip, dummy)) { + if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip, + dummy)) { addrs.push_back(ip); } } @@ -6647,6 +6690,49 @@ inline bool ClientImpl::process_request(Stream &strm, Request &req, return true; } +inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider( + const std::string &boundary, const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + size_t cur_item = 0, cur_start = 0; + // cur_item and cur_start are copied to within the std::function and maintain + // state between successive calls + return [&, cur_item, cur_start](size_t offset, + DataSink &sink) mutable -> bool { + if (!offset && items.size()) { + sink.os << detail::serialize_multipart_formdata(items, boundary, false); + return true; + } else if (cur_item < provider_items.size()) { + if (!cur_start) { + const auto &begin = detail::serialize_multipart_formdata_item_begin( + provider_items[cur_item], boundary); + offset += begin.size(); + cur_start = offset; + sink.os << begin; + } + + DataSink cur_sink; + bool has_data = true; + cur_sink.write = sink.write; + cur_sink.done = [&]() { has_data = false; }; + cur_sink.is_writable = sink.is_writable; + + if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) + return false; + + if (!has_data) { + sink.os << detail::serialize_multipart_formdata_item_end(); + cur_item++; + cur_start = 0; + } + return true; + } else { + sink.os << detail::serialize_multipart_formdata_finish(boundary); + sink.done(); + return true; + } + }; +} + inline bool ClientImpl::process_socket(const Socket &socket, std::function callback) { @@ -6869,9 +6955,10 @@ inline Result ClientImpl::Post(const std::string &path, inline Result ClientImpl::Post(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } @@ -6882,12 +6969,25 @@ inline Result ClientImpl::Post(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Post(path, headers, body, content_type.c_str()); } +inline Result +ClientImpl::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "POST", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} + inline Result ClientImpl::Put(const std::string &path) { return Put(path, std::string(), std::string()); } @@ -6964,9 +7064,10 @@ inline Result ClientImpl::Put(const std::string &path, inline Result ClientImpl::Put(const std::string &path, const Headers &headers, const MultipartFormDataItems &items) { - std::string content_type; - const auto &body = detail::serialize_multipart_formdata( - items, detail::make_multipart_data_boundary(), content_type); + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } @@ -6977,12 +7078,24 @@ inline Result ClientImpl::Put(const std::string &path, const Headers &headers, return Result{nullptr, Error::UnsupportedMultipartBoundaryChars}; } - std::string content_type; - const auto &body = - detail::serialize_multipart_formdata(items, boundary, content_type); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + const auto &body = detail::serialize_multipart_formdata(items, boundary); return Put(path, headers, body, content_type); } +inline Result +ClientImpl::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + const auto &boundary = detail::make_multipart_data_boundary(); + const auto &content_type = + detail::serialize_multipart_formdata_get_content_type(boundary); + return send_with_content_provider( + "PUT", path, headers, nullptr, 0, nullptr, + get_multipart_content_provider(boundary, items, provider_items), + content_type); +} inline Result ClientImpl::Patch(const std::string &path) { return Patch(path, std::string(), std::string()); } @@ -7443,7 +7556,7 @@ inline void SSLSocketStream::get_remote_ip_and_port(std::string &ip, } inline void SSLSocketStream::get_local_ip_and_port(std::string &ip, - int &port) const { + int &port) const { detail::get_local_ip_and_port(sock_, ip, port); } @@ -8147,6 +8260,12 @@ inline Result Client::Post(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Post(path, headers, items, boundary); } +inline Result +Client::Post(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Post(path, headers, items, provider_items); +} inline Result Client::Put(const std::string &path) { return cli_->Put(path); } inline Result Client::Put(const std::string &path, const char *body, size_t content_length, @@ -8210,6 +8329,12 @@ inline Result Client::Put(const std::string &path, const Headers &headers, const std::string &boundary) { return cli_->Put(path, headers, items, boundary); } +inline Result +Client::Put(const std::string &path, const Headers &headers, + const MultipartFormDataItems &items, + const MultipartFormDataProviderItems &provider_items) { + return cli_->Put(path, headers, items, provider_items); +} inline Result Client::Patch(const std::string &path) { return cli_->Patch(path); } diff --git a/test/test.cc b/test/test.cc index 88937ce..e592ed1 100644 --- a/test/test.cc +++ b/test/test.cc @@ -5146,6 +5146,204 @@ TEST(MultipartFormDataTest, LargeData) { t.join(); } +TEST(MultipartFormDataTest, DataProviderItems) { + + std::random_device seed_gen; + std::mt19937 random(seed_gen()); + + std::string rand1; + rand1.resize(1000); + std::generate(rand1.begin(), rand1.end(), [&]() { return random(); }); + + std::string rand2; + rand2.resize(3000); + std::generate(rand2.begin(), rand2.end(), [&]() { return random(); }); + + SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE); + + svr.Post("/post-none", [&](const Request &req, Response & /*res*/, + const ContentReader &content_reader) { + ASSERT_FALSE(req.is_multipart_form_data()); + + std::string body; + content_reader([&](const char *data, size_t data_length) { + body.append(data, data_length); + return true; + }); + + EXPECT_EQ(body, ""); + }); + + svr.Post("/post-items", [&](const Request &req, Response & /*res*/, + const ContentReader &content_reader) { + ASSERT_TRUE(req.is_multipart_form_data()); + MultipartFormDataItems files; + content_reader( + [&](const MultipartFormData &file) { + files.push_back(file); + return true; + }, + [&](const char *data, size_t data_length) { + files.back().content.append(data, data_length); + return true; + }); + + ASSERT_TRUE(files.size() == 2); + + EXPECT_EQ(std::string(files[0].name), "name1"); + EXPECT_EQ(files[0].content, "Testing123"); + EXPECT_EQ(files[0].filename, "filename1"); + EXPECT_EQ(files[0].content_type, "application/octet-stream"); + + EXPECT_EQ(files[1].name, "name2"); + EXPECT_EQ(files[1].content, "Testing456"); + EXPECT_EQ(files[1].filename, ""); + EXPECT_EQ(files[1].content_type, ""); + }); + + svr.Post("/post-providers", [&](const Request &req, Response & /*res*/, + const ContentReader &content_reader) { + ASSERT_TRUE(req.is_multipart_form_data()); + MultipartFormDataItems files; + content_reader( + [&](const MultipartFormData &file) { + files.push_back(file); + return true; + }, + [&](const char *data, size_t data_length) { + files.back().content.append(data, data_length); + return true; + }); + + ASSERT_TRUE(files.size() == 2); + + EXPECT_EQ(files[0].name, "name3"); + EXPECT_EQ(files[0].content, rand1); + EXPECT_EQ(files[0].filename, "filename3"); + EXPECT_EQ(files[0].content_type, ""); + + EXPECT_EQ(files[1].name, "name4"); + EXPECT_EQ(files[1].content, rand2); + EXPECT_EQ(files[1].filename, "filename4"); + EXPECT_EQ(files[1].content_type, ""); + }); + + svr.Post("/post-both", [&](const Request &req, Response & /*res*/, + const ContentReader &content_reader) { + ASSERT_TRUE(req.is_multipart_form_data()); + MultipartFormDataItems files; + content_reader( + [&](const MultipartFormData &file) { + files.push_back(file); + return true; + }, + [&](const char *data, size_t data_length) { + files.back().content.append(data, data_length); + return true; + }); + + ASSERT_TRUE(files.size() == 4); + + EXPECT_EQ(std::string(files[0].name), "name1"); + EXPECT_EQ(files[0].content, "Testing123"); + EXPECT_EQ(files[0].filename, "filename1"); + EXPECT_EQ(files[0].content_type, "application/octet-stream"); + + EXPECT_EQ(files[1].name, "name2"); + EXPECT_EQ(files[1].content, "Testing456"); + EXPECT_EQ(files[1].filename, ""); + EXPECT_EQ(files[1].content_type, ""); + + EXPECT_EQ(files[2].name, "name3"); + EXPECT_EQ(files[2].content, rand1); + EXPECT_EQ(files[2].filename, "filename3"); + EXPECT_EQ(files[2].content_type, ""); + + EXPECT_EQ(files[3].name, "name4"); + EXPECT_EQ(files[3].content, rand2); + EXPECT_EQ(files[3].filename, "filename4"); + EXPECT_EQ(files[3].content_type, ""); + }); + + auto t = std::thread([&]() { svr.listen("localhost", 8080); }); + while (!svr.is_running()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + + { + Client cli("https://localhost:8080"); + cli.enable_server_certificate_verification(false); + + MultipartFormDataItems items{ + {"name1", "Testing123", "filename1", "application/octet-stream"}, + {"name2", "Testing456", "", ""}, // not a file + }; + + { + auto res = cli.Post("/post-none", {}, {}, {}); + ASSERT_TRUE(res); + ASSERT_EQ(200, res->status); + } + + MultipartFormDataProviderItems providers; + + { + auto res = + cli.Post("/post-items", {}, items, providers); // empty providers + ASSERT_TRUE(res); + ASSERT_EQ(200, res->status); + } + + providers.push_back({"name3", + [&](size_t offset, httplib::DataSink &sink) -> bool { + // test the offset is given correctly at each step + if (!offset) + sink.os.write(rand1.data(), 30); + else if (offset == 30) + sink.os.write(rand1.data() + 30, 300); + else if (offset == 330) + sink.os.write(rand1.data() + 330, 670); + else if (offset == rand1.size()) + sink.done(); + return true; + }, + "filename3", + {}}); + + providers.push_back({"name4", + [&](size_t offset, httplib::DataSink &sink) -> bool { + // test the offset is given correctly at each step + if (!offset) + sink.os.write(rand2.data(), 2000); + else if (offset == 2000) + sink.os.write(rand2.data() + 2000, 1); + else if (offset == 2001) + sink.os.write(rand2.data() + 2001, 999); + else if (offset == rand2.size()) + sink.done(); + return true; + }, + "filename4", + {}}); + + { + auto res = cli.Post("/post-providers", {}, {}, providers); + ASSERT_TRUE(res); + ASSERT_EQ(200, res->status); + } + + { + auto res = cli.Post("/post-both", {}, items, providers); + ASSERT_TRUE(res); + ASSERT_EQ(200, res->status); + } + } + + svr.stop(); + t.join(); +} + TEST(MultipartFormDataTest, WithPreamble) { Server svr; svr.Post("/post", [&](const Request & /*req*/, Response &res) {