Fixed problem with proxy support and added unit tests

.gitignore

@@ -9,6 +9,7 @@ example/redirect
 example/upload
 example/*.pem
 test/test
+test/test_proxy
 test/test.xcodeproj/xcuser*
 test/test.xcodeproj/*/xcuser*
 test/*.pem

httplib.h

@@ -2476,7 +2476,8 @@ inline std::pair<std::string, std::string> make_range_header(Ranges ranges) {
 
 inline std::pair<std::string, std::string>
 make_basic_authentication_header(const std::string &username,
-                                 const std::string &password, bool is_proxy = false) {
+                                 const std::string &password,
+                                 bool is_proxy = false) {
   auto field = "Basic " + detail::base64_encode(username + ":" + password);
   auto key = is_proxy ? "Proxy-Authorization" : "Authorization";
   return std::make_pair(key, field);
@@ -3500,31 +3501,36 @@ inline bool Client::send(const Request &req, Response &res) {
       return false;
     }
 
-    if (res2.status == 407 && !proxy_digest_auth_username_.empty() &&
-        !proxy_digest_auth_password_.empty()) {
-      std::map<std::string, std::string> auth;
-      if (parse_www_authenticate(res2, auth, true)) {
-        detail::close_socket(sock);
-        sock = create_client_socket();
-        if (sock == INVALID_SOCKET) { return false; }
+    if (res2.status == 407) {
+      if (!proxy_digest_auth_username_.empty() &&
+          !proxy_digest_auth_password_.empty()) {
+        std::map<std::string, std::string> auth;
+        if (parse_www_authenticate(res2, auth, true)) {
+          detail::close_socket(sock);
+          sock = create_client_socket();
+          if (sock == INVALID_SOCKET) { return false; }
 
-        Response res2;
-        if (!detail::process_socket(
-                true, sock, 1, read_timeout_sec_, read_timeout_usec_,
-                [&](Stream &strm, bool /*last_connection*/,
-                    bool &connection_close) {
-                  Request req2;
-                  req2.method = "CONNECT";
-                  req2.path = host_and_port_;
-                  req2.headers.insert(make_digest_authentication_header(
-                      req2, auth, 1, random_string(10),
-                      proxy_digest_auth_username_, proxy_digest_auth_password_,
-                      true));
-                  return process_request(strm, req2, res2, false,
-                                         connection_close);
-                })) {
-          return false;
+          Response res2;
+          if (!detail::process_socket(
+                  true, sock, 1, read_timeout_sec_, read_timeout_usec_,
+                  [&](Stream &strm, bool /*last_connection*/,
+                      bool &connection_close) {
+                    Request req2;
+                    req2.method = "CONNECT";
+                    req2.path = host_and_port_;
+                    req2.headers.insert(make_digest_authentication_header(
+                        req2, auth, 1, random_string(10),
+                        proxy_digest_auth_username_,
+                        proxy_digest_auth_password_, true));
+                    return process_request(strm, req2, res2, false,
+                                           connection_close);
+                  })) {
+            return false;
+          }
         }
+      } else {
+        res = res2;
+        return true;
       }
     }
   }
@@ -3890,7 +3896,8 @@ inline std::shared_ptr<Response> Client::Get(const char *path,
 inline std::shared_ptr<Response> Client::Get(const char *path,
                                              ContentReceiver content_receiver,
                                              Progress progress) {
-  return Get(path, Headers(), nullptr, std::move(content_receiver), std::move(progress));
+  return Get(path, Headers(), nullptr, std::move(content_receiver),
+             std::move(progress));
 }
 
 inline std::shared_ptr<Response> Client::Get(const char *path,
@@ -3903,14 +3910,16 @@ inline std::shared_ptr<Response> Client::Get(const char *path,
                                              const Headers &headers,
                                              ContentReceiver content_receiver,
                                              Progress progress) {
-  return Get(path, headers, nullptr, std::move(content_receiver), std::move(progress));
+  return Get(path, headers, nullptr, std::move(content_receiver),
+             std::move(progress));
 }
 
 inline std::shared_ptr<Response> Client::Get(const char *path,
                                              const Headers &headers,
                                              ResponseHandler response_handler,
                                              ContentReceiver content_receiver) {
-  return Get(path, headers, std::move(response_handler), content_receiver, Progress());
+  return Get(path, headers, std::move(response_handler), content_receiver,
+             Progress());
 }
 
 inline std::shared_ptr<Response> Client::Get(const char *path,

test/Makefile

@@ -11,6 +11,9 @@ all : test
 test : test.cc ../httplib.h Makefile cert.pem
 	$(CXX) -o test $(CXXFLAGS) test.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) -pthread
 
+test_proxy : test_proxy.cc ../httplib.h Makefile cert.pem
+	$(CXX) -o test_proxy $(CXXFLAGS) test_proxy.cc gtest/gtest-all.cc gtest/gtest_main.cc $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) -pthread
+
 cert.pem:
 	openssl genrsa 2048 > key.pem
 	openssl req -new -batch -config test.conf -key key.pem | openssl x509 -days 3650 -req -signkey key.pem > cert.pem
@@ -21,4 +24,4 @@ cert.pem:
 	#c_rehash .
 
 clean:
-	rm -f test *.pem *.0 *.1 *.srl
+	rm -f test test_proxy pem *.0 *.1 *.srl

test/test_proxy.cc

@@ -0,0 +1,211 @@
+#include <future>
+#include <gtest/gtest.h>
+#include <httplib.h>
+
+using namespace std;
+using namespace httplib;
+
+void ProxyTest(Client& cli, bool basic) {
+  cli.set_proxy("localhost", basic ? 3128 : 3129);
+  auto res = cli.Get("/get");
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(407, res->status);
+}
+
+TEST(ProxyTest, NoSSLBasic) {
+  Client cli("httpbin.org");
+  ProxyTest(cli, true);
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(ProxyTest, SSLBasic) {
+  SSLClient cli("httpbin.org");
+  ProxyTest(cli, true);
+}
+
+TEST(ProxyTest, NoSSLDigest) {
+  Client cli("httpbin.org");
+  ProxyTest(cli, false);
+}
+
+TEST(ProxyTest, SSLDigest) {
+  SSLClient cli("httpbin.org");
+  ProxyTest(cli, false);
+}
+#endif
+
+// ----------------------------------------------------------------------------
+
+void RedirectTestHTTPBin(Client& cli, const char *path, bool basic) {
+  cli.set_proxy("localhost", basic ? 3128 : 3129);
+  if (basic) {
+    cli.set_proxy_basic_auth("hello", "world");
+  } else {
+    cli.set_proxy_digest_auth("hello", "world");
+  }
+  cli.set_follow_location(true);
+
+  auto res = cli.Get(path);
+  ASSERT_TRUE(res != nullptr);
+  EXPECT_EQ(200, res->status);
+}
+
+TEST(RedirectTest, HTTPBinNoSSLBasic) {
+  Client cli("httpbin.org");
+  RedirectTestHTTPBin(cli, "/redirect/2", true);
+}
+
+TEST(RedirectTest, HTTPBinNoSSLDigest) {
+  Client cli("httpbin.org");
+  RedirectTestHTTPBin(cli, "/redirect/2", false);
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(RedirectTest, HTTPBinSSLBasic) {
+  SSLClient cli("httpbin.org");
+  RedirectTestHTTPBin(cli, "/redirect/2", true);
+}
+
+TEST(RedirectTest, HTTPBinSSLDigest) {
+  SSLClient cli("httpbin.org");
+  RedirectTestHTTPBin(cli, "/redirect/2", false);
+}
+#endif
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(RedirectTest, YouTubeNoSSLBasic) {
+  Client cli("youtube.com");
+  RedirectTestHTTPBin(cli, "/", true);
+}
+
+TEST(RedirectTest, YouTubeNoSSLDigest) {
+  Client cli("youtube.com");
+  RedirectTestHTTPBin(cli, "/", false);
+}
+
+TEST(RedirectTest, YouTubeSSLBasic) {
+  SSLClient cli("youtube.com");
+  RedirectTestHTTPBin(cli, "/", true);
+}
+
+TEST(RedirectTest, YouTubeSSLDigest) {
+  SSLClient cli("youtube.com");
+  RedirectTestHTTPBin(cli, "/", false);
+}
+#endif
+
+// ----------------------------------------------------------------------------
+
+void BaseAuthTestFromHTTPWatch(Client& cli) {
+  cli.set_proxy("localhost", 3128);
+  cli.set_proxy_basic_auth("hello", "world");
+
+  {
+    auto res = cli.Get("/basic-auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(401, res->status);
+  }
+
+  {
+    auto res =
+        cli.Get("/basic-auth/hello/world",
+                {make_basic_authentication_header("hello", "world")});
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(res->body,
+              "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+    EXPECT_EQ(200, res->status);
+  }
+
+  {
+    cli.set_basic_auth("hello", "world");
+    auto res = cli.Get("/basic-auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(res->body,
+              "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+    EXPECT_EQ(200, res->status);
+  }
+
+  {
+    cli.set_basic_auth("hello", "bad");
+    auto res = cli.Get("/basic-auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(401, res->status);
+  }
+
+  {
+    cli.set_basic_auth("bad", "world");
+    auto res = cli.Get("/basic-auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(401, res->status);
+  }
+}
+
+TEST(BaseAuthTest, NoSSL) {
+  Client cli("httpbin.org");
+  BaseAuthTestFromHTTPWatch(cli);
+}
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+TEST(BaseAuthTest, SSL) {
+  SSLClient cli("httpbin.org");
+  BaseAuthTestFromHTTPWatch(cli);
+}
+
+// ----------------------------------------------------------------------------
+
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+void DigestAuthTestFromHTTPWatch(Client& cli) {
+  cli.set_proxy("localhost", 3129);
+  cli.set_proxy_digest_auth("hello", "world");
+
+  {
+    auto res = cli.Get("/digest-auth/auth/hello/world");
+    ASSERT_TRUE(res != nullptr);
+    EXPECT_EQ(401, res->status);
+  }
+
+  {
+    std::vector<std::string> paths = {
+        "/digest-auth/auth/hello/world/MD5",
+        "/digest-auth/auth/hello/world/SHA-256",
+        "/digest-auth/auth/hello/world/SHA-512",
+        "/digest-auth/auth-init/hello/world/MD5",
+        "/digest-auth/auth-int/hello/world/MD5",
+    };
+
+    cli.set_digest_auth("hello", "world");
+    for (auto path : paths) {
+      auto res = cli.Get(path.c_str());
+      ASSERT_TRUE(res != nullptr);
+      EXPECT_EQ(res->body,
+                "{\n  \"authenticated\": true, \n  \"user\": \"hello\"\n}\n");
+      EXPECT_EQ(200, res->status);
+    }
+
+    cli.set_digest_auth("hello", "bad");
+    for (auto path : paths) {
+      auto res = cli.Get(path.c_str());
+      ASSERT_TRUE(res != nullptr);
+      EXPECT_EQ(400, res->status);
+    }
+
+    cli.set_digest_auth("bad", "world");
+    for (auto path : paths) {
+      auto res = cli.Get(path.c_str());
+      ASSERT_TRUE(res != nullptr);
+      EXPECT_EQ(400, res->status);
+    }
+  }
+}
+#endif
+
+TEST(DigestAuthTest, SSL) {
+  SSLClient cli("httpbin.org");
+  DigestAuthTestFromHTTPWatch(cli);
+}
+
+TEST(DigestAuthTest, NoSSL) {
+  Client cli("httpbin.org");
+  DigestAuthTestFromHTTPWatch(cli);
+}
+#endif

test/test_proxy_docker/Dockerfile

@@ -0,0 +1,13 @@
+FROM centos
+LABEL maintainer="yuji.hirose.bug@gmail.com"
+ARG auth="basic"
+ARG port="3128"
+
+RUN yum install -y squid
+
+COPY ./${auth}_squid.conf /etc/squid/squid.conf
+COPY ./${auth}_passwd /etc/squid/passwd
+
+EXPOSE ${port}
+
+CMD ["/usr/sbin/squid", "-N"]

test/test_proxy_docker/basic_passwd

@@ -0,0 +1 @@
+hello:$apr1$O6S28OBL$8dr3ixl4Mohf97hgsYvLy/

test/test_proxy_docker/basic_squid.conf

@@ -0,0 +1,81 @@
+#
+# Recommended minimum configuration:
+#
+
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl localnet src 0.0.0.1-0.255.255.255	# RFC 1122 "this" network (LAN)
+acl localnet src 10.0.0.0/8		# RFC 1918 local private network (LAN)
+acl localnet src 100.64.0.0/10		# RFC 6598 shared address space (CGN)
+acl localnet src 169.254.0.0/16 	# RFC 3927 link-local (directly plugged) machines
+acl localnet src 172.16.0.0/12		# RFC 1918 local private network (LAN)
+acl localnet src 192.168.0.0/16		# RFC 1918 local private network (LAN)
+acl localnet src fc00::/7       	# RFC 4193 local private network range
+acl localnet src fe80::/10      	# RFC 4291 link-local (directly plugged) machines
+
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl CONNECT method CONNECT
+
+auth_param basic program /usr/lib64/squid/basic_ncsa_auth /etc/squid/passwd
+auth_param basic realm proxy
+acl authenticated proxy_auth REQUIRED
+http_access allow authenticated
+
+#
+# Recommended minimum Access Permission configuration:
+#
+# Deny requests to certain unsafe ports
+http_access deny !Safe_ports
+
+# Deny CONNECT to other than secure SSL ports
+http_access deny CONNECT !SSL_ports
+
+# Only allow cachemgr access from localhost
+http_access allow localhost manager
+http_access deny manager
+
+# We strongly recommend the following be uncommented to protect innocent
+# web applications running on the proxy server who think the only
+# one who can access services on "localhost" is a local user
+#http_access deny to_localhost
+
+#
+# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
+#
+
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+
+# And finally deny all other access to this proxy
+http_access deny all
+
+# Squid normally listens to port 3128
+http_port 3128
+
+# Uncomment and adjust the following to add a disk cache directory.
+#cache_dir ufs /var/spool/squid 100 16 256
+
+# Leave coredumps in the first cache dir
+coredump_dir /var/spool/squid
+
+#
+# Add any of your own refresh_pattern entries above these.
+#
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern .		0	20%	4320

test/test_proxy_docker/digest_passwd

@@ -0,0 +1 @@
+hello:world

test/test_proxy_docker/digest_squid.conf

@@ -0,0 +1,81 @@
+#
+# Recommended minimum configuration:
+#
+
+# Example rule allowing access from your local networks.
+# Adapt to list your (internal) IP networks from where browsing
+# should be allowed
+acl localnet src 0.0.0.1-0.255.255.255	# RFC 1122 "this" network (LAN)
+acl localnet src 10.0.0.0/8		# RFC 1918 local private network (LAN)
+acl localnet src 100.64.0.0/10		# RFC 6598 shared address space (CGN)
+acl localnet src 169.254.0.0/16 	# RFC 3927 link-local (directly plugged) machines
+acl localnet src 172.16.0.0/12		# RFC 1918 local private network (LAN)
+acl localnet src 192.168.0.0/16		# RFC 1918 local private network (LAN)
+acl localnet src fc00::/7       	# RFC 4193 local private network range
+acl localnet src fe80::/10      	# RFC 4291 link-local (directly plugged) machines
+
+acl SSL_ports port 443
+acl Safe_ports port 80		# http
+acl Safe_ports port 21		# ftp
+acl Safe_ports port 443		# https
+acl Safe_ports port 70		# gopher
+acl Safe_ports port 210		# wais
+acl Safe_ports port 1025-65535	# unregistered ports
+acl Safe_ports port 280		# http-mgmt
+acl Safe_ports port 488		# gss-http
+acl Safe_ports port 591		# filemaker
+acl Safe_ports port 777		# multiling http
+acl CONNECT method CONNECT
+
+auth_param digest program /usr/lib64/squid/digest_file_auth /etc/squid/passwd
+auth_param digest realm proxy
+acl authenticated proxy_auth REQUIRED
+http_access allow authenticated
+
+#
+# Recommended minimum Access Permission configuration:
+#
+# Deny requests to certain unsafe ports
+http_access deny !Safe_ports
+
+# Deny CONNECT to other than secure SSL ports
+http_access deny CONNECT !SSL_ports
+
+# Only allow cachemgr access from localhost
+http_access allow localhost manager
+http_access deny manager
+
+# We strongly recommend the following be uncommented to protect innocent
+# web applications running on the proxy server who think the only
+# one who can access services on "localhost" is a local user
+#http_access deny to_localhost
+
+#
+# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
+#
+
+# Example rule allowing access from your local networks.
+# Adapt localnet in the ACL section to list your (internal) IP networks
+# from where browsing should be allowed
+http_access allow localnet
+http_access allow localhost
+
+# And finally deny all other access to this proxy
+http_access deny all
+
+# Squid normally listens to port 3128
+http_port 3129
+
+# Uncomment and adjust the following to add a disk cache directory.
+#cache_dir ufs /var/spool/squid 100 16 256
+
+# Leave coredumps in the first cache dir
+coredump_dir /var/spool/squid
+
+#
+# Add any of your own refresh_pattern entries above these.
+#
+refresh_pattern ^ftp:		1440	20%	10080
+refresh_pattern ^gopher:	1440	0%	1440
+refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
+refresh_pattern .		0	20%	4320

test/test_proxy_docker/down.sh

@@ -0,0 +1,4 @@
+docker stop squid_basic
+docker rmi squid_basic
+docker stop squid_digest
+docker rmi squid_digest

test/test_proxy_docker/up.sh

@@ -0,0 +1,5 @@
+docker build -t squid_basic --build-arg auth=basic .
+docker run -dt --name squid_basic -p 3128:3128 --rm squid_basic
+
+docker build -t squid_digest --build-arg auth=digest .
+docker run -dt --name squid_digest -p 3129:3129 --rm squid_digest