The Samba-Bugzilla – Attachment 18276 Details for
Bug 15611
http library doesn't support 'chunked transfer encoding'
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
backport of master patch to 4.20, 4.19 & 4.18
http_chunk-v418.patch (text/plain), 36.02 KB, created by
Noel Power
on 2024-03-27 10:32:14 UTC
(
hide
)
Description:
backport of master patch to 4.20, 4.19 & 4.18
Filename:
MIME Type:
Creator:
Noel Power
Created:
2024-03-27 10:32:14 UTC
Size:
36.02 KB
patch
obsolete
>From 819836e61f3e55ab1b153decf3dc6eaa0ac4f940 Mon Sep 17 00:00:00 2001 >From: Noel Power <noel.power@suse.com> >Date: Mon, 25 Mar 2024 19:21:54 +0000 >Subject: [PATCH 1/5] Add simple http_client for use in black box tests (in > following commits) > >BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 >Signed-off-by: Noel Power <noel.power@suse.com> >Reviewed-by: Andrew Bartlett <abartlet@samba.org> >(cherry picked from commit cd6c075476c820b4fe8bdc10a24d8fc8ac74e9c9) >--- > source4/client/http_test.c | 401 +++++++++++++++++++++++++++++++++++++ > source4/wscript_build | 5 + > 2 files changed, 406 insertions(+) > create mode 100644 source4/client/http_test.c > >diff --git a/source4/client/http_test.c b/source4/client/http_test.c >new file mode 100644 >index 00000000000..ca542ba1c6b >--- /dev/null >+++ b/source4/client/http_test.c >@@ -0,0 +1,401 @@ >+#include "includes.h" >+#include "version.h" >+#include "libcli/libcli.h" >+#include "lib/events/events.h" >+#include "libcli/resolve/resolve.h" >+#include "param/param.h" >+#include "libcli/raw/raw_proto.h" >+#include "libcli/http/http.h" >+#include "credentials.h" >+#include "util/tevent_ntstatus.h" >+#include "lib/tls/tls.h" >+#include "lib/cmdline/cmdline.h" >+ >+ >+struct http_client_info { >+ struct http_conn *http_conn; >+ uint16_t server_port; >+ const char *server_addr; >+ struct tstream_tls_params *tls_params; >+ struct cli_credentials *creds; >+ struct loadparm_context *lp_ctx; >+ const char *uri; >+}; >+ >+static bool send_http_request(TALLOC_CTX *mem_ctx, >+ struct tevent_context *ev_ctx, >+ struct http_client_info* es, >+ size_t response_size, >+ NTSTATUS *pstatus) >+{ >+ struct http_request *http_req = NULL; >+ struct tevent_req *req = NULL; >+ char *uri = NULL; >+ struct http_request *http_response = NULL; >+ NTSTATUS status; >+ >+ http_req = talloc_zero(mem_ctx, struct http_request); >+ if (!http_req) { >+ DBG_ERR("no memory\n"); >+ return false; >+ } >+ >+ uri = talloc_strdup(mem_ctx, es->uri); >+ >+ http_req->type = HTTP_REQ_POST; >+ http_req->uri = uri; >+ http_req->body = data_blob_null; >+ http_req->major = '1'; >+ http_req->minor = '1'; >+ >+ http_add_header(mem_ctx, &http_req->headers, >+ "User-Agent", "Samba/http_test"); >+ http_add_header(mem_ctx, &http_req->headers, >+ "Accept", "*/*"); >+ >+ req = http_send_auth_request_send(mem_ctx, >+ ev_ctx, >+ es->http_conn, >+ http_req, >+ es->creds, >+ es->lp_ctx, >+ HTTP_AUTH_BASIC); >+ if (!tevent_req_set_endtime(req, ev_ctx, timeval_current_ofs(10, 0))) { >+ DBG_ERR("Failed to set timeout\n"); >+ return false; >+ } >+ >+ if (!tevent_req_poll_ntstatus(req, ev_ctx, pstatus)) { >+ DBG_ERR("Failed to connect: %s\n", nt_errstr(*pstatus)); >+ return false; >+ } >+ >+ status = http_send_auth_request_recv(req); >+ if (!NT_STATUS_IS_OK(status)) { >+ DBG_ERR("Auth request failed: %s\n", nt_errstr(status)); >+ return false; >+ } >+ >+ req = http_read_response_send(mem_ctx, >+ ev_ctx, >+ es->http_conn, >+ response_size); >+ if (!req) { >+ DBG_ERR("no memory\n"); >+ return -1; >+ } >+ >+ if (!tevent_req_set_endtime(req, ev_ctx, timeval_current_ofs(10, 0))) { >+ DBG_ERR("Failed to set timeout\n"); >+ return false; >+ } >+ >+ if (!tevent_req_poll_ntstatus(req, ev_ctx, pstatus)) { >+ DBG_ERR("Failed to read_resonse: %s\n", nt_errstr(*pstatus)); >+ return false; >+ } >+ >+ *pstatus = http_read_response_recv(req, mem_ctx, &http_response); >+ >+ if (!NT_STATUS_IS_OK(*pstatus)) { >+ DBG_ERR("Failed to receive response: %s\n", nt_errstr(*pstatus)); >+ return false; >+ } >+ /* following are not 'hard' errors */ >+ if (http_response->response_code != 200) { >+ fprintf(stdout, "HTTP server response: %u\n", >+ http_response->response_code); >+ fflush(stdout); >+ return false; >+ >+ } >+ if (http_response->body.length == 0) { >+ fprintf(stdout, "unexpected 0 len response\n"); >+ fflush(stdout); >+ return false; >+ } >+ DBG_ERR("response: len (%d)\n%s\n", >+ (int)http_response->body.length, >+ talloc_strndup(mem_ctx, >+ (char *)http_response->body.data, >+ http_response->body.length)); >+ fprintf(stdout,"%s", talloc_strndup(mem_ctx, >+ (char *)http_response->body.data, >+ http_response->body.length)); >+ fflush(stdout); >+ return true; >+} >+ >+int main(int argc, const char *argv[]) >+ >+{ >+ TALLOC_CTX *mem_ctx; >+ struct tevent_context *ev_ctx; >+ int retries = 4; >+ int count = 0; >+ struct http_client_info *http_info = NULL; >+ bool use_tls = false; >+ int res; >+ NTSTATUS status; >+ struct tevent_req *req = NULL; >+ bool connected = false; >+ poptContext pc; >+ const char **const_argv = discard_const_p(const char *, argv); >+ int opt; >+ bool ok; >+ const char *ca_file = NULL; >+ int port = 0; >+ size_t response_size = 8192000; >+ struct cli_credentials *cli_creds; >+ >+ struct poptOption long_options[] = { >+ POPT_AUTOHELP >+ >+ { >+ .longName = "usetls", >+ .shortName = 't', >+ .argInfo = POPT_ARG_NONE, >+ .arg = NULL, >+ .val = 't', >+ .descrip = "Use tls", >+ .argDescrip = "enable tls", >+ }, >+ { >+ .longName = "ip-address", >+ .shortName = 'I', >+ .argInfo = POPT_ARG_STRING, >+ .arg = NULL, >+ .val = 'I', >+ .descrip = "Use this IP to connect to", >+ .argDescrip = "IP", >+ }, >+ { >+ .longName = "port", >+ .shortName = 'p', >+ .argInfo = POPT_ARG_INT, >+ .arg = &port, >+ .val = 'p', >+ .descrip = "port to connect to", >+ .argDescrip = "port", >+ }, >+ { >+ .longName = "cacart", >+ .shortName = 'c', >+ .argInfo = POPT_ARG_STRING, >+ .arg = NULL, >+ .val = 'c', >+ .descrip = "CA certificate to verify peer against", >+ .argDescrip = "ca cert", >+ }, >+ { >+ .longName = "uri", >+ .shortName = 'u', >+ .argInfo = POPT_ARG_STRING, >+ .arg = NULL, >+ .val = 'u', >+ .descrip = "uri to send as part of http request", >+ .argDescrip = "uri", >+ }, >+ { >+ .longName = "rsize", >+ .argInfo = POPT_ARG_LONG, >+ .arg = &response_size, >+ .descrip = "response size", >+ }, >+ POPT_COMMON_SAMBA >+ POPT_COMMON_CREDENTIALS >+ POPT_TABLEEND >+ }; >+ >+ mem_ctx = talloc_init("http_test"); >+ >+ if (!mem_ctx) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ >+ http_info = talloc_zero(mem_ctx, struct http_client_info); >+ >+ if (http_info == NULL) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ >+ ok = samba_cmdline_init(mem_ctx, >+ SAMBA_CMDLINE_CONFIG_CLIENT, >+ false /* require_smbconf */); >+ if (!ok) { >+ DBG_ERR("Failed to init cmdline parser!\n"); >+ res = -1; >+ goto done; >+ } >+ >+ pc = samba_popt_get_context(getprogname(), >+ argc, >+ const_argv, >+ long_options, >+ 0); >+ if (pc == NULL) { >+ DBG_ERR("Failed to setup popt context!\n"); >+ res = -1; >+ goto done; >+ } >+ >+ /* some defaults */ >+ >+ http_info->server_addr = "localhost"; >+ http_info->uri = "/_search?pretty"; >+ >+ while ((opt = poptGetNextOpt(pc)) != -1) { >+ switch (opt) { >+ case 't': >+ use_tls = true; >+ break; >+ case 'c': { >+ ca_file = talloc_strdup(mem_ctx, >+ poptGetOptArg(pc)); >+ if (ca_file == NULL) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ break; >+ } >+ case 'I': { >+ http_info->server_addr = talloc_strdup(mem_ctx, >+ poptGetOptArg(pc)); >+ if (http_info->server_addr == NULL) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ break; >+ } >+ case 'u': { >+ http_info->uri = talloc_strdup(mem_ctx, >+ poptGetOptArg(pc)); >+ if (http_info->uri == NULL) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ break; >+ } >+ } >+ } >+ >+ if (use_tls && ca_file == NULL) { >+ DBG_ERR("No cacert\n"); >+ res = -1; >+ poptPrintUsage(pc, stderr, 0); >+ goto done; >+ } >+ >+ if (!port) { >+ port = 8080; >+ } >+ http_info->server_port = port; >+ >+ ev_ctx = s4_event_context_init(mem_ctx); >+ if (!ev_ctx) { >+ DBG_ERR("Not enough memory\n"); >+ res = -1; >+ goto done; >+ } >+ >+ >+ cli_creds = samba_cmdline_get_creds(); >+ if (!cli_credentials_is_anonymous(cli_creds)) { >+ http_info->creds = cli_credentials_init(mem_ctx); >+ cli_credentials_set_username( >+ http_info->creds, >+ cli_credentials_get_username(cli_creds), >+ CRED_SPECIFIED); >+ cli_credentials_set_password(http_info->creds, >+ cli_credentials_get_password(cli_creds), >+ CRED_SPECIFIED); >+ } else { >+ DBG_DEBUG("Anonymous creds!!!\n"); >+ http_info->creds = cli_creds; >+ } >+ if (http_info->creds == NULL) { >+ DBG_ERR("Failed to create creds\n"); >+ res = -1; >+ goto done; >+ } >+ http_info->lp_ctx = samba_cmdline_get_lp_ctx(); >+ >+ DBG_ERR("retries = %d/%d, Using server %s, port %d, using tls %s\n", >+ count, retries, >+ http_info->server_addr, >+ http_info->server_port, >+ use_tls ? "true" : "false"); >+ >+ while (count < retries) { >+ int error; >+ DBG_ERR("Connecting to HTTP [%s] port [%"PRIu16"]%s\n", >+ http_info->server_addr, http_info->server_port, >+ use_tls ? " with tls" : " without tls"); >+ if (use_tls) { >+ const char *crl_file = NULL; >+ const char *tls_priority = "NORMAL:-VERS-SSL3.0"; >+ enum tls_verify_peer_state verify_peer = >+ TLS_VERIFY_PEER_CA_ONLY; >+ >+ status = tstream_tls_params_client(mem_ctx, >+ ca_file, >+ crl_file, >+ tls_priority, >+ verify_peer, >+ http_info->server_addr, >+ &http_info->tls_params); >+ if (!NT_STATUS_IS_OK(status)) { >+ DBG_ERR("Failed tstream_tls_params_client - %s\n", >+ nt_errstr(status)); >+ res = -1; >+ goto done; >+ } >+ } >+ >+ req = http_connect_send(mem_ctx, >+ ev_ctx, >+ http_info->server_addr, >+ http_info->server_port, >+ http_info->creds, >+ http_info->tls_params); >+ if (!tevent_req_poll_ntstatus(req, ev_ctx, &status)) { >+ res = -1; >+ goto done; >+ } >+ >+ error = http_connect_recv(req, >+ mem_ctx, >+ &http_info->http_conn); >+ if (error != 0) { >+ count++; >+ DBG_ERR("HTTP connection failed retry %d/%d: %s\n", count, retries, strerror(error)); >+ } else { >+ DBG_ERR("HTTP connection succeeded\n"); >+ connected = true; >+ break; >+ } >+ } >+ >+ if (!connected) { >+ DBG_ERR("Leaving early\n"); >+ res = -1; >+ goto done; >+ } >+ >+ if (!send_http_request(mem_ctx, ev_ctx, http_info, response_size, &status)) { >+ DBG_ERR("Failure\n"); >+ res = -1; >+ goto done; >+ } >+ res = 0; >+done: >+ TALLOC_FREE(mem_ctx); >+ return res; >+} >diff --git a/source4/wscript_build b/source4/wscript_build >index d20444135eb..db2ad3eda5b 100644 >--- a/source4/wscript_build >+++ b/source4/wscript_build >@@ -6,6 +6,11 @@ bld.SAMBA_BINARY('client/smbclient' + bld.env.suffix4, > install=False > ) > >+bld.SAMBA_BINARY('client/http_test', >+ source='client/http_test.c', >+ deps='samba-hostconfig SMBREADLINE samba-util LIBCLI_SMB RPC_NDR_SRVSVC LIBCLI_LSA popt CMDLINE_S4', >+ for_selftest=True, >+ ) > > bld.SAMBA_BINARY('client/cifsdd', > source='client/cifsdd.c client/cifsddio.c', >-- >2.35.3 > > >From c9c0890c8b263f30a5a7243a3e952d5d0b2828f4 Mon Sep 17 00:00:00 2001 >From: Noel Power <noel.power@suse.com> >Date: Mon, 25 Mar 2024 16:25:55 +0000 >Subject: [PATCH 2/5] selftest: Add basic content-lenght http tests > >very simple test of basic http request/response plus some checks to >ensure http response doesn't exceed the response max length set by >the client call. > >BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 >Signed-off-by: Noel Power <noel.power@suse.com> >Reviewed-by: Andrew Bartlett <abartlet@samba.org> >(cherry picked from commit 74cdebeae3d1bc35eea96b51b9491f6c52844b10) >--- > python/samba/tests/blackbox/http_content.py | 95 +++++++++++++++++++++ > selftest/tests.py | 1 + > 2 files changed, 96 insertions(+) > create mode 100644 python/samba/tests/blackbox/http_content.py > >diff --git a/python/samba/tests/blackbox/http_content.py b/python/samba/tests/blackbox/http_content.py >new file mode 100644 >index 00000000000..9ecb6ffe279 >--- /dev/null >+++ b/python/samba/tests/blackbox/http_content.py >@@ -0,0 +1,95 @@ >+# Blackbox tests for http_test >+# >+# Copyright (C) Noel Power noel.power@suse.com >+# >+# This program is free software; you can redistribute it and/or modify >+# it under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# This program is distributed in the hope that it will be useful, >+# but WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+# along with this program. If not, see <http://www.gnu.org/licenses/>. >+ >+import os >+import time >+import threading >+import logging >+import json >+from http.server import HTTPServer, BaseHTTPRequestHandler >+from samba.logger import get_samba_logger >+from samba.tests import BlackboxTestCase, BlackboxProcessError >+ >+logger = get_samba_logger(name=__name__) >+COMMAND = "bin/http_test" >+ >+# simple handler, spits back the 'path' passed in >+# GET or POST and a chunked encoded http response >+# where the chunk size is 10 octets >+class ContentHTTPRequestHandler(BaseHTTPRequestHandler): >+ def handle_req(self): >+ msg = bytes(self.path, encoding="utf-8") >+ >+ self.send_response(200) >+ self.send_header('content-type', 'application/json; charset=UTF-8') >+ self.send_header('content-length', len(msg)) >+ self.end_headers() >+ self.wfile.write(msg) >+ >+ def do_POST(self): >+ self.handle_req() >+ def do_GET(self): >+ self.handle_req() >+ >+class HttpContentBlackboxTests(BlackboxTestCase): >+ def setUp(self): >+ self.server = HTTPServer((os.getenv("SERVER_IP", "localhost"), 8080), >+ ContentHTTPRequestHandler, >+ bind_and_activate=False) >+ self.t = threading.Thread(target=HttpContentBlackboxTests.http_server, args=(self,)) >+ self.t.setDaemon(True) >+ self.t.start() >+ time.sleep(1) >+ >+ def tearDown(self): >+ super().tearDown() >+ >+ def http_server(self): >+ self.server.server_bind() >+ self.server.server_activate() >+ self.server.serve_forever() >+ >+ def notest_simple_msg(self): >+ try: >+ msg = "simplemessage" >+ resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.assertEqual(msg, resp.decode('utf-8')) >+ except BlackboxProcessError as e: >+ print("Failed with: %s" % e) >+ self.fail(str(e)) >+ >+ def test_exceed_request_size(self): >+ try: >+ msg = "012345678" # 9 bytes >+ # limit response to 8 bytes >+ resp = self.check_output("%s -d11 -U%% -I%s --rsize 8 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.fail(str(e)) >+ except BlackboxProcessError as e: >+ if "unexpected 0 len response" not in e.stdout.decode('utf-8'): >+ self.fail(str(e)) >+ if "http_parse_headers: Skipping body for code 200" not in e.stderr.decode('utf-8'): >+ self.fail(str(e)) >+ >+ def test_exact_request_size(self): >+ try: >+ msg = "012345678" # 9 bytes >+ # limit response to 9 bytes >+ resp = self.check_output("%s -U%% -I%s --rsize 9 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.assertEqual(msg, resp.decode('utf-8')) >+ except BlackboxProcessError as e: >+ print("Failed with: %s" % e) >+ self.fail(str(e)) >diff --git a/selftest/tests.py b/selftest/tests.py >index 2cafe2faa4e..1efa8e79b09 100644 >--- a/selftest/tests.py >+++ b/selftest/tests.py >@@ -220,6 +220,7 @@ plantestsuite( > "samba4.blackbox.test_special_group", "none", > cmdline('test_special_group.sh', '$PREFIX_ABS/provision')) > >+planpythontestsuite("fileserver", "samba.tests.blackbox.http_content") > planpythontestsuite("none", "samba.tests.upgradeprovision") > planpythontestsuite("none", "samba.tests.xattr") > planpythontestsuite("none", "samba.tests.ntacls") >-- >2.35.3 > > >From 60bb165fa119fa427d42e204ab8335606304b06c Mon Sep 17 00:00:00 2001 >From: Noel Power <noel.power@suse.com> >Date: Fri, 22 Mar 2024 08:55:49 +0000 >Subject: [PATCH 3/5] libcli/http: Optimise reading for content-length > >Instead of reading byte-by-byte we know the content length we >want to read so lets use it. > >Signed-off-by: Noel Power <noel.power@suse.com> >Reviewed-by: Andrew Bartlett <abartlet@samba.org> >BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 >(cherry picked from commit 5f03d84e3b52bf5a31a0f885cb83bdcb48ec96f7) >--- > libcli/http/http.c | 53 ++++++++++++++++++++++++++++++++++++++++------ > 1 file changed, 46 insertions(+), 7 deletions(-) > >diff --git a/libcli/http/http.c b/libcli/http/http.c >index d20fc25f9e2..00bbde534ea 100644 >--- a/libcli/http/http.c >+++ b/libcli/http/http.c >@@ -527,20 +527,59 @@ static int http_read_response_next_vector(struct tstream_context *stream, > *_count = 1; > } > break; >- case HTTP_MORE_DATA_EXPECTED: >- /* TODO Optimize, allocating byte by byte */ >- state->buffer.data = talloc_realloc(state, state->buffer.data, >- uint8_t, state->buffer.length + 1); >+ case HTTP_MORE_DATA_EXPECTED: { >+ size_t toread = 1; >+ size_t total; >+ if (state->parser_state == HTTP_READING_BODY) { >+ struct http_request *resp = state->response; >+ toread = resp->remaining_content_length - >+ state->buffer.length; >+ } >+ >+ total = toread + state->buffer.length; >+ >+ if (total < state->buffer.length) { >+ DBG_ERR("adding %zu to buf len %zu " >+ "will overflow\n", >+ toread, >+ state->buffer.length); >+ return -1; >+ } >+ >+ /* >+ * test if content-length message exceeds the >+ * specified max_content_length >+ * Note: This check wont be hit at the moment >+ * due to an existing check in parse_headers >+ * which will skip the body. Check is here >+ * for completeness and to cater for future >+ * code changes. >+ */ >+ if (state->parser_state == HTTP_READING_BODY) { >+ if (total > state->max_content_length) { >+ DBG_ERR("content size %zu exceeds " >+ "max content len %zu\n", >+ total, >+ state->max_content_length); >+ return -1; >+ } >+ } >+ >+ state->buffer.data = >+ talloc_realloc(state, state->buffer.data, >+ uint8_t, >+ state->buffer.length + toread); > if (!state->buffer.data) { > return -1; > } >- state->buffer.length++; >+ state->buffer.length += toread; > vector[0].iov_base = (void *)(state->buffer.data + >- state->buffer.length - 1); >- vector[0].iov_len = 1; >+ state->buffer.length - toread); >+ vector[0].iov_len = toread; > *_vector = vector; > *_count = 1; > break; >+ } > case HTTP_DATA_CORRUPTED: > case HTTP_REQUEST_CANCELED: > case HTTP_DATA_TOO_LONG: >-- >2.35.3 > > >From 1c7ecdbfbb77e191f513e6646410d320049b62d2 Mon Sep 17 00:00:00 2001 >From: Noel Power <noel.power@suse.com> >Date: Thu, 23 Sep 2021 12:18:22 +0100 >Subject: [PATCH 4/5] tests: add test for chunked encoding with http cli > library > >Adds http test client to excercise the http client library >and a blackbox test to run the client. This client is built >only with selftest > >also adds a knownfail for the test > >Signed-off-by: Noel Power <noel.power@suse.com> >Reviewed-by: Andrew Bartlett <abartlet@samba.org> >BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 >(cherry picked from commit 30acd609f560352d3edb0c931b9a864110025b2c) >--- > python/samba/tests/blackbox/http_chunk.py | 116 ++++++++++++++++++++++ > selftest/knownfail.d/http_chunks | 5 + > selftest/tests.py | 1 + > 3 files changed, 122 insertions(+) > create mode 100644 python/samba/tests/blackbox/http_chunk.py > create mode 100644 selftest/knownfail.d/http_chunks > >diff --git a/python/samba/tests/blackbox/http_chunk.py b/python/samba/tests/blackbox/http_chunk.py >new file mode 100644 >index 00000000000..1b5cc174788 >--- /dev/null >+++ b/python/samba/tests/blackbox/http_chunk.py >@@ -0,0 +1,116 @@ >+# Blackbox tests for http_test >+# >+# Copyright (C) Noel Power noel.power@suse.com >+# >+# This program is free software; you can redistribute it and/or modify >+# it under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# This program is distributed in the hope that it will be useful, >+# but WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+# along with this program. If not, see <http://www.gnu.org/licenses/>. >+ >+import os >+import time >+import threading >+import logging >+import json >+from http.server import HTTPServer, BaseHTTPRequestHandler >+from samba.logger import get_samba_logger >+from samba.tests import BlackboxTestCase, BlackboxProcessError >+ >+logger = get_samba_logger(name=__name__) >+COMMAND = "bin/http_test" >+def make_chunks(msg, chunk_size): >+ chunks = [] >+ while len(msg) > chunk_size: >+ chunk = msg[:chunk_size] >+ chunks.append(chunk) >+ msg = msg[chunk_size:] >+ if len(msg): >+ chunks.append(msg) >+ return chunks >+ >+# simple handler, spits back the 'path' passed in >+# GET or POST and a chunked encoded http response >+# where the chunk size is 10 octets >+class ChunkHTTPRequestHandler(BaseHTTPRequestHandler): >+ def handle_req(self): >+ msg = bytes(self.path, encoding="utf-8") >+ chunks = make_chunks(msg, 10) >+ >+ self.send_response(200) >+ self.send_header('content-type', 'application/json; charset=UTF-8') >+ self.send_header('Transfer-Encoding', 'chunked') >+ self.end_headers() >+ resp = bytes() >+ for chunk in chunks: >+ resp = resp + ("%x" % len(chunk)).encode("utf-8") + b'\r\n' + chunk + b'\r\n' >+ resp += b'0\r\n\r\n' >+ self.wfile.write(resp) >+ >+ def do_POST(self): >+ self.handle_req() >+ def do_GET(self): >+ self.handle_req() >+ >+class HttpChunkBlackboxTests(BlackboxTestCase): >+ def setUp(self): >+ self.server = HTTPServer((os.getenv("SERVER_IP", "localhost"), 8080), >+ ChunkHTTPRequestHandler, >+ bind_and_activate=False) >+ self.t = threading.Thread(target=HttpChunkBlackboxTests.http_server, args=(self,)) >+ self.t.setDaemon(True) >+ self.t.start() >+ time.sleep(1) >+ >+ def tearDown(self): >+ super().tearDown() >+ >+ def http_server(self): >+ self.server.server_bind() >+ self.server.server_activate() >+ self.server.serve_forever() >+ >+ def test_single_chunk(self): >+ try: >+ msg = "one_chunk" >+ resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.assertEqual(msg, resp.decode('utf-8')) >+ except BlackboxProcessError as e: >+ print("Failed with: %s" % e) >+ self.fail(str(e)) >+ >+ def test_multi_chunks(self): >+ try: >+ msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" >+ resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.assertEqual(msg, resp.decode('utf-8')) >+ except BlackboxProcessError as e: >+ print("Failed with: %s" % e) >+ self.fail(str(e)) >+ >+ def test_exceed_request_size(self): >+ try: >+ msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" >+ resp = self.check_output("%s -d11 -U%% -I%s --rsize 49 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.fail(str(e)) >+ except BlackboxProcessError as e: >+ if "http_read_chunk: size 50 exceeds max content len 49 skipping body" not in e.stderr.decode('utf-8'): >+ self.fail(str(e)) >+ if "unexpected 0 len response" not in e.stdout.decode('utf-8'): >+ self.fail(str(e)) >+ >+ def test_exact_request_size(self): >+ try: >+ msg = "snglechunksnglechunksnglechunksnglechunksnglechunk" >+ resp = self.check_output("%s -U%% -I%s --rsize 50 --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >+ self.assertEqual(msg, resp.decode('utf-8')) >+ except BlackboxProcessError as e: >+ print("Failed with: %s" % e) >+ self.fail(str(e)) >diff --git a/selftest/knownfail.d/http_chunks b/selftest/knownfail.d/http_chunks >new file mode 100644 >index 00000000000..e58e002abdb >--- /dev/null >+++ b/selftest/knownfail.d/http_chunks >@@ -0,0 +1,5 @@ >+^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_multi_chunks >+^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_single_chunk >+^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exact_request_size >+^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exceed_request_size >+ >diff --git a/selftest/tests.py b/selftest/tests.py >index 1efa8e79b09..76950f447eb 100644 >--- a/selftest/tests.py >+++ b/selftest/tests.py >@@ -221,6 +221,7 @@ plantestsuite( > cmdline('test_special_group.sh', '$PREFIX_ABS/provision')) > > planpythontestsuite("fileserver", "samba.tests.blackbox.http_content") >+planpythontestsuite("fileserver", "samba.tests.blackbox.http_chunk") > planpythontestsuite("none", "samba.tests.upgradeprovision") > planpythontestsuite("none", "samba.tests.xattr") > planpythontestsuite("none", "samba.tests.ntacls") >-- >2.35.3 > > >From 1720632775afbe719931ea52a03b8271f29c5fb1 Mon Sep 17 00:00:00 2001 >From: Noel Power <noel.power@suse.com> >Date: Mon, 25 Mar 2024 19:44:10 +0000 >Subject: [PATCH 5/5] libcli/http: Handle http chunked transfer encoding > >Also removes the knownfail for the chunked transfer test > >Signed-off-by: Noel Power <noel.power@suse.com> >Reviewed-by: Andrew Bartlett <abartlet@samba.org> >BUG: https://bugzilla.samba.org/show_bug.cgi?id=15611 >(cherry picked from commit 03240c91fb6ffcf5afe47c14a1ba7a8bc12f2348) >--- > libcli/http/http.c | 218 +++++++++++++++++++++- > libcli/http/http_internal.h | 4 + > python/samba/tests/blackbox/http_chunk.py | 2 +- > selftest/knownfail.d/http_chunks | 5 - > 4 files changed, 220 insertions(+), 9 deletions(-) > delete mode 100644 selftest/knownfail.d/http_chunks > >diff --git a/libcli/http/http.c b/libcli/http/http.c >index 00bbde534ea..95ca3fad9cb 100644 >--- a/libcli/http/http.c >+++ b/libcli/http/http.c >@@ -45,6 +45,12 @@ static int http_response_needs_body(struct http_request *req) > char c; > unsigned long long v; > >+ cmp = strcasecmp(h->key, "Transfer-Encoding"); >+ if (cmp == 0) { >+ cmp = strcasecmp(h->value, "chunked"); >+ return 2; >+ } >+ > cmp = strcasecmp(h->key, "Content-Length"); > if (cmp != 0) { > continue; >@@ -66,6 +72,11 @@ static int http_response_needs_body(struct http_request *req) > > return 0; > } >+struct http_chunk >+{ >+ struct http_chunk *prev, *next; >+ DATA_BLOB blob; >+}; > > struct http_read_response_state { > enum http_parser_state parser_state; >@@ -73,6 +84,7 @@ struct http_read_response_state { > uint64_t max_content_length; > DATA_BLOB buffer; > struct http_request *response; >+ struct http_chunk *chunks; > }; > > /** >@@ -119,6 +131,11 @@ static enum http_read_status http_parse_headers(struct http_read_response_state > > ret = http_response_needs_body(state->response); > switch (ret) { >+ case 2: >+ DEBUG(11, ("%s: need to process chunks... %d\n", __func__, >+ state->response->response_code)); >+ state->parser_state = HTTP_READING_CHUNK_SIZE; >+ break; > case 1: > if (state->response->remaining_content_length <= state->max_content_length) { > DEBUG(11, ("%s: Start of read body\n", __func__)); >@@ -162,6 +179,141 @@ error: > return status; > } > >+static bool http_response_process_chunks(struct http_read_response_state *state) >+{ >+ struct http_chunk *chunk = NULL; >+ struct http_request *resp = state->response; >+ >+ for (chunk = state->chunks; chunk; chunk = chunk->next) { >+ DBG_DEBUG("processing chunk of size %zi\n", >+ chunk->blob.length); >+ if (resp->body.data == NULL) { >+ resp->body = chunk->blob; >+ chunk->blob = data_blob_null; >+ talloc_steal(resp, resp->body.data); >+ continue; >+ } >+ >+ resp->body.data = >+ talloc_realloc(resp, >+ resp->body.data, >+ uint8_t, >+ resp->body.length + chunk->blob.length); >+ if (!resp->body.data) { >+ return false; >+ } >+ memcpy(resp->body.data + resp->body.length, >+ chunk->blob.data, >+ chunk->blob.length); >+ >+ resp->body.length += chunk->blob.length; >+ >+ TALLOC_FREE(chunk->blob.data); >+ chunk->blob = data_blob_null; >+ } >+ return true; >+} >+ >+static enum http_read_status http_read_chunk_term(struct http_read_response_state *state) >+{ >+ enum http_read_status status = HTTP_ALL_DATA_READ; >+ char *ptr = NULL; >+ char *line = NULL; >+ >+ /* Sanity checks */ >+ if (!state || !state->response) { >+ DBG_ERR("%s: Invalid Parameter\n", __func__); >+ return HTTP_DATA_CORRUPTED; >+ } >+ >+ line = talloc_strndup(state, (char *)state->buffer.data, state->buffer.length); >+ if (!line) { >+ DBG_ERR("%s: Memory error\n", __func__); >+ return HTTP_DATA_CORRUPTED; >+ } >+ ptr = strstr(line, "\r\n"); >+ if (ptr == NULL) { >+ TALLOC_FREE(line); >+ return HTTP_MORE_DATA_EXPECTED; >+ } >+ >+ if (strncmp(line, "\r\n", 2) == 0) { >+ /* chunk terminator */ >+ if (state->parser_state == HTTP_READING_FINAL_CHUNK_TERM) { >+ if (http_response_process_chunks(state) == false) { >+ status = HTTP_DATA_CORRUPTED; >+ goto out; >+ } >+ state->parser_state = HTTP_READING_DONE; >+ } else { >+ state->parser_state = HTTP_READING_CHUNK_SIZE; >+ } >+ status = HTTP_ALL_DATA_READ; >+ goto out; >+ } >+ >+ status = HTTP_DATA_CORRUPTED; >+out: >+ TALLOC_FREE(line); >+ return status; >+} >+ >+static enum http_read_status http_read_chunk_size(struct http_read_response_state *state) >+{ >+ enum http_read_status status = HTTP_ALL_DATA_READ; >+ char *ptr = NULL; >+ char *line = NULL; >+ char *value = NULL; >+ int n = 0; >+ unsigned long long v; >+ >+ /* Sanity checks */ >+ if (!state || !state->response) { >+ DBG_ERR("%s: Invalid Parameter\n", __func__); >+ return HTTP_DATA_CORRUPTED; >+ } >+ >+ line = talloc_strndup(state, (char *)state->buffer.data, state->buffer.length); >+ if (!line) { >+ DBG_ERR("%s: Memory error\n", __func__); >+ return HTTP_DATA_CORRUPTED; >+ } >+ ptr = strstr(line, "\r\n"); >+ if (ptr == NULL) { >+ TALLOC_FREE(line); >+ return HTTP_MORE_DATA_EXPECTED; >+ } >+ >+ n = sscanf(line, "%m[^\r\n]\r\n", &value); >+ if (n != 1) { >+ DBG_ERR("%s: Error parsing chunk size '%s'\n", __func__, line); >+ status = HTTP_DATA_CORRUPTED; >+ goto out; >+ } >+ >+ DBG_DEBUG("Got chunk size string %s\n", value); >+ n = sscanf(value, "%llx", &v); >+ if (n != 1) { >+ DBG_ERR("%s: Error parsing chunk size '%s'\n", __func__, line); >+ status = HTTP_DATA_CORRUPTED; >+ goto out; >+ } >+ DBG_DEBUG("Got chunk size %llu 0x%llx\n", v, v); >+ if (v == 0) { >+ state->parser_state = HTTP_READING_FINAL_CHUNK_TERM; >+ } else { >+ state->parser_state = HTTP_READING_CHUNK; >+ } >+ state->response->remaining_content_length = v; >+ status = HTTP_ALL_DATA_READ; >+out: >+ if (value) { >+ free(value); >+ } >+ TALLOC_FREE(line); >+ return status; >+} >+ > /** > * Parses the first line of a HTTP response > */ >@@ -301,6 +453,55 @@ static enum http_read_status http_read_body(struct http_read_response_state *sta > return HTTP_ALL_DATA_READ; > } > >+static enum http_read_status http_read_chunk(struct http_read_response_state *state) >+{ >+ struct http_request *resp = state->response; >+ struct http_chunk *chunk = NULL; >+ size_t total = 0; >+ size_t prev = 0; >+ >+ if (state->buffer.length < resp->remaining_content_length) { >+ return HTTP_MORE_DATA_EXPECTED; >+ } >+ >+ for (chunk = state->chunks; chunk; chunk = chunk->next) { >+ total += chunk->blob.length; >+ } >+ >+ prev = total; >+ total = total + state->buffer.length; >+ if (total < prev) { >+ DBG_ERR("adding chunklen %zu to buf len %zu " >+ "will overflow\n", >+ state->buffer.length, >+ prev); >+ return HTTP_DATA_CORRUPTED; >+ } >+ if (total > state->max_content_length) { >+ DBG_DEBUG("size %zu exceeds " >+ "max content len %"PRIu64" skipping body\n", >+ total, >+ state->max_content_length); >+ state->parser_state = HTTP_READING_DONE; >+ goto out; >+ } >+ >+ /* chunk read */ >+ chunk = talloc_zero(state, struct http_chunk); >+ if (chunk == NULL) { >+ DBG_ERR("%s: Memory error\n", __func__); >+ return HTTP_DATA_CORRUPTED; >+ } >+ chunk->blob = state->buffer; >+ talloc_steal(chunk, chunk->blob.data); >+ DLIST_ADD_END(state->chunks, chunk); >+ state->parser_state = HTTP_READING_CHUNK_TERM; >+out: >+ state->buffer = data_blob_null; >+ resp->remaining_content_length = 0; >+ return HTTP_ALL_DATA_READ; >+} >+ > static enum http_read_status http_read_trailer(struct http_read_response_state *state) > { > enum http_read_status status = HTTP_DATA_CORRUPTED; >@@ -323,6 +524,16 @@ static enum http_read_status http_parse_buffer(struct http_read_response_state * > case HTTP_READING_BODY: > return http_read_body(state); > break; >+ case HTTP_READING_FINAL_CHUNK_TERM: >+ case HTTP_READING_CHUNK_TERM: >+ return http_read_chunk_term(state); >+ break; >+ case HTTP_READING_CHUNK_SIZE: >+ return http_read_chunk_size(state); >+ break; >+ case HTTP_READING_CHUNK: >+ return http_read_chunk(state); >+ break; > case HTTP_READING_TRAILER: > return http_read_trailer(state); > break; >@@ -530,7 +741,8 @@ static int http_read_response_next_vector(struct tstream_context *stream, > case HTTP_MORE_DATA_EXPECTED: { > size_t toread = 1; > size_t total; >- if (state->parser_state == HTTP_READING_BODY) { >+ if (state->parser_state == HTTP_READING_BODY || >+ state->parser_state == HTTP_READING_CHUNK) { > struct http_request *resp = state->response; > toread = resp->remaining_content_length - > state->buffer.length; >@@ -549,7 +761,7 @@ static int http_read_response_next_vector(struct tstream_context *stream, > /* > * test if content-length message exceeds the > * specified max_content_length >- * Note: This check wont be hit at the moment >+ * Note: This check won't be hit at the moment > * due to an existing check in parse_headers > * which will skip the body. Check is here > * for completeness and to cater for future >@@ -558,7 +770,7 @@ static int http_read_response_next_vector(struct tstream_context *stream, > if (state->parser_state == HTTP_READING_BODY) { > if (total > state->max_content_length) { > DBG_ERR("content size %zu exceeds " >- "max content len %zu\n", >+ "max content len %"PRIu64"\n", > total, > state->max_content_length); > return -1; >diff --git a/libcli/http/http_internal.h b/libcli/http/http_internal.h >index ec17f7e2850..786ace62d84 100644 >--- a/libcli/http/http_internal.h >+++ b/libcli/http/http_internal.h >@@ -28,6 +28,10 @@ enum http_parser_state { > HTTP_READING_BODY, > HTTP_READING_TRAILER, > HTTP_READING_DONE, >+ HTTP_READING_CHUNK_SIZE, >+ HTTP_READING_CHUNK, >+ HTTP_READING_CHUNK_TERM, >+ HTTP_READING_FINAL_CHUNK_TERM, > }; > > enum http_read_status { >diff --git a/python/samba/tests/blackbox/http_chunk.py b/python/samba/tests/blackbox/http_chunk.py >index 1b5cc174788..175c60d98a2 100644 >--- a/python/samba/tests/blackbox/http_chunk.py >+++ b/python/samba/tests/blackbox/http_chunk.py >@@ -81,7 +81,7 @@ class HttpChunkBlackboxTests(BlackboxTestCase): > try: > msg = "one_chunk" > resp = self.check_output("%s -U%% -I%s --uri %s" % (COMMAND, os.getenv("SERVER_IP", "localhost"), msg)) >- self.assertEqual(msg, resp.decode('utf-8')) >+ self.assertEqual(msg,resp.decode('utf-8')) > except BlackboxProcessError as e: > print("Failed with: %s" % e) > self.fail(str(e)) >diff --git a/selftest/knownfail.d/http_chunks b/selftest/knownfail.d/http_chunks >deleted file mode 100644 >index e58e002abdb..00000000000 >--- a/selftest/knownfail.d/http_chunks >+++ /dev/null >@@ -1,5 +0,0 @@ >-^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_multi_chunks >-^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_single_chunk >-^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exact_request_size >-^samba.tests.blackbox.http_chunk.samba.tests.blackbox.http_chunk.HttpChunkBlackboxTests.test_exceed_request_size >- >-- >2.35.3 >
<b>You cannot view the attachment while viewing its details because your browser does not support IFRAMEs. <a href="attachment.cgi?id=18276">View the attachment on a separate page</a>.</b>
View Attachment As Raw
Actions:
View
Attachments on
bug 15611
:
18276
|
18282