Attachment 18008 Details for Bug 15342 – Patch for 4.17 and 4.18 cherry-picked from master

[patch] Patch for 4.17 and 4.18 cherry-picked from master

bug15342-v417,v418.patch (text/plain), 10.15 KB, created by Ralph Böhme on 2023-07-28 13:12:12 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ralph Böhme

Created: 2023-07-28 13:12:12 UTC

Size: 10.15 KB

patch

obsolete

>From b1cc7430f0a1d43f92e98991ae81f004b370a35b Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Wed, 19 Apr 2023 14:38:45 +0200
>Subject: [PATCH 1/6] mdssvc: set query state for continued queries to
> SLQ_STATE_RUNNING
>
>SLQ_STATE_RESULTS implies that there are already results attached to the slq
>which is not the case. Instead the backend will start processing from where it
>left off when it hits the maximum result limit and had set the state to
>SLQ_STATE_FULL.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 4149ef97e5906604be1587622f390f121db183e2)
>---
> source3/rpc_server/mdssvc/mdssvc.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/source3/rpc_server/mdssvc/mdssvc.c b/source3/rpc_server/mdssvc/mdssvc.c
>index 22cc7abd1530..1f077bee23dd 100644
>--- a/source3/rpc_server/mdssvc/mdssvc.c
>+++ b/source3/rpc_server/mdssvc/mdssvc.c
>@@ -1126,7 +1126,7 @@ static bool slrpc_fetch_query_results(struct mds_ctx *mds_ctx,
> 			goto error;
> 		}
> 		if (slq->state == SLQ_STATE_FULL) {
>-			slq->state = SLQ_STATE_RESULTS;
>+			slq->state = SLQ_STATE_RUNNING;
> 			slq->mds_ctx->backend->search_cont(slq);
> 		}
> 		break;
>-- 
>2.41.0
>
>
>From 0372b9bcbdaedeb8c9122701dc3c58fbea500579 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Thu, 23 Mar 2023 16:39:11 +0100
>Subject: [PATCH 2/6] mdssvc: fix long running backend queries
>
>If a query is still running in the backend and we have no results yet, returning
>0 triggers a search termination by the client in latest macOS releases. macOS
>returns 0x23 in this case.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 925fefae20e52a3c89a56bdd0cd5b98cc951db5f)
>---
> source3/rpc_server/mdssvc/mdssvc.c | 13 ++++++++++++-
> 1 file changed, 12 insertions(+), 1 deletion(-)
>
>diff --git a/source3/rpc_server/mdssvc/mdssvc.c b/source3/rpc_server/mdssvc/mdssvc.c
>index 1f077bee23dd..c394a180b1ac 100644
>--- a/source3/rpc_server/mdssvc/mdssvc.c
>+++ b/source3/rpc_server/mdssvc/mdssvc.c
>@@ -306,10 +306,21 @@ static bool create_result_handle(struct sl_query *slq)
> static bool add_results(sl_array_t *array, struct sl_query *slq)
> {
> 	sl_filemeta_t *fm;
>-	uint64_t status = 0;
>+	uint64_t status;
> 	int result;
> 	bool ok;
> 
>+	/*
>+	 * Taken from a network trace against a macOS SMB Spotlight server. If
>+	 * the first fetch-query-results has no results yet because the search
>+	 * is still running, macOS returns 0x23, otherwise 0x0.
>+	 */
>+	if (slq->state >= SLQ_STATE_RESULTS ) {
>+		status = 0;
>+	} else {
>+		status = 0x23;
>+	}
>+
> 	/* FileMeta */
> 	fm = dalloc_zero(array, sl_filemeta_t);
> 	if (fm == NULL) {
>-- 
>2.41.0
>
>
>From 814dcce64f93ed89cd8fc3edaf6847e55978a8e9 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Thu, 20 Apr 2023 17:24:30 +0200
>Subject: [PATCH 3/6] mdssvc: add and use SL_PAGESIZE
>
>SL_PAGESIZE is the number of entries we want to process per paged search result
>set. This is different from MAX_SL_RESULTS which ought to be a default maximum
>value for total number of results returned for a search query.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 086c2602d074d4dc0d44f5534857e5f59a8690b2)
>---
> source3/rpc_server/mdssvc/mdssvc.h    | 1 +
> source3/rpc_server/mdssvc/mdssvc_es.c | 6 +++---
> 2 files changed, 4 insertions(+), 3 deletions(-)
>
>diff --git a/source3/rpc_server/mdssvc/mdssvc.h b/source3/rpc_server/mdssvc/mdssvc.h
>index 3b2ce250f1f1..75e6fa9f3200 100644
>--- a/source3/rpc_server/mdssvc/mdssvc.h
>+++ b/source3/rpc_server/mdssvc/mdssvc.h
>@@ -36,6 +36,7 @@
> 
> #define MAX_SL_FRAGMENT_SIZE 0xFFFFF
> #define MAX_SL_RESULTS 100
>+#define SL_PAGESIZE 100
> #define MAX_SL_RUNTIME 30
> #define MDS_TRACKER_ASYNC_TIMEOUT_MS 250
> 
>diff --git a/source3/rpc_server/mdssvc/mdssvc_es.c b/source3/rpc_server/mdssvc/mdssvc_es.c
>index dafb42610fae..c240a5381695 100644
>--- a/source3/rpc_server/mdssvc/mdssvc_es.c
>+++ b/source3/rpc_server/mdssvc/mdssvc_es.c
>@@ -398,7 +398,7 @@ static bool mds_es_search(struct sl_query *slq)
> 		.ev = mds_es_ctx->mdssvc_es_ctx->mdssvc_ctx->ev_ctx,
> 		.mds_es_ctx = mds_es_ctx,
> 		.slq = slq,
>-		.size = MAX_SL_RESULTS,
>+		.size = SL_PAGESIZE,
> 	};
> 
> 	/* 0 would mean no limit */
>@@ -502,7 +502,7 @@ static void mds_es_search_done(struct tevent_req *subreq)
> 		goto trigger;
> 	}
> 
>-	if (slq->query_results->num_results >= MAX_SL_RESULTS) {
>+	if (slq->query_results->num_results >= SL_PAGESIZE) {
> 		slq->state = SLQ_STATE_FULL;
> 		goto trigger;
> 	}
>@@ -693,7 +693,7 @@ static void mds_es_search_http_send_done(struct tevent_req *subreq)
> 	subreq = http_read_response_send(state,
> 					 state->ev,
> 					 state->s->mds_es_ctx->http_conn,
>-					 MAX_SL_RESULTS * 8192);
>+					 SL_PAGESIZE * 8192);
> 	if (tevent_req_nomem(subreq, req)) {
> 		return;
> 	}
>-- 
>2.41.0
>
>
>From ba4e30872fb98a11f3c4a31d289eb941df1603a8 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Thu, 20 Apr 2023 17:58:38 +0200
>Subject: [PATCH 4/6] mdssvc: fix enforcement of "elasticsearch:max results"
>
>This wasn't enforced at all thus a query would return all available matches
>without limit.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit d8fa5c8e2a1794ea8dc663485315ebd9401b2628)
>---
> source3/rpc_server/mdssvc/mdssvc_es.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/source3/rpc_server/mdssvc/mdssvc_es.c b/source3/rpc_server/mdssvc/mdssvc_es.c
>index c240a5381695..8460b48b80aa 100644
>--- a/source3/rpc_server/mdssvc/mdssvc_es.c
>+++ b/source3/rpc_server/mdssvc/mdssvc_es.c
>@@ -800,7 +800,7 @@ static void mds_es_search_http_read_done(struct tevent_req *subreq)
> 	}
> 	DBG_DEBUG("Hits: %zu\n", hits);
> 
>-	for (i = 0; i < hits; i++) {
>+	for (i = 0; i < hits && s->from + i < s->max; i++) {
> 		const char *path = NULL;
> 
> 		match = json_array_get(matches, i);
>-- 
>2.41.0
>
>
>From 7face699e18d3b558932a6f4da4e64ba2acc8757 Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Fri, 21 Apr 2023 07:07:13 +0200
>Subject: [PATCH 5/6] tests/mdssvc: match hits:total:value to be the actual
> amount of entries in hits
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 7f5e4edf64f7e4175f652bf8762d4edc110ad6b1)
>---
> python/samba/tests/dcerpc/mdssvc.py | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/python/samba/tests/dcerpc/mdssvc.py b/python/samba/tests/dcerpc/mdssvc.py
>index 5002e5d26d64..985f35aa725d 100644
>--- a/python/samba/tests/dcerpc/mdssvc.py
>+++ b/python/samba/tests/dcerpc/mdssvc.py
>@@ -166,7 +166,7 @@ testfiles = [
>         }'''
>         fake_json_response = r'''{
>           "hits" : {
>-            "total" : {"value" : 2},
>+            "total" : {"value" : 9},
>             "hits" : [
>               {"_source" : {"path" : {"real" : "%BASEPATH%/x+x"}}},
>               {"_source" : {"path" : {"real" : "%BASEPATH%/x*x"}}},
>-- 
>2.41.0
>
>
>From 4c74fae6b5e710915384f7f8249ff7ca5e4d066c Mon Sep 17 00:00:00 2001
>From: Ralph Boehme <slow@samba.org>
>Date: Thu, 20 Apr 2023 17:27:20 +0200
>Subject: [PATCH 6/6] mdssvc: reduce pagesize to 50
>
>Lastest macOS queries additional file metadata per search result, which causes
>the mashalled paged result set including metadata to exceed the 64 KB result
>fragment buffer.
>
>Lacking fragementation support in mdssvc (it's supported by the protocol), for
>now just reduce the maximum number of results per search page.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342
>
>Signed-off-by: Ralph Boehme <slow@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 724a0518c901589fe1171d94648391832e056f4d)
>---
> python/samba/tests/blackbox/mdsearch.py | 2 +-
> python/samba/tests/dcerpc/mdssvc.py     | 4 ++--
> source3/rpc_server/mdssvc/mdssvc.h      | 2 +-
> 3 files changed, 4 insertions(+), 4 deletions(-)
>
>diff --git a/python/samba/tests/blackbox/mdsearch.py b/python/samba/tests/blackbox/mdsearch.py
>index c8e75661f151..46af6f94624d 100644
>--- a/python/samba/tests/blackbox/mdsearch.py
>+++ b/python/samba/tests/blackbox/mdsearch.py
>@@ -100,7 +100,7 @@ testfiles = [
>         config = os.environ["SMB_CONF_PATH"]
> 
>         json_in = r'''{
>-          "from": 0, "size": 100, "_source": ["path.real"],
>+          "from": 0, "size": 50, "_source": ["path.real"],
>           "query": {
>             "query_string": {
>               "query": "(samba*) AND path.real.fulltext:\"%BASEPATH%\""
>diff --git a/python/samba/tests/dcerpc/mdssvc.py b/python/samba/tests/dcerpc/mdssvc.py
>index 985f35aa725d..7d09a5559366 100644
>--- a/python/samba/tests/dcerpc/mdssvc.py
>+++ b/python/samba/tests/dcerpc/mdssvc.py
>@@ -125,7 +125,7 @@ testfiles = [
> 
>     def test_mdscli_search(self):
>         exp_json_query = r'''{
>-          "from": 0, "size": 100, "_source": ["path.real"],
>+          "from": 0, "size": 50, "_source": ["path.real"],
>           "query": {
>             "query_string": {
>               "query": "(samba*) AND path.real.fulltext:\"%BASEPATH%\""
>@@ -157,7 +157,7 @@ testfiles = [
>             r'kMDItemFSName=="x\\x"'
>         )
>         exp_json_query = r'''{
>-          "from": 0, "size": 100, "_source": ["path.real"],
>+          "from": 0, "size": 50, "_source": ["path.real"],
>           "query": {
>             "query_string": {
>               "query": "(file.filename:x\\+x OR file.filename:x\\*x OR file.filename:x=x OR file.filename:x'x OR file.filename:x\\?x OR file.filename:x\\ x OR file.filename:x\\(x OR file.filename:x\\\"x OR file.filename:x\\\\x) AND path.real.fulltext:\"%BASEPATH%\""
>diff --git a/source3/rpc_server/mdssvc/mdssvc.h b/source3/rpc_server/mdssvc/mdssvc.h
>index 75e6fa9f3200..8aabf5b86e5c 100644
>--- a/source3/rpc_server/mdssvc/mdssvc.h
>+++ b/source3/rpc_server/mdssvc/mdssvc.h
>@@ -36,7 +36,7 @@
> 
> #define MAX_SL_FRAGMENT_SIZE 0xFFFFF
> #define MAX_SL_RESULTS 100
>-#define SL_PAGESIZE 100
>+#define SL_PAGESIZE 50
> #define MAX_SL_RUNTIME 30
> #define MDS_TRACKER_ASYNC_TIMEOUT_MS 250
> 
>-- 
>2.41.0
>

Flags:

jra: review+

Actions: View

Attachments on bug 15342: 17844 | 18008