From b1cc7430f0a1d43f92e98991ae81f004b370a35b Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Wed, 19 Apr 2023 14:38:45 +0200 Subject: [PATCH 1/6] mdssvc: set query state for continued queries to SLQ_STATE_RUNNING SLQ_STATE_RESULTS implies that there are already results attached to the slq which is not the case. Instead the backend will start processing from where it left off when it hits the maximum result limit and had set the state to SLQ_STATE_FULL. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 4149ef97e5906604be1587622f390f121db183e2) --- source3/rpc_server/mdssvc/mdssvc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/rpc_server/mdssvc/mdssvc.c b/source3/rpc_server/mdssvc/mdssvc.c index 22cc7abd1530..1f077bee23dd 100644 --- a/source3/rpc_server/mdssvc/mdssvc.c +++ b/source3/rpc_server/mdssvc/mdssvc.c @@ -1126,7 +1126,7 @@ static bool slrpc_fetch_query_results(struct mds_ctx *mds_ctx, goto error; } if (slq->state == SLQ_STATE_FULL) { - slq->state = SLQ_STATE_RESULTS; + slq->state = SLQ_STATE_RUNNING; slq->mds_ctx->backend->search_cont(slq); } break; -- 2.41.0 From 0372b9bcbdaedeb8c9122701dc3c58fbea500579 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 23 Mar 2023 16:39:11 +0100 Subject: [PATCH 2/6] mdssvc: fix long running backend queries If a query is still running in the backend and we have no results yet, returning 0 triggers a search termination by the client in latest macOS releases. macOS returns 0x23 in this case. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 925fefae20e52a3c89a56bdd0cd5b98cc951db5f) --- source3/rpc_server/mdssvc/mdssvc.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source3/rpc_server/mdssvc/mdssvc.c b/source3/rpc_server/mdssvc/mdssvc.c index 1f077bee23dd..c394a180b1ac 100644 --- a/source3/rpc_server/mdssvc/mdssvc.c +++ b/source3/rpc_server/mdssvc/mdssvc.c @@ -306,10 +306,21 @@ static bool create_result_handle(struct sl_query *slq) static bool add_results(sl_array_t *array, struct sl_query *slq) { sl_filemeta_t *fm; - uint64_t status = 0; + uint64_t status; int result; bool ok; + /* + * Taken from a network trace against a macOS SMB Spotlight server. If + * the first fetch-query-results has no results yet because the search + * is still running, macOS returns 0x23, otherwise 0x0. + */ + if (slq->state >= SLQ_STATE_RESULTS ) { + status = 0; + } else { + status = 0x23; + } + /* FileMeta */ fm = dalloc_zero(array, sl_filemeta_t); if (fm == NULL) { -- 2.41.0 From 814dcce64f93ed89cd8fc3edaf6847e55978a8e9 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 20 Apr 2023 17:24:30 +0200 Subject: [PATCH 3/6] mdssvc: add and use SL_PAGESIZE SL_PAGESIZE is the number of entries we want to process per paged search result set. This is different from MAX_SL_RESULTS which ought to be a default maximum value for total number of results returned for a search query. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 086c2602d074d4dc0d44f5534857e5f59a8690b2) --- source3/rpc_server/mdssvc/mdssvc.h | 1 + source3/rpc_server/mdssvc/mdssvc_es.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source3/rpc_server/mdssvc/mdssvc.h b/source3/rpc_server/mdssvc/mdssvc.h index 3b2ce250f1f1..75e6fa9f3200 100644 --- a/source3/rpc_server/mdssvc/mdssvc.h +++ b/source3/rpc_server/mdssvc/mdssvc.h @@ -36,6 +36,7 @@ #define MAX_SL_FRAGMENT_SIZE 0xFFFFF #define MAX_SL_RESULTS 100 +#define SL_PAGESIZE 100 #define MAX_SL_RUNTIME 30 #define MDS_TRACKER_ASYNC_TIMEOUT_MS 250 diff --git a/source3/rpc_server/mdssvc/mdssvc_es.c b/source3/rpc_server/mdssvc/mdssvc_es.c index dafb42610fae..c240a5381695 100644 --- a/source3/rpc_server/mdssvc/mdssvc_es.c +++ b/source3/rpc_server/mdssvc/mdssvc_es.c @@ -398,7 +398,7 @@ static bool mds_es_search(struct sl_query *slq) .ev = mds_es_ctx->mdssvc_es_ctx->mdssvc_ctx->ev_ctx, .mds_es_ctx = mds_es_ctx, .slq = slq, - .size = MAX_SL_RESULTS, + .size = SL_PAGESIZE, }; /* 0 would mean no limit */ @@ -502,7 +502,7 @@ static void mds_es_search_done(struct tevent_req *subreq) goto trigger; } - if (slq->query_results->num_results >= MAX_SL_RESULTS) { + if (slq->query_results->num_results >= SL_PAGESIZE) { slq->state = SLQ_STATE_FULL; goto trigger; } @@ -693,7 +693,7 @@ static void mds_es_search_http_send_done(struct tevent_req *subreq) subreq = http_read_response_send(state, state->ev, state->s->mds_es_ctx->http_conn, - MAX_SL_RESULTS * 8192); + SL_PAGESIZE * 8192); if (tevent_req_nomem(subreq, req)) { return; } -- 2.41.0 From ba4e30872fb98a11f3c4a31d289eb941df1603a8 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 20 Apr 2023 17:58:38 +0200 Subject: [PATCH 4/6] mdssvc: fix enforcement of "elasticsearch:max results" This wasn't enforced at all thus a query would return all available matches without limit. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit d8fa5c8e2a1794ea8dc663485315ebd9401b2628) --- source3/rpc_server/mdssvc/mdssvc_es.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/rpc_server/mdssvc/mdssvc_es.c b/source3/rpc_server/mdssvc/mdssvc_es.c index c240a5381695..8460b48b80aa 100644 --- a/source3/rpc_server/mdssvc/mdssvc_es.c +++ b/source3/rpc_server/mdssvc/mdssvc_es.c @@ -800,7 +800,7 @@ static void mds_es_search_http_read_done(struct tevent_req *subreq) } DBG_DEBUG("Hits: %zu\n", hits); - for (i = 0; i < hits; i++) { + for (i = 0; i < hits && s->from + i < s->max; i++) { const char *path = NULL; match = json_array_get(matches, i); -- 2.41.0 From 7face699e18d3b558932a6f4da4e64ba2acc8757 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Fri, 21 Apr 2023 07:07:13 +0200 Subject: [PATCH 5/6] tests/mdssvc: match hits:total:value to be the actual amount of entries in hits BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 7f5e4edf64f7e4175f652bf8762d4edc110ad6b1) --- python/samba/tests/dcerpc/mdssvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/tests/dcerpc/mdssvc.py b/python/samba/tests/dcerpc/mdssvc.py index 5002e5d26d64..985f35aa725d 100644 --- a/python/samba/tests/dcerpc/mdssvc.py +++ b/python/samba/tests/dcerpc/mdssvc.py @@ -166,7 +166,7 @@ testfiles = [ }''' fake_json_response = r'''{ "hits" : { - "total" : {"value" : 2}, + "total" : {"value" : 9}, "hits" : [ {"_source" : {"path" : {"real" : "%BASEPATH%/x+x"}}}, {"_source" : {"path" : {"real" : "%BASEPATH%/x*x"}}}, -- 2.41.0 From 4c74fae6b5e710915384f7f8249ff7ca5e4d066c Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 20 Apr 2023 17:27:20 +0200 Subject: [PATCH 6/6] mdssvc: reduce pagesize to 50 Lastest macOS queries additional file metadata per search result, which causes the mashalled paged result set including metadata to exceed the 64 KB result fragment buffer. Lacking fragementation support in mdssvc (it's supported by the protocol), for now just reduce the maximum number of results per search page. BUG: https://bugzilla.samba.org/show_bug.cgi?id=15342 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 724a0518c901589fe1171d94648391832e056f4d) --- python/samba/tests/blackbox/mdsearch.py | 2 +- python/samba/tests/dcerpc/mdssvc.py | 4 ++-- source3/rpc_server/mdssvc/mdssvc.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/samba/tests/blackbox/mdsearch.py b/python/samba/tests/blackbox/mdsearch.py index c8e75661f151..46af6f94624d 100644 --- a/python/samba/tests/blackbox/mdsearch.py +++ b/python/samba/tests/blackbox/mdsearch.py @@ -100,7 +100,7 @@ testfiles = [ config = os.environ["SMB_CONF_PATH"] json_in = r'''{ - "from": 0, "size": 100, "_source": ["path.real"], + "from": 0, "size": 50, "_source": ["path.real"], "query": { "query_string": { "query": "(samba*) AND path.real.fulltext:\"%BASEPATH%\"" diff --git a/python/samba/tests/dcerpc/mdssvc.py b/python/samba/tests/dcerpc/mdssvc.py index 985f35aa725d..7d09a5559366 100644 --- a/python/samba/tests/dcerpc/mdssvc.py +++ b/python/samba/tests/dcerpc/mdssvc.py @@ -125,7 +125,7 @@ testfiles = [ def test_mdscli_search(self): exp_json_query = r'''{ - "from": 0, "size": 100, "_source": ["path.real"], + "from": 0, "size": 50, "_source": ["path.real"], "query": { "query_string": { "query": "(samba*) AND path.real.fulltext:\"%BASEPATH%\"" @@ -157,7 +157,7 @@ testfiles = [ r'kMDItemFSName=="x\\x"' ) exp_json_query = r'''{ - "from": 0, "size": 100, "_source": ["path.real"], + "from": 0, "size": 50, "_source": ["path.real"], "query": { "query_string": { "query": "(file.filename:x\\+x OR file.filename:x\\*x OR file.filename:x=x OR file.filename:x'x OR file.filename:x\\?x OR file.filename:x\\ x OR file.filename:x\\(x OR file.filename:x\\\"x OR file.filename:x\\\\x) AND path.real.fulltext:\"%BASEPATH%\"" diff --git a/source3/rpc_server/mdssvc/mdssvc.h b/source3/rpc_server/mdssvc/mdssvc.h index 75e6fa9f3200..8aabf5b86e5c 100644 --- a/source3/rpc_server/mdssvc/mdssvc.h +++ b/source3/rpc_server/mdssvc/mdssvc.h @@ -36,7 +36,7 @@ #define MAX_SL_FRAGMENT_SIZE 0xFFFFF #define MAX_SL_RESULTS 100 -#define SL_PAGESIZE 100 +#define SL_PAGESIZE 50 #define MAX_SL_RUNTIME 30 #define MDS_TRACKER_ASYNC_TIMEOUT_MS 250 -- 2.41.0