From de692970345c7b7e47b9d6bb8f6dfae70c44aecb Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 12 Aug 2020 15:48:01 +0200 Subject: [PATCH 01/14] build: Wrap a long line There will be another entry in the next commit Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit c8c2f8ba73324ba43ccef9f6d1c0c726d7ec0d25) --- source4/torture/wscript_build | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source4/torture/wscript_build b/source4/torture/wscript_build index 573710d9180f..ea7653454890 100644 --- a/source4/torture/wscript_build +++ b/source4/torture/wscript_build @@ -253,7 +253,17 @@ bld.SAMBA_MODULE('TORTURE_UNIX', bld.SAMBA_MODULE('TORTURE_LDAP', - source='ldap/common.c ldap/basic.c ldap/schema.c ldap/uptodatevector.c ldap/cldap.c ldap/netlogon.c ldap/cldapbench.c ldap/ldap_sort.c ldap/nested_search.c', + source=''' + ldap/common.c + ldap/basic.c + ldap/schema.c + ldap/uptodatevector.c + ldap/cldap.c + ldap/netlogon.c + ldap/cldapbench.c + ldap/ldap_sort.c + ldap/nested_search.c + ''', subsystem='smbtorture', deps='cli-ldap cli_cldap samdb popt POPT_CREDENTIALS torture ldbsamba', internal_module=True, -- 2.20.1 From 1fa9f78d6336c493510067f78f55006fef59947d Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 12 Aug 2020 15:50:58 +0200 Subject: [PATCH 02/14] torture: Test ldap session expiry LDAP connections should time out when the kerberos ticket used to authenticate expires. Windows does this with a RFC4511 section 4.4.1 message (that as of August 2020 is encoded not according to the RFC) followed by a TCP disconnect. ldb sees the section 4.4.1 as a protocol violation and returns LDB_ERR_PROTOCOL_ERROR. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit 35c4bb0b0c55a65490fe199edb1a534548104e95) --- selftest/knownfail.d/ldap | 1 + source4/selftest/tests.py | 7 +- source4/torture/ldap/common.c | 2 + source4/torture/ldap/session_expiry.c | 121 ++++++++++++++++++++++++++ source4/torture/wscript_build | 1 + 5 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 source4/torture/ldap/session_expiry.c diff --git a/selftest/knownfail.d/ldap b/selftest/knownfail.d/ldap index 0331d3687d41..21c1aca7e6e7 100644 --- a/selftest/knownfail.d/ldap +++ b/selftest/knownfail.d/ldap @@ -1,3 +1,4 @@ # the attributes too long test returns the wrong error ^samba4.ldap.python.+test_attribute_ranges_too_long samba4.ldap.python\(ad_dc_default\).*__main__.BasicTests.test_ldapSearchNoAttributes +^samba4.ldap.session-expiry.session-expiry\(ad_dc_default\) diff --git a/source4/selftest/tests.py b/source4/selftest/tests.py index 73f273d00452..c819a61418ff 100755 --- a/source4/selftest/tests.py +++ b/source4/selftest/tests.py @@ -152,7 +152,12 @@ for options in ['-U"$USERNAME%$PASSWORD"']: "%s/test_ldb.sh ldapi $PREFIX_ABS/ad_dc_ntvfs/private/ldapi %s" % (bbdir, options)) for t in smbtorture4_testsuites("ldap."): - plansmbtorture4testsuite(t, "ad_dc_default", '-U"$USERNAME%$PASSWORD" //$SERVER_IP/_none_') + if t == "ldap.session-expiry": + # This requires kerberos and thus the server name + plansmbtorture4testsuite( + t, "ad_dc_default", '-U"$USERNAME%$PASSWORD" //$DC_SERVER/_none_') + else: + plansmbtorture4testsuite(t, "ad_dc_default", '-U"$USERNAME%$PASSWORD" //$SERVER_IP/_none_') for t in smbtorture4_testsuites("dsdb."): plansmbtorture4testsuite(t, "ad_dc:local", "localhost") diff --git a/source4/torture/ldap/common.c b/source4/torture/ldap/common.c index 9482e0392bcc..d16f611fa771 100644 --- a/source4/torture/ldap/common.c +++ b/source4/torture/ldap/common.c @@ -124,6 +124,8 @@ NTSTATUS torture_ldap_init(TALLOC_CTX *ctx) torture_suite_add_simple_test(suite, "schema", torture_ldap_schema); torture_suite_add_simple_test(suite, "uptodatevector", torture_ldap_uptodatevector); torture_suite_add_simple_test(suite, "nested-search", test_ldap_nested_search); + torture_suite_add_simple_test( + suite, "session-expiry", torture_ldap_session_expiry); suite->description = talloc_strdup(suite, "LDAP and CLDAP tests"); diff --git a/source4/torture/ldap/session_expiry.c b/source4/torture/ldap/session_expiry.c new file mode 100644 index 000000000000..35dda439b17f --- /dev/null +++ b/source4/torture/ldap/session_expiry.c @@ -0,0 +1,121 @@ +/* + * Unix SMB/CIFS implementation. + * + * Test LDB attribute functions + * + * Copyright (C) Andrew Bartlet 2008-2009 + * Copyright (C) Matthieu Patou 2009 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "includes.h" +#include "lib/events/events.h" +#include +#include +#include "ldb_wrap.h" +#include "param/param.h" +#include "lib/cmdline/popt_common.h" +#include "auth/credentials/credentials.h" +#include "libcli/ldap/ldap_client.h" +#include "torture/smbtorture.h" +#include "torture/ldap/proto.h" + +bool torture_ldap_session_expiry(struct torture_context *torture) +{ + const char *host = torture_setting_string(torture, "host", NULL); + struct cli_credentials *credentials = popt_get_cmdline_credentials(); + struct ldb_context *ldb = NULL; + const char *url = NULL; + bool ret = false; + bool ok; + struct ldb_dn *rootdn = NULL; + struct ldb_result *result = NULL; + int rc = LDB_SUCCESS; + + /* + * Further down we request a ticket lifetime of 4 + * seconds. Give the server 10 seconds for this to kick in + */ + const struct timeval endtime = timeval_current_ofs(10, 0); + + url = talloc_asprintf(torture, "ldap://%s/", host); + torture_assert_goto( + torture, url!=NULL, ret, fail, "talloc_asprintf failed"); + + cli_credentials_set_kerberos_state( + credentials, CRED_MUST_USE_KERBEROS); + + ok = lpcfg_set_option( + torture->lp_ctx, "gensec_gssapi:requested_life_time=4"); + torture_assert_goto( + torture, ok, ret, fail, "lpcfg_set_option failed"); + + ldb = ldb_wrap_connect( + torture, + torture->ev, + torture->lp_ctx, + url, + NULL, + credentials, + 0); + torture_assert_goto( + torture, ldb!=NULL, ret, fail, "ldb_wrap_connect failed"); + + rootdn = ldb_dn_new(ldb, ldb, NULL); + torture_assert_goto( + torture, rootdn!=NULL, ret, fail, "ldb_dn_new failed"); + + rc = ldb_search( + ldb, /* ldb */ + ldb, /* mem_ctx */ + &result, /* result */ + rootdn, /* base */ + LDB_SCOPE_BASE, /* scope */ + NULL, /* attrs */ + "(objectclass=*)"); /* exp_fmt */ + torture_assert_goto( + torture, rc==LDB_SUCCESS, ret, fail, "1st ldb_search failed"); + + do { + smb_msleep(1000); + + rc = ldb_search( + ldb, /* ldb */ + ldb, /* mem_ctx */ + &result, /* result */ + rootdn, /* base */ + LDB_SCOPE_BASE, /* scope */ + NULL, /* attrs */ + "(objectclass=*)"); /* exp_fmt */ + printf("ldb_search returned %s\n", ldb_strerror(rc)); + TALLOC_FREE(result); + + if (rc != LDB_SUCCESS) { + break; + } + } while (!timeval_expired(&endtime)); + + torture_assert_goto( + torture, + rc==LDB_ERR_PROTOCOL_ERROR, + ret, + fail, + "expected LDB_ERR_PROTOCOL_ERROR after 4 seconds"); + + ret = true; +fail: + TALLOC_FREE(ldb); + return ret; +} diff --git a/source4/torture/wscript_build b/source4/torture/wscript_build index ea7653454890..9149d24ac118 100644 --- a/source4/torture/wscript_build +++ b/source4/torture/wscript_build @@ -263,6 +263,7 @@ bld.SAMBA_MODULE('TORTURE_LDAP', ldap/cldapbench.c ldap/ldap_sort.c ldap/nested_search.c + ldap/session_expiry.c ''', subsystem='smbtorture', deps='cli-ldap cli_cldap samdb popt POPT_CREDENTIALS torture ldbsamba', -- 2.20.1 From 2f68ee4d44ff03d8c100063be963b1c86bbccd23 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Fri, 7 Aug 2020 13:40:58 +0200 Subject: [PATCH 03/14] ldap_server: Add the krb5 expiry to conn->limits Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit 77f72fb01faba45babfe6080f805361492ce49e5) --- source4/ldap_server/ldap_bind.c | 15 +++++++++++++++ source4/ldap_server/ldap_server.c | 4 ++++ source4/ldap_server/ldap_server.h | 1 + 3 files changed, 20 insertions(+) diff --git a/source4/ldap_server/ldap_bind.c b/source4/ldap_server/ldap_bind.c index 5cddecd79be4..54493d040fff 100644 --- a/source4/ldap_server/ldap_bind.c +++ b/source4/ldap_server/ldap_bind.c @@ -29,6 +29,7 @@ #include "auth/gensec/gensec_tstream.h" #include "param/param.h" #include "../lib/util/tevent_ntstatus.h" +#include "lib/util/time_basic.h" static char *ldapsrv_bind_error_msg(TALLOC_CTX *mem_ctx, HRESULT hresult, @@ -483,6 +484,7 @@ static void ldapsrv_BindSASL_done(struct tevent_req *subreq) const char *errstr = NULL; char *ldb_errstring = NULL; DATA_BLOB output = data_blob_null; + NTTIME expire_time_nt; status = gensec_update_recv(subreq, call, &output); TALLOC_FREE(subreq); @@ -602,6 +604,19 @@ static void ldapsrv_BindSASL_done(struct tevent_req *subreq) goto do_reply; } + expire_time_nt = gensec_expire_time(conn->gensec); + if (expire_time_nt != GENSEC_EXPIRE_TIME_INFINITY) { + struct timeval_buf buf; + + nttime_to_timeval(&conn->limits.expire_time, expire_time_nt); + + DBG_DEBUG("Setting connection expire_time to %s\n", + timeval_str_buf(&conn->limits.expire_time, + false, + true, + &buf)); + } + if (context != NULL) { const void *ptr = NULL; diff --git a/source4/ldap_server/ldap_server.c b/source4/ldap_server/ldap_server.c index a9b162b284e6..3a37ddbcaf95 100644 --- a/source4/ldap_server/ldap_server.c +++ b/source4/ldap_server/ldap_server.c @@ -47,6 +47,7 @@ #include "../lib/util/tevent_ntstatus.h" #include "../libcli/util/tstream.h" #include "libds/common/roles.h" +#include "lib/util/time.h" static void ldapsrv_terminate_connection_done(struct tevent_req *subreq); @@ -178,6 +179,9 @@ static int ldapsrv_load_limits(struct ldapsrv_connection *conn) conn->limits.max_page_size = 1000; conn->limits.max_notifications = 5; conn->limits.search_timeout = 120; + conn->limits.expire_time = (struct timeval) { + .tv_sec = get_time_t_max(), + }; tmp_ctx = talloc_new(conn); diff --git a/source4/ldap_server/ldap_server.h b/source4/ldap_server/ldap_server.h index e1efe8a49438..74c19fd2fbc4 100644 --- a/source4/ldap_server/ldap_server.h +++ b/source4/ldap_server/ldap_server.h @@ -61,6 +61,7 @@ struct ldapsrv_connection { int max_notifications; int search_timeout; struct timeval endtime; + struct timeval expire_time; /* Krb5 ticket expiry */ const char *reason; } limits; -- 2.20.1 From 0d88075111f02c29f40c138698bfd221027de2fa Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Mon, 10 Aug 2020 16:24:04 +0200 Subject: [PATCH 04/14] ldap_server: Terminate LDAP connections on krb ticket expiry See RFC4511 section 4.4.1 and https://lists.samba.org/archive/cifs-protocol/2020-August/003515.html for details: Windows terminates LDAP connections when the krb5 ticket expires, Samba should do the same. This patch slightly deviates from Windows behaviour by sending a LDAP exop response with msgid 0 that is ASN1-encoded conforming to RFC4511. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit eb72f887b0bf91c050fd5d911f58a1b3ff9b8bcc) --- selftest/knownfail.d/ldap | 1 - source4/ldap_server/ldap_backend.c | 37 ++++++++++++++++++ source4/ldap_server/ldap_server.c | 62 ++++++++++++++++++++++++++++++ source4/ldap_server/ldap_server.h | 1 + 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/selftest/knownfail.d/ldap b/selftest/knownfail.d/ldap index 21c1aca7e6e7..0331d3687d41 100644 --- a/selftest/knownfail.d/ldap +++ b/selftest/knownfail.d/ldap @@ -1,4 +1,3 @@ # the attributes too long test returns the wrong error ^samba4.ldap.python.+test_attribute_ranges_too_long samba4.ldap.python\(ad_dc_default\).*__main__.BasicTests.test_ldapSearchNoAttributes -^samba4.ldap.session-expiry.session-expiry\(ad_dc_default\) diff --git a/source4/ldap_server/ldap_backend.c b/source4/ldap_server/ldap_backend.c index bf724335a250..2839082daefd 100644 --- a/source4/ldap_server/ldap_backend.c +++ b/source4/ldap_server/ldap_backend.c @@ -1384,11 +1384,48 @@ static NTSTATUS ldapsrv_AbandonRequest(struct ldapsrv_call *call) return NT_STATUS_OK; } +static NTSTATUS ldapsrv_expired(struct ldapsrv_call *call) +{ + struct ldapsrv_reply *reply = NULL; + struct ldap_ExtendedResponse *r = NULL; + + DBG_DEBUG("Sending connection expired message\n"); + + reply = ldapsrv_init_reply(call, LDAP_TAG_ExtendedResponse); + if (reply == NULL) { + return NT_STATUS_NO_MEMORY; + } + + /* + * According to RFC4511 section 4.4.1 this has a msgid of 0 + */ + reply->msg->messageid = 0; + + r = &reply->msg->r.ExtendedResponse; + r->response.resultcode = LDB_ERR_UNAVAILABLE; + r->response.errormessage = "The server has timed out this connection"; + r->oid = "1.3.6.1.4.1.1466.20036"; /* see rfc4511 section 4.4.1 */ + + ldapsrv_queue_reply(call, reply); + return NT_STATUS_OK; +} + NTSTATUS ldapsrv_do_call(struct ldapsrv_call *call) { unsigned int i; struct ldap_message *msg = call->request; + struct ldapsrv_connection *conn = call->conn; NTSTATUS status; + bool expired; + + expired = timeval_expired(&conn->limits.expire_time); + if (expired) { + status = ldapsrv_expired(call); + if (!NT_STATUS_IS_OK(status)) { + return status; + } + return NT_STATUS_NETWORK_SESSION_EXPIRED; + } /* Check for undecoded critical extensions */ for (i=0; msg->controls && msg->controls[i]; i++) { diff --git a/source4/ldap_server/ldap_server.c b/source4/ldap_server/ldap_server.c index 3a37ddbcaf95..fd2309b7ce3b 100644 --- a/source4/ldap_server/ldap_server.c +++ b/source4/ldap_server/ldap_server.c @@ -69,6 +69,7 @@ static void ldapsrv_terminate_connection(struct ldapsrv_connection *conn, tevent_queue_stop(conn->sockets.send_queue); TALLOC_FREE(conn->sockets.read_req); + TALLOC_FREE(conn->deferred_expire_disconnect); if (conn->active_call) { tevent_req_cancel(conn->active_call); conn->active_call = NULL; @@ -1016,16 +1017,62 @@ static struct tevent_req *ldapsrv_process_call_send(TALLOC_CTX *mem_ctx, return req; } +static void ldapsrv_disconnect_ticket_expired(struct tevent_req *subreq); + static void ldapsrv_process_call_trigger(struct tevent_req *req, void *private_data) { struct ldapsrv_process_call_state *state = tevent_req_data(req, struct ldapsrv_process_call_state); + struct ldapsrv_connection *conn = state->call->conn; NTSTATUS status; + if (conn->deferred_expire_disconnect != NULL) { + /* + * Just drop this on the floor + */ + tevent_req_done(req); + return; + } + /* make the call */ status = ldapsrv_do_call(state->call); + + if (NT_STATUS_EQUAL(status, NT_STATUS_NETWORK_SESSION_EXPIRED)) { + /* + * For testing purposes, defer the TCP disconnect + * after having sent the msgid 0 + * 1.3.6.1.4.1.1466.20036 exop response. LDAP clients + * should not wait for the TCP connection to close but + * handle this packet equivalent to a TCP + * disconnect. This delay enables testing both cases + * in LDAP client libraries. + */ + + int defer_msec = lpcfg_parm_int( + conn->lp_ctx, + NULL, + "ldap_server", + "delay_expire_disconnect", + 0); + + conn->deferred_expire_disconnect = tevent_wakeup_send( + conn, + conn->connection->event.ctx, + timeval_current_ofs_msec(defer_msec)); + if (tevent_req_nomem(conn->deferred_expire_disconnect, req)) { + return; + } + tevent_req_set_callback( + conn->deferred_expire_disconnect, + ldapsrv_disconnect_ticket_expired, + conn); + + tevent_req_done(req); + return; + } + if (!NT_STATUS_IS_OK(status)) { tevent_req_nterror(req, status); return; @@ -1034,6 +1081,21 @@ static void ldapsrv_process_call_trigger(struct tevent_req *req, tevent_req_done(req); } +static void ldapsrv_disconnect_ticket_expired(struct tevent_req *subreq) +{ + struct ldapsrv_connection *conn = tevent_req_callback_data( + subreq, struct ldapsrv_connection); + bool ok; + + ok = tevent_wakeup_recv(subreq); + TALLOC_FREE(subreq); + if (!ok) { + DBG_WARNING("tevent_wakeup_recv failed\n"); + } + conn->deferred_expire_disconnect = NULL; + ldapsrv_terminate_connection(conn, "network session expired"); +} + static NTSTATUS ldapsrv_process_call_recv(struct tevent_req *req) { NTSTATUS status; diff --git a/source4/ldap_server/ldap_server.h b/source4/ldap_server/ldap_server.h index 74c19fd2fbc4..c94bd914b9b7 100644 --- a/source4/ldap_server/ldap_server.h +++ b/source4/ldap_server/ldap_server.h @@ -66,6 +66,7 @@ struct ldapsrv_connection { } limits; struct tevent_req *active_call; + struct tevent_req *deferred_expire_disconnect; struct ldapsrv_call *pending_calls; }; -- 2.20.1 From 815af91887a60da0867fa5c574a064ac3a4e9a22 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 16:16:12 +0200 Subject: [PATCH 05/14] tldap: Only free() ld->pending if "req" is part of it Best reviewed with "git show -U10". We need to check that "req" is actually the last request that is being freed before freeing the whole array. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit b85dbc9ccf80d8c19aff33c1da83954e5d6a37ef) --- source3/lib/tldap.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index bf5fc05d785e..9da0e0c086e6 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -492,11 +492,6 @@ static void tldap_msg_unset_pending(struct tevent_req *req) tevent_req_set_cleanup_fn(req, NULL); - if (num_pending == 1) { - TALLOC_FREE(ld->pending); - return; - } - for (i=0; ipending[i]) { break; @@ -511,6 +506,11 @@ static void tldap_msg_unset_pending(struct tevent_req *req) return; } + if (num_pending == 1) { + TALLOC_FREE(ld->pending); + return; + } + /* * Remove ourselves from the cli->pending array */ -- 2.20.1 From d34b4fbdd6b0e5bbab123285d593924496eb42a4 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 16:54:34 +0200 Subject: [PATCH 06/14] tldap: Fix tldap_msg_received() The callback of "req" might have destroyed "ld", we can't reference this anymore after calling tevent_req_done(req). Defer calling the callbacks, which also means that the callbacks can't have added anything to ld->pending. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit f816ccb8f4d212fe7f6bf36f90cbb9297c899786) --- source3/lib/tldap.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index 9da0e0c086e6..9274db126c46 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -677,21 +677,13 @@ static void tldap_msg_received(struct tevent_req *subreq) tldap_msg_unset_pending(req); num_pending = talloc_array_length(ld->pending); + tevent_req_defer_callback(req, state->ev); tevent_req_done(req); done: if (num_pending == 0) { return; } - if (talloc_array_length(ld->pending) > num_pending) { - /* - * The callback functions called from tevent_req_done() above - * have put something on the pending queue. We don't have to - * trigger the read_ldap_send(), tldap_msg_set_pending() has - * done it for us already. - */ - return; - } state = tevent_req_data(ld->pending[0], struct tldap_msg_state); subreq = read_ldap_send(ld->pending, state->ev, ld->conn); -- 2.20.1 From 20788ea6c50b4355c20ad3c82d0313df0a05df1b Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Mon, 17 Aug 2020 21:59:48 +0200 Subject: [PATCH 07/14] tldap: Always remove ourselves from ld->pending at cleanup time Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit 14f6d1996ec38620b1c05a3b6c0e26dd21801fac) --- source3/lib/tldap.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index 9274db126c46..09f8446677d7 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -529,14 +529,7 @@ static void tldap_msg_unset_pending(struct tevent_req *req) static void tldap_msg_cleanup(struct tevent_req *req, enum tevent_req_state req_state) { - switch (req_state) { - case TEVENT_REQ_USER_ERROR: - case TEVENT_REQ_RECEIVED: - tldap_msg_unset_pending(req); - return; - default: - return; - } + tldap_msg_unset_pending(req); } static bool tldap_msg_set_pending(struct tevent_req *req) -- 2.20.1 From c5c6a361459dd84bdc4d05ae999365e69fd84ff6 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 17:14:14 +0200 Subject: [PATCH 08/14] tldap: Maintain the ldap read request in tldap_context Required for proper connection rundown, we need to TALLOC_FREE() the read request before shutting down the tstream Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit cb852c9dc0d0fa1d3e7473082ad6b460106b314b) --- source3/lib/tldap.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index 09f8446677d7..3bf58d0df3e6 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -87,6 +87,7 @@ struct tldap_context { int msgid; struct tevent_queue *outgoing; struct tevent_req **pending; + struct tevent_req *read_req; /* For the sync wrappers we need something like get_last_error... */ struct tldap_message *last_msg; @@ -539,7 +540,6 @@ static bool tldap_msg_set_pending(struct tevent_req *req) struct tldap_context *ld; struct tevent_req **pending; int num_pending; - struct tevent_req *subreq; ld = state->ld; num_pending = tldap_pending_reqs(ld); @@ -553,7 +553,7 @@ static bool tldap_msg_set_pending(struct tevent_req *req) ld->pending = pending; tevent_req_set_cleanup_fn(req, tldap_msg_cleanup); - if (num_pending > 0) { + if (ld->read_req != NULL) { return true; } @@ -561,12 +561,12 @@ static bool tldap_msg_set_pending(struct tevent_req *req) * We're the first one, add the read_ldap request that waits for the * answer from the server */ - subreq = read_ldap_send(ld->pending, state->ev, ld->conn); - if (subreq == NULL) { + ld->read_req = read_ldap_send(ld->pending, state->ev, ld->conn); + if (ld->read_req == NULL) { tldap_msg_unset_pending(req); return false; } - tevent_req_set_callback(subreq, tldap_msg_received, ld); + tevent_req_set_callback(ld->read_req, tldap_msg_received, ld); return true; } @@ -619,6 +619,7 @@ static void tldap_msg_received(struct tevent_req *subreq) received = read_ldap_recv(subreq, talloc_tos(), &inbuf, &err); TALLOC_FREE(subreq); + ld->read_req = NULL; if (received == -1) { ld->server_down = true; status = TLDAP_SERVER_DOWN; @@ -679,12 +680,12 @@ static void tldap_msg_received(struct tevent_req *subreq) } state = tevent_req_data(ld->pending[0], struct tldap_msg_state); - subreq = read_ldap_send(ld->pending, state->ev, ld->conn); - if (subreq == NULL) { + ld->read_req = read_ldap_send(ld->pending, state->ev, ld->conn); + if (ld->read_req == NULL) { status = TLDAP_NO_MEMORY; goto fail; } - tevent_req_set_callback(subreq, tldap_msg_received, ld); + tevent_req_set_callback(ld->read_req, tldap_msg_received, ld); return; fail: -- 2.20.1 From 194bdd78f5b353743bf3f70e7833663b389c9797 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 17:30:22 +0200 Subject: [PATCH 09/14] tldap: Centralize connection rundown on error Whenever send or recv return -1, we have to cancel all pending requests and our transport stream is no longer usable: Discard it upon such an error. To avoid duplicate state, tldap_connection_ok() now looks at whether we have a tstream_context around. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit f745f5b12560dbcb7be6f3ffb3bc10704c87149c) --- source3/lib/tldap.c | 72 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index 3bf58d0df3e6..de9f0c70ff11 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -83,7 +83,6 @@ struct tldap_ctx_attribute { struct tldap_context { int ld_version; struct tstream_context *conn; - bool server_down; int msgid; struct tevent_queue *outgoing; struct tevent_req **pending; @@ -186,10 +185,22 @@ struct tldap_context *tldap_context_create(TALLOC_CTX *mem_ctx, int fd) bool tldap_connection_ok(struct tldap_context *ld) { + int ret; + if (ld == NULL) { return false; } - return !ld->server_down; + + if (ld->conn == NULL) { + return false; + } + + ret = tstream_pending_bytes(ld->conn); + if (ret == -1) { + return false; + } + + return true; } static size_t tldap_pending_reqs(struct tldap_context *ld) @@ -425,6 +436,43 @@ static bool tldap_push_controls(struct asn1_data *data, return asn1_pop_tag(data); /* ASN1_CONTEXT(0) */ } +#define tldap_context_disconnect(ld, status) \ + _tldap_context_disconnect(ld, status, __location__) + +static void _tldap_context_disconnect(struct tldap_context *ld, + TLDAPRC status, + const char *location) +{ + if (ld->conn == NULL) { + /* + * We don't need to tldap_debug() on + * a potential 2nd run. + * + * The rest of the function would just + * be a noop for the 2nd run anyway. + */ + return; + } + + tldap_debug(ld, TLDAP_DEBUG_WARNING, + "tldap_context_disconnect: %s at %s\n", + tldap_rc2string(status), + location); + tevent_queue_stop(ld->outgoing); + TALLOC_FREE(ld->read_req); + TALLOC_FREE(ld->conn); + + while (talloc_array_length(ld->pending) > 0) { + struct tevent_req *req = NULL; + struct tldap_msg_state *state = NULL; + + req = ld->pending[0]; + state = tevent_req_data(req, struct tldap_msg_state); + tevent_req_defer_callback(req, state->ev); + tevent_req_ldap_error(req, status); + } +} + static void tldap_msg_sent(struct tevent_req *subreq); static void tldap_msg_received(struct tevent_req *subreq); @@ -438,6 +486,7 @@ static struct tevent_req *tldap_msg_send(TALLOC_CTX *mem_ctx, struct tevent_req *req, *subreq; struct tldap_msg_state *state; DATA_BLOB blob; + bool ok; tldap_debug(ld, TLDAP_DEBUG_TRACE, "tldap_msg_send: sending msg %d\n", id); @@ -450,7 +499,8 @@ static struct tevent_req *tldap_msg_send(TALLOC_CTX *mem_ctx, state->ev = ev; state->id = id; - if (state->ld->server_down) { + ok = tldap_connection_ok(ld); + if (!ok) { tevent_req_ldap_error(req, TLDAP_SERVER_DOWN); return tevent_req_post(req, ev); } @@ -582,7 +632,7 @@ static void tldap_msg_sent(struct tevent_req *subreq) nwritten = tstream_writev_queue_recv(subreq, &err); TALLOC_FREE(subreq); if (nwritten == -1) { - state->ld->server_down = true; + tldap_context_disconnect(state->ld, TLDAP_SERVER_DOWN); tevent_req_ldap_error(req, TLDAP_SERVER_DOWN); return; } @@ -612,7 +662,7 @@ static void tldap_msg_received(struct tevent_req *subreq) ssize_t received; size_t num_pending; int i, err; - TLDAPRC status; + TLDAPRC status = TLDAP_PROTOCOL_ERROR; int id; uint8_t type; bool ok; @@ -621,13 +671,16 @@ static void tldap_msg_received(struct tevent_req *subreq) TALLOC_FREE(subreq); ld->read_req = NULL; if (received == -1) { - ld->server_down = true; status = TLDAP_SERVER_DOWN; goto fail; } data = asn1_init(talloc_tos(), ASN1_MAX_TREE_DEPTH); if (data == NULL) { + /* + * We have to disconnect all, we can't tell which of + * the requests this reply is for. + */ status = TLDAP_NO_MEMORY; goto fail; } @@ -689,12 +742,7 @@ static void tldap_msg_received(struct tevent_req *subreq) return; fail: - while (talloc_array_length(ld->pending) > 0) { - req = ld->pending[0]; - state = tevent_req_data(req, struct tldap_msg_state); - tevent_req_defer_callback(req, state->ev); - tevent_req_ldap_error(req, status); - } + tldap_context_disconnect(ld, status); } static TLDAPRC tldap_msg_recv(struct tevent_req *req, TALLOC_CTX *mem_ctx, -- 2.20.1 From df8feb164a6e044e76f417a4209f4636e3688332 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 17:44:42 +0200 Subject: [PATCH 10/14] tldap: Make sure all requests are cancelled on rundown Put messages into the ld->pending array before sending them out, not after they have been sent. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit 2a2a6b27cccb2409d321c7e03feb8baa047d1bf4) --- source3/lib/tldap.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index de9f0c70ff11..52012ab9a55e 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -29,6 +29,7 @@ #include "../lib/util/tevent_unix.h" static TLDAPRC tldap_simple_recv(struct tevent_req *req); +static bool tldap_msg_set_pending(struct tevent_req *req); #define TEVENT_TLDAP_RC_MAGIC (0x87bcd26e) @@ -521,6 +522,11 @@ static struct tevent_req *tldap_msg_send(TALLOC_CTX *mem_ctx, return tevent_req_post(req, ev); } + if (!tldap_msg_set_pending(req)) { + tevent_req_oom(req); + return tevent_req_post(req, ev);; + } + state->iov.iov_base = (void *)blob.data; state->iov.iov_len = blob.length; @@ -633,12 +639,6 @@ static void tldap_msg_sent(struct tevent_req *subreq) TALLOC_FREE(subreq); if (nwritten == -1) { tldap_context_disconnect(state->ld, TLDAP_SERVER_DOWN); - tevent_req_ldap_error(req, TLDAP_SERVER_DOWN); - return; - } - - if (!tldap_msg_set_pending(req)) { - tevent_req_oom(req); return; } } -- 2.20.1 From 4257dae23207f8930f17880c68fe7f71c274944d Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 12 Aug 2020 13:26:18 +0200 Subject: [PATCH 11/14] tldap: Add PRINTF_ATTRIBUTE declaration to tldap_debug() Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit a2b281bed022c04427ef478529462ff84fe42908) --- source3/lib/tldap.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index 52012ab9a55e..af6959ee895a 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -133,6 +133,11 @@ void tldap_set_debug(struct tldap_context *ld, ld->log_private = log_private; } +static void tldap_debug( + struct tldap_context *ld, + enum tldap_debug_level level, + const char *fmt, ...) PRINTF_ATTRIBUTE(3,4); + static void tldap_debug(struct tldap_context *ld, enum tldap_debug_level level, const char *fmt, ...) -- 2.20.1 From f6f623cfa7e6e3c52d533b3e34bd385cd214cd6e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 11 Aug 2020 18:24:39 +0200 Subject: [PATCH 12/14] idmap_ad: Pass tldap debug messages on to DEBUG() Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Stefan Metzmacher Reviewed-by: Volker Lendecke (cherry picked from commit 7af2df01dff62d6d9ca572f320ef60dea41d6064) --- source3/winbindd/idmap_ad.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source3/winbindd/idmap_ad.c b/source3/winbindd/idmap_ad.c index 6d879cdf4d7b..3bfeeee2d74b 100644 --- a/source3/winbindd/idmap_ad.c +++ b/source3/winbindd/idmap_ad.c @@ -254,6 +254,41 @@ static TLDAPRC get_posix_schema_names(struct tldap_context *ld, return TLDAP_SUCCESS; } +static void idmap_ad_tldap_debug(void *log_private, + enum tldap_debug_level level, + const char *fmt, + va_list ap) +{ + int samba_level = -1; + + switch (level) { + case TLDAP_DEBUG_FATAL: + samba_level = DBGLVL_ERR; + break; + case TLDAP_DEBUG_ERROR: + samba_level = DBGLVL_ERR; + break; + case TLDAP_DEBUG_WARNING: + samba_level = DBGLVL_WARNING; + break; + case TLDAP_DEBUG_TRACE: + samba_level = DBGLVL_DEBUG; + break; + } + + if (CHECK_DEBUGLVL(samba_level)) { + char *s = NULL; + int ret; + + ret = vasprintf(&s, fmt, ap); + if (ret == -1) { + return; + } + DEBUG(samba_level, ("idmap_ad_tldap: %s", s)); + free(s); + } +} + static NTSTATUS idmap_ad_get_tldap_ctx(TALLOC_CTX *mem_ctx, const char *domname, struct tldap_context **pld) @@ -307,6 +342,7 @@ static NTSTATUS idmap_ad_get_tldap_ctx(TALLOC_CTX *mem_ctx, TALLOC_FREE(dcinfo); return NT_STATUS_NO_MEMORY; } + tldap_set_debug(ld, idmap_ad_tldap_debug, NULL); /* * Here we use or own machine account as -- 2.20.1 From 7fffedae3c6d9c427b8c0868af1811882e5c0d5c Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Thu, 13 Aug 2020 14:59:58 +0200 Subject: [PATCH 13/14] test: Test winbind idmap_ad ticket expiry behaviour We need to make sure that winbind's idmap_ad deals fine with an expired krb ticket used to connect to AD via LDAP. In a customer situation we have seen the RFC4511 section 4.4.1 unsolicited ldap exop response coming through, but the TCP disconnect that Windows seems to do after that did not make it. Winbind deals fine with a TCP disconnect, but right now it does not handle just the section 4.4.1 response properly: It completely hangs. This test requests a ticket valid for 5 seconds and makes the LDAP server postpone the TCP disconnect after the ticket expiry for 10 seconds. The tests that winbind reacts to the ticket expiry exop response by making sure in this situation the wbinfo call running into the issue takes less than 8 seconds. If it did not look at the expiry exop response, it would take more than 10 seconds. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher (cherry picked from commit a4ecd112e7754ab25bcae749594952a28c4c8905) --- nsswitch/tests/test_ticket_expiry.sh | 74 ++++++++++++++++++++++++++++ selftest/knownfail.d/ticket_expiry | 1 + selftest/target/Samba3.pm | 1 + selftest/target/Samba4.pm | 6 ++- source3/selftest/tests.py | 5 ++ 5 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 nsswitch/tests/test_ticket_expiry.sh create mode 100644 selftest/knownfail.d/ticket_expiry diff --git a/nsswitch/tests/test_ticket_expiry.sh b/nsswitch/tests/test_ticket_expiry.sh new file mode 100755 index 000000000000..3b98b0fe87af --- /dev/null +++ b/nsswitch/tests/test_ticket_expiry.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Test winbind ad backend behaviour when the kerberos ticket expires + +if [ $# -ne 1 ]; then + echo Usage: $0 DOMAIN + exit 1 +fi + +DOMAIN="$1" + +wbinfo="$VALGRIND $BINDIR/wbinfo" +net="$VALGRIND $BINDIR/net" + +failed=0 + +. `dirname $0`/../../testprogs/blackbox/subunit.sh + +DOMAIN_SID=$($wbinfo -n "$DOMAIN/" | cut -f 1 -d " ") +if [ $? -ne 0 ] ; then + echo "Could not find domain SID" | subunit_fail_test "test_idmap_ad" + exit 1 +fi +ADMINS_SID="$DOMAIN_SID-512" + +# Previous tests might have put in a mapping +$net cache del IDMAP/SID2XID/"$ADMINS_SID" + +# Trigger a winbind ad connection with a 5-second ticket lifetime, +# see the smb.conf for the ad_member_idmap_ad environment we're in +# +# We expect failure here because there are no mappings in AD. In this +# test we are only interested in the winbind LDAP connection as such, +# we don't really care whether idmap_ad works fine. This is done in +# different tests. And a negative lookup also triggers the LDAP +# connection. + +testit_expect_failure "Deleting0 IDMAP/SID2XID/$ADMINS_SID" $net cache del IDMAP/SID2XID/"$ADMINS_SID" || + failed=$(expr $failed + 1) + +testit_expect_failure "Expecting failure1, no mapping in AD" $wbinfo --sid-to-gid "$ADMINS_SID" || + failed=$(expr $failed + 1) + +testit "Deleting1 IDMAP/SID2XID/$ADMINS_SID" $net cache del IDMAP/SID2XID/"$ADMINS_SID" || + failed=$(expr $failed + 1) + +# allow our kerberos ticket to expire +testit "Sleeping for 6 seconds" sleep 6 || failed=$(expr $failed + 1) + +# Try again, check how long it took to recover from ticket expiry +# +# On the LDAP connection two things happen: First we get an +# unsolicited exop response telling us the network session was +# abandoned, and secondly the LDAP server will kill the TCP +# connection. Our ldap server is configured to defer the TCP +# disconnect by 10 seconds. We need to make sure that winbind already +# reacts to the unsolicited exop reply, discarding the connection. The +# only way is to make sure the following wbinfo does not take too +# long. + +# We need to do the test command in this funny way as on gitlab we're +# using the bash builtin + +START=$(date +%s) +testit_expect_failure "Expecting failure2, no mapping in AD" $wbinfo --sid-to-gid "$ADMINS_SID" || + failed=$(expr $failed + 1) +END=$(date +%s) +DURATION=$(expr $END - $START) +testit "timeout DURATION[$DURATION] < 8" test "$DURATION" -le 8 || + failed=$(expr $failed + 1) + +testit "Deleting2 IDMAP/SID2XID/$ADMINS_SID" $net cache del IDMAP/SID2XID/"$ADMINS_SID" || + failed=$(expr $failed + 1) + +exit $failed diff --git a/selftest/knownfail.d/ticket_expiry b/selftest/knownfail.d/ticket_expiry new file mode 100644 index 000000000000..04e508a9f8e6 --- /dev/null +++ b/selftest/knownfail.d/ticket_expiry @@ -0,0 +1 @@ +idmap_ad.ticket_expiry.timeout.DURATION.*\(ad_member_idmap_ad:local\) diff --git a/selftest/target/Samba3.pm b/selftest/target/Samba3.pm index a15979199f0b..c22c567cd654 100755 --- a/selftest/target/Samba3.pm +++ b/selftest/target/Samba3.pm @@ -728,6 +728,7 @@ sub setup_ad_member_idmap_ad idmap config $dcvars->{DOMAIN} : range = 2000000-2999999 idmap config $dcvars->{TRUST_DOMAIN} : backend = ad idmap config $dcvars->{TRUST_DOMAIN} : range = 2000000-2999999 + gensec_gssapi:requested_life_time = 5 "; my $ret = $self->provision($prefix, $dcvars->{DOMAIN}, diff --git a/selftest/target/Samba4.pm b/selftest/target/Samba4.pm index a599099c59db..4758a5513d14 100755 --- a/selftest/target/Samba4.pm +++ b/selftest/target/Samba4.pm @@ -1709,7 +1709,11 @@ sub provision_fl2008r2dc($$$) my ($self, $prefix, $dcvars) = @_; print "PROVISIONING DC WITH FOREST LEVEL 2008r2...\n"; - my $extra_conf_options = "ldap server require strong auth = no"; + my $extra_conf_options = " + ldap server require strong auth = no + # delay by 10 seconds, 10^7 usecs + ldap_server:delay_expire_disconnect = 10000 +"; my $extra_provision_options = ["--use-ntvfs", "--base-schema=2008_R2"]; my $ret = $self->provision($prefix, "domain controller", diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py index 817fc783062a..430a7e327f2a 100755 --- a/source3/selftest/tests.py +++ b/source3/selftest/tests.py @@ -719,6 +719,11 @@ for t in tests: plansmbtorture4testsuite(t, "nt4_dc", '//$SERVER_IP/tmp -U$USERNAME%$PASSWORD') plansmbtorture4testsuite(t, "ad_dc", '//$SERVER/tmp -U$USERNAME%$PASSWORD') +plantestsuite( + "idmap_ad.ticket_expiry", + "ad_member_idmap_ad:local", + [os.path.join(samba3srcdir, "../nsswitch/tests/test_ticket_expiry.sh"), + '$DOMAIN']) test = 'rpc.lsa.lookupsids' auth_options = ["", "ntlm", "spnego", "spnego,ntlm", "spnego,smb1", "spnego,smb2"] -- 2.20.1 From c5e3fd2bacd39aafb8c42617bba396f0b48324e8 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 11 Aug 2020 18:09:14 +0200 Subject: [PATCH 14/14] tldap: Receiving "msgid == 0" means the connection is dead We never use msgid=0, see tldap_next_msgid(). RFC4511 section 4.4.1 says that the unsolicited disconnect response uses msgid 0. We don't parse this message, which supposedly is an extended response: Windows up to 2019 sends an extended response in an ASN.1 encoding that does not match RFC4511. Bug: https://bugzilla.samba.org/show_bug.cgi?id=14465 Signed-off-by: Volker Lendecke Reviewed-by: Stefan Metzmacher Autobuild-User(master): Volker Lendecke Autobuild-Date(master): Fri Aug 21 20:37:25 UTC 2020 on sn-devel-184 (cherry picked from commit ccaf661f7c75717341140e3fbfb2a48f96ea952c) --- selftest/knownfail.d/ticket_expiry | 1 - source3/lib/tldap.c | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) delete mode 100644 selftest/knownfail.d/ticket_expiry diff --git a/selftest/knownfail.d/ticket_expiry b/selftest/knownfail.d/ticket_expiry deleted file mode 100644 index 04e508a9f8e6..000000000000 --- a/selftest/knownfail.d/ticket_expiry +++ /dev/null @@ -1 +0,0 @@ -idmap_ad.ticket_expiry.timeout.DURATION.*\(ad_member_idmap_ad:local\) diff --git a/source3/lib/tldap.c b/source3/lib/tldap.c index af6959ee895a..0e39a3077285 100644 --- a/source3/lib/tldap.c +++ b/source3/lib/tldap.c @@ -704,6 +704,17 @@ static void tldap_msg_received(struct tevent_req *subreq) tldap_debug(ld, TLDAP_DEBUG_TRACE, "tldap_msg_received: got msg %d " "type %d\n", id, (int)type); + if (id == 0) { + tldap_debug( + ld, + TLDAP_DEBUG_WARNING, + "tldap_msg_received: got msgid 0 of " + "type %"PRIu8", disconnecting\n", + type); + tldap_context_disconnect(ld, TLDAP_SERVER_DOWN); + return; + } + num_pending = talloc_array_length(ld->pending); for (i=0; i