From eb8c51633a9037c2cb2a8b686c5db6d26a245e54 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 11 Aug 2017 11:31:05 +1200 Subject: [PATCH 01/23] ldb: Add LDB_UNPACK_DATA_FLAG_NO_ATTRS This will allow us to avoid a full unpack in situations where we just want to confirm if the DN exists Signed-off-by: Andrew Bartlett Reviewed-by: Garming Sam (cherry picked from commit bff81a2c9cc43a2cfec822dde94944d0295dd87f) --- lib/ldb/common/ldb_pack.c | 5 +++++ lib/ldb/include/ldb_module.h | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/lib/ldb/common/ldb_pack.c b/lib/ldb/common/ldb_pack.c index 1f1688a6c50..448c577ae1b 100644 --- a/lib/ldb/common/ldb_pack.c +++ b/lib/ldb/common/ldb_pack.c @@ -301,6 +301,11 @@ int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb, goto failed; } + + if (flags & LDB_UNPACK_DATA_FLAG_NO_ATTRS) { + return 0; + } + if (message->num_elements == 0) { return 0; } diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h index 8ad212a1e08..71b407496f0 100644 --- a/lib/ldb/include/ldb_module.h +++ b/lib/ldb/include/ldb_module.h @@ -518,6 +518,10 @@ int ldb_unpack_data(struct ldb_context *ldb, * LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC is also specified. * * Likewise if LDB_UNPACK_DATA_FLAG_NO_DN is specified, the DN is omitted. + * + * If LDB_UNPACK_DATA_FLAG_NO_ATTRS is specified, then no attributes + * are unpacked or returned. + * */ int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb, const struct ldb_val *data, @@ -530,6 +534,7 @@ int ldb_unpack_data_only_attr_list_flags(struct ldb_context *ldb, #define LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC 0x0001 #define LDB_UNPACK_DATA_FLAG_NO_DN 0x0002 #define LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC 0x0004 +#define LDB_UNPACK_DATA_FLAG_NO_ATTRS 0x0008 /** Forces a specific ldb handle to use the global event context. -- 2.11.0 From d962dbe682ed769abdbc1e7d1f8ecc7589dd0ecf Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 16 Aug 2017 12:51:09 +1200 Subject: [PATCH 02/23] ldb_tdb: Rework ltdb_modify_internal() to use ltdb_search_dn1() internally This avoids duplicate code and allows us to use the allocation-avoiding LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC flag. We can not use LDB_UNPACK_DATA_FLAG_NO_VALUES_ALLOC as el2->values is talloc_realloc()ed in the routine. Signed-off-by: Andrew Bartlett Reviewed-by: Garming Sam Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Tue Aug 29 11:13:50 CEST 2017 on sn-devel-144 (cherry picked from commit 3164c0ac54685d6ae430e2cb3bb50a9ad7f3e7fc) --- lib/ldb/ldb_tdb/ldb_tdb.c | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c index 2ac1967ee15..bc8780ad7ee 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/lib/ldb/ldb_tdb/ldb_tdb.c @@ -686,52 +686,34 @@ int ltdb_modify_internal(struct ldb_module *module, struct ldb_request *req) { struct ldb_context *ldb = ldb_module_get_ctx(module); - void *data = ldb_module_get_private(module); - struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private); - TDB_DATA tdb_key, tdb_data; - struct ldb_val ldb_data; struct ldb_message *msg2; unsigned int i, j; int ret = LDB_SUCCESS, idx; struct ldb_control *control_permissive = NULL; + TALLOC_CTX *mem_ctx = talloc_new(req); + if (mem_ctx == NULL) { + return ldb_module_oom(module); + } + if (req) { control_permissive = ldb_request_get_control(req, LDB_CONTROL_PERMISSIVE_MODIFY_OID); } - tdb_key = ltdb_key(module, msg->dn); - if (!tdb_key.dptr) { - return LDB_ERR_OTHER; - } - - tdb_data = tdb_fetch(ltdb->tdb, tdb_key); - if (!tdb_data.dptr) { - talloc_free(tdb_key.dptr); - return ltdb_err_map(tdb_error(ltdb->tdb)); - } - - msg2 = ldb_msg_new(tdb_key.dptr); + msg2 = ldb_msg_new(mem_ctx); if (msg2 == NULL) { - free(tdb_data.dptr); ret = LDB_ERR_OTHER; goto done; } - ldb_data.data = tdb_data.dptr; - ldb_data.length = tdb_data.dsize; - - ret = ldb_unpack_data(ldb_module_get_ctx(module), &ldb_data, msg2); - free(tdb_data.dptr); - if (ret == -1) { - ret = LDB_ERR_OTHER; + ret = ltdb_search_dn1(module, msg->dn, + msg2, + LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC); + if (ret != LDB_SUCCESS) { goto done; } - if (!msg2->dn) { - msg2->dn = msg->dn; - } - for (i=0; inum_elements; i++) { struct ldb_message_element *el = &msg->elements[i], *el2; struct ldb_val *vals; @@ -1018,7 +1000,7 @@ int ltdb_modify_internal(struct ldb_module *module, } done: - talloc_free(tdb_key.dptr); + TALLOC_FREE(mem_ctx); return ret; } -- 2.11.0 From 7ccdd213dc6737507bc04eadacbc010b5fb3009c Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 24 Aug 2017 16:29:58 +1200 Subject: [PATCH 03/23] ldb: Fix tests to call the parent tearDown(), not setUp in tearDown Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 841e763a2b1dba1c0557d4f47a3674d6a555cab0) --- lib/ldb/tests/python/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py index 0c3b20e28ad..64f6989b1ae 100755 --- a/lib/ldb/tests/python/api.py +++ b/lib/ldb/tests/python/api.py @@ -54,7 +54,7 @@ class SimpleLdb(TestCase): def tearDown(self): shutil.rmtree(self.testdir) - super(SimpleLdb, self).setUp() + super(SimpleLdb, self).tearDown() def test_connect(self): ldb.Ldb(self.filename) @@ -616,7 +616,7 @@ class DnTests(TestCase): def tearDown(self): shutil.rmtree(self.testdir) - super(DnTests, self).setUp() + super(DnTests, self).tearDown() def test_set_dn_invalid(self): x = ldb.Message() -- 2.11.0 From 1fe36c8b9707e3162cc4fbf2bc3d9880edd10f45 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 24 Aug 2017 17:01:14 +1200 Subject: [PATCH 04/23] ldb: Add tests for indexed and unindexed search expressions Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Wed Aug 30 14:58:32 CEST 2017 on sn-devel-144 (cherry picked from commit a5a2243f073f00bbfd7692fb2fe68ea79830ae0d) --- lib/ldb/tests/python/api.py | 299 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) diff --git a/lib/ldb/tests/python/api.py b/lib/ldb/tests/python/api.py index 64f6989b1ae..b95266b0384 100755 --- a/lib/ldb/tests/python/api.py +++ b/lib/ldb/tests/python/api.py @@ -605,6 +605,305 @@ class SimpleLdb(TestCase): l = ldb.Ldb(self.filename) self.assertRaises(ldb.LdbError,lambda: l.search("", ldb.SCOPE_SUBTREE, "&(dc=*)(dn=*)", ["dc"])) +class SearchTests(TestCase): + def tearDown(self): + shutil.rmtree(self.testdir) + super(SearchTests, self).tearDown() + + def setUp(self): + super(SearchTests, self).setUp() + self.testdir = tempdir() + self.filename = os.path.join(self.testdir, "search_test.ldb") + self.l = ldb.Ldb(self.filename, options=["modules:rdn_name"]) + + self.l.add({"dn": "DC=SAMBA,DC=ORG", "name": b"samba.org"}) + self.l.add({"dn": "OU=ADMIN,DC=SAMBA,DC=ORG", + "name": b"Admins", + "x": "z", "y": "a"}) + self.l.add({"dn": "OU=USERS,DC=SAMBA,DC=ORG", + "name": b"Users", + "x": "z", "y": "a"}) + self.l.add({"dn": "OU=OU1,DC=SAMBA,DC=ORG", + "name": b"OU #1", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU2,DC=SAMBA,DC=ORG", + "name": b"OU #2", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU3,DC=SAMBA,DC=ORG", + "name": b"OU #3", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU4,DC=SAMBA,DC=ORG", + "name": b"OU #4", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU5,DC=SAMBA,DC=ORG", + "name": b"OU #5", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU6,DC=SAMBA,DC=ORG", + "name": b"OU #6", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU7,DC=SAMBA,DC=ORG", + "name": b"OU #7", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU8,DC=SAMBA,DC=ORG", + "name": b"OU #8", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU9,DC=SAMBA,DC=ORG", + "name": b"OU #9", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU10,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU11,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "y", "y": "a"}) + self.l.add({"dn": "OU=OU12,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "y", "y": "b"}) + self.l.add({"dn": "OU=OU13,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU14,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU15,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU16,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU17,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU18,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU19,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU20,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "b"}) + self.l.add({"dn": "OU=OU21,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "c"}) + self.l.add({"dn": "OU=OU22,DC=SAMBA,DC=ORG", + "name": b"OU #10", + "x": "x", "y": "c"}) + + def test_base(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE) + self.assertEqual(len(res11), 1) + + def test_base_or(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(|(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 1) + + def test_base_or2(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(|(x=y)(y=b))") + self.assertEqual(len(res11), 1) + + def test_base_and(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(&(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 0) + + def test_base_and2(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(&(x=y)(y=a))") + self.assertEqual(len(res11), 1) + + def test_base_false(self): + """Testing a search""" + + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(|(ou=ou13)(ou=ou12))") + self.assertEqual(len(res11), 0) + + def test_check_base_false(self): + """Testing a search""" + res11 = self.l.search(base="OU=OU11,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(|(ou=ou13)(ou=ou12))") + self.assertEqual(len(res11), 0) + + def test_check_base_error(self): + """Testing a search""" + self.l.add({"dn": "@OPTIONS", "checkBaseOnSearch": b"TRUE"}) + + try: + res11 = self.l.search(base="OU=OU11x,DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_BASE, + expression="(|(ou=ou13)(ou=ou12))") + self.fail("Should have failed on missing base") + except ldb.LdbError as err: + enum = err.args[0] + self.assertEqual(enum, ldb.ERR_NO_SUCH_OBJECT) + + def test_subtree_and(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 0) + + def test_subtree_and2(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(x=y)(|(y=b)(y=c)))") + self.assertEqual(len(res11), 1) + + def test_subtree_or(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(|(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 2) + + def test_subtree_or2(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(|(x=y)(y=b))") + self.assertEqual(len(res11), 20) + + def test_subtree_or3(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(|(x=y)(y=b)(y=c))") + self.assertEqual(len(res11), 22) + + def test_one_and(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_ONELEVEL, + expression="(&(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 0) + + def test_one_and2(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_ONELEVEL, + expression="(&(x=y)(y=b))") + self.assertEqual(len(res11), 1) + + def test_one_or(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_ONELEVEL, + expression="(|(ou=ou11)(ou=ou12))") + self.assertEqual(len(res11), 2) + + def test_one_or2(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_ONELEVEL, + expression="(|(x=y)(y=b))") + self.assertEqual(len(res11), 20) + + def test_subtree_and_or(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(|(x=z)(y=b))(x=x)(y=c))") + self.assertEqual(len(res11), 0) + + def test_subtree_and_or2(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(x=x)(y=c)(|(x=z)(y=b)))") + self.assertEqual(len(res11), 0) + + def test_subtree_and_or3(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(|(ou=ou11)(ou=ou10))(|(x=y)(y=b)(y=c)))") + self.assertEqual(len(res11), 2) + + def test_subtree_and_or4(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(|(x=y)(y=b)(y=c))(|(ou=ou11)(ou=ou10)))") + self.assertEqual(len(res11), 2) + + def test_subtree_and_or5(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(|(x=y)(y=b)(y=c))(ou=ou11))") + self.assertEqual(len(res11), 1) + + def test_subtree_or_and(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(|(x=x)(y=c)(&(x=z)(y=b)))") + self.assertEqual(len(res11), 10) + + def test_subtree_large_and_unique(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(ou=ou10)(y=a))") + self.assertEqual(len(res11), 1) + + def test_subtree_and_none(self): + """Testing a search""" + + res11 = self.l.search(base="DC=SAMBA,DC=ORG", + scope=ldb.SCOPE_SUBTREE, + expression="(&(ou=ouX)(y=a))") + self.assertEqual(len(res11), 0) + + +class IndexedSearchTests(SearchTests): + """Test searches using the index, to ensure the index doesn't + break things""" + def setUp(self): + super(IndexedSearchTests, self).setUp() + self.l.add({"dn": "@INDEXLIST", + "@IDXATTR": [b"x", b"y", b"ou"], + "@IDXONE": [b"1"]}) + + class DnTests(TestCase): -- 2.11.0 From bd1c4d1c5d48119cc82d5d7e7431d5e247acbe5d Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 15 Aug 2017 14:25:59 +1200 Subject: [PATCH 05/23] ldb_tdb: Use memcmp() to compare TDB keys in re_index() The keys may not always be a null terminated string, they could well be a binary GUID in a future revision, for efficiency.. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13016 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit da1e23418a1d0e6b77dcf24bf6cbbc18b0fe020b) --- lib/ldb/ldb_tdb/ldb_index.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index 232bb4c16d9..fbc05bd6f1a 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1620,7 +1620,8 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * talloc_free(msg); return 0; } - if (strcmp((char *)key2.dptr, (char *)key.dptr) != 0) { + if (key.dsize != key2.dsize || + (memcmp(key.dptr, key2.dptr, key.dsize) != 0)) { tdb_delete(tdb, key); tdb_store(tdb, key2, data, 0); } -- 2.11.0 From 9f704930ccbfe80996eae6059a527c0ab416f0ae Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 18 Aug 2017 17:01:07 +1200 Subject: [PATCH 06/23] ldb_tdb: Do not trigger the unique index check during a re-index, use another pass We want to rename the objects, then scan looking for the index values. This avoids a DB modify during the index scan traverse (the index values are actually added to an in-memory TDB, written in prepare_commit()). This allows us to remove the "this might already exist" case in the index handling, we now know that the entry did not exist in the index before we add it. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13015 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 2d0007ee5a658d199029f7e81200e206ba0d89e0) --- lib/ldb/ldb_tdb/ldb_index.c | 112 ++++++++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 29 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index fbc05bd6f1a..29f59f3003f 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1147,8 +1147,7 @@ int ltdb_search_indexed(struct ltdb_context *ac, uint32_t *match_count) * @return An ldb error code */ static int ltdb_index_add1(struct ldb_module *module, const char *dn, - struct ldb_message_element *el, int v_idx, - bool is_new) + struct ldb_message_element *el, int v_idx) { struct ldb_context *ldb; struct ldb_dn *dn_key; @@ -1198,16 +1197,6 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn, return LDB_ERR_ENTRY_ALREADY_EXISTS; } - /* If we are doing an ADD, then this can not already be in the index, - as it was not already in the database, and this has already been - checked because the store succeeded */ - if (! is_new) { - if (ltdb_dn_list_find_str(list, dn) != -1) { - talloc_free(list); - return LDB_SUCCESS; - } - } - /* overallocate the list a bit, to reduce the number of * realloc trigered copies */ alloc_len = ((list->count+1)+7) & ~7; @@ -1231,11 +1220,11 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn, add index entries for one elements in a message */ static int ltdb_index_add_el(struct ldb_module *module, const char *dn, - struct ldb_message_element *el, bool is_new) + struct ldb_message_element *el) { unsigned int i; for (i = 0; i < el->num_values; i++) { - int ret = ltdb_index_add1(module, dn, el, i, is_new); + int ret = ltdb_index_add1(module, dn, el, i); if (ret != LDB_SUCCESS) { return ret; } @@ -1249,8 +1238,7 @@ static int ltdb_index_add_el(struct ldb_module *module, const char *dn, */ static int ltdb_index_add_all(struct ldb_module *module, const char *dn, struct ldb_message_element *elements, - unsigned int num_el, - bool is_new) + unsigned int num_el) { struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private); unsigned int i; @@ -1269,7 +1257,7 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn, if (!ltdb_is_indexed(module, ltdb, elements[i].name)) { continue; } - ret = ltdb_index_add_el(module, dn, &elements[i], is_new); + ret = ltdb_index_add_el(module, dn, &elements[i]); if (ret != LDB_SUCCESS) { struct ldb_context *ldb = ldb_module_get_ctx(module); ldb_asprintf_errstring(ldb, @@ -1286,9 +1274,11 @@ static int ltdb_index_add_all(struct ldb_module *module, const char *dn, /* insert a one level index for a message */ -static int ltdb_index_onelevel(struct ldb_module *module, const struct ldb_message *msg, int add) -{ - struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), struct ltdb_private); +static int ltdb_index_onelevel(struct ldb_module *module, + const struct ldb_message *msg, int add) +{ + struct ltdb_private *ltdb = talloc_get_type(ldb_module_get_private(module), + struct ltdb_private); struct ldb_message_element el; struct ldb_val val; struct ldb_dn *pdn; @@ -1323,7 +1313,7 @@ static int ltdb_index_onelevel(struct ldb_module *module, const struct ldb_messa el.num_values = 1; if (add) { - ret = ltdb_index_add1(module, dn, &el, 0, add); + ret = ltdb_index_add1(module, dn, &el, 0); } else { /* delete */ ret = ltdb_index_del_value(module, msg->dn, &el, 0); } @@ -1347,7 +1337,7 @@ int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn, if (!ltdb_is_indexed(module, ltdb, el->name)) { return LDB_SUCCESS; } - return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el, true); + return ltdb_index_add_el(module, ldb_dn_get_linearized(dn), el); } /* @@ -1367,8 +1357,7 @@ int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg) return LDB_ERR_OPERATIONS_ERROR; } - ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements, - true); + ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements); if (ret != LDB_SUCCESS) { return ret; } @@ -1571,7 +1560,7 @@ struct ltdb_reindex_context { /* traversal function that adds @INDEX records during a re index */ -static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state) +static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state) { struct ldb_context *ldb; struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state; @@ -1582,7 +1571,6 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * .data = data.dptr, .length = data.dsize, }; - const char *dn = NULL; int ret; TDB_DATA key2; @@ -1627,6 +1615,52 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * } talloc_free(key2.dptr); + talloc_free(msg); + + return 0; +} + +/* + traversal function that adds @INDEX records during a re index +*/ +static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *state) +{ + struct ldb_context *ldb; + struct ltdb_reindex_context *ctx = (struct ltdb_reindex_context *)state; + struct ldb_module *module = ctx->module; + struct ldb_message *msg; + const char *dn = NULL; + unsigned int nb_elements_in_db; + const struct ldb_val val = { + .data = data.dptr, + .length = data.dsize, + }; + int ret; + + ldb = ldb_module_get_ctx(module); + + if (strncmp((char *)key.dptr, "DN=@", 4) == 0 || + strncmp((char *)key.dptr, "DN=", 3) != 0) { + return 0; + } + + msg = ldb_msg_new(module); + if (msg == NULL) { + return -1; + } + + ret = ldb_unpack_data_only_attr_list_flags(ldb, &val, + msg, + NULL, 0, + LDB_UNPACK_DATA_FLAG_NO_DATA_ALLOC, + &nb_elements_in_db); + if (ret != 0) { + ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %s\n", + ldb_dn_get_linearized(msg->dn)); + talloc_free(msg); + return -1; + } + if (msg->dn == NULL) { dn = (char *)key.dptr + 3; } else { @@ -1642,8 +1676,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * return -1; } - ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements, - false); + ret = ltdb_index_add_all(module, dn, msg->elements, msg->num_elements); if (ret != LDB_SUCCESS) { ctx->error = ret; @@ -1686,6 +1719,9 @@ int ltdb_reindex(struct ldb_module *module) */ ret = tdb_traverse(ltdb->tdb, delete_index, module); if (ret < 0) { + struct ldb_context *ldb = ldb_module_get_ctx(module); + ldb_asprintf_errstring(ldb, "index deletion traverse failed: %s", + ldb_errstring(ldb)); return LDB_ERR_OPERATIONS_ERROR; } @@ -1698,10 +1734,28 @@ int ltdb_reindex(struct ldb_module *module) ctx.error = 0; /* now traverse adding any indexes for normal LDB records */ + ret = tdb_traverse(ltdb->tdb, re_key, &ctx); + if (ret < 0) { + struct ldb_context *ldb = ldb_module_get_ctx(module); + ldb_asprintf_errstring(ldb, "key correction traverse failed: %s", + ldb_errstring(ldb)); + return LDB_ERR_OPERATIONS_ERROR; + } + + if (ctx.error != LDB_SUCCESS) { + struct ldb_context *ldb = ldb_module_get_ctx(module); + ldb_asprintf_errstring(ldb, "reindexing failed: %s", ldb_errstring(ldb)); + return ctx.error; + } + + ctx.error = 0; + + /* now traverse adding any indexes for normal LDB records */ ret = tdb_traverse(ltdb->tdb, re_index, &ctx); if (ret < 0) { struct ldb_context *ldb = ldb_module_get_ctx(module); - ldb_asprintf_errstring(ldb, "reindexing traverse failed: %s", ldb_errstring(ldb)); + ldb_asprintf_errstring(ldb, "reindexing traverse failed: %s", + ldb_errstring(ldb)); return LDB_ERR_OPERATIONS_ERROR; } -- 2.11.0 From 1c1ec753a210c76aa8831ad7d4baad3feb29161f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 1 Sep 2017 14:35:08 +1200 Subject: [PATCH 07/23] ldb_tdb: Create a common ltdb_key_is_record() allowing multiple key forms If backported, this allows old ldb versions to full-search and re-index newer databases and in current code allows the transition to and from a GUID or incrementing ID based index BUG: https://bugzilla.samba.org/show_bug.cgi?id=13016 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit fec666b334ad90408843d8fbfd4c2f62434831e2) --- lib/ldb/ldb_tdb/ldb_index.c | 24 ++++++++++++++++++------ lib/ldb/ldb_tdb/ldb_search.c | 3 +-- lib/ldb/ldb_tdb/ldb_tdb.c | 29 +++++++++++++++++++++++++++++ lib/ldb/ldb_tdb/ldb_tdb.h | 5 +++++ 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index 29f59f3003f..dbed867f94d 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1573,14 +1573,20 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st }; int ret; TDB_DATA key2; - + bool is_record; + ldb = ldb_module_get_ctx(module); - if (strncmp((char *)key.dptr, "DN=@", 4) == 0 || - strncmp((char *)key.dptr, "DN=", 3) != 0) { + if (key.dsize > 4 && + strncmp((char *)key.dptr, "DN=@", 4) == 0) { return 0; } + is_record = ltdb_key_is_record(key); + if (is_record == false) { + return 0; + } + msg = ldb_msg_new(module); if (msg == NULL) { return -1; @@ -1636,14 +1642,20 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * .length = data.dsize, }; int ret; - + bool is_record; + ldb = ldb_module_get_ctx(module); - if (strncmp((char *)key.dptr, "DN=@", 4) == 0 || - strncmp((char *)key.dptr, "DN=", 3) != 0) { + if (key.dsize > 4 && + strncmp((char *)key.dptr, "DN=@", 4) == 0) { return 0; } + is_record = ltdb_key_is_record(key); + if (is_record == false) { + return 0; + } + msg = ldb_msg_new(module); if (msg == NULL) { return -1; diff --git a/lib/ldb/ldb_tdb/ldb_search.c b/lib/ldb/ldb_tdb/ldb_search.c index 53355e04f6b..a6c408a53ce 100644 --- a/lib/ldb/ldb_tdb/ldb_search.c +++ b/lib/ldb/ldb_tdb/ldb_search.c @@ -410,8 +410,7 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi ac = talloc_get_type(state, struct ltdb_context); ldb = ldb_module_get_ctx(ac->module); - if (key.dsize < 4 || - strncmp((char *)key.dptr, "DN=", 3) != 0) { + if (ltdb_key_is_record(key) == false) { return 0; } diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c index bc8780ad7ee..316a487bfe4 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/lib/ldb/ldb_tdb/ldb_tdb.c @@ -127,6 +127,35 @@ int ltdb_unlock_read(struct ldb_module *module) } +/* + * Determine if this key could hold a record. We allow the new GUID + * index, the old DN index and a possible future ID= + */ +bool ltdb_key_is_record(TDB_DATA key) +{ + if (key.dsize < 4) { + return false; + } + + if (strncmp((char *)key.dptr, "DN=", 3) == 0) { + return true; + } + + if (strncmp((char *)key.dptr, "ID=", 3) == 0) { + return true; + } + + if (key.dsize < 6) { + return false; + } + + if (strncmp((char *)key.dptr, "GUID=", 5) == 0) { + return true; + } + + return false; +} + /* form a TDB_DATA for a record key caller frees diff --git a/lib/ldb/ldb_tdb/ldb_tdb.h b/lib/ldb/ldb_tdb/ldb_tdb.h index 26ae68e89c8..a391606dd16 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.h +++ b/lib/ldb/ldb_tdb/ldb_tdb.h @@ -110,6 +110,11 @@ int ltdb_search(struct ltdb_context *ctx); /* The following definitions come from lib/ldb/ldb_tdb/ldb_tdb.c */ int ltdb_lock_read(struct ldb_module *module); int ltdb_unlock_read(struct ldb_module *module); +/* + * Determine if this key could hold a record. We allow the new GUID + * index, the old DN index and a possible future ID= + */ +bool ltdb_key_is_record(TDB_DATA key); TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn); int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs); int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg, struct ldb_request *req); -- 2.11.0 From 91abe2220caaf3e113d5488e6dcc245135d19911 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 7 Sep 2017 16:23:43 +1200 Subject: [PATCH 08/23] ldb_tdb: Use memcmp rather than strncmp() in ltdb_key_is_record(), re_key() and re_index() BUG: https://bugzilla.samba.org/show_bug.cgi?id=13016 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 3ce80cfb60d86a80efb6b66205f6d8d683791f6c) --- lib/ldb/ldb_tdb/ldb_index.c | 4 ++-- lib/ldb/ldb_tdb/ldb_tdb.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index dbed867f94d..eb432d2d0f8 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1578,7 +1578,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st ldb = ldb_module_get_ctx(module); if (key.dsize > 4 && - strncmp((char *)key.dptr, "DN=@", 4) == 0) { + memcmp(key.dptr, "DN=@", 4) == 0) { return 0; } @@ -1647,7 +1647,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * ldb = ldb_module_get_ctx(module); if (key.dsize > 4 && - strncmp((char *)key.dptr, "DN=@", 4) == 0) { + memcmp(key.dptr, "DN=@", 4) == 0) { return 0; } diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c index 316a487bfe4..ccad8168a6e 100644 --- a/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/lib/ldb/ldb_tdb/ldb_tdb.c @@ -137,11 +137,11 @@ bool ltdb_key_is_record(TDB_DATA key) return false; } - if (strncmp((char *)key.dptr, "DN=", 3) == 0) { + if (memcmp(key.dptr, "DN=", 3) == 0) { return true; } - if (strncmp((char *)key.dptr, "ID=", 3) == 0) { + if (memcmp(key.dptr, "ID=", 3) == 0) { return true; } @@ -149,7 +149,7 @@ bool ltdb_key_is_record(TDB_DATA key) return false; } - if (strncmp((char *)key.dptr, "GUID=", 5) == 0) { + if (memcmp(key.dptr, "GUID=", 5) == 0) { return true; } -- 2.11.0 From 8cfe37cbbccb307437ce081916030bece9e13e65 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 23 Aug 2017 15:38:01 +1200 Subject: [PATCH 09/23] ldb_tdb: Check for errors during tdb operations in ltdb_reindex() BUG: https://bugzilla.samba.org/show_bug.cgi?id=13015 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit ffc8023a0e66cf9693665565b1692ea7c882d437) --- lib/ldb/ldb_tdb/ldb_index.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index eb432d2d0f8..3670323c44c 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1600,6 +1600,7 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st if (ret != 0) { ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %s\n", ldb_dn_get_linearized(msg->dn)); + ctx->error = ret; talloc_free(msg); return -1; } @@ -1616,8 +1617,32 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st } if (key.dsize != key2.dsize || (memcmp(key.dptr, key2.dptr, key.dsize) != 0)) { - tdb_delete(tdb, key); - tdb_store(tdb, key2, data, 0); + int tdb_ret; + tdb_ret = tdb_delete(tdb, key); + if (tdb_ret != 0) { + ldb_debug(ldb, LDB_DEBUG_ERROR, + "Failed to delete %*.*s " + "for rekey as %*.*s: %s", + (int)key.dsize, (int)key.dsize, + (const char *)key.dptr, + (int)key2.dsize, (int)key2.dsize, + (const char *)key.dptr, + tdb_errorstr(tdb)); + ctx->error = ltdb_err_map(tdb_error(tdb)); + return -1; + } + tdb_ret = tdb_store(tdb, key2, data, 0); + if (tdb_ret != 0) { + ldb_debug(ldb, LDB_DEBUG_ERROR, + "Failed to rekey %*.*s as %*.*s: %s", + (int)key.dsize, (int)key.dsize, + (const char *)key.dptr, + (int)key2.dsize, (int)key2.dsize, + (const char *)key.dptr, + tdb_errorstr(tdb)); + ctx->error = ltdb_err_map(tdb_error(tdb)); + return -1; + } } talloc_free(key2.dptr); @@ -1669,6 +1694,7 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * if (ret != 0) { ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid data for index %s\n", ldb_dn_get_linearized(msg->dn)); + ctx->error = ret; talloc_free(msg); return -1; } -- 2.11.0 From 176ba11e82952afd456e6440056275c91cc0ff1f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 10 Aug 2017 14:44:27 +1200 Subject: [PATCH 10/23] ldb_tdb: Check for talloc_strdup() failure in ltdb_index_add1() BUG: https://bugzilla.samba.org/show_bug.cgi?id=13015 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 6246c326a747382c728288ce7d3a8faa0b741ede) --- lib/ldb/ldb_tdb/ldb_index.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index 3670323c44c..a5ad8e20fbf 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1205,7 +1205,13 @@ static int ltdb_index_add1(struct ldb_module *module, const char *dn, talloc_free(list); return LDB_ERR_OPERATIONS_ERROR; } - list->dn[list->count].data = (uint8_t *)talloc_strdup(list->dn, dn); + + list->dn[list->count].data + = (uint8_t *)talloc_strdup(list->dn, dn); + if (list->dn[list->count].data == NULL) { + talloc_free(list); + return LDB_ERR_OPERATIONS_ERROR; + } list->dn[list->count].length = strlen(dn); list->count++; -- 2.11.0 From a96b3ec3214b12fe0719c432a45078b141632749 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 10 Aug 2017 16:09:31 +1200 Subject: [PATCH 11/23] ldb_tdb: Use braces in ltdb_dn_list_find_val() BUG: https://bugzilla.samba.org/show_bug.cgi?id=13015 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 54636011223ff350e172d52de302bcfcc5a0114a) --- lib/ldb/ldb_tdb/ldb_index.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index a5ad8e20fbf..a599b2602f0 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -83,7 +83,9 @@ static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_va { unsigned int i; for (i=0; icount; i++) { - if (dn_list_cmp(&list->dn[i], v) == 0) return i; + if (dn_list_cmp(&list->dn[i], v) == 0) { + return i; + } } return -1; } -- 2.11.0 From 60bdd56ba848a483be7bb7ff01f1f62a20fca12f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 10 Aug 2017 17:08:54 +1200 Subject: [PATCH 12/23] ldb_tdb: Refuse to re-index very old database with no DN in the record These are not found on any AD DC, and would segfault previous LDB versions. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13015 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 01c49b1a3561decdb10b5273d96d68d8e3d2b178) --- lib/ldb/ldb_tdb/ldb_index.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index a599b2602f0..3510dd9b1bc 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -1613,6 +1613,16 @@ static int re_key(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *st return -1; } + if (msg->dn == NULL) { + ldb_debug(ldb, LDB_DEBUG_ERROR, + "Refusing to re-index as GUID " + "key %*.*s with no DN\n", + (int)key.dsize, (int)key.dsize, + (char *)key.dptr); + talloc_free(msg); + return -1; + } + /* check if the DN key has changed, perhaps due to the case insensitivity of an element changing */ key2 = ltdb_key(module, msg->dn); @@ -1708,7 +1718,13 @@ static int re_index(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void * } if (msg->dn == NULL) { - dn = (char *)key.dptr + 3; + ldb_debug(ldb, LDB_DEBUG_ERROR, + "Refusing to re-index as GUID " + "key %*.*s with no DN\n", + (int)key.dsize, (int)key.dsize, + (char *)key.dptr); + talloc_free(msg); + return -1; } else { dn = ldb_dn_get_linearized(msg->dn); } -- 2.11.0 From 38daf2c11e91544a7c8a72abb6f5a3ce0dfe9319 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Tue, 5 Sep 2017 14:05:43 +1200 Subject: [PATCH 13/23] ldb: Add new ldb_ldif_message_redacted_string() with tests This is designed to be a drop in replacement for ldb_ldif_message_string() while better protecting privacy. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 37bb62990b96b266ba4a3e90cadd414e6fee5ddf) --- lib/ldb/common/ldb_ldif.c | 21 ++++++++++ lib/ldb/include/ldb.h | 40 +++++++++++++++--- lib/ldb/tests/ldb_mod_op_test.c | 91 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/lib/ldb/common/ldb_ldif.c b/lib/ldb/common/ldb_ldif.c index 0aeda94beac..b90d27eea59 100644 --- a/lib/ldb/common/ldb_ldif.c +++ b/lib/ldb/common/ldb_ldif.c @@ -1080,3 +1080,24 @@ char *ldb_ldif_message_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, return ldb_ldif_write_string(ldb, mem_ctx, &ldif); } + +/* + * convenient function to turn a ldb_message into a string. Useful for + * debugging but also safer if some of the LDIF could be sensitive. + * + * The secret attributes are specified in a 'const char * const *' within + * the LDB_SECRET_ATTRIBUTE_LIST opaque set on the ldb + * + */ +char *ldb_ldif_message_redacted_string(struct ldb_context *ldb, + TALLOC_CTX *mem_ctx, + enum ldb_changetype changetype, + const struct ldb_message *msg) +{ + struct ldb_ldif ldif; + + ldif.changetype = changetype; + ldif.msg = discard_const_p(struct ldb_message, msg); + + return ldb_ldif_write_redacted_trace_string(ldb, mem_ctx, &ldif); +} diff --git a/lib/ldb/include/ldb.h b/lib/ldb/include/ldb.h index 14cec0e0b93..9918b4e69d9 100644 --- a/lib/ldb/include/ldb.h +++ b/lib/ldb/include/ldb.h @@ -1724,16 +1724,46 @@ char * ldb_ldif_write_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, const struct ldb_ldif *msg); -/* - Produce a string form of an ldb message +/** + Write an LDB message to a string - convenient function to turn a ldb_message into a string. Useful for - debugging - */ + \param ldb the ldb context (from ldb_init()) + \param mem_ctx the talloc context on which to attach the string) + \param changetype LDB_CHANGETYPE_ADD or LDB_CHANGETYPE_MODIFY + \param msg the message to write out + + \return the string containing the LDIF, or NULL on error + + \sa ldb_ldif_message_redacted_string for a safer version of this + function +*/ char *ldb_ldif_message_string(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, enum ldb_changetype changetype, const struct ldb_message *msg); +/** + Write an LDB message to a string + + \param ldb the ldb context (from ldb_init()) + \param mem_ctx the talloc context on which to attach the string) + \param changetype LDB_CHANGETYPE_ADD or LDB_CHANGETYPE_MODIFY + \param msg the message to write out + + \return the string containing the LDIF, or NULL on error, but + with secret attributes redacted + + \note The secret attributes are specified in a + 'const char * const *' within the LDB_SECRET_ATTRIBUTE_LIST + opaque set on the ldb + + \sa ldb_ldif_message_string for an exact representiation of the + message as LDIF +*/ +char *ldb_ldif_message_redacted_string(struct ldb_context *ldb, + TALLOC_CTX *mem_ctx, + enum ldb_changetype changetype, + const struct ldb_message *msg); + /** Base64 encode a buffer diff --git a/lib/ldb/tests/ldb_mod_op_test.c b/lib/ldb/tests/ldb_mod_op_test.c index 96d2dd27df2..5e439f8d151 100644 --- a/lib/ldb/tests/ldb_mod_op_test.c +++ b/lib/ldb/tests/ldb_mod_op_test.c @@ -114,6 +114,91 @@ static void test_connect(void **state) assert_int_equal(ret, 0); } +static struct ldb_message *get_test_ldb_message(TALLOC_CTX *mem_ctx, + struct ldb_context *ldb) +{ + struct ldb_message *msg = ldb_msg_new(mem_ctx); + int ret; + assert_non_null(msg); + + msg->dn = ldb_dn_new(msg, ldb, "dc=samba,dc=org"); + assert_non_null(msg->dn); + ret = ldb_msg_add_string(msg, "public", "key"); + assert_int_equal(ret, LDB_SUCCESS); + ret = ldb_msg_add_string(msg, "supersecret", "password"); + assert_int_equal(ret, LDB_SUCCESS); + ret = ldb_msg_add_string(msg, "binary", "\xff\xff\0"); + assert_int_equal(ret, LDB_SUCCESS); + return msg; +} + +static void test_ldif_message(void **state) +{ + struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state, + struct ldbtest_ctx); + char *got_ldif; + const char *expected_ldif = + "dn: dc=samba,dc=org\n" + "changetype: add\n" + "public: key\n" + "supersecret: password\n" + "binary:: //8=\n" + "\n"; + + struct ldb_message *msg = get_test_ldb_message(test_ctx, + test_ctx->ldb); + + got_ldif = ldb_ldif_message_string(test_ctx->ldb, + test_ctx, + LDB_CHANGETYPE_ADD, + msg); + assert_string_equal(got_ldif, expected_ldif); + TALLOC_FREE(got_ldif); +} + +static void test_ldif_message_redacted(void **state) +{ + struct ldbtest_ctx *test_ctx = talloc_get_type_abort(*state, + struct ldbtest_ctx); + int ret; + char *got_ldif; + const char *expected_ldif = + "dn: dc=samba,dc=org\n" + "changetype: add\n" + "public: key\n" + "# supersecret::: REDACTED SECRET ATTRIBUTE\n" + "binary:: //8=\n" + "\n"; + + const char *secret_attrs[] = { + "supersecret", + NULL + }; + + struct ldb_message *msg = ldb_msg_new(test_ctx); + + ldb_set_opaque(test_ctx->ldb, + LDB_SECRET_ATTRIBUTE_LIST_OPAQUE, + secret_attrs); + + assert_non_null(msg); + + msg->dn = ldb_dn_new(msg, test_ctx->ldb, "dc=samba,dc=org"); + ret = ldb_msg_add_string(msg, "public", "key"); + assert_int_equal(ret, LDB_SUCCESS); + ret = ldb_msg_add_string(msg, "supersecret", "password"); + assert_int_equal(ret, LDB_SUCCESS); + ret = ldb_msg_add_string(msg, "binary", "\xff\xff\0"); + assert_int_equal(ret, LDB_SUCCESS); + got_ldif = ldb_ldif_message_redacted_string(test_ctx->ldb, + test_ctx, + LDB_CHANGETYPE_ADD, + msg); + assert_string_equal(got_ldif, expected_ldif); + TALLOC_FREE(got_ldif); + assert_int_equal(ret, 0); +} + static int ldbtest_setup(void **state) { struct ldbtest_ctx *test_ctx; @@ -2782,6 +2867,12 @@ int main(int argc, const char **argv) cmocka_unit_test_setup_teardown(test_connect, ldbtest_noconn_setup, ldbtest_noconn_teardown), + cmocka_unit_test_setup_teardown(test_ldif_message, + ldbtest_noconn_setup, + ldbtest_noconn_teardown), + cmocka_unit_test_setup_teardown(test_ldif_message_redacted, + ldbtest_noconn_setup, + ldbtest_noconn_teardown), cmocka_unit_test_setup_teardown(test_ldb_add, ldbtest_setup, ldbtest_teardown), -- 2.11.0 From eb56c0d83f36389e499ec1f474cbd1ee860d2282 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 14:26:25 +1200 Subject: [PATCH 14/23] ldb: version 1.2.2 * Bug #13017: Add ldb_ldif_message_redacted_string() to allow debug of redacted log messages, avoiding showing secret values * Bug #13015: Allow re-index of newer databases with binary GUID TDB keys (this officially removes support for re-index of the original pack format 0, rather than simply segfaulting). * Avoid memory allocation and so make modify of records in ldb_tdb faster BUG: https://bugzilla.samba.org/show_bug.cgi?id=13023 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit ba54816875d43be66000977c6dd068e3165d7a46) --- lib/ldb/ABI/ldb-1.2.2.sigs | 277 ++++++++++++++++++++++++++++++++++ lib/ldb/ABI/pyldb-util-1.2.2.sigs | 2 + lib/ldb/ABI/pyldb-util.py3-1.2.2.sigs | 2 + lib/ldb/wscript | 2 +- 4 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 lib/ldb/ABI/ldb-1.2.2.sigs create mode 100644 lib/ldb/ABI/pyldb-util-1.2.2.sigs create mode 100644 lib/ldb/ABI/pyldb-util.py3-1.2.2.sigs diff --git a/lib/ldb/ABI/ldb-1.2.2.sigs b/lib/ldb/ABI/ldb-1.2.2.sigs new file mode 100644 index 00000000000..9dc61cda019 --- /dev/null +++ b/lib/ldb/ABI/ldb-1.2.2.sigs @@ -0,0 +1,277 @@ +ldb_add: int (struct ldb_context *, const struct ldb_message *) +ldb_any_comparison: int (struct ldb_context *, void *, ldb_attr_handler_t, const struct ldb_val *, const struct ldb_val *) +ldb_asprintf_errstring: void (struct ldb_context *, const char *, ...) +ldb_attr_casefold: char *(TALLOC_CTX *, const char *) +ldb_attr_dn: int (const char *) +ldb_attr_in_list: int (const char * const *, const char *) +ldb_attr_list_copy: const char **(TALLOC_CTX *, const char * const *) +ldb_attr_list_copy_add: const char **(TALLOC_CTX *, const char * const *, const char *) +ldb_base64_decode: int (char *) +ldb_base64_encode: char *(TALLOC_CTX *, const char *, int) +ldb_binary_decode: struct ldb_val (TALLOC_CTX *, const char *) +ldb_binary_encode: char *(TALLOC_CTX *, struct ldb_val) +ldb_binary_encode_string: char *(TALLOC_CTX *, const char *) +ldb_build_add_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_del_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_extended_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const char *, void *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_mod_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_rename_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, struct ldb_dn *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_search_req: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, const char *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_build_search_req_ex: int (struct ldb_request **, struct ldb_context *, TALLOC_CTX *, struct ldb_dn *, enum ldb_scope, struct ldb_parse_tree *, const char * const *, struct ldb_control **, void *, ldb_request_callback_t, struct ldb_request *) +ldb_casefold: char *(struct ldb_context *, TALLOC_CTX *, const char *, size_t) +ldb_casefold_default: char *(void *, TALLOC_CTX *, const char *, size_t) +ldb_check_critical_controls: int (struct ldb_control **) +ldb_comparison_binary: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *) +ldb_comparison_fold: int (struct ldb_context *, void *, const struct ldb_val *, const struct ldb_val *) +ldb_connect: int (struct ldb_context *, const char *, unsigned int, const char **) +ldb_control_to_string: char *(TALLOC_CTX *, const struct ldb_control *) +ldb_controls_except_specified: struct ldb_control **(struct ldb_control **, TALLOC_CTX *, struct ldb_control *) +ldb_debug: void (struct ldb_context *, enum ldb_debug_level, const char *, ...) +ldb_debug_add: void (struct ldb_context *, const char *, ...) +ldb_debug_end: void (struct ldb_context *, enum ldb_debug_level) +ldb_debug_set: void (struct ldb_context *, enum ldb_debug_level, const char *, ...) +ldb_delete: int (struct ldb_context *, struct ldb_dn *) +ldb_dn_add_base: bool (struct ldb_dn *, struct ldb_dn *) +ldb_dn_add_base_fmt: bool (struct ldb_dn *, const char *, ...) +ldb_dn_add_child: bool (struct ldb_dn *, struct ldb_dn *) +ldb_dn_add_child_fmt: bool (struct ldb_dn *, const char *, ...) +ldb_dn_alloc_casefold: char *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_alloc_linearized: char *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_canonical_ex_string: char *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_canonical_string: char *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_check_local: bool (struct ldb_module *, struct ldb_dn *) +ldb_dn_check_special: bool (struct ldb_dn *, const char *) +ldb_dn_compare: int (struct ldb_dn *, struct ldb_dn *) +ldb_dn_compare_base: int (struct ldb_dn *, struct ldb_dn *) +ldb_dn_copy: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_escape_value: char *(TALLOC_CTX *, struct ldb_val) +ldb_dn_extended_add_syntax: int (struct ldb_context *, unsigned int, const struct ldb_dn_extended_syntax *) +ldb_dn_extended_filter: void (struct ldb_dn *, const char * const *) +ldb_dn_extended_syntax_by_name: const struct ldb_dn_extended_syntax *(struct ldb_context *, const char *) +ldb_dn_from_ldb_val: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const struct ldb_val *) +ldb_dn_get_casefold: const char *(struct ldb_dn *) +ldb_dn_get_comp_num: int (struct ldb_dn *) +ldb_dn_get_component_name: const char *(struct ldb_dn *, unsigned int) +ldb_dn_get_component_val: const struct ldb_val *(struct ldb_dn *, unsigned int) +ldb_dn_get_extended_comp_num: int (struct ldb_dn *) +ldb_dn_get_extended_component: const struct ldb_val *(struct ldb_dn *, const char *) +ldb_dn_get_extended_linearized: char *(TALLOC_CTX *, struct ldb_dn *, int) +ldb_dn_get_ldb_context: struct ldb_context *(struct ldb_dn *) +ldb_dn_get_linearized: const char *(struct ldb_dn *) +ldb_dn_get_parent: struct ldb_dn *(TALLOC_CTX *, struct ldb_dn *) +ldb_dn_get_rdn_name: const char *(struct ldb_dn *) +ldb_dn_get_rdn_val: const struct ldb_val *(struct ldb_dn *) +ldb_dn_has_extended: bool (struct ldb_dn *) +ldb_dn_is_null: bool (struct ldb_dn *) +ldb_dn_is_special: bool (struct ldb_dn *) +ldb_dn_is_valid: bool (struct ldb_dn *) +ldb_dn_map_local: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *) +ldb_dn_map_rebase_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *) +ldb_dn_map_remote: struct ldb_dn *(struct ldb_module *, void *, struct ldb_dn *) +ldb_dn_minimise: bool (struct ldb_dn *) +ldb_dn_new: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *) +ldb_dn_new_fmt: struct ldb_dn *(TALLOC_CTX *, struct ldb_context *, const char *, ...) +ldb_dn_remove_base_components: bool (struct ldb_dn *, unsigned int) +ldb_dn_remove_child_components: bool (struct ldb_dn *, unsigned int) +ldb_dn_remove_extended_components: void (struct ldb_dn *) +ldb_dn_replace_components: bool (struct ldb_dn *, struct ldb_dn *) +ldb_dn_set_component: int (struct ldb_dn *, int, const char *, const struct ldb_val) +ldb_dn_set_extended_component: int (struct ldb_dn *, const char *, const struct ldb_val *) +ldb_dn_update_components: int (struct ldb_dn *, const struct ldb_dn *) +ldb_dn_validate: bool (struct ldb_dn *) +ldb_dump_results: void (struct ldb_context *, struct ldb_result *, FILE *) +ldb_error_at: int (struct ldb_context *, int, const char *, const char *, int) +ldb_errstring: const char *(struct ldb_context *) +ldb_extended: int (struct ldb_context *, const char *, void *, struct ldb_result **) +ldb_extended_default_callback: int (struct ldb_request *, struct ldb_reply *) +ldb_filter_from_tree: char *(TALLOC_CTX *, const struct ldb_parse_tree *) +ldb_get_config_basedn: struct ldb_dn *(struct ldb_context *) +ldb_get_create_perms: unsigned int (struct ldb_context *) +ldb_get_default_basedn: struct ldb_dn *(struct ldb_context *) +ldb_get_event_context: struct tevent_context *(struct ldb_context *) +ldb_get_flags: unsigned int (struct ldb_context *) +ldb_get_opaque: void *(struct ldb_context *, const char *) +ldb_get_root_basedn: struct ldb_dn *(struct ldb_context *) +ldb_get_schema_basedn: struct ldb_dn *(struct ldb_context *) +ldb_global_init: int (void) +ldb_handle_get_event_context: struct tevent_context *(struct ldb_handle *) +ldb_handle_new: struct ldb_handle *(TALLOC_CTX *, struct ldb_context *) +ldb_handle_use_global_event_context: void (struct ldb_handle *) +ldb_handler_copy: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *) +ldb_handler_fold: int (struct ldb_context *, void *, const struct ldb_val *, struct ldb_val *) +ldb_init: struct ldb_context *(TALLOC_CTX *, struct tevent_context *) +ldb_ldif_message_redacted_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *) +ldb_ldif_message_string: char *(struct ldb_context *, TALLOC_CTX *, enum ldb_changetype, const struct ldb_message *) +ldb_ldif_parse_modrdn: int (struct ldb_context *, const struct ldb_ldif *, TALLOC_CTX *, struct ldb_dn **, struct ldb_dn **, bool *, struct ldb_dn **, struct ldb_dn **) +ldb_ldif_read: struct ldb_ldif *(struct ldb_context *, int (*)(void *), void *) +ldb_ldif_read_file: struct ldb_ldif *(struct ldb_context *, FILE *) +ldb_ldif_read_file_state: struct ldb_ldif *(struct ldb_context *, struct ldif_read_file_state *) +ldb_ldif_read_free: void (struct ldb_context *, struct ldb_ldif *) +ldb_ldif_read_string: struct ldb_ldif *(struct ldb_context *, const char **) +ldb_ldif_write: int (struct ldb_context *, int (*)(void *, const char *, ...), void *, const struct ldb_ldif *) +ldb_ldif_write_file: int (struct ldb_context *, FILE *, const struct ldb_ldif *) +ldb_ldif_write_redacted_trace_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *) +ldb_ldif_write_string: char *(struct ldb_context *, TALLOC_CTX *, const struct ldb_ldif *) +ldb_load_modules: int (struct ldb_context *, const char **) +ldb_map_add: int (struct ldb_module *, struct ldb_request *) +ldb_map_delete: int (struct ldb_module *, struct ldb_request *) +ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const struct ldb_map_objectclass *, const char * const *, const char *, const char *) +ldb_map_modify: int (struct ldb_module *, struct ldb_request *) +ldb_map_rename: int (struct ldb_module *, struct ldb_request *) +ldb_map_search: int (struct ldb_module *, struct ldb_request *) +ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope) +ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *) +ldb_match_msg_objectclass: int (const struct ldb_message *, const char *) +ldb_mod_register_control: int (struct ldb_module *, const char *) +ldb_modify: int (struct ldb_context *, const struct ldb_message *) +ldb_modify_default_callback: int (struct ldb_request *, struct ldb_reply *) +ldb_module_call_chain: char *(struct ldb_request *, TALLOC_CTX *) +ldb_module_connect_backend: int (struct ldb_context *, const char *, const char **, struct ldb_module **) +ldb_module_done: int (struct ldb_request *, struct ldb_control **, struct ldb_extended *, int) +ldb_module_flags: uint32_t (struct ldb_context *) +ldb_module_get_ctx: struct ldb_context *(struct ldb_module *) +ldb_module_get_name: const char *(struct ldb_module *) +ldb_module_get_ops: const struct ldb_module_ops *(struct ldb_module *) +ldb_module_get_private: void *(struct ldb_module *) +ldb_module_init_chain: int (struct ldb_context *, struct ldb_module *) +ldb_module_load_list: int (struct ldb_context *, const char **, struct ldb_module *, struct ldb_module **) +ldb_module_new: struct ldb_module *(TALLOC_CTX *, struct ldb_context *, const char *, const struct ldb_module_ops *) +ldb_module_next: struct ldb_module *(struct ldb_module *) +ldb_module_popt_options: struct poptOption **(struct ldb_context *) +ldb_module_send_entry: int (struct ldb_request *, struct ldb_message *, struct ldb_control **) +ldb_module_send_referral: int (struct ldb_request *, char *) +ldb_module_set_next: void (struct ldb_module *, struct ldb_module *) +ldb_module_set_private: void (struct ldb_module *, void *) +ldb_modules_hook: int (struct ldb_context *, enum ldb_module_hook_type) +ldb_modules_list_from_string: const char **(struct ldb_context *, TALLOC_CTX *, const char *) +ldb_modules_load: int (const char *, const char *) +ldb_msg_add: int (struct ldb_message *, const struct ldb_message_element *, int) +ldb_msg_add_empty: int (struct ldb_message *, const char *, int, struct ldb_message_element **) +ldb_msg_add_fmt: int (struct ldb_message *, const char *, const char *, ...) +ldb_msg_add_linearized_dn: int (struct ldb_message *, const char *, struct ldb_dn *) +ldb_msg_add_steal_string: int (struct ldb_message *, const char *, char *) +ldb_msg_add_steal_value: int (struct ldb_message *, const char *, struct ldb_val *) +ldb_msg_add_string: int (struct ldb_message *, const char *, const char *) +ldb_msg_add_value: int (struct ldb_message *, const char *, const struct ldb_val *, struct ldb_message_element **) +ldb_msg_canonicalize: struct ldb_message *(struct ldb_context *, const struct ldb_message *) +ldb_msg_check_string_attribute: int (const struct ldb_message *, const char *, const char *) +ldb_msg_copy: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *) +ldb_msg_copy_attr: int (struct ldb_message *, const char *, const char *) +ldb_msg_copy_shallow: struct ldb_message *(TALLOC_CTX *, const struct ldb_message *) +ldb_msg_diff: struct ldb_message *(struct ldb_context *, struct ldb_message *, struct ldb_message *) +ldb_msg_difference: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message *, struct ldb_message *, struct ldb_message **) +ldb_msg_element_compare: int (struct ldb_message_element *, struct ldb_message_element *) +ldb_msg_element_compare_name: int (struct ldb_message_element *, struct ldb_message_element *) +ldb_msg_element_equal_ordered: bool (const struct ldb_message_element *, const struct ldb_message_element *) +ldb_msg_find_attr_as_bool: int (const struct ldb_message *, const char *, int) +ldb_msg_find_attr_as_dn: struct ldb_dn *(struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, const char *) +ldb_msg_find_attr_as_double: double (const struct ldb_message *, const char *, double) +ldb_msg_find_attr_as_int: int (const struct ldb_message *, const char *, int) +ldb_msg_find_attr_as_int64: int64_t (const struct ldb_message *, const char *, int64_t) +ldb_msg_find_attr_as_string: const char *(const struct ldb_message *, const char *, const char *) +ldb_msg_find_attr_as_uint: unsigned int (const struct ldb_message *, const char *, unsigned int) +ldb_msg_find_attr_as_uint64: uint64_t (const struct ldb_message *, const char *, uint64_t) +ldb_msg_find_common_values: int (struct ldb_context *, TALLOC_CTX *, struct ldb_message_element *, struct ldb_message_element *, uint32_t) +ldb_msg_find_duplicate_val: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message_element *, struct ldb_val **, uint32_t) +ldb_msg_find_element: struct ldb_message_element *(const struct ldb_message *, const char *) +ldb_msg_find_ldb_val: const struct ldb_val *(const struct ldb_message *, const char *) +ldb_msg_find_val: struct ldb_val *(const struct ldb_message_element *, struct ldb_val *) +ldb_msg_new: struct ldb_message *(TALLOC_CTX *) +ldb_msg_normalize: int (struct ldb_context *, TALLOC_CTX *, const struct ldb_message *, struct ldb_message **) +ldb_msg_remove_attr: void (struct ldb_message *, const char *) +ldb_msg_remove_element: void (struct ldb_message *, struct ldb_message_element *) +ldb_msg_rename_attr: int (struct ldb_message *, const char *, const char *) +ldb_msg_sanity_check: int (struct ldb_context *, const struct ldb_message *) +ldb_msg_sort_elements: void (struct ldb_message *) +ldb_next_del_trans: int (struct ldb_module *) +ldb_next_end_trans: int (struct ldb_module *) +ldb_next_init: int (struct ldb_module *) +ldb_next_prepare_commit: int (struct ldb_module *) +ldb_next_read_lock: int (struct ldb_module *) +ldb_next_read_unlock: int (struct ldb_module *) +ldb_next_remote_request: int (struct ldb_module *, struct ldb_request *) +ldb_next_request: int (struct ldb_module *, struct ldb_request *) +ldb_next_start_trans: int (struct ldb_module *) +ldb_op_default_callback: int (struct ldb_request *, struct ldb_reply *) +ldb_options_find: const char *(struct ldb_context *, const char **, const char *) +ldb_pack_data: int (struct ldb_context *, const struct ldb_message *, struct ldb_val *) +ldb_parse_control_from_string: struct ldb_control *(struct ldb_context *, TALLOC_CTX *, const char *) +ldb_parse_control_strings: struct ldb_control **(struct ldb_context *, TALLOC_CTX *, const char **) +ldb_parse_tree: struct ldb_parse_tree *(TALLOC_CTX *, const char *) +ldb_parse_tree_attr_replace: void (struct ldb_parse_tree *, const char *, const char *) +ldb_parse_tree_copy_shallow: struct ldb_parse_tree *(TALLOC_CTX *, const struct ldb_parse_tree *) +ldb_parse_tree_walk: int (struct ldb_parse_tree *, int (*)(struct ldb_parse_tree *, void *), void *) +ldb_qsort: void (void * const, size_t, size_t, void *, ldb_qsort_cmp_fn_t) +ldb_register_backend: int (const char *, ldb_connect_fn, bool) +ldb_register_extended_match_rule: int (struct ldb_context *, const struct ldb_extended_match_rule *) +ldb_register_hook: int (ldb_hook_fn) +ldb_register_module: int (const struct ldb_module_ops *) +ldb_rename: int (struct ldb_context *, struct ldb_dn *, struct ldb_dn *) +ldb_reply_add_control: int (struct ldb_reply *, const char *, bool, void *) +ldb_reply_get_control: struct ldb_control *(struct ldb_reply *, const char *) +ldb_req_get_custom_flags: uint32_t (struct ldb_request *) +ldb_req_is_untrusted: bool (struct ldb_request *) +ldb_req_location: const char *(struct ldb_request *) +ldb_req_mark_trusted: void (struct ldb_request *) +ldb_req_mark_untrusted: void (struct ldb_request *) +ldb_req_set_custom_flags: void (struct ldb_request *, uint32_t) +ldb_req_set_location: void (struct ldb_request *, const char *) +ldb_request: int (struct ldb_context *, struct ldb_request *) +ldb_request_add_control: int (struct ldb_request *, const char *, bool, void *) +ldb_request_done: int (struct ldb_request *, int) +ldb_request_get_control: struct ldb_control *(struct ldb_request *, const char *) +ldb_request_get_status: int (struct ldb_request *) +ldb_request_replace_control: int (struct ldb_request *, const char *, bool, void *) +ldb_request_set_state: void (struct ldb_request *, int) +ldb_reset_err_string: void (struct ldb_context *) +ldb_save_controls: int (struct ldb_control *, struct ldb_request *, struct ldb_control ***) +ldb_schema_attribute_add: int (struct ldb_context *, const char *, unsigned int, const char *) +ldb_schema_attribute_add_with_syntax: int (struct ldb_context *, const char *, unsigned int, const struct ldb_schema_syntax *) +ldb_schema_attribute_by_name: const struct ldb_schema_attribute *(struct ldb_context *, const char *) +ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, const char *, unsigned int, const struct ldb_schema_syntax *, struct ldb_schema_attribute *) +ldb_schema_attribute_remove: void (struct ldb_context *, const char *) +ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int) +ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *) +ldb_schema_set_override_indexlist: void (struct ldb_context *, bool) +ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...) +ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *) +ldb_sequence_number: int (struct ldb_context *, enum ldb_sequence_type, uint64_t *) +ldb_set_create_perms: void (struct ldb_context *, unsigned int) +ldb_set_debug: int (struct ldb_context *, void (*)(void *, enum ldb_debug_level, const char *, va_list), void *) +ldb_set_debug_stderr: int (struct ldb_context *) +ldb_set_default_dns: void (struct ldb_context *) +ldb_set_errstring: void (struct ldb_context *, const char *) +ldb_set_event_context: void (struct ldb_context *, struct tevent_context *) +ldb_set_flags: void (struct ldb_context *, unsigned int) +ldb_set_modules_dir: void (struct ldb_context *, const char *) +ldb_set_opaque: int (struct ldb_context *, const char *, void *) +ldb_set_require_private_event_context: void (struct ldb_context *) +ldb_set_timeout: int (struct ldb_context *, struct ldb_request *, int) +ldb_set_timeout_from_prev_req: int (struct ldb_context *, struct ldb_request *, struct ldb_request *) +ldb_set_utf8_default: void (struct ldb_context *) +ldb_set_utf8_fns: void (struct ldb_context *, void *, char *(*)(void *, void *, const char *, size_t)) +ldb_setup_wellknown_attributes: int (struct ldb_context *) +ldb_should_b64_encode: int (struct ldb_context *, const struct ldb_val *) +ldb_standard_syntax_by_name: const struct ldb_schema_syntax *(struct ldb_context *, const char *) +ldb_strerror: const char *(int) +ldb_string_to_time: time_t (const char *) +ldb_string_utc_to_time: time_t (const char *) +ldb_timestring: char *(TALLOC_CTX *, time_t) +ldb_timestring_utc: char *(TALLOC_CTX *, time_t) +ldb_transaction_cancel: int (struct ldb_context *) +ldb_transaction_cancel_noerr: int (struct ldb_context *) +ldb_transaction_commit: int (struct ldb_context *) +ldb_transaction_prepare_commit: int (struct ldb_context *) +ldb_transaction_start: int (struct ldb_context *) +ldb_unpack_data: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *) +ldb_unpack_data_only_attr_list: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int *) +ldb_unpack_data_only_attr_list_flags: int (struct ldb_context *, const struct ldb_val *, struct ldb_message *, const char * const *, unsigned int, unsigned int, unsigned int *) +ldb_val_dup: struct ldb_val (TALLOC_CTX *, const struct ldb_val *) +ldb_val_equal_exact: int (const struct ldb_val *, const struct ldb_val *) +ldb_val_map_local: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *) +ldb_val_map_remote: struct ldb_val (struct ldb_module *, void *, const struct ldb_map_attribute *, const struct ldb_val *) +ldb_val_string_cmp: int (const struct ldb_val *, const char *) +ldb_val_to_time: int (const struct ldb_val *, time_t *) +ldb_valid_attr_name: int (const char *) +ldb_vdebug: void (struct ldb_context *, enum ldb_debug_level, const char *, va_list) +ldb_wait: int (struct ldb_handle *, enum ldb_wait_type) diff --git a/lib/ldb/ABI/pyldb-util-1.2.2.sigs b/lib/ldb/ABI/pyldb-util-1.2.2.sigs new file mode 100644 index 00000000000..74d6719d2bc --- /dev/null +++ b/lib/ldb/ABI/pyldb-util-1.2.2.sigs @@ -0,0 +1,2 @@ +pyldb_Dn_FromDn: PyObject *(struct ldb_dn *) +pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **) diff --git a/lib/ldb/ABI/pyldb-util.py3-1.2.2.sigs b/lib/ldb/ABI/pyldb-util.py3-1.2.2.sigs new file mode 100644 index 00000000000..74d6719d2bc --- /dev/null +++ b/lib/ldb/ABI/pyldb-util.py3-1.2.2.sigs @@ -0,0 +1,2 @@ +pyldb_Dn_FromDn: PyObject *(struct ldb_dn *) +pyldb_Object_AsDn: bool (TALLOC_CTX *, PyObject *, struct ldb_context *, struct ldb_dn **) diff --git a/lib/ldb/wscript b/lib/ldb/wscript index 6f3b32665d3..bd17b7b9edc 100644 --- a/lib/ldb/wscript +++ b/lib/ldb/wscript @@ -1,7 +1,7 @@ #!/usr/bin/env python APPNAME = 'ldb' -VERSION = '1.2.1' +VERSION = '1.2.2' blddir = 'bin' -- 2.11.0 From 8202d0d064218969002744db0a897dca27ea80dd Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 15:38:14 +1200 Subject: [PATCH 15/23] repl_meta_data: Use ldb_ldif_message_redacted_string() to avoid printing secrets in logs This avoids printing un-encrypted secret values in logs BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit cc78de55810ac20858066a25159ad390e7644f07) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 0ed24cb593e..2a8e689eb46 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4980,7 +4980,9 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar) if (DEBUGLVL(4)) { struct GUID_txt_buf guid_txt; - char *s = ldb_ldif_message_string(ldb, ar, LDB_CHANGETYPE_ADD, msg); + char *s = ldb_ldif_message_redacted_string(ldb, ar, + LDB_CHANGETYPE_ADD, + msg); DEBUG(4, ("DRS replication add message of %s:\n%s\n", GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), s)); @@ -5552,7 +5554,8 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar) if (DEBUGLVL(5)) { struct GUID_txt_buf guid_txt; - char *s = ldb_ldif_message_string(ldb, ar, LDB_CHANGETYPE_MODIFY, msg); + char *s = ldb_ldif_message_redacted_string(ldb, ar, + LDB_CHANGETYPE_MODIFY, msg); DEBUG(5, ("Initial DRS replication modify message of %s is:\n%s\n" "%s\n" "%s\n", @@ -5828,7 +5831,9 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar) if (DEBUGLVL(4)) { struct GUID_txt_buf guid_txt; - char *s = ldb_ldif_message_string(ldb, ar, LDB_CHANGETYPE_MODIFY, msg); + char *s = ldb_ldif_message_redacted_string(ldb, ar, + LDB_CHANGETYPE_MODIFY, + msg); DEBUG(4, ("Final DRS replication modify message of %s:\n%s\n", GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), s)); @@ -6428,7 +6433,9 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a nrf_el->flags = LDB_FLAG_MOD_REPLACE; if (CHECK_DEBUGLVL(4)) { - char *s = ldb_ldif_message_string(ldb, ar, LDB_CHANGETYPE_MODIFY, msg); + char *s = ldb_ldif_message_redacted_string(ldb, ar, + LDB_CHANGETYPE_MODIFY, + msg); DEBUG(4, ("DRS replication uptodate modify message:\n%s\n", s)); talloc_free(s); } @@ -7046,7 +7053,10 @@ linked_attributes[0]: if (ret != LDB_SUCCESS) { ldb_debug(ldb, LDB_DEBUG_WARNING, "Failed to apply linked attribute change '%s'\n%s\n", ldb_errstring(ldb), - ldb_ldif_message_string(ldb, tmp_ctx, LDB_CHANGETYPE_MODIFY, msg)); + ldb_ldif_message_redacted_string(ldb, + tmp_ctx, + LDB_CHANGETYPE_MODIFY, + msg)); talloc_free(tmp_ctx); return ret; } -- 2.11.0 From 4d10d9848b0a6355467d1c669dabca95cb6d7a5c Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 15:56:59 +1200 Subject: [PATCH 16/23] linked_attributes: Use ldb_ldif_message_redacted_string() for consistency This avoids printing un-encrypted secret values in logs, and while links are not likely secret, this avoids a future copy and paste using ldb_ldif_message_string() again. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 7cfaf706942308c3682d9a37ac778cfbbdf49919) --- source4/dsdb/samdb/ldb_modules/linked_attributes.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/linked_attributes.c b/source4/dsdb/samdb/ldb_modules/linked_attributes.c index 2016524a2d4..c6beb25e58b 100644 --- a/source4/dsdb/samdb/ldb_modules/linked_attributes.c +++ b/source4/dsdb/samdb/ldb_modules/linked_attributes.c @@ -1136,14 +1136,18 @@ static int la_do_op_request(struct ldb_module *module, struct la_context *ac, st if (DEBUGLVL(4)) { DEBUG(4,("Applying linked attribute change:\n%s\n", - ldb_ldif_message_string(ldb, op, LDB_CHANGETYPE_MODIFY, new_msg))); + ldb_ldif_message_redacted_string(ldb, op, + LDB_CHANGETYPE_MODIFY, + new_msg))); } ret = dsdb_module_modify(module, new_msg, DSDB_FLAG_NEXT_MODULE, ac->req); if (ret != LDB_SUCCESS) { ldb_debug(ldb, LDB_DEBUG_WARNING, __location__ ": failed to apply linked attribute change '%s'\n%s\n", ldb_errstring(ldb), - ldb_ldif_message_string(ldb, op, LDB_CHANGETYPE_MODIFY, new_msg)); + ldb_ldif_message_redacted_string(ldb, op, + LDB_CHANGETYPE_MODIFY, + new_msg)); } return ret; -- 2.11.0 From 399975241d7e08478025e5376cadf8a31824f731 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 16:24:35 +1200 Subject: [PATCH 17/23] repl_meta_data: Re-work printing of replicated entries This re-work of our LDIF printing avoids some of the privacy issue from printing the full LDIF at level 4, while showing the entry that actually fails. Instead, we print the DN only at level 4, then the full message at 8. While all of the DRS replication data is potentially sensitive the passwords are most sensitive, and are now not printed unencrypted. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit e3988f8f74f4a11e8f26a548e0a33d20f4e863f7) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 37 +++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 2a8e689eb46..56c7218d76e 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -4977,18 +4977,22 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar) } } - if (DEBUGLVL(4)) { + if (DEBUGLVL(8)) { struct GUID_txt_buf guid_txt; char *s = ldb_ldif_message_redacted_string(ldb, ar, LDB_CHANGETYPE_ADD, msg); - DEBUG(4, ("DRS replication add message of %s:\n%s\n", + DEBUG(8, ("DRS replication add message of %s:\n%s\n", GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), s)); talloc_free(s); + } else if (DEBUGLVL(4)) { + struct GUID_txt_buf guid_txt; + DEBUG(4, ("DRS replication add DN of %s is %s\n", + GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), + ldb_dn_get_linearized(msg->dn))); } - remote_isDeleted = ldb_msg_find_attr_as_bool(msg, "isDeleted", false); @@ -5551,12 +5555,12 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar) } } - if (DEBUGLVL(5)) { + if (DEBUGLVL(8)) { struct GUID_txt_buf guid_txt; char *s = ldb_ldif_message_redacted_string(ldb, ar, LDB_CHANGETYPE_MODIFY, msg); - DEBUG(5, ("Initial DRS replication modify message of %s is:\n%s\n" + DEBUG(8, ("Initial DRS replication modify message of %s is:\n%s\n" "%s\n" "%s\n", GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), @@ -5570,8 +5574,15 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar) "incoming replPropertyMetaData", rmd))); talloc_free(s); - } + } else if (DEBUGLVL(4)) { + struct GUID_txt_buf guid_txt; + DEBUG(4, ("Initial DRS replication modify DN of %s is: %s\n", + GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, + &guid_txt), + ldb_dn_get_linearized(msg->dn))); + } + local_isDeleted = ldb_msg_find_attr_as_bool(ar->search_msg, "isDeleted", false); remote_isDeleted = ldb_msg_find_attr_as_bool(msg, @@ -5828,16 +5839,24 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar) } } - if (DEBUGLVL(4)) { + if (DEBUGLVL(8)) { struct GUID_txt_buf guid_txt; char *s = ldb_ldif_message_redacted_string(ldb, ar, LDB_CHANGETYPE_MODIFY, msg); - DEBUG(4, ("Final DRS replication modify message of %s:\n%s\n", - GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, &guid_txt), + DEBUG(8, ("Final DRS replication modify message of %s:\n%s\n", + GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, + &guid_txt), s)); talloc_free(s); + } else if (DEBUGLVL(4)) { + struct GUID_txt_buf guid_txt; + + DEBUG(4, ("Final DRS replication modify DN of %s is %s\n", + GUID_buf_string(&ar->objs->objects[ar->index_current].object_guid, + &guid_txt), + ldb_dn_get_linearized(msg->dn))); } ret = ldb_build_mod_req(&change_req, -- 2.11.0 From 5ea534430921cc3e705bc90f66d0caf472dc932f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 16:37:34 +1200 Subject: [PATCH 18/23] debug: Add new debug class "drs_repl" for DRS replication processing This is used in the client and in the server BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 51289a6f9bf25189386dd3f66b5b547f02348508) --- lib/util/debug.c | 1 + lib/util/debug.h | 1 + source4/dsdb/repl/drepl_extended.c | 3 +++ source4/dsdb/repl/drepl_fsmo.c | 3 +++ source4/dsdb/repl/drepl_notify.c | 3 +++ source4/dsdb/repl/drepl_out_helpers.c | 3 +++ source4/dsdb/repl/drepl_out_pull.c | 3 +++ source4/dsdb/repl/drepl_partitions.c | 3 +++ source4/dsdb/repl/drepl_periodic.c | 3 +++ source4/dsdb/repl/drepl_replica.c | 3 +++ source4/dsdb/repl/drepl_ridalloc.c | 3 +++ source4/dsdb/repl/drepl_secret.c | 3 +++ source4/dsdb/repl/drepl_service.c | 3 +++ source4/dsdb/repl/replicated_objects.c | 3 +++ source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 3 +++ source4/libnet/libnet_vampire.c | 3 +++ source4/rpc_server/drsuapi/addentry.c | 3 +++ source4/rpc_server/drsuapi/dcesrv_drsuapi.c | 3 +++ source4/rpc_server/drsuapi/drsutil.c | 3 +++ source4/rpc_server/drsuapi/getncchanges.c | 3 +++ source4/rpc_server/drsuapi/updaterefs.c | 3 +++ source4/rpc_server/drsuapi/writespn.c | 3 +++ 22 files changed, 62 insertions(+) diff --git a/lib/util/debug.c b/lib/util/debug.c index d30b1a9fae5..8b28002501f 100644 --- a/lib/util/debug.c +++ b/lib/util/debug.c @@ -540,6 +540,7 @@ static const char *default_classname_table[] = { [DBGC_AUTH_AUDIT] = "auth_audit", [DBGC_AUTH_AUDIT_JSON] = "auth_json_audit", [DBGC_KERBEROS] = "kerberos", + [DBGC_DRS_REPL] = "drs_repl", }; /* diff --git a/lib/util/debug.h b/lib/util/debug.h index 71d8ed69a42..e82553aaf71 100644 --- a/lib/util/debug.h +++ b/lib/util/debug.h @@ -92,6 +92,7 @@ bool dbghdr( int level, const char *location, const char *func); #define DBGC_AUTH_AUDIT 24 #define DBGC_AUTH_AUDIT_JSON 25 #define DBGC_KERBEROS 26 +#define DBGC_DRS_REPL 27 /* So you can define DBGC_CLASS before including debug.h */ #ifndef DBGC_CLASS diff --git a/source4/dsdb/repl/drepl_extended.c b/source4/dsdb/repl/drepl_extended.c index de53028b7a4..caed06b0922 100644 --- a/source4/dsdb/repl/drepl_extended.c +++ b/source4/dsdb/repl/drepl_extended.c @@ -31,6 +31,9 @@ #include "dsdb/repl/drepl_service.h" #include "param/param.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* create the role owner source dsa structure diff --git a/source4/dsdb/repl/drepl_fsmo.c b/source4/dsdb/repl/drepl_fsmo.c index 7a107da6a3d..a83ac4f6c2f 100644 --- a/source4/dsdb/repl/drepl_fsmo.c +++ b/source4/dsdb/repl/drepl_fsmo.c @@ -31,6 +31,9 @@ #include "dsdb/repl/drepl_service.h" #include "param/param.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + struct fsmo_role_state { struct irpc_message *msg; struct drepl_takeFSMORole *r; diff --git a/source4/dsdb/repl/drepl_notify.c b/source4/dsdb/repl/drepl_notify.c index 980806b89c1..f21ad7f94e0 100644 --- a/source4/dsdb/repl/drepl_notify.c +++ b/source4/dsdb/repl/drepl_notify.c @@ -35,6 +35,9 @@ #include "libcli/composite/composite.h" #include "../lib/util/tevent_ntstatus.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + struct dreplsrv_op_notify_state { struct tevent_context *ev; diff --git a/source4/dsdb/repl/drepl_out_helpers.c b/source4/dsdb/repl/drepl_out_helpers.c index 079edc8ba46..80762f03130 100644 --- a/source4/dsdb/repl/drepl_out_helpers.c +++ b/source4/dsdb/repl/drepl_out_helpers.c @@ -36,6 +36,9 @@ #include "../lib/util/tevent_ntstatus.h" #include "libcli/security/security.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + struct dreplsrv_out_drsuapi_state { struct tevent_context *ev; diff --git a/source4/dsdb/repl/drepl_out_pull.c b/source4/dsdb/repl/drepl_out_pull.c index 8af64124e89..9a9388ce076 100644 --- a/source4/dsdb/repl/drepl_out_pull.c +++ b/source4/dsdb/repl/drepl_out_pull.c @@ -33,6 +33,9 @@ #include "libcli/composite/composite.h" #include "libcli/security/security.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* update repsFrom/repsTo error information */ diff --git a/source4/dsdb/repl/drepl_partitions.c b/source4/dsdb/repl/drepl_partitions.c index 65b4843a5f3..586da1983dc 100644 --- a/source4/dsdb/repl/drepl_partitions.c +++ b/source4/dsdb/repl/drepl_partitions.c @@ -34,6 +34,9 @@ #include "param/param.h" #include "dsdb/common/util.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* load the partitions list based on replicated NC attributes in our NTDSDSA object diff --git a/source4/dsdb/repl/drepl_periodic.c b/source4/dsdb/repl/drepl_periodic.c index d6b94679e0e..9bb00696939 100644 --- a/source4/dsdb/repl/drepl_periodic.c +++ b/source4/dsdb/repl/drepl_periodic.c @@ -31,6 +31,9 @@ #include "librpc/gen_ndr/ndr_drsuapi.h" #include "librpc/gen_ndr/ndr_drsblobs.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + static void dreplsrv_periodic_run(struct dreplsrv_service *service); static void dreplsrv_periodic_handler_te(struct tevent_context *ev, struct tevent_timer *te, diff --git a/source4/dsdb/repl/drepl_replica.c b/source4/dsdb/repl/drepl_replica.c index 2f344a4b93a..adb0e971eba 100644 --- a/source4/dsdb/repl/drepl_replica.c +++ b/source4/dsdb/repl/drepl_replica.c @@ -28,6 +28,9 @@ #include "param/param.h" #include "librpc/gen_ndr/ndr_drsuapi.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* implement DsReplicaAdd (forwarded from DRS server) */ diff --git a/source4/dsdb/repl/drepl_ridalloc.c b/source4/dsdb/repl/drepl_ridalloc.c index bd3a62b14b1..3615859ed9b 100644 --- a/source4/dsdb/repl/drepl_ridalloc.c +++ b/source4/dsdb/repl/drepl_ridalloc.c @@ -30,6 +30,9 @@ #include "dsdb/repl/drepl_service.h" #include "param/param.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* called when a rid allocation request has completed */ diff --git a/source4/dsdb/repl/drepl_secret.c b/source4/dsdb/repl/drepl_secret.c index b7f36805a59..64075194009 100644 --- a/source4/dsdb/repl/drepl_secret.c +++ b/source4/dsdb/repl/drepl_secret.c @@ -28,6 +28,9 @@ #include "dsdb/repl/drepl_service.h" #include "param/param.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + struct repl_secret_state { const char *user_dn; }; diff --git a/source4/dsdb/repl/drepl_service.c b/source4/dsdb/repl/drepl_service.c index 39791b44aa8..b02717ae154 100644 --- a/source4/dsdb/repl/drepl_service.c +++ b/source4/dsdb/repl/drepl_service.c @@ -35,6 +35,9 @@ #include "param/param.h" #include "libds/common/roles.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /** * Call-back data for _drepl_replica_sync_done_cb() */ diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index e5fec1fc094..3f1a7e86e62 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -31,6 +31,9 @@ #include "libcli/auth/libcli_auth.h" #include "param/param.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + static WERROR dsdb_repl_merge_working_schema(struct ldb_context *ldb, struct dsdb_schema *dest_schema, const struct dsdb_schema *ref_schema) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 56c7218d76e..e4c3cda8cda 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -51,6 +51,9 @@ #include "dsdb/samdb/ldb_modules/util.h" #include "lib/util/tsort.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* * It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 * Deleted Objects Container diff --git a/source4/libnet/libnet_vampire.c b/source4/libnet/libnet_vampire.c index 7f25a3aa078..25e261e5e2d 100644 --- a/source4/libnet/libnet_vampire.c +++ b/source4/libnet/libnet_vampire.c @@ -42,6 +42,9 @@ #include "libcli/security/security.h" #include "dsdb/common/util.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* List of tasks vampire.py must perform: - Domain Join diff --git a/source4/rpc_server/drsuapi/addentry.c b/source4/rpc_server/drsuapi/addentry.c index 7cfb3fdf99f..ff23c525518 100644 --- a/source4/rpc_server/drsuapi/addentry.c +++ b/source4/rpc_server/drsuapi/addentry.c @@ -30,6 +30,9 @@ #include "rpc_server/drsuapi/dcesrv_drsuapi.h" #include "librpc/gen_ndr/ndr_drsuapi.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* add special SPNs needed for DRS replication to machine accounts when an AddEntry is done to create a nTDSDSA object diff --git a/source4/rpc_server/drsuapi/dcesrv_drsuapi.c b/source4/rpc_server/drsuapi/dcesrv_drsuapi.c index 21de6477a74..026098dcb9f 100644 --- a/source4/rpc_server/drsuapi/dcesrv_drsuapi.c +++ b/source4/rpc_server/drsuapi/dcesrv_drsuapi.c @@ -33,6 +33,9 @@ #include "param/param.h" #include "lib/messaging/irpc.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + #define DRSUAPI_UNSUPPORTED(fname) do { \ DEBUG(1,(__location__ ": Unsupported DRS call %s\n", #fname)); \ if (DEBUGLVL(2)) NDR_PRINT_IN_DEBUG(fname, r); \ diff --git a/source4/rpc_server/drsuapi/drsutil.c b/source4/rpc_server/drsuapi/drsutil.c index 2762a4e5cc2..53bcb16bd8e 100644 --- a/source4/rpc_server/drsuapi/drsutil.c +++ b/source4/rpc_server/drsuapi/drsutil.c @@ -28,6 +28,9 @@ #include "auth/session.h" #include "rpc_server/drsuapi/dcesrv_drsuapi.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + int drsuapi_search_with_extended_dn(struct ldb_context *ldb, TALLOC_CTX *mem_ctx, struct ldb_result **_res, diff --git a/source4/rpc_server/drsuapi/getncchanges.c b/source4/rpc_server/drsuapi/getncchanges.c index 32580bd23e0..fc37a0db44b 100644 --- a/source4/rpc_server/drsuapi/getncchanges.c +++ b/source4/rpc_server/drsuapi/getncchanges.c @@ -42,6 +42,9 @@ #include "lib/dbwrap/dbwrap_rbt.h" #include "librpc/gen_ndr/ndr_misc.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* state of a partially completed getncchanges call */ struct drsuapi_getncchanges_state { struct db_context *anc_cache; diff --git a/source4/rpc_server/drsuapi/updaterefs.c b/source4/rpc_server/drsuapi/updaterefs.c index 9cf3556f497..c92add744ec 100644 --- a/source4/rpc_server/drsuapi/updaterefs.c +++ b/source4/rpc_server/drsuapi/updaterefs.c @@ -30,6 +30,9 @@ #include "librpc/gen_ndr/ndr_irpc_c.h" #include "lib/messaging/irpc.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + struct repsTo { uint32_t count; struct repsFromToBlob *r; diff --git a/source4/rpc_server/drsuapi/writespn.c b/source4/rpc_server/drsuapi/writespn.c index b3903d315b4..991efccae38 100644 --- a/source4/rpc_server/drsuapi/writespn.c +++ b/source4/rpc_server/drsuapi/writespn.c @@ -32,6 +32,9 @@ #include "librpc/gen_ndr/ndr_drsuapi.h" #include "auth/session.h" +#undef DBGC_CLASS +#define DBGC_CLASS DBGC_DRS_REPL + /* check that the SPN update should be allowed as an override via sam_ctx_system -- 2.11.0 From 48882294a80f15186fcbe2454eb902dedd2bd744 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 6 Sep 2017 16:40:05 +1200 Subject: [PATCH 19/23] drs repl: Only print raw DRS replication traffic at level 9 This can be sensitive even with the passwords still encrypted. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit dc48fa982251292a0d46d701c6e912620caf7c72) --- python/samba/drs_utils.py | 2 +- python/samba/join.py | 4 ++-- python/samba/netcmd/rodc.py | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/samba/drs_utils.py b/python/samba/drs_utils.py index 8624f3f2238..7221ca34446 100644 --- a/python/samba/drs_utils.py +++ b/python/samba/drs_utils.py @@ -46,7 +46,7 @@ def drsuapi_connect(server, lp, creds): """ binding_options = "seal" - if lp.log_level() >= 5: + if lp.log_level() >= 9: binding_options += ",print" binding_string = "ncacn_ip_tcp:%s[%s]" % (server, binding_options) try: diff --git a/python/samba/join.py b/python/samba/join.py index fa87f0bb32f..3aefb7baa97 100644 --- a/python/samba/join.py +++ b/python/samba/join.py @@ -437,7 +437,7 @@ class dc_join(object): def drsuapi_connect(ctx): '''make a DRSUAPI connection to the naming master''' binding_options = "seal" - if ctx.lp.log_level() >= 4: + if ctx.lp.log_level() >= 9: binding_options += ",print" binding_string = "ncacn_ip_tcp:%s[%s]" % (ctx.server, binding_options) ctx.drsuapi = drsuapi.drsuapi(binding_string, ctx.lp, ctx.creds) @@ -910,7 +910,7 @@ class dc_join(object): repl_creds = ctx.creds binding_options = "seal" - if ctx.lp.log_level() >= 5: + if ctx.lp.log_level() >= 9: binding_options += ",print" repl = drs_utils.drs_Replicate( "ncacn_ip_tcp:%s[%s]" % (ctx.server, binding_options), diff --git a/python/samba/netcmd/rodc.py b/python/samba/netcmd/rodc.py index e7fbcdc57b0..41a7a85b712 100644 --- a/python/samba/netcmd/rodc.py +++ b/python/samba/netcmd/rodc.py @@ -111,7 +111,11 @@ class cmd_rodc_preload(Command): destination_dsa_guid = misc.GUID(local_samdb.get_ntds_GUID()) - repl = drs_Replicate("ncacn_ip_tcp:%s[seal,print]" % server, lp, creds, + binding_options = "seal" + if lp.log_level() >= 9: + binding_options += ",print" + repl = drs_Replicate("ncacn_ip_tcp:%s[%s]" % (server, binding_options), + lp, creds, local_samdb, destination_dsa_guid) errors = [] -- 2.11.0 From 6edeec8869a60d46380a8e8e5ab399083f801e55 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 7 Sep 2017 11:13:17 +1200 Subject: [PATCH 20/23] dsdb: Add missing \n to debug BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 8d8d31eb2bce6fa25485c4e989e6df372fb0e66e) --- source4/dsdb/common/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source4/dsdb/common/util.c b/source4/dsdb/common/util.c index 7a12c71799b..cecef81e8cf 100644 --- a/source4/dsdb/common/util.c +++ b/source4/dsdb/common/util.c @@ -3972,7 +3972,7 @@ int dsdb_find_nc_root(struct ldb_context *samdb, TALLOC_CTX *mem_ctx, struct ldb if ((el == NULL) || (el->num_values < 3)) { struct ldb_message *tmp_msg; - DEBUG(5,("dsdb_find_nc_root: Finding a valid 'namingContexts' element in the RootDSE failed. Using a temporary list.")); + DEBUG(5,("dsdb_find_nc_root: Finding a valid 'namingContexts' element in the RootDSE failed. Using a temporary list.\n")); /* This generates a temporary list of NCs in order to let the * provisioning work. */ -- 2.11.0 From 5115e5462b0aae7bfa9d4e08a7153401fa458c5f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 7 Sep 2017 11:19:01 +1200 Subject: [PATCH 21/23] libcli/security: Move debug message to DBG_DEBUG() This message shows up a lot at level 6 for no particularly good reason BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 544084d4a2e19958982e6003b1b1290315099b34) --- libcli/security/create_descriptor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcli/security/create_descriptor.c b/libcli/security/create_descriptor.c index 03c0658485e..ef60d847033 100644 --- a/libcli/security/create_descriptor.c +++ b/libcli/security/create_descriptor.c @@ -253,7 +253,7 @@ static struct security_acl *process_user_acl(TALLOC_CTX *mem_ctx, return NULL; tmp_acl->revision = acl->revision; - DEBUG(6,(__location__ ": acl revision %d\n", acl->revision)); + DBG_DEBUG("acl revision %d\n", acl->revision); for (i=0; i < acl->num_aces; i++){ struct security_ace *ace = &acl->aces[i]; -- 2.11.0 From f1747cbc7db230aa70fa7e8b25dde2d01f27a683 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 7 Sep 2017 11:20:27 +1200 Subject: [PATCH 22/23] librpc/dceprc_util.c: Move debug message to DBG_DEBUG() This message shows up a lot (every packet) at level 6 for the succesful case BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit cbb3dcf2c67dd6ddbb419fff04112e3c345c2108) --- librpc/rpc/dcerpc_util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/librpc/rpc/dcerpc_util.c b/librpc/rpc/dcerpc_util.c index b09e36a3121..0808131dd14 100644 --- a/librpc/rpc/dcerpc_util.c +++ b/librpc/rpc/dcerpc_util.c @@ -267,8 +267,8 @@ NTSTATUS dcerpc_pull_auth_trailer(const struct ncacn_packet *pkt, return NT_STATUS_RPC_PROTOCOL_ERROR; } - DEBUG(6,(__location__ ": auth_pad_length %u\n", - (unsigned)auth->auth_pad_length)); + DBG_DEBUG("auth_pad_length %u\n", + (unsigned)auth->auth_pad_length); talloc_steal(mem_ctx, auth->credentials.data); talloc_free(ndr); -- 2.11.0 From bfdb51b77b99dd5e112ae20d67fe2220637e0005 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 7 Sep 2017 11:26:04 +1200 Subject: [PATCH 23/23] python: Allow debug classes to be specified on the command line for python tools Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall BUG: https://bugzilla.samba.org/show_bug.cgi?id=13017 Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu Sep 7 10:43:33 CEST 2017 on sn-devel-144 (cherry picked from commit c938f61d332de0323cb135b201367f90f08d76a8) --- python/samba/getopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/samba/getopt.py b/python/samba/getopt.py index 9e1fb83167a..f166469e77f 100644 --- a/python/samba/getopt.py +++ b/python/samba/getopt.py @@ -40,7 +40,7 @@ class SambaOptions(optparse.OptionGroup): type=str, metavar="FILE", help="Configuration file", callback=self._load_configfile) self.add_option("-d", "--debuglevel", action="callback", - type=int, metavar="DEBUGLEVEL", help="debug level", + type=str, metavar="DEBUGLEVEL", help="debug level", callback=self._set_debuglevel) self.add_option("--option", action="callback", type=str, metavar="OPTION", @@ -64,8 +64,8 @@ class SambaOptions(optparse.OptionGroup): if arg < 0: raise optparse.OptionValueError("invalid %s option value: %s" % (opt_str, arg)) - self._lp.set('debug level', str(arg)) - parser.values.debuglevel = int(arg) + self._lp.set('debug level', arg) + parser.values.debuglevel = arg def _set_realm(self, option, opt_str, arg, parser): self._lp.set('realm', arg) -- 2.11.0