From 9285a536960c9c4d435e909e4cf87ce9ee33ce9f Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 7 Sep 2018 12:42:19 +0100 Subject: [PATCH 01/18] python/samba: PY3 port for ridalloc_exop test to work Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett (cherry picked from commit fc13a1268a4a9de94efd312a8309aa55d331ae19) --- python/samba/dbchecker.py | 9 ++++----- python/samba/remove_dc.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 2619b9bc72e3..6a44a46c61e6 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1979,8 +1979,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) raise else: instancetype |= dsdb.INSTANCE_TYPE_NC_ABOVE - - if self.write_ncs is not None and str(nc_root) in self.write_ncs: + if self.write_ncs is not None and str(nc_root) in [str(x) for x in self.write_ncs]: instancetype |= dsdb.INSTANCE_TYPE_WRITE return instancetype @@ -2100,10 +2099,10 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" % (len(obj[attrname]), attrname, str(obj.dn))) else: - object_rdn_val = obj[attrname][0] + object_rdn_val = str(obj[attrname][0]) if str(attrname).lower() == 'isdeleted': - if obj[attrname][0] != "FALSE": + if str(obj[attrname][0]) != "FALSE": isDeleted = True if str(attrname).lower() == 'systemflags': @@ -2278,7 +2277,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if str(attrname).lower() == "instancetype": calculated_instancetype = self.calculate_instancetype(dn) - if len(obj["instanceType"]) != 1 or obj["instanceType"][0] != str(calculated_instancetype): + if len(obj["instanceType"]) != 1 or int(obj["instanceType"][0]) != calculated_instancetype: error_count += 1 self.err_wrong_instancetype(obj, calculated_instancetype) diff --git a/python/samba/remove_dc.py b/python/samba/remove_dc.py index 4c8ee8924643..84360ed15039 100644 --- a/python/samba/remove_dc.py +++ b/python/samba/remove_dc.py @@ -221,7 +221,7 @@ def offline_remove_server(samdb, logger, computer_dn = None try: - dnsHostName = msgs[0]["dnsHostName"][0] + dnsHostName = str(msgs[0]["dnsHostName"][0]) except KeyError: dnsHostName = None @@ -251,7 +251,7 @@ def offline_remove_server(samdb, logger, samdb.delete(computer_dn, ["tree_delete:0"]) if "dnsHostName" in msgs[0]: - dnsHostName = msgs[0]["dnsHostName"][0] + dnsHostName = str(msgs[0]["dnsHostName"][0]) if remove_dns_account: res = samdb.search(expression="(&(objectclass=user)(cn=dns-%s)(servicePrincipalName=DNS/%s))" % -- 2.17.1 From 306b932443b91a03536600dd24728198eb4bf78e Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 7 Sep 2018 14:38:54 +0100 Subject: [PATCH 02/18] python/samba: extra ndr_unpack needs bytes function (cherry picked from commit 8db43696e70d7c4cb21172b7e7461cf6a72914a2) --- python/samba/dbchecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 6a44a46c61e6..f7310b0b8e4b 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2320,7 +2320,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container expectedTimeDo = 2650466015990000000 - originating = self.get_originating_time(obj["replPropertyMetaData"], isDeletedAttId) + originating = self.get_originating_time(obj["replPropertyMetaData"][0], isDeletedAttId) if originating != expectedTimeDo: if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'): nmsg = ldb.Message() -- 2.17.1 From 2b95b374d8609789fec77df56b747519b3d96794 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 27 Feb 2019 08:22:09 +0100 Subject: [PATCH 03/18] selftest: force running with TZ=UTC Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Wed Feb 27 11:24:59 UTC 2019 on sn-devel-144 (cherry picked from commit 4f307f2302b0fe8fd0fc6379eb8e6491faf8520c) --- selftest/selftest.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 0e56e6a13ef1..e35e67798ed3 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -286,6 +286,9 @@ unless (defined($ENV{VALGRIND})) { # make all our python scripts unbuffered $ENV{PYTHONUNBUFFERED} = 1; +# do not depend on the users setup +$ENV{TZ} = "UTC"; + my $bindir_abs = abs_path($bindir); # Backwards compatibility: -- 2.17.1 From 4b1ee454b4a413a93622730f278c25e8fb14991a Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 14:52:57 +0100 Subject: [PATCH 04/18] blackbox/dbcheck-links.sh: reproduce lost deleted object problem When a parent object is removed during the tombstone garbage collection before a child object and samba-tool dbcheck runs at the same time, the following can happen: - If the object child had DISALLOW_MOVE_ON_DELETE in systemFlags, samba-tool dbcheck moves the object under the LostAndFound[Config] object (as an originating update!) - The lastKnownParent attribute is removed (as an originating update!) These originating updates cause the object to have an extended time as tombstone. And these changes are replicated to other DCs, which very likely already removed the object completely! This means the destination DC of replication has no chance to handle the object it gets from the source DC with just 2 attributes (name, lastKnownParent). The destination logs something like: No objectClass found in replPropertyMetaData BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5357f591accffbf8c62335c308b985811b66f0b5) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user1.txt | 14 +++ testprogs/blackbox/dbcheck-links.sh | 113 ++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..676281faba58 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt new file mode 100644 index 000000000000..db18b9b188b6 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -0,0 +1,14 @@ +Checking 232 objects +WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' +Remove stale DN link? [YES] +Removed deleted DN on attribute lastKnownParent +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] +Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 9798813004c5..4bd075b5b480 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -238,6 +238,114 @@ dbcheck_missing_link_sid_corruption() { return $? } +add_lost_deleted_user1() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user1.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user1() { + dbcheck "-lost-deleted-user1" "1" "" + return $? +} + +remove_lost_deleted_user1() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -454,6 +562,11 @@ if [ -d $release_dir ]; then testit "missing_link_sid_corruption" missing_link_sid_corruption testit "dbcheck_missing_link_sid_corruption" dbcheck_missing_link_sid_corruption testit "missing_link_sid_clean" dbcheck_clean + testit "add_lost_deleted_user1" add_lost_deleted_user1 + testit "dbcheck_lost_deleted_user1" dbcheck_lost_deleted_user1 + testit "lost_deleted_user1_clean_A" dbcheck_clean + testit "remove_lost_deleted_user1" remove_lost_deleted_user1 + testit "lost_deleted_user1_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From 44433a652cb0c91b1e6d35ab5c6074a5e25879fc Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:38:38 +0100 Subject: [PATCH 05/18] dsdb:repl_meta_data: allow CONTROL_DBCHECK_FIX_LINK_DN_NAME to by pass rename We need a way to rename an object without updating the replication meta data. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 3e8a435d27da899d0e3dab7cbc0a1c738067eba3) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index c2eafd0a5215..b8e03d7ca154 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -3697,6 +3697,7 @@ static int replmd_rename_callback(struct ldb_request *req, struct ldb_reply *are static int replmd_rename(struct ldb_module *module, struct ldb_request *req) { struct ldb_context *ldb; + struct ldb_control *fix_dn_name_control = NULL; struct replmd_replicated_request *ac; int ret; struct ldb_request *down_req; @@ -3706,6 +3707,12 @@ static int replmd_rename(struct ldb_module *module, struct ldb_request *req) return ldb_next_request(module, req); } + fix_dn_name_control = ldb_request_get_control(req, + DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME); + if (fix_dn_name_control != NULL) { + return ldb_next_request(module, req); + } + ldb = ldb_module_get_ctx(module); ldb_debug(ldb, LDB_DEBUG_TRACE, "replmd_rename\n"); -- 2.17.1 From a08f16a4d35fe7ffcdbb8bb3c32e33fd75f522d9 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:45:46 +0100 Subject: [PATCH 06/18] dbcheck: use DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME when renaming deleted objects We should never do originating updates on deleted objects. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 07a8326746f0c444eedf3860b178fc29d84e8d16) --- python/samba/dbchecker.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index f7310b0b8e4b..75b934b3a59f 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -853,7 +853,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: self.samdb.transaction_cancel() - def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val): + def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val, controls): '''handle a wrong dn''' new_rdn = ldb.Dn(self.samdb, str(new_dn)) @@ -870,7 +870,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Not renaming %s to %s" % (obj.dn, new_dn)) return - if self.do_rename(obj.dn, new_rdn, new_parent, ["show_recycled:1", "relax:0"], + if self.do_rename(obj.dn, new_rdn, new_parent, controls, "Failed to rename object %s into %s" % (obj.dn, new_dn)): self.report("Renamed %s into %s" % (obj.dn, new_dn)) @@ -2295,9 +2295,11 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if name_val is not None: parent_dn = None + controls = ["show_recycled:1", "relax:0"] if isDeleted: if not (systemFlags & samba.dsdb.SYSTEM_FLAG_DISALLOW_MOVE_ON_DELETE): parent_dn = deleted_objects_dn + controls += ["local_oid:%s:1" % dsdb.DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME] if parent_dn is None: parent_dn = obj.dn.parent() expected_dn = ldb.Dn(self.samdb, "RDN=RDN,%s" % (parent_dn)) @@ -2308,7 +2310,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if expected_dn != obj.dn: error_count += 1 - self.err_wrong_dn(obj, expected_dn, object_rdn_attr, object_rdn_val, name_val) + self.err_wrong_dn(obj, expected_dn, object_rdn_attr, + object_rdn_val, name_val, controls) elif obj.dn.get_rdn_value() != object_rdn_val: error_count += 1 self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) -- 2.17.1 From 96320b0075c7f8e7826e7fbe6421ccf999d24a2d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:09:36 +0100 Subject: [PATCH 07/18] dbcheck: do isDeleted, systemFlags and replPropertyMetaData detection first BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 9afcd5331ce567bd80d35175f8e4e21c506e9347) --- python/samba/dbchecker.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 75b934b3a59f..a7edb5ace533 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2058,7 +2058,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count = 0 set_attrs_from_md = set() set_attrs_seen = set() - got_repl_property_meta_data = False got_objectclass = False nc_dn = self.samdb.get_nc_root(obj.dn) @@ -2076,6 +2075,18 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) name_val = None isDeleted = False systemFlags = 0 + repl_meta_data_val = None + + for attrname in obj: + if str(attrname).lower() == 'isdeleted': + if str(obj[attrname][0]) != "FALSE": + isDeleted = True + + if str(attrname).lower() == 'systemflags': + systemFlags = int(obj[attrname][0]) + + if str(attrname).lower() == 'replpropertymetadata': + repl_meta_data_val = obj[attrname][0] for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": @@ -2101,13 +2112,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: object_rdn_val = str(obj[attrname][0]) - if str(attrname).lower() == 'isdeleted': - if str(obj[attrname][0]) != "FALSE": - isDeleted = True - - if str(attrname).lower() == 'systemflags': - systemFlags = int(obj[attrname][0]) - if str(attrname).lower() == 'replpropertymetadata': if self.has_replmetadata_zero_invocationid(dn, obj[attrname]): error_count += 1 @@ -2137,7 +2141,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing incorrect inital attributeID in '%s' on '%s', it should be objectClass" % (attrname, str(dn))) - got_repl_property_meta_data = True continue if str(attrname).lower() == 'ntsecuritydescriptor': @@ -2317,13 +2320,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) show_dn = True - if got_repl_property_meta_data: + if repl_meta_data_val: if obj.dn == deleted_objects_dn: isDeletedAttId = 131120 # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container expectedTimeDo = 2650466015990000000 - originating = self.get_originating_time(obj["replPropertyMetaData"][0], isDeletedAttId) + originating = self.get_originating_time(repl_meta_data_val, isDeletedAttId) if originating != expectedTimeDo: if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'): nmsg = ldb.Message() -- 2.17.1 From 82d67a08979d58c63da0c40a070d20cd0aeedeca Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 08/18] dbcheck: don't move already deleted objects to LostAndFound This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6d50ee74920c39cdb18b427bfaaf200775bf2d73) --- python/samba/dbchecker.py | 9 +++++++-- selftest/knownfail.d/dbcheck-list-deleted | 1 - .../expected-dbcheck-link-output-lost-deleted-user1.txt | 8 +++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index a7edb5ace533..560ad5485a2f 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2360,8 +2360,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) controls=["show_recycled:1", "show_deleted:1"]) except ldb.LdbError, (enum, estr): if enum == ldb.ERR_NO_SUCH_OBJECT: - self.err_missing_parent(obj) - error_count += 1 + if isDeleted: + self.report("WARNING: parent object not found for %s" % (obj.dn)) + self.report("Not moving to LostAndFound " + "(tombstone garbage collection in progress?)") + else: + self.err_missing_parent(obj) + error_count += 1 else: raise diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted index 676281faba58..a8fcb0a223f0 100644 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -1,2 +1 @@ ^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index db18b9b188b6..cfc2644b3cbb 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -7,8 +7,6 @@ Removed deleted DN on attribute lastKnownParent ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] -Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) -- 2.17.1 From 97e8b60155eb2b1e9a5d14f58c0ccf76699112ac Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 09/18] dbcheck: don't remove dangling one-way links on already deleted objects This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. In this case the lastKnownParent attributes points a non existing object. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit e388e599495b6d7c38b8b6966332e27f8b958783) --- python/samba/dbchecker.py | 13 +++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 1 - ...ected-dbcheck-link-output-lost-deleted-user1.txt | 7 ++----- 3 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 560ad5485a2f..a11e76038e55 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -544,6 +544,19 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) def err_missing_target_dn_or_GUID(self, dn, attrname, val, dsdb_dn): """handle a missing target DN (if specified, GUID form can't be found, and otherwise DN string form can't be found)""" + + # Don't change anything if the object itself is deleted + if str(dn).find('\\0ADEL') != -1: + # We don't bump the error count as Samba produces these + # in normal operation + self.report("WARNING: no target object found for GUID " + "component link %s in deleted object " + "%s - %s" % (attrname, dn, val)) + self.report("Not removing dangling one-way " + "link on deleted object " + "(tombstone garbage collection in progress?)") + return 0 + # check if its a backlink linkID, _ = self.get_attr_linkID_and_reverse_name(attrname) if (linkID & 1 == 0) and str(dsdb_dn).find('\\0ADEL') == -1: diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index a8fcb0a223f0..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index cfc2644b3cbb..3c55de8fa01f 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -1,9 +1,6 @@ Checking 232 objects -WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' -Remove stale DN link? [YES] -Removed deleted DN on attribute lastKnownParent +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -- 2.17.1 From 34546d97f0b0e88128639f75571599f5212b314a Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2019 18:16:27 +0100 Subject: [PATCH 10/18] dbcheck: add find_repl_attid() helper function BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 598e38d2a5e0832429ba65b4e55bf7127618f894) --- python/samba/dbchecker.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index a11e76038e55..4b21cab80b75 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1454,6 +1454,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) return error_count + def find_repl_attid(self, repl, attid): + for o in repl.ctr.array: + if o.attid == attid: + return o + + return None def get_originating_time(self, val, attid): '''Read metadata properties and return the originating time for @@ -1463,12 +1469,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) ''' repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, str(val)) - obj = repl.ctr - - for o in repl.ctr.array: - if o.attid == attid: - return o.originating_change_time - + o = self.find_repl_attid(repl, attid) + if o is not None: + return o.originating_change_time return 0 def process_metadata(self, dn, val): -- 2.17.1 From d99c5a3d73df9f1dcbc42ab88c32bf667b309ab7 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 23:14:02 +0100 Subject: [PATCH 11/18] blackbox/dbcheck-links.sh: add regression test for lost deleted object repair BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 1ccc21a34d295be3bb2ab481a5918003eae88bf4) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user2.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 100 ++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..670e42b747c6 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt new file mode 100644 index 000000000000..dfb7422ac0bf --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -0,0 +1,9 @@ +Checking 232 objects +ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 4bd075b5b480..161cdd6bf5ed 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -346,6 +346,103 @@ remove_lost_deleted_user1() { return 0 } +add_lost_deleted_user2() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user2.ldif + cat > $ldif < Date: Thu, 28 Feb 2019 18:22:18 +0100 Subject: [PATCH 12/18] dbcheck: detect the change after deletion bug Old versions of 'samba-tool dbcheck' could reanimate deleted objects, when running at the same time as the tombstone garbage collection. When the (deleted) parent of a deleted object (with the DISALLOW_MOVE_ON_DELETE bit in systemFlags), is removed before the object itself, dbcheck moved it in the LostAndFound[Config] subtree of the partition as an originating change. That means that the object will be in tombstone state again for 180 days on the local DC. And other DCs fail to replicate the object as it's already removed completely there and the replication only gives the name and lastKnownParent attributes, because all other attributes should already be known to the other DC. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit a1658b306d85452407388b91a745078c9c1f7dc7) --- python/samba/dbchecker.py | 109 ++++++++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 2 - ...dbcheck-link-output-lost-deleted-user2.txt | 15 ++- 3 files changed, 116 insertions(+), 10 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 4b21cab80b75..98bd5776b20a 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -100,6 +100,7 @@ class dbcheck(object): self.fix_missing_deleted_objects = False self.fix_replica_locations = False self.fix_missing_rid_set_master = False + self.fix_changes_after_deletion_bug = False self.dn_set = set() self.link_id_cache = {} @@ -187,6 +188,14 @@ class dbcheck(object): else: self.rid_set_dn = None + ntds_service_dn = "CN=Directory Service,CN=Windows NT,CN=Services,%s" % \ + self.samdb.get_config_basedn().get_linearized() + res = samdb.search(base=ntds_service_dn, + scope=ldb.SCOPE_BASE, + expression="(objectClass=nTDSService)", + attrs=["tombstoneLifetime"]) + self.tombstoneLifetime = int(res[0]["tombstoneLifetime"][0]) + self.compatibleFeatures = [] self.requiredFeatures = [] @@ -1724,6 +1733,100 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def find_changes_after_deletion(self, repl_val): + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + found = [] + for o in repl.ctr.array: + if o.attid == drsuapi.DRSUAPI_ATTID_isDeleted: + continue + + if o.local_usn <= isDeleted.local_usn: + continue + + if o.originating_change_time <= isDeleted.originating_change_time: + continue + + change_time = samba.nttime2unix(o.originating_change_time) + + delta = change_time - delete_time + if delta <= tombstone_delta: + continue + + # If the modification happened after the tombstone lifetime + # has passed, we have a bug as the object might be deleted + # already on other DCs and won't be able to replicate + # back + found.append(o) + + return found, isDeleted + + def has_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + if len(found) == 0: + return False + + def report_attid(o): + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + + self.report("%s: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + attname, o.attid, o.version, + o.originating_invocation_id, + o.originating_usn, + o.local_usn, + time.ctime(samba.nttime2unix(o.originating_change_time)))) + + self.report("ERROR: object %s, has changes after deletion" % dn) + report_attid(isDeleted) + for o in found: + report_attid(o) + + return True + + def err_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + + in_schema_nc = dn.is_child_of(self.schema_dn) + rdn_attr = dn.get_rdn_name() + rdn_attid = self.samdb_schema.get_attid_from_lDAPDisplayName(rdn_attr, + is_schema_nc=in_schema_nc) + + unexpected = [] + for o in found: + if o.attid == rdn_attid: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_name: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_lastKnownParent: + continue + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + unexpected.append(attname) + + if len(unexpected) > 0: + self.report('Unexpeted attributes: %s' % ",".join(unexpected)) + self.report('Not fixing changes after deletion bug') + return + + if not self.confirm_all('Delete broken tombstone object %s deleted %s days ago?' % ( + dn, self.tombstoneLifetime), 'fix_changes_after_deletion_bug'): + self.report('Not fixing changes after deletion bug') + return + + if self.do_delete(dn, ["relax:0"], + "Failed to remove DN %s" % dn): + self.report("Removed DN %s" % dn) def has_replmetadata_zero_invocationid(self, dn, repl_meta_data): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, @@ -2104,6 +2207,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if str(attrname).lower() == 'replpropertymetadata': repl_meta_data_val = obj[attrname][0] + if isDeleted and repl_meta_data_val: + if self.has_changes_after_deletion(dn, repl_meta_data_val): + error_count += 1 + self.err_changes_after_deletion(dn, repl_meta_data_val) + return error_count + for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": continue diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index 670e42b747c6..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1,2 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt index dfb7422ac0bf..9b87ca10c57e 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -1,9 +1,8 @@ Checking 232 objects -ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +ERROR: object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp, has changes after deletion +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +name: attid=0x00090001 version=4 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3772 (local=3772) at Mon Mar 11 13:28:24 2019 +lastKnownParent: attid=0x0009030d version=3 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3773 (local=3773) at Mon Mar 11 13:28:24 2019 +Delete broken tombstone object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp deleted 180 days ago? [YES] +Removed DN CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (1 errors) -- 2.17.1 From 618deea7aa076a10679bb50a03616f7ccc8fe26d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 10:25:40 +0100 Subject: [PATCH 13/18] python/samba/netcmd: provide SUPPRESS_HELP via Option class BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b61d580fb7dba8ff94e9e98c958e324865cd2f1d) --- python/samba/netcmd/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/samba/netcmd/__init__.py b/python/samba/netcmd/__init__.py index 05ecc432ea6b..9dd6748447f7 100644 --- a/python/samba/netcmd/__init__.py +++ b/python/samba/netcmd/__init__.py @@ -23,6 +23,7 @@ import sys, traceback import textwrap class Option(optparse.Option): + SUPPRESS_HELP = optparse.SUPPRESS_HELP pass # This help formatter does text wrapping and preserves newlines -- 2.17.1 From e03b25537ad7603cab60d7cf451b837497fb2493 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:02:18 +0100 Subject: [PATCH 14/18] dbcheck: add --selftest-check-expired-tombstones cmdline option This will be used by dbcheck tests which operate on static/old provision dumps in the following commits. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6f9c5ed8de47bb98e21e8064d8e90f963f2f71ca) --- python/samba/netcmd/dbcheck.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index 2217366cc8db..efc25c2b33a6 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -74,13 +74,18 @@ class cmd_dbcheck(Command): Option("--reset-well-known-acls", dest="reset_well_known_acls", default=False, action="store_true", help="reset ACLs on objects with well known default ACL values to the default"), Option("-H", "--URL", help="LDB URL for database or target server (defaults to local SAM database)", type=str, metavar="URL", dest="H"), - ] + Option("--selftest-check-expired-tombstones", + dest="selftest_check_expired_tombstones", default=False, action="store_true", + help=Option.SUPPRESS_HELP), # This is only used by tests + ] def run(self, DN=None, H=None, verbose=False, fix=False, yes=False, cross_ncs=False, quiet=False, scope="SUB", credopts=None, sambaopts=None, versionopts=None, attrs=None, reindex=False, force_modules=False, - reset_well_known_acls=False, yes_rules=[]): + reset_well_known_acls=False, + selftest_check_expired_tombstones=False, + yes_rules=[]): lp = sambaopts.get_loadparm() -- 2.17.1 From 593e4a0b31e18eda894ded0ac1939611b10c2cd2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:04:33 +0100 Subject: [PATCH 15/18] blackbox/dbcheck*.sh: pass --selftest-check-expired-tombstones to dbcheck These tests operate on provision dumps created long ago, they still want to run tests on deleted objects, when the next commits remove processing expired tombstone objects in dbcheck. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5fccc4e9044d2e57be33471f5e6b9be7cc37ac3a) --- testprogs/blackbox/dbcheck-links.sh | 18 +++++++++--------- testprogs/blackbox/dbcheck-oldrelease.sh | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 161cdd6bf5ed..acb800d33fdb 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -42,12 +42,12 @@ dbcheck() { } dbcheck_dangling() { - dbcheck "" "1" "" + dbcheck "" "1" "--selftest-check-expired-tombstones" return $? } dbcheck_one_way() { - dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp" + dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp --selftest-check-expired-tombstones" return $? } @@ -118,7 +118,7 @@ duplicate_member() { } dbcheck_duplicate_member() { - dbcheck "_duplicate_member" "1" "" + dbcheck "_duplicate_member" "1" "--selftest-check-expired-tombstones" return $? } @@ -234,7 +234,7 @@ EOF } dbcheck_missing_link_sid_corruption() { - dbcheck "-missing-link-sid-corruption" "1" "" + dbcheck "-missing-link-sid-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -332,7 +332,7 @@ EOF } dbcheck_lost_deleted_user1() { - dbcheck "-lost-deleted-user1" "1" "" + dbcheck "-lost-deleted-user1" "1" "--selftest-check-expired-tombstones" return $? } @@ -439,7 +439,7 @@ EOF } dbcheck_lost_deleted_user2() { - dbcheck "-lost-deleted-user2" "1" "" + dbcheck "-lost-deleted-user2" "1" "--selftest-check-expired-tombstones" return $? } @@ -504,7 +504,7 @@ EOF } dbcheck_forward_link_corruption() { - dbcheck "-forward-link-corruption" "1" "" + dbcheck "-forward-link-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -565,7 +565,7 @@ EOF } dbcheck_oneway_link_corruption() { - dbcheck "-oneway-link-corruption" "0" "" + dbcheck "-oneway-link-corruption" "0" "--selftest-check-expired-tombstones" return $? } @@ -580,7 +580,7 @@ check_expected_after_dbcheck_oneway_link_corruption() { dbcheck_dangling_multi_valued() { - $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --fix --yes + $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --selftest-check-expired-tombstones --fix --yes if [ "$?" != "1" ]; then return 1 fi diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh index a525f547bc93..6e0725709f69 100755 --- a/testprogs/blackbox/dbcheck-oldrelease.sh +++ b/testprogs/blackbox/dbcheck-oldrelease.sh @@ -195,7 +195,7 @@ check_expected_before_values() { # This should 'fail', because it returns the number of modified records dbcheck_objectclass() { if [ x$RELEASE = x"release-4-1-6-partial-object" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ else return 1 fi @@ -203,7 +203,7 @@ dbcheck_objectclass() { # This should 'fail', because it returns the number of modified records dbcheck() { - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ } check_expected_after_values() { @@ -273,7 +273,7 @@ check_forced_duplicate_values() { # This should 'fail', because it returns the number of modified records dbcheck_after_dup() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ else return 1 fi @@ -316,7 +316,7 @@ dbcheck_acl_reset_clean() { # This should 'fail', because it returns the number of modified records dbcheck2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -324,7 +324,7 @@ dbcheck2() { # But having fixed it all up, this should pass dbcheck_clean2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } @@ -341,7 +341,7 @@ rm_deleted_objects() { # This should 'fail', because it returns the number of modified records dbcheck3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -349,7 +349,7 @@ dbcheck3() { # But having fixed it all up, this should pass dbcheck_clean3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } -- 2.17.1 From 7736265801f90c1db4839c178e449c3b875f2b83 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:38:22 +0100 Subject: [PATCH 16/18] blackbox/dbcheck-links.sh: prepare regression test for skipping expired tombstones BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b096a3117ed9249fd6f65f3221a26c88efbba3b8) --- ...dbcheck-link-output-lost-deleted-user3.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 115 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt new file mode 100644 index 000000000000..67ca493c44f7 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -0,0 +1,9 @@ +Checking 232 objects +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index acb800d33fdb..49f9b1e21d49 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -443,6 +443,116 @@ dbcheck_lost_deleted_user2() { return $? } +add_lost_deleted_user3() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user3.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user3() { + # here we don't pass --selftest-check-expired-tombstones + # as we want to test the default + dbcheck "-lost-deleted-user3" "1" "" + return $? +} + +remove_lost_deleted_user3() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -667,6 +777,11 @@ if [ -d $release_dir ]; then testit "add_lost_deleted_user2" add_lost_deleted_user2 testit "dbcheck_lost_deleted_user2" dbcheck_lost_deleted_user2 testit "lost_deleted_user2_clean" dbcheck_clean + testit "add_lost_deleted_user3" add_lost_deleted_user3 + testit "dbcheck_lost_deleted_user3" dbcheck_lost_deleted_user3 + testit "lost_deleted_user3_clean_A" dbcheck_clean + testit "remove_lost_deleted_user3" remove_lost_deleted_user3 + testit "lost_deleted_user3_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From ab34efeb7253ece5ac4fff0ae9c4ab2e3ef8d60c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:41:01 +0100 Subject: [PATCH 17/18] dbcheck: don't check expired tombstone objects by default anymore These will be removed anyway and any change on them risks to be an originating update that causes replication problems. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu Mar 14 03:12:27 UTC 2019 on sn-devel-144 (cherry picked from commit a2c5f8cf41c2dfdc4f122e8427d1dfeabb6ba311) --- python/samba/dbchecker.py | 45 ++++++++++++++++++- python/samba/netcmd/dbcheck.py | 6 ++- ...dbcheck-link-output-lost-deleted-user3.txt | 26 +++++++---- testprogs/blackbox/dbcheck-links.sh | 2 +- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 98bd5776b20a..5d568a739696 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -40,7 +40,8 @@ class dbcheck(object): def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, yes=False, quiet=False, in_transaction=False, - reset_well_known_acls=False): + reset_well_known_acls=False, + check_expired_tombstones=False): self.samdb = samdb self.dict_oid_name = None self.samdb_schema = (samdb_schema or samdb) @@ -87,6 +88,8 @@ class dbcheck(object): self.fix_doubled_userparameters = False self.fix_sid_rid_set_conflict = False self.reset_well_known_acls = reset_well_known_acls + self.check_expired_tombstones = check_expired_tombstones + self.expired_tombstones = 0 self.reset_all_well_known_acls = False self.in_transaction = in_transaction self.infrastructure_dn = ldb.Dn(samdb, "CN=Infrastructure," + samdb.domain_dn()) @@ -230,6 +233,13 @@ class dbcheck(object): if DN is None: error_count += self.check_rootdse() + if self.expired_tombstones > 0: + self.report("NOTICE: found %d expired tombstones, " + "'samba' will remove them daily, " + "'samba-tool domain tombstones expunge' " + "would do that immediately." % ( + self.expired_tombstones)) + if error_count != 0 and not self.fix: self.report("Please use --fix to fix these errors") @@ -1733,6 +1743,37 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def is_expired_tombstone(self, dn, repl_val): + if self.check_expired_tombstones: + # This is not the default, it's just + # used to keep dbcheck tests work with + # old static provision dumps + return False + + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + current_time = time.time() + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + delta = current_time - delete_time + if delta <= tombstone_delta: + return False + + self.report("SKIPING: object %s is an expired tombstone" % dn) + self.report("isDeleted: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + isDeleted.attid, + isDeleted.version, + isDeleted.originating_invocation_id, + isDeleted.originating_usn, + isDeleted.local_usn, + time.ctime(samba.nttime2unix(isDeleted.originating_change_time)))) + self.expired_tombstones += 1 + return True + def find_changes_after_deletion(self, repl_val): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) @@ -2212,6 +2253,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count += 1 self.err_changes_after_deletion(dn, repl_meta_data_val) return error_count + if self.is_expired_tombstone(dn, repl_meta_data_val): + return error_count for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index efc25c2b33a6..57030274cad7 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -136,8 +136,10 @@ class cmd_dbcheck(Command): started_transaction = True try: chk = dbcheck(samdb, samdb_schema=samdb_schema, verbose=verbose, - fix=fix, yes=yes, quiet=quiet, in_transaction=started_transaction, - reset_well_known_acls=reset_well_known_acls) + fix=fix, yes=yes, quiet=quiet, + in_transaction=started_transaction, + reset_well_known_acls=reset_well_known_acls, + check_expired_tombstones=selftest_check_expired_tombstones) for option in yes_rules: if hasattr(chk, option): diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt index 67ca493c44f7..d014bfacae2c 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -1,9 +1,19 @@ Checking 232 objects -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not moving to LostAndFound (tombstone garbage collection in progress?) -Checked 232 objects (1 errors) +SKIPING: object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=fred\0ADEL:2301a64c-5b42-4ca8-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=dsg\0ADEL:6d66d0ef-cad7-4e5d-b1b6-4a233a21c269,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3734 (local=3734) at Wed Jun 29 04:34:32 2016 +SKIPING: object CN=udg\0ADEL:7cff5537-51b1-4d26-a295-0225dbea8525,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3739 (local=3739) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=usg\0ADEL:d012e8f5-a4bd-40ea-a2a1-68ff2508847d,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3736 (local=3736) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=ddg\0ADEL:fb8c2fe3-5448-43de-99f9-e1d3b9357cfc,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3737 (local=3737) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=gsg\0ADEL:91aa85cc-fc19-4b8c-9fc7-aaba425439c7,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3735 (local=3735) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=gdg\0ADEL:e0f581e7-14ee-4fc2-839c-8f46f581c72a,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3738 (local=3738) at Wed Jun 29 04:34:34 2016 +NOTICE: found 8 expired tombstones, 'samba' will remove them daily, 'samba-tool domain tombstones expunge' would do that immediately. +Checked 232 objects (0 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 49f9b1e21d49..851c0dc9a1fb 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -539,7 +539,7 @@ EOF dbcheck_lost_deleted_user3() { # here we don't pass --selftest-check-expired-tombstones # as we want to test the default - dbcheck "-lost-deleted-user3" "1" "" + dbcheck "-lost-deleted-user3" "0" "" return $? } -- 2.17.1 From 793ec6aa5182cae9c740da031cc909e7c5d9526c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 19 Mar 2019 13:05:16 +0100 Subject: [PATCH 18/18] dbcheck: use the str() value of the "name" attribute We do the same with the rdn attribute value and we need the same logic on both in order to check they are the same. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Noel Power (cherry picked from commit dd6f0dad218ec1d5aa38ea8aa6848ec81035cb3f) --- python/samba/dbchecker.py | 2 +- .../expected-dbcheck-link-output-lost-deleted-user1.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 5d568a739696..f17ff39ae02b 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2269,7 +2269,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" % (len(obj[attrname]), attrname, str(obj.dn))) else: - name_val = obj[attrname][0] + name_val = str(obj[attrname][0]) if str(attrname).lower() == str(obj.dn.get_rdn_name()).lower(): object_rdn_attr = attrname diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index 3c55de8fa01f..1f5f2272bc10 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -1,7 +1,7 @@ Checking 232 objects WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] name='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -- 2.17.1