From b92d67b67a254ffcd9b5a76a662f4c680d699d25 Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 7 Sep 2018 12:42:19 +0100 Subject: [PATCH 01/18] python/samba: PY3 port for ridalloc_exop test to work Signed-off-by: Noel Power Reviewed-by: Andrew Bartlett (cherry picked from commit fc13a1268a4a9de94efd312a8309aa55d331ae19) --- python/samba/dbchecker.py | 9 ++++----- python/samba/remove_dc.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index a35c4075a43f..6fdd59a81375 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1989,8 +1989,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) raise else: instancetype |= dsdb.INSTANCE_TYPE_NC_ABOVE - - if self.write_ncs is not None and str(nc_root) in self.write_ncs: + if self.write_ncs is not None and str(nc_root) in [str(x) for x in self.write_ncs]: instancetype |= dsdb.INSTANCE_TYPE_WRITE return instancetype @@ -2111,10 +2110,10 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" % (len(obj[attrname]), attrname, str(obj.dn))) else: - object_rdn_val = obj[attrname][0] + object_rdn_val = str(obj[attrname][0]) if str(attrname).lower() == 'isdeleted': - if obj[attrname][0] != "FALSE": + if str(obj[attrname][0]) != "FALSE": isDeleted = True if str(attrname).lower() == 'systemflags': @@ -2289,7 +2288,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if str(attrname).lower() == "instancetype": calculated_instancetype = self.calculate_instancetype(dn) - if len(obj["instanceType"]) != 1 or obj["instanceType"][0] != str(calculated_instancetype): + if len(obj["instanceType"]) != 1 or int(obj["instanceType"][0]) != calculated_instancetype: error_count += 1 self.err_wrong_instancetype(obj, calculated_instancetype) diff --git a/python/samba/remove_dc.py b/python/samba/remove_dc.py index e6513ae04faf..b6cde0675028 100644 --- a/python/samba/remove_dc.py +++ b/python/samba/remove_dc.py @@ -236,7 +236,7 @@ def offline_remove_server(samdb, logger, computer_dn = None try: - dnsHostName = msgs[0]["dnsHostName"][0] + dnsHostName = str(msgs[0]["dnsHostName"][0]) except KeyError: dnsHostName = None @@ -267,7 +267,7 @@ def offline_remove_server(samdb, logger, samdb.delete(computer_dn, ["tree_delete:0"]) if "dnsHostName" in msgs[0]: - dnsHostName = msgs[0]["dnsHostName"][0] + dnsHostName = str(msgs[0]["dnsHostName"][0]) if remove_dns_account: res = samdb.search(expression="(&(objectclass=user)(cn=dns-%s)(servicePrincipalName=DNS/%s))" % -- 2.17.1 From 258baf4e3b6dab07106ad66bf19ebce358c1167d Mon Sep 17 00:00:00 2001 From: Noel Power Date: Fri, 7 Sep 2018 14:38:54 +0100 Subject: [PATCH 02/18] python/samba: extra ndr_unpack needs bytes function (cherry picked from commit 8db43696e70d7c4cb21172b7e7461cf6a72914a2) --- python/samba/dbchecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 6fdd59a81375..b8d22f9c503a 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2331,7 +2331,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container expectedTimeDo = 2650466015990000000 - originating = self.get_originating_time(obj["replPropertyMetaData"], isDeletedAttId) + originating = self.get_originating_time(obj["replPropertyMetaData"][0], isDeletedAttId) if originating != expectedTimeDo: if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'): nmsg = ldb.Message() -- 2.17.1 From 9208096cbff2dc714eb7eafe610b087225e35f90 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 27 Feb 2019 08:22:09 +0100 Subject: [PATCH 03/18] selftest: force running with TZ=UTC Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Wed Feb 27 11:24:59 UTC 2019 on sn-devel-144 (cherry picked from commit 4f307f2302b0fe8fd0fc6379eb8e6491faf8520c) --- selftest/selftest.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index bd72841199f4..a6051deee9e5 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -301,6 +301,9 @@ unless (defined($ENV{VALGRIND})) { # make all our python scripts unbuffered $ENV{PYTHONUNBUFFERED} = 1; +# do not depend on the users setup +$ENV{TZ} = "UTC"; + my $bindir_abs = abs_path($bindir); # Backwards compatibility: -- 2.17.1 From 79e05754c43a0beb984b900ecc2b89a3f16473b5 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 14:52:57 +0100 Subject: [PATCH 04/18] blackbox/dbcheck-links.sh: reproduce lost deleted object problem When a parent object is removed during the tombstone garbage collection before a child object and samba-tool dbcheck runs at the same time, the following can happen: - If the object child had DISALLOW_MOVE_ON_DELETE in systemFlags, samba-tool dbcheck moves the object under the LostAndFound[Config] object (as an originating update!) - The lastKnownParent attribute is removed (as an originating update!) These originating updates cause the object to have an extended time as tombstone. And these changes are replicated to other DCs, which very likely already removed the object completely! This means the destination DC of replication has no chance to handle the object it gets from the source DC with just 2 attributes (name, lastKnownParent). The destination logs something like: No objectClass found in replPropertyMetaData BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5357f591accffbf8c62335c308b985811b66f0b5) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user1.txt | 14 +++ testprogs/blackbox/dbcheck-links.sh | 113 ++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..676281faba58 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt new file mode 100644 index 000000000000..db18b9b188b6 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -0,0 +1,14 @@ +Checking 232 objects +WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' +Remove stale DN link? [YES] +Removed deleted DN on attribute lastKnownParent +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] +Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 9798813004c5..4bd075b5b480 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -238,6 +238,114 @@ dbcheck_missing_link_sid_corruption() { return $? } +add_lost_deleted_user1() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user1.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user1() { + dbcheck "-lost-deleted-user1" "1" "" + return $? +} + +remove_lost_deleted_user1() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -454,6 +562,11 @@ if [ -d $release_dir ]; then testit "missing_link_sid_corruption" missing_link_sid_corruption testit "dbcheck_missing_link_sid_corruption" dbcheck_missing_link_sid_corruption testit "missing_link_sid_clean" dbcheck_clean + testit "add_lost_deleted_user1" add_lost_deleted_user1 + testit "dbcheck_lost_deleted_user1" dbcheck_lost_deleted_user1 + testit "lost_deleted_user1_clean_A" dbcheck_clean + testit "remove_lost_deleted_user1" remove_lost_deleted_user1 + testit "lost_deleted_user1_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From 94e4545d00683e5ce317d5daf501eef762f90641 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:38:38 +0100 Subject: [PATCH 05/18] dsdb:repl_meta_data: allow CONTROL_DBCHECK_FIX_LINK_DN_NAME to by pass rename We need a way to rename an object without updating the replication meta data. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 3e8a435d27da899d0e3dab7cbc0a1c738067eba3) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index a24b2638d483..be4ea5592c47 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -3701,6 +3701,7 @@ static int replmd_rename_callback(struct ldb_request *req, struct ldb_reply *are static int replmd_rename(struct ldb_module *module, struct ldb_request *req) { struct ldb_context *ldb; + struct ldb_control *fix_dn_name_control = NULL; struct replmd_replicated_request *ac; int ret; struct ldb_request *down_req; @@ -3710,6 +3711,12 @@ static int replmd_rename(struct ldb_module *module, struct ldb_request *req) return ldb_next_request(module, req); } + fix_dn_name_control = ldb_request_get_control(req, + DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME); + if (fix_dn_name_control != NULL) { + return ldb_next_request(module, req); + } + ldb = ldb_module_get_ctx(module); ldb_debug(ldb, LDB_DEBUG_TRACE, "replmd_rename\n"); -- 2.17.1 From ccea9c00b8854eb929da3f467bf4031f31a50b7c Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:45:46 +0100 Subject: [PATCH 06/18] dbcheck: use DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME when renaming deleted objects We should never do originating updates on deleted objects. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 07a8326746f0c444eedf3860b178fc29d84e8d16) --- python/samba/dbchecker.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index b8d22f9c503a..1cdf9048ad9d 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -859,7 +859,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: self.samdb.transaction_cancel() - def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val): + def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val, controls): '''handle a wrong dn''' new_rdn = ldb.Dn(self.samdb, str(new_dn)) @@ -876,7 +876,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Not renaming %s to %s" % (obj.dn, new_dn)) return - if self.do_rename(obj.dn, new_rdn, new_parent, ["show_recycled:1", "relax:0"], + if self.do_rename(obj.dn, new_rdn, new_parent, controls, "Failed to rename object %s into %s" % (obj.dn, new_dn)): self.report("Renamed %s into %s" % (obj.dn, new_dn)) @@ -2306,9 +2306,11 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if name_val is not None: parent_dn = None + controls = ["show_recycled:1", "relax:0"] if isDeleted: if not (systemFlags & samba.dsdb.SYSTEM_FLAG_DISALLOW_MOVE_ON_DELETE): parent_dn = deleted_objects_dn + controls += ["local_oid:%s:1" % dsdb.DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME] if parent_dn is None: parent_dn = obj.dn.parent() expected_dn = ldb.Dn(self.samdb, "RDN=RDN,%s" % (parent_dn)) @@ -2319,7 +2321,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if expected_dn != obj.dn: error_count += 1 - self.err_wrong_dn(obj, expected_dn, object_rdn_attr, object_rdn_val, name_val) + self.err_wrong_dn(obj, expected_dn, object_rdn_attr, + object_rdn_val, name_val, controls) elif obj.dn.get_rdn_value() != object_rdn_val: error_count += 1 self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) -- 2.17.1 From f5f7dcc59ae5770788e86e823b9ca63211040f3b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:09:36 +0100 Subject: [PATCH 07/18] dbcheck: do isDeleted, systemFlags and replPropertyMetaData detection first BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 9afcd5331ce567bd80d35175f8e4e21c506e9347) --- python/samba/dbchecker.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 1cdf9048ad9d..a89773de53c5 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2069,7 +2069,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count = 0 set_attrs_from_md = set() set_attrs_seen = set() - got_repl_property_meta_data = False got_objectclass = False nc_dn = self.samdb.get_nc_root(obj.dn) @@ -2087,6 +2086,18 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) name_val = None isDeleted = False systemFlags = 0 + repl_meta_data_val = None + + for attrname in obj: + if str(attrname).lower() == 'isdeleted': + if str(obj[attrname][0]) != "FALSE": + isDeleted = True + + if str(attrname).lower() == 'systemflags': + systemFlags = int(obj[attrname][0]) + + if str(attrname).lower() == 'replpropertymetadata': + repl_meta_data_val = obj[attrname][0] for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": @@ -2112,13 +2123,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: object_rdn_val = str(obj[attrname][0]) - if str(attrname).lower() == 'isdeleted': - if str(obj[attrname][0]) != "FALSE": - isDeleted = True - - if str(attrname).lower() == 'systemflags': - systemFlags = int(obj[attrname][0]) - if str(attrname).lower() == 'replpropertymetadata': if self.has_replmetadata_zero_invocationid(dn, obj[attrname]): error_count += 1 @@ -2148,7 +2152,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing incorrect inital attributeID in '%s' on '%s', it should be objectClass" % (attrname, str(dn))) - got_repl_property_meta_data = True continue if str(attrname).lower() == 'ntsecuritydescriptor': @@ -2328,13 +2331,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) show_dn = True - if got_repl_property_meta_data: + if repl_meta_data_val: if obj.dn == deleted_objects_dn: isDeletedAttId = 131120 # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container expectedTimeDo = 2650466015990000000 - originating = self.get_originating_time(obj["replPropertyMetaData"][0], isDeletedAttId) + originating = self.get_originating_time(repl_meta_data_val, isDeletedAttId) if originating != expectedTimeDo: if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'): nmsg = ldb.Message() -- 2.17.1 From b6d6ed6e2e3d6e8697667372f24713d6dbcd7aea Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 08/18] dbcheck: don't move already deleted objects to LostAndFound This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6d50ee74920c39cdb18b427bfaaf200775bf2d73) --- python/samba/dbchecker.py | 9 +++++++-- selftest/knownfail.d/dbcheck-list-deleted | 1 - .../expected-dbcheck-link-output-lost-deleted-user1.txt | 8 +++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index a89773de53c5..ecb9ac7b94ac 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2372,8 +2372,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) except ldb.LdbError as e11: (enum, estr) = e11.args if enum == ldb.ERR_NO_SUCH_OBJECT: - self.err_missing_parent(obj) - error_count += 1 + if isDeleted: + self.report("WARNING: parent object not found for %s" % (obj.dn)) + self.report("Not moving to LostAndFound " + "(tombstone garbage collection in progress?)") + else: + self.err_missing_parent(obj) + error_count += 1 else: raise diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted index 676281faba58..a8fcb0a223f0 100644 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -1,2 +1 @@ ^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index db18b9b188b6..cfc2644b3cbb 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -7,8 +7,6 @@ Removed deleted DN on attribute lastKnownParent ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] -Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) -- 2.17.1 From 8ef6c41f92b4edd596b148caa30ac05645d56944 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 09/18] dbcheck: don't remove dangling one-way links on already deleted objects This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. In this case the lastKnownParent attributes points a non existing object. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit e388e599495b6d7c38b8b6966332e27f8b958783) --- python/samba/dbchecker.py | 13 +++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 1 - ...ected-dbcheck-link-output-lost-deleted-user1.txt | 7 ++----- 3 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index ecb9ac7b94ac..d609be6604e3 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -549,6 +549,19 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) def err_missing_target_dn_or_GUID(self, dn, attrname, val, dsdb_dn): """handle a missing target DN (if specified, GUID form can't be found, and otherwise DN string form can't be found)""" + + # Don't change anything if the object itself is deleted + if str(dn).find('\\0ADEL') != -1: + # We don't bump the error count as Samba produces these + # in normal operation + self.report("WARNING: no target object found for GUID " + "component link %s in deleted object " + "%s - %s" % (attrname, dn, val)) + self.report("Not removing dangling one-way " + "link on deleted object " + "(tombstone garbage collection in progress?)") + return 0 + # check if its a backlink linkID, _ = self.get_attr_linkID_and_reverse_name(attrname) if (linkID & 1 == 0) and str(dsdb_dn).find('\\0ADEL') == -1: diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index a8fcb0a223f0..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index cfc2644b3cbb..3c55de8fa01f 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -1,9 +1,6 @@ Checking 232 objects -WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' -Remove stale DN link? [YES] -Removed deleted DN on attribute lastKnownParent +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -- 2.17.1 From 6ea4695f8caa3d422a8552111752cf5b63844ff2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2019 18:16:27 +0100 Subject: [PATCH 10/18] dbcheck: add find_repl_attid() helper function BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 598e38d2a5e0832429ba65b4e55bf7127618f894) --- python/samba/dbchecker.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index d609be6604e3..471c66c00d23 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1463,6 +1463,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) return error_count + def find_repl_attid(self, repl, attid): + for o in repl.ctr.array: + if o.attid == attid: + return o + + return None def get_originating_time(self, val, attid): '''Read metadata properties and return the originating time for @@ -1472,12 +1478,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) ''' repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, str(val)) - obj = repl.ctr - - for o in repl.ctr.array: - if o.attid == attid: - return o.originating_change_time - + o = self.find_repl_attid(repl, attid) + if o is not None: + return o.originating_change_time return 0 def process_metadata(self, dn, val): -- 2.17.1 From c0324f3f5450971d6da14adba9ff358e9aedbfcc Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 23:14:02 +0100 Subject: [PATCH 11/18] blackbox/dbcheck-links.sh: add regression test for lost deleted object repair BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 1ccc21a34d295be3bb2ab481a5918003eae88bf4) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user2.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 100 ++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..670e42b747c6 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt new file mode 100644 index 000000000000..dfb7422ac0bf --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -0,0 +1,9 @@ +Checking 232 objects +ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 4bd075b5b480..161cdd6bf5ed 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -346,6 +346,103 @@ remove_lost_deleted_user1() { return 0 } +add_lost_deleted_user2() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user2.ldif + cat > $ldif < Date: Thu, 28 Feb 2019 18:22:18 +0100 Subject: [PATCH 12/18] dbcheck: detect the change after deletion bug Old versions of 'samba-tool dbcheck' could reanimate deleted objects, when running at the same time as the tombstone garbage collection. When the (deleted) parent of a deleted object (with the DISALLOW_MOVE_ON_DELETE bit in systemFlags), is removed before the object itself, dbcheck moved it in the LostAndFound[Config] subtree of the partition as an originating change. That means that the object will be in tombstone state again for 180 days on the local DC. And other DCs fail to replicate the object as it's already removed completely there and the replication only gives the name and lastKnownParent attributes, because all other attributes should already be known to the other DC. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit a1658b306d85452407388b91a745078c9c1f7dc7) --- python/samba/dbchecker.py | 109 ++++++++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 2 - ...dbcheck-link-output-lost-deleted-user2.txt | 15 ++- 3 files changed, 116 insertions(+), 10 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 471c66c00d23..df58e298c380 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -101,6 +101,7 @@ class dbcheck(object): self.fix_missing_deleted_objects = False self.fix_replica_locations = False self.fix_missing_rid_set_master = False + self.fix_changes_after_deletion_bug = False self.dn_set = set() self.link_id_cache = {} @@ -189,6 +190,14 @@ class dbcheck(object): else: self.rid_set_dn = None + ntds_service_dn = "CN=Directory Service,CN=Windows NT,CN=Services,%s" % \ + self.samdb.get_config_basedn().get_linearized() + res = samdb.search(base=ntds_service_dn, + scope=ldb.SCOPE_BASE, + expression="(objectClass=nTDSService)", + attrs=["tombstoneLifetime"]) + self.tombstoneLifetime = int(res[0]["tombstoneLifetime"][0]) + self.compatibleFeatures = [] self.requiredFeatures = [] @@ -1733,6 +1742,100 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def find_changes_after_deletion(self, repl_val): + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + found = [] + for o in repl.ctr.array: + if o.attid == drsuapi.DRSUAPI_ATTID_isDeleted: + continue + + if o.local_usn <= isDeleted.local_usn: + continue + + if o.originating_change_time <= isDeleted.originating_change_time: + continue + + change_time = samba.nttime2unix(o.originating_change_time) + + delta = change_time - delete_time + if delta <= tombstone_delta: + continue + + # If the modification happened after the tombstone lifetime + # has passed, we have a bug as the object might be deleted + # already on other DCs and won't be able to replicate + # back + found.append(o) + + return found, isDeleted + + def has_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + if len(found) == 0: + return False + + def report_attid(o): + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + + self.report("%s: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + attname, o.attid, o.version, + o.originating_invocation_id, + o.originating_usn, + o.local_usn, + time.ctime(samba.nttime2unix(o.originating_change_time)))) + + self.report("ERROR: object %s, has changes after deletion" % dn) + report_attid(isDeleted) + for o in found: + report_attid(o) + + return True + + def err_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + + in_schema_nc = dn.is_child_of(self.schema_dn) + rdn_attr = dn.get_rdn_name() + rdn_attid = self.samdb_schema.get_attid_from_lDAPDisplayName(rdn_attr, + is_schema_nc=in_schema_nc) + + unexpected = [] + for o in found: + if o.attid == rdn_attid: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_name: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_lastKnownParent: + continue + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + unexpected.append(attname) + + if len(unexpected) > 0: + self.report('Unexpeted attributes: %s' % ",".join(unexpected)) + self.report('Not fixing changes after deletion bug') + return + + if not self.confirm_all('Delete broken tombstone object %s deleted %s days ago?' % ( + dn, self.tombstoneLifetime), 'fix_changes_after_deletion_bug'): + self.report('Not fixing changes after deletion bug') + return + + if self.do_delete(dn, ["relax:0"], + "Failed to remove DN %s" % dn): + self.report("Removed DN %s" % dn) def has_replmetadata_zero_invocationid(self, dn, repl_meta_data): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, @@ -2115,6 +2218,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if str(attrname).lower() == 'replpropertymetadata': repl_meta_data_val = obj[attrname][0] + if isDeleted and repl_meta_data_val: + if self.has_changes_after_deletion(dn, repl_meta_data_val): + error_count += 1 + self.err_changes_after_deletion(dn, repl_meta_data_val) + return error_count + for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": continue diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index 670e42b747c6..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1,2 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt index dfb7422ac0bf..9b87ca10c57e 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -1,9 +1,8 @@ Checking 232 objects -ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +ERROR: object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp, has changes after deletion +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +name: attid=0x00090001 version=4 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3772 (local=3772) at Mon Mar 11 13:28:24 2019 +lastKnownParent: attid=0x0009030d version=3 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3773 (local=3773) at Mon Mar 11 13:28:24 2019 +Delete broken tombstone object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp deleted 180 days ago? [YES] +Removed DN CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (1 errors) -- 2.17.1 From c005f1986f8e7d3cbdf2578006dd803a4a4deed5 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 10:25:40 +0100 Subject: [PATCH 13/18] python/samba/netcmd: provide SUPPRESS_HELP via Option class BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b61d580fb7dba8ff94e9e98c958e324865cd2f1d) --- python/samba/netcmd/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/samba/netcmd/__init__.py b/python/samba/netcmd/__init__.py index 9b144d97f43f..8032f05ea280 100644 --- a/python/samba/netcmd/__init__.py +++ b/python/samba/netcmd/__init__.py @@ -24,6 +24,7 @@ import sys, traceback import textwrap class Option(optparse.Option): + SUPPRESS_HELP = optparse.SUPPRESS_HELP pass # This help formatter does text wrapping and preserves newlines -- 2.17.1 From b9c84588277b91efa257264bb16c2c5364c9a603 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:02:18 +0100 Subject: [PATCH 14/18] dbcheck: add --selftest-check-expired-tombstones cmdline option This will be used by dbcheck tests which operate on static/old provision dumps in the following commits. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6f9c5ed8de47bb98e21e8064d8e90f963f2f71ca) --- python/samba/netcmd/dbcheck.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index 210d4a26a1df..5e791ad48b61 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -74,13 +74,18 @@ class cmd_dbcheck(Command): Option("--reset-well-known-acls", dest="reset_well_known_acls", default=False, action="store_true", help="reset ACLs on objects with well known default ACL values to the default"), Option("-H", "--URL", help="LDB URL for database or target server (defaults to local SAM database)", type=str, metavar="URL", dest="H"), - ] + Option("--selftest-check-expired-tombstones", + dest="selftest_check_expired_tombstones", default=False, action="store_true", + help=Option.SUPPRESS_HELP), # This is only used by tests + ] def run(self, DN=None, H=None, verbose=False, fix=False, yes=False, cross_ncs=False, quiet=False, scope="SUB", credopts=None, sambaopts=None, versionopts=None, attrs=None, reindex=False, force_modules=False, - reset_well_known_acls=False, yes_rules=[]): + reset_well_known_acls=False, + selftest_check_expired_tombstones=False, + yes_rules=[]): lp = sambaopts.get_loadparm() -- 2.17.1 From a320d9b526ae47eff0bea8cee790ef40246b183f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:04:33 +0100 Subject: [PATCH 15/18] blackbox/dbcheck*.sh: pass --selftest-check-expired-tombstones to dbcheck These tests operate on provision dumps created long ago, they still want to run tests on deleted objects, when the next commits remove processing expired tombstone objects in dbcheck. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5fccc4e9044d2e57be33471f5e6b9be7cc37ac3a) --- testprogs/blackbox/dbcheck-links.sh | 18 +++++++++--------- testprogs/blackbox/dbcheck-oldrelease.sh | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 161cdd6bf5ed..acb800d33fdb 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -42,12 +42,12 @@ dbcheck() { } dbcheck_dangling() { - dbcheck "" "1" "" + dbcheck "" "1" "--selftest-check-expired-tombstones" return $? } dbcheck_one_way() { - dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp" + dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp --selftest-check-expired-tombstones" return $? } @@ -118,7 +118,7 @@ duplicate_member() { } dbcheck_duplicate_member() { - dbcheck "_duplicate_member" "1" "" + dbcheck "_duplicate_member" "1" "--selftest-check-expired-tombstones" return $? } @@ -234,7 +234,7 @@ EOF } dbcheck_missing_link_sid_corruption() { - dbcheck "-missing-link-sid-corruption" "1" "" + dbcheck "-missing-link-sid-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -332,7 +332,7 @@ EOF } dbcheck_lost_deleted_user1() { - dbcheck "-lost-deleted-user1" "1" "" + dbcheck "-lost-deleted-user1" "1" "--selftest-check-expired-tombstones" return $? } @@ -439,7 +439,7 @@ EOF } dbcheck_lost_deleted_user2() { - dbcheck "-lost-deleted-user2" "1" "" + dbcheck "-lost-deleted-user2" "1" "--selftest-check-expired-tombstones" return $? } @@ -504,7 +504,7 @@ EOF } dbcheck_forward_link_corruption() { - dbcheck "-forward-link-corruption" "1" "" + dbcheck "-forward-link-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -565,7 +565,7 @@ EOF } dbcheck_oneway_link_corruption() { - dbcheck "-oneway-link-corruption" "0" "" + dbcheck "-oneway-link-corruption" "0" "--selftest-check-expired-tombstones" return $? } @@ -580,7 +580,7 @@ check_expected_after_dbcheck_oneway_link_corruption() { dbcheck_dangling_multi_valued() { - $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --fix --yes + $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --selftest-check-expired-tombstones --fix --yes if [ "$?" != "1" ]; then return 1 fi diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh index 62f8d7ecead1..4a5acd928e97 100755 --- a/testprogs/blackbox/dbcheck-oldrelease.sh +++ b/testprogs/blackbox/dbcheck-oldrelease.sh @@ -195,7 +195,7 @@ check_expected_before_values() { # This should 'fail', because it returns the number of modified records dbcheck_objectclass() { if [ x$RELEASE = x"release-4-1-6-partial-object" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ else return 1 fi @@ -203,7 +203,7 @@ dbcheck_objectclass() { # This should 'fail', because it returns the number of modified records dbcheck() { - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ } check_expected_after_values() { @@ -273,7 +273,7 @@ check_forced_duplicate_values() { # This should 'fail', because it returns the number of modified records dbcheck_after_dup() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ else return 1 fi @@ -316,7 +316,7 @@ dbcheck_acl_reset_clean() { # This should 'fail', because it returns the number of modified records dbcheck2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -324,7 +324,7 @@ dbcheck2() { # But having fixed it all up, this should pass dbcheck_clean2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } @@ -341,7 +341,7 @@ rm_deleted_objects() { # This should 'fail', because it returns the number of modified records dbcheck3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -349,7 +349,7 @@ dbcheck3() { # But having fixed it all up, this should pass dbcheck_clean3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } -- 2.17.1 From 595757a035108290655b85d5e26eab82775fbf1f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:38:22 +0100 Subject: [PATCH 16/18] blackbox/dbcheck-links.sh: prepare regression test for skipping expired tombstones BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b096a3117ed9249fd6f65f3221a26c88efbba3b8) --- ...dbcheck-link-output-lost-deleted-user3.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 115 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt new file mode 100644 index 000000000000..67ca493c44f7 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -0,0 +1,9 @@ +Checking 232 objects +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index acb800d33fdb..49f9b1e21d49 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -443,6 +443,116 @@ dbcheck_lost_deleted_user2() { return $? } +add_lost_deleted_user3() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user3.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user3() { + # here we don't pass --selftest-check-expired-tombstones + # as we want to test the default + dbcheck "-lost-deleted-user3" "1" "" + return $? +} + +remove_lost_deleted_user3() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -667,6 +777,11 @@ if [ -d $release_dir ]; then testit "add_lost_deleted_user2" add_lost_deleted_user2 testit "dbcheck_lost_deleted_user2" dbcheck_lost_deleted_user2 testit "lost_deleted_user2_clean" dbcheck_clean + testit "add_lost_deleted_user3" add_lost_deleted_user3 + testit "dbcheck_lost_deleted_user3" dbcheck_lost_deleted_user3 + testit "lost_deleted_user3_clean_A" dbcheck_clean + testit "remove_lost_deleted_user3" remove_lost_deleted_user3 + testit "lost_deleted_user3_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From 2dd063722abe6cab066212aeee3782abda7e0926 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:41:01 +0100 Subject: [PATCH 17/18] dbcheck: don't check expired tombstone objects by default anymore These will be removed anyway and any change on them risks to be an originating update that causes replication problems. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu Mar 14 03:12:27 UTC 2019 on sn-devel-144 (cherry picked from commit a2c5f8cf41c2dfdc4f122e8427d1dfeabb6ba311) --- python/samba/dbchecker.py | 45 ++++++++++++++++++- python/samba/netcmd/dbcheck.py | 6 ++- ...dbcheck-link-output-lost-deleted-user3.txt | 26 +++++++---- testprogs/blackbox/dbcheck-links.sh | 2 +- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index df58e298c380..41999a7129c4 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -41,7 +41,8 @@ class dbcheck(object): def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, yes=False, quiet=False, in_transaction=False, - reset_well_known_acls=False): + reset_well_known_acls=False, + check_expired_tombstones=False): self.samdb = samdb self.dict_oid_name = None self.samdb_schema = (samdb_schema or samdb) @@ -88,6 +89,8 @@ class dbcheck(object): self.fix_doubled_userparameters = False self.fix_sid_rid_set_conflict = False self.reset_well_known_acls = reset_well_known_acls + self.check_expired_tombstones = check_expired_tombstones + self.expired_tombstones = 0 self.reset_all_well_known_acls = False self.in_transaction = in_transaction self.infrastructure_dn = ldb.Dn(samdb, "CN=Infrastructure," + samdb.domain_dn()) @@ -233,6 +236,13 @@ class dbcheck(object): if DN is None: error_count += self.check_rootdse() + if self.expired_tombstones > 0: + self.report("NOTICE: found %d expired tombstones, " + "'samba' will remove them daily, " + "'samba-tool domain tombstones expunge' " + "would do that immediately." % ( + self.expired_tombstones)) + if error_count != 0 and not self.fix: self.report("Please use --fix to fix these errors") @@ -1742,6 +1752,37 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def is_expired_tombstone(self, dn, repl_val): + if self.check_expired_tombstones: + # This is not the default, it's just + # used to keep dbcheck tests work with + # old static provision dumps + return False + + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + current_time = time.time() + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + delta = current_time - delete_time + if delta <= tombstone_delta: + return False + + self.report("SKIPING: object %s is an expired tombstone" % dn) + self.report("isDeleted: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + isDeleted.attid, + isDeleted.version, + isDeleted.originating_invocation_id, + isDeleted.originating_usn, + isDeleted.local_usn, + time.ctime(samba.nttime2unix(isDeleted.originating_change_time)))) + self.expired_tombstones += 1 + return True + def find_changes_after_deletion(self, repl_val): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) @@ -2223,6 +2264,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count += 1 self.err_changes_after_deletion(dn, repl_meta_data_val) return error_count + if self.is_expired_tombstone(dn, repl_meta_data_val): + return error_count for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index 5e791ad48b61..12a7bbe2cf17 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -136,8 +136,10 @@ class cmd_dbcheck(Command): started_transaction = True try: chk = dbcheck(samdb, samdb_schema=samdb_schema, verbose=verbose, - fix=fix, yes=yes, quiet=quiet, in_transaction=started_transaction, - reset_well_known_acls=reset_well_known_acls) + fix=fix, yes=yes, quiet=quiet, + in_transaction=started_transaction, + reset_well_known_acls=reset_well_known_acls, + check_expired_tombstones=selftest_check_expired_tombstones) for option in yes_rules: if hasattr(chk, option): diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt index 67ca493c44f7..d014bfacae2c 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -1,9 +1,19 @@ Checking 232 objects -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not moving to LostAndFound (tombstone garbage collection in progress?) -Checked 232 objects (1 errors) +SKIPING: object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=fred\0ADEL:2301a64c-5b42-4ca8-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=dsg\0ADEL:6d66d0ef-cad7-4e5d-b1b6-4a233a21c269,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3734 (local=3734) at Wed Jun 29 04:34:32 2016 +SKIPING: object CN=udg\0ADEL:7cff5537-51b1-4d26-a295-0225dbea8525,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3739 (local=3739) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=usg\0ADEL:d012e8f5-a4bd-40ea-a2a1-68ff2508847d,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3736 (local=3736) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=ddg\0ADEL:fb8c2fe3-5448-43de-99f9-e1d3b9357cfc,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3737 (local=3737) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=gsg\0ADEL:91aa85cc-fc19-4b8c-9fc7-aaba425439c7,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3735 (local=3735) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=gdg\0ADEL:e0f581e7-14ee-4fc2-839c-8f46f581c72a,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3738 (local=3738) at Wed Jun 29 04:34:34 2016 +NOTICE: found 8 expired tombstones, 'samba' will remove them daily, 'samba-tool domain tombstones expunge' would do that immediately. +Checked 232 objects (0 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 49f9b1e21d49..851c0dc9a1fb 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -539,7 +539,7 @@ EOF dbcheck_lost_deleted_user3() { # here we don't pass --selftest-check-expired-tombstones # as we want to test the default - dbcheck "-lost-deleted-user3" "1" "" + dbcheck "-lost-deleted-user3" "0" "" return $? } -- 2.17.1 From e9d877b7aa805559fab8377d3a0d0518683958da Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 19 Mar 2019 13:05:16 +0100 Subject: [PATCH 18/18] dbcheck: use the str() value of the "name" attribute We do the same with the rdn attribute value and we need the same logic on both in order to check they are the same. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Noel Power (cherry picked from commit dd6f0dad218ec1d5aa38ea8aa6848ec81035cb3f) --- python/samba/dbchecker.py | 2 +- .../expected-dbcheck-link-output-lost-deleted-user1.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 41999a7129c4..bd43667b99fd 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2280,7 +2280,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" % (len(obj[attrname]), attrname, str(obj.dn))) else: - name_val = obj[attrname][0] + name_val = str(obj[attrname][0]) if str(attrname).lower() == str(obj.dn.get_rdn_name()).lower(): object_rdn_attr = attrname diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index 3c55de8fa01f..1f5f2272bc10 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -1,7 +1,7 @@ Checking 232 objects WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] name='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -- 2.17.1