From 8fafea9aaad11a3b3c6d6c1c6d8afc8c9c22ca14 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Fri, 11 Dec 2020 15:37:04 +1300 Subject: [PATCH 1/2] selftest: Confirm that we fix any errors on the Deleted Objects container itself BUG: https://bugzilla.samba.org/show_bug.cgi?id=14593 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall (cherry picked from commit 1ec1c35a3ae422720df491f5555c9bc787c9944c) --- selftest/knownfail.d/dbcheck-deleted-objects | 2 ++ testprogs/blackbox/dbcheck-oldrelease.sh | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-deleted-objects diff --git a/selftest/knownfail.d/dbcheck-deleted-objects b/selftest/knownfail.d/dbcheck-deleted-objects new file mode 100644 index 00000000000..9b32ddbb50a --- /dev/null +++ b/selftest/knownfail.d/dbcheck-deleted-objects @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck.alpha13.dbcheck_deleted_objects +^samba4.blackbox.dbcheck.alpha13.quick.dbcheck_deleted_objects diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh index 9e9924654be..64c08c57981 100755 --- a/testprogs/blackbox/dbcheck-oldrelease.sh +++ b/testprogs/blackbox/dbcheck-oldrelease.sh @@ -297,6 +297,17 @@ dbcheck_objectclass() { fi } +# This should 'fail', because it returns the number of wrong records, which it must if we did not skip the deleted objects +dbcheck_deleted_objects() { + if [ x$RELEASE = x"alpha13" ]; then + basedn=$($ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb -s base -b "" defaultNamingContext| grep -i defaultNamingContext| cut -d\ -f 2) + + $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "cn=deleted objects,$basedn" --scope base $@ + else + return 1 + fi +} + # This should 'fail', because it returns the number of modified records dbcheck() { $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ @@ -488,6 +499,7 @@ testit $RELEASE undump || failed=`expr $failed + 1` testit "reindex" reindex || failed=`expr $failed + 1` testit "current_version_mod" do_current_version_mod || failed=`expr $failed + 1` testit "check_expected_before_values" check_expected_before_values || failed=`expr $failed + 1` +testit_expect_failure "dbcheck_deleted_objects" dbcheck_deleted_objects || failed=`expr $failed + 1` testit_expect_failure "dbcheck_objectclass" dbcheck_objectclass || failed=`expr $failed + 1` testit_expect_failure "dbcheck" dbcheck || failed=`expr $failed + 1` testit "check_expected_after_values" check_expected_after_values || failed=`expr $failed + 1` -- 2.25.1 From 7491f0d34b1773bacad4ac6fdd0770da31d1d27f Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Thu, 10 Dec 2020 16:03:49 +1300 Subject: [PATCH 2/2] dbcheck: Check Deleted Objects and reduce noise in reports about expired tombstones These reports (about recently deleted objects) create concern about a perfectly normal part of DB operation. We must not operate on objects that are expired or we might reanimate them, but we must fix "Deleted Objects" if it is wrong (mostly it is set as being deleted in 9999, but in alpha19 we got this wrong). BUG: https://bugzilla.samba.org/show_bug.cgi?id=14593 Signed-off-by: Andrew Bartlett Reviewed-by: Douglas Bagnall Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Wed Feb 3 05:29:11 UTC 2021 on sn-devel-184 (cherry picked from commit da627106cdbf8d375b25fa3338a717447f3dbb6e) --- python/samba/dbchecker.py | 25 ++++++++++++++++++- selftest/knownfail.d/dbcheck-deleted-objects | 2 -- ...dbcheck-link-output-lost-deleted-user3.txt | 16 ++++++------ testprogs/blackbox/dbcheck-links.sh | 2 +- 4 files changed, 33 insertions(+), 12 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-deleted-objects diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 593aa8cf6d2..d12833d9390 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1819,6 +1819,11 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) # old static provision dumps return False + if dn in self.deleted_objects_containers: + # The Deleted Objects container will look like an expired + # tombstone + return False + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) @@ -1832,7 +1837,25 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if delta <= tombstone_delta: return False - self.report("SKIPING: object %s is an expired tombstone" % dn) + expunge_time = delete_time + tombstone_delta + + delta_days = delta / (24 * 60 * 60) + + if delta_days <= 2: + self.report("SKIPPING additional checks on object " + "%s which very recently " + "became an expired tombstone (normal)" % dn) + self.report("INFO: it is expected this will be expunged " + "by the next daily task some time after %s, " + "%d hours ago" + % (time.ctime(expunge_time), delta // (60 * 60))) + else: + self.report("SKIPPING: object %s is an expired tombstone" % dn) + self.report("INFO: it was expected this object would have " + "been expunged soon after" + "%s, %d days ago" + % (time.ctime(expunge_time), delta_days)) + self.report("isDeleted: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( isDeleted.attid, isDeleted.version, diff --git a/selftest/knownfail.d/dbcheck-deleted-objects b/selftest/knownfail.d/dbcheck-deleted-objects deleted file mode 100644 index 9b32ddbb50a..00000000000 --- a/selftest/knownfail.d/dbcheck-deleted-objects +++ /dev/null @@ -1,2 +0,0 @@ -^samba4.blackbox.dbcheck.alpha13.dbcheck_deleted_objects -^samba4.blackbox.dbcheck.alpha13.quick.dbcheck_deleted_objects diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt index d014bfacae2..ea9b630df08 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -1,19 +1,19 @@ Checking 232 objects -SKIPING: object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 -SKIPING: object CN=fred\0ADEL:2301a64c-5b42-4ca8-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=fred\0ADEL:2301a64c-5b42-4ca8-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 -SKIPING: object CN=dsg\0ADEL:6d66d0ef-cad7-4e5d-b1b6-4a233a21c269,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=dsg\0ADEL:6d66d0ef-cad7-4e5d-b1b6-4a233a21c269,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3734 (local=3734) at Wed Jun 29 04:34:32 2016 -SKIPING: object CN=udg\0ADEL:7cff5537-51b1-4d26-a295-0225dbea8525,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=udg\0ADEL:7cff5537-51b1-4d26-a295-0225dbea8525,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3739 (local=3739) at Wed Jun 29 04:34:34 2016 -SKIPING: object CN=usg\0ADEL:d012e8f5-a4bd-40ea-a2a1-68ff2508847d,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=usg\0ADEL:d012e8f5-a4bd-40ea-a2a1-68ff2508847d,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3736 (local=3736) at Wed Jun 29 04:34:33 2016 -SKIPING: object CN=ddg\0ADEL:fb8c2fe3-5448-43de-99f9-e1d3b9357cfc,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=ddg\0ADEL:fb8c2fe3-5448-43de-99f9-e1d3b9357cfc,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3737 (local=3737) at Wed Jun 29 04:34:34 2016 -SKIPING: object CN=gsg\0ADEL:91aa85cc-fc19-4b8c-9fc7-aaba425439c7,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=gsg\0ADEL:91aa85cc-fc19-4b8c-9fc7-aaba425439c7,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3735 (local=3735) at Wed Jun 29 04:34:33 2016 -SKIPING: object CN=gdg\0ADEL:e0f581e7-14ee-4fc2-839c-8f46f581c72a,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +SKIPPING: object CN=gdg\0ADEL:e0f581e7-14ee-4fc2-839c-8f46f581c72a,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3738 (local=3738) at Wed Jun 29 04:34:34 2016 NOTICE: found 8 expired tombstones, 'samba' will remove them daily, 'samba-tool domain tombstones expunge' would do that immediately. Checked 232 objects (0 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index ead59d691e0..f00fe46c2de 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -42,7 +42,7 @@ dbcheck() { if [ "$?" != "$2" ]; then return 1 fi - sort $tmpfile > $tmpfile.sorted + sort $tmpfile | grep -v "^INFO:" > $tmpfile.sorted sort $release_dir/expected-dbcheck-link-output${1}.txt > $tmpfile.expected diff -u $tmpfile.sorted $tmpfile.expected if [ "$?" != "0" ]; then -- 2.25.1