From 967272037f6f89830129a34755839b9eb899fc9b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Wed, 27 Feb 2019 08:22:09 +0100 Subject: [PATCH 01/16] selftest: force running with TZ=UTC Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider Autobuild-User(master): Stefan Metzmacher Autobuild-Date(master): Wed Feb 27 11:24:59 UTC 2019 on sn-devel-144 (cherry picked from commit 4f307f2302b0fe8fd0fc6379eb8e6491faf8520c) --- selftest/selftest.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selftest/selftest.pl b/selftest/selftest.pl index 3ee266c4d0ac..45eb51fa3c16 100755 --- a/selftest/selftest.pl +++ b/selftest/selftest.pl @@ -301,6 +301,9 @@ unless (defined($ENV{VALGRIND})) { # make all our python scripts unbuffered $ENV{PYTHONUNBUFFERED} = 1; +# do not depend on the users setup +$ENV{TZ} = "UTC"; + my $bindir_abs = abs_path($bindir); # Backwards compatibility: -- 2.17.1 From fb0914549e875e63e3d9bf3ba6f512358c26c516 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 10:36:49 +0100 Subject: [PATCH 02/16] blackbox/*.sh: pass -u to 'diff' This is what we work with every day... BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 8ba6f1c895ee9b6b592578f21e7f79ed36236bef) --- testprogs/blackbox/dbcheck-links.sh | 14 ++++++------ testprogs/blackbox/dbcheck-oldrelease.sh | 28 ++++++++++++------------ testprogs/blackbox/tombstones-expunge.sh | 14 ++++++------ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 9798813004c5..7b18e11feb3d 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -63,7 +63,7 @@ dbcheck_clean() { tmpldif2=$PREFIX_ABS/$RELEASE/expected-dbcheck-output2.txt.tmp2 TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb -s base -b '' | grep highestCommittedUSN > $tmpldif2 - diff $tmpldif1 $tmpldif2 + diff -u $tmpldif1 $tmpldif2 if [ "$?" != "0" ]; then return 1 fi @@ -72,7 +72,7 @@ dbcheck_clean() { check_expected_after_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-links-after-link-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=swimmers)(cn=leaders)(cn=helpers))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --sorted member > $tmpldif - diff $tmpldif $release_dir/expected-links-after-link-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-links-after-link-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -81,7 +81,7 @@ check_expected_after_links() { check_expected_after_deleted_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-deleted-links-after-link-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=swimmers)(cn=leaders)(cn=helpers))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted member > $tmpldif - diff $tmpldif $release_dir/expected-deleted-links-after-link-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-deleted-links-after-link-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -90,7 +90,7 @@ check_expected_after_deleted_links() { check_expected_after_objects() { tmpldif=$PREFIX_ABS/$RELEASE/expected-objects-after-link-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(samaccountname=fred)(samaccountname=ddg)(samaccountname=usg)(samaccountname=user1)(samaccountname=user1x)(samaccountname=user2))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted samAccountName | grep sAMAccountName > $tmpldif - diff $tmpldif $release_dir/expected-objects-after-link-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-objects-after-link-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -125,7 +125,7 @@ dbcheck_duplicate_member() { check_expected_after_duplicate_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-duplicates-after-link-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=administrator)(cn=enterprise admins))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --sorted memberOf member > $tmpldif - diff $tmpldif $release_dir/expected-duplicates-after-link-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-duplicates-after-link-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -306,7 +306,7 @@ dbcheck_forward_link_corruption() { check_expected_after_dbcheck_forward_link_corruption() { tmpldif=$PREFIX_ABS/$RELEASE/expected-after-dbcheck-forward-link-corruption.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=dangling)(cn=enterprise admins))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --sorted memberOf member > $tmpldif - diff $tmpldif $release_dir/expected-after-dbcheck-forward-link-corruption.ldif + diff -u $tmpldif $release_dir/expected-after-dbcheck-forward-link-corruption.ldif if [ "$?" != "0" ]; then return 1 fi @@ -367,7 +367,7 @@ dbcheck_oneway_link_corruption() { check_expected_after_dbcheck_oneway_link_corruption() { tmpldif=$PREFIX_ABS/$RELEASE/expected-after-dbcheck-oneway-link-corruption.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(ou=dangling-ou)(ou=dangling-ou2)(ou=dangling-from))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --sorted seeAlso > $tmpldif - diff $tmpldif $release_dir/expected-after-dbcheck-oneway-link-corruption.ldif + diff -u $tmpldif $release_dir/expected-after-dbcheck-oneway-link-corruption.ldif if [ "$?" != "0" ]; then return 1 fi diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh index e36379621c44..67fd6a49b61d 100755 --- a/testprogs/blackbox/dbcheck-oldrelease.sh +++ b/testprogs/blackbox/dbcheck-oldrelease.sh @@ -146,7 +146,7 @@ check_expected_userparameters() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then tmpldif=$PREFIX_ABS/$RELEASE/expected-userParameters-after-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb userParameters=* -s sub -b DC=release-4-1-0rc3,DC=samba,DC=corp userParameters --sorted | grep -v \# > $tmpldif - diff $tmpldif $release_dir/expected-userParameters-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-userParameters-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -173,7 +173,7 @@ check_expected_before_values() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then tmpldif=$PREFIX_ABS/$RELEASE/expected-replpropertymetadata-before-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=ops_run_anything -s one -b OU=SUDOers,DC=release-4-1-0rc3,DC=samba,DC=corp \* replpropertymetadata --sorted --show-binary > $tmpldif - diff $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -183,20 +183,20 @@ check_expected_before_values() { # Here we remove originating_change_time and whenChanged as # these are time-dependent, caused by the ldbmodify above. - diff $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck2.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck2.ldif if [ "$?" != "0" ]; then return 1 fi TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=ops_run_anything3 -s one -b OU=SUDOers,DC=release-4-1-0rc3,DC=samba,DC=corp \* replpropertymetadata --sorted --show-binary > $tmpldif - diff $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck3.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-before-dbcheck3.ldif if [ "$?" != "0" ]; then return 1 fi elif [ x$RELEASE = x"release-4-5-0-pre1" ]; then tmpldif=$PREFIX_ABS/$RELEASE/rootdse-version.initial.txt.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb -s base -b '' | grep highestCommittedUSN > $tmpldif - diff $tmpldif $release_dir/rootdse-version.initial.txt + diff -u $tmpldif $release_dir/rootdse-version.initial.txt if [ "$?" != "0" ]; then return 1 fi @@ -222,30 +222,30 @@ check_expected_after_values() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then tmpldif=$PREFIX_ABS/$RELEASE/expected-replpropertymetadata-after-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=ops_run_anything -s one -b OU=SUDOers,DC=release-4-1-0rc3,DC=samba,DC=corp \* replpropertymetadata --sorted --show-binary > $tmpldif - diff $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=ops_run_anything2 -s one -b OU=SUDOers,DC=release-4-1-0rc3,DC=samba,DC=corp \* replpropertymetadata --sorted --show-binary | grep -v originating_change_time| grep -v whenChanged > $tmpldif - diff $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck2.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck2.ldif if [ "$?" != "0" ]; then return 1 fi TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=ops_run_anything3 -s one -b OU=SUDOers,DC=release-4-1-0rc3,DC=samba,DC=corp \* replpropertymetadata --sorted --show-binary > $tmpldif - diff $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck3.ldif + diff -u $tmpldif $release_dir/expected-replpropertymetadata-after-dbcheck3.ldif if [ "$?" != "0" ]; then return 1 fi # Check DomainDNS partition for replica locations tmpldif=$PREFIX_ABS/$RELEASE/expected-replica-locations-after-dbcheck.ldif.tmp $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=49a69498-9a85-48af-9be4-aa0b3e0054f9 -s one -b CN=Partitions,CN=Configuration,DC=release-4-1-0rc3,DC=samba,DC=corp msDS-NC-Replica-Locations > $tmpldif - diff $tmpldif $release_dir/expected-replica-locations-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-replica-locations-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi # Check ForestDNS partition for replica locations $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=7d2a15af-c0d4-487c-847e-e036292bcc65 -s one -b CN=Partitions,CN=Configuration,DC=release-4-1-0rc3,DC=samba,DC=corp msDS-NC-Replica-Locations > $tmpldif - diff $tmpldif $release_dir/expected-replica-locations-after-dbcheck2.ldif + diff -u $tmpldif $release_dir/expected-replica-locations-after-dbcheck2.ldif if [ "$?" != "0" ]; then return 1 fi @@ -253,7 +253,7 @@ check_expected_after_values() { echo $RELEASE checking after values tmpldif=$PREFIX_ABS/$RELEASE/expected-links-after-dbcheck.ldif.tmp $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --show-recycled --show-deleted --show-deactivated-link --reveal member memberOf lastKnownParent objectCategory lastKnownParent wellKnownObjects legacyExchangeDN sAMAccountType uSNChanged --sorted > $tmpldif - diff $tmpldif $release_dir/expected-links-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-links-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -262,7 +262,7 @@ check_expected_after_values() { # this test will fail and can be removed. tmpversion=$PREFIX_ABS/$RELEASE/rootdse-version.final.txt.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb -s base -b '' | grep highestCommittedUSN > $tmpversion - diff $tmpversion $release_dir/rootdse-version.final.txt + diff -u $tmpversion $release_dir/rootdse-version.final.txt if [ "$?" != "0" ]; then return 1 fi @@ -295,7 +295,7 @@ check_expected_after_dup_values() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then tmpldif=$PREFIX_ABS/$RELEASE/expected-otherphone-after-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator -s base -b cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp otherHomePhone --sorted --show-binary | grep -v \# | sort > $tmpldif - diff $tmpldif $release_dir/expected-otherphone-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-otherphone-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi @@ -369,7 +369,7 @@ check_expected_after_deleted_objects() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then tmpldif=$PREFIX_ABS/$RELEASE/expected-deleted_objects-after-dbcheck.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=deleted\ objects -s base -b cn=deleted\ objects,DC=release-4-1-0rc3,DC=samba,DC=corp objectClass description isDeleted isCriticalSystemObject objectGUID showInAdvancedViewOnly systemFlags --sorted --show-binary --show-deleted | grep -v \# | sort > $tmpldif - diff $tmpldif $release_dir/expected-deleted_objects-after-dbcheck.ldif + diff -u $tmpldif $release_dir/expected-deleted_objects-after-dbcheck.ldif if [ "$?" != "0" ]; then return 1 fi diff --git a/testprogs/blackbox/tombstones-expunge.sh b/testprogs/blackbox/tombstones-expunge.sh index d03547f85cd7..aa37cfe278fa 100755 --- a/testprogs/blackbox/tombstones-expunge.sh +++ b/testprogs/blackbox/tombstones-expunge.sh @@ -54,7 +54,7 @@ tombstones_expunge() { if [ "$?" != "0" ]; then return 1 fi - diff $tmpfile $release_dir/expected-expunge-output.txt + diff -u $tmpfile $release_dir/expected-expunge-output.txt if [ "$?" != "0" ]; then return 1 fi @@ -62,7 +62,7 @@ tombstones_expunge() { tmpldif2=$PREFIX_ABS/$RELEASE/expected-expunge-output2.txt.tmp2 TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb -s base -b '' | grep highestCommittedUSN > $tmpldif2 - diff $tmpldif1 $tmpldif2 + diff -u $tmpldif1 $tmpldif2 if [ "$?" != "0" ]; then return 1 fi @@ -124,7 +124,7 @@ remove_one_user() { check_match_rule_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-match-rule-links.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(member:1.3.6.1.4.1.7165.4.5.2:=131139216000000000)' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted no_attrs > $tmpldif - diff $tmpldif $release_dir/expected-match-rule-links.ldif + diff -u $tmpldif $release_dir/expected-match-rule-links.ldif if [ "$?" != "0" ]; then return 1 fi @@ -165,7 +165,7 @@ check_match_rule_links_notlink() { check_expected_after_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-links-after-expunge.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=swimmers)(cn=leaders)(cn=helpers))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --sorted member > $tmpldif - diff $tmpldif $release_dir/expected-links-after-expunge.ldif + diff -u $tmpldif $release_dir/expected-links-after-expunge.ldif if [ "$?" != "0" ]; then return 1 fi @@ -174,7 +174,7 @@ check_expected_after_links() { check_expected_after_deleted_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-deleted-links-after-expunge.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(cn=swimmers)(cn=leaders)(cn=helpers))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted member > $tmpldif - diff $tmpldif $release_dir/expected-deleted-links-after-expunge.ldif + diff -u $tmpldif $release_dir/expected-deleted-links-after-expunge.ldif if [ "$?" != "0" ]; then return 1 fi @@ -183,7 +183,7 @@ check_expected_after_deleted_links() { check_expected_after_objects() { tmpldif=$PREFIX_ABS/$RELEASE/expected-objects-after-expunge.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(|(samaccountname=fred)(samaccountname=ddg)(samaccountname=usg)(samaccountname=user1)(samaccountname=user2))' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted samAccountName | grep sAMAccountName > $tmpldif - diff $tmpldif $release_dir/expected-objects-after-expunge.ldif + diff -u $tmpldif $release_dir/expected-objects-after-expunge.ldif if [ "$?" != "0" ]; then return 1 fi @@ -192,7 +192,7 @@ check_expected_after_objects() { check_expected_unsorted_links() { tmpldif=$PREFIX_ABS/$RELEASE/expected-unsorted-links-after-expunge.ldif.tmp TZ=UTC $ldbsearch -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb '(name=unsorted-g)' -s sub -b DC=release-4-5-0-pre1,DC=samba,DC=corp --show-deleted --reveal --sorted member > $tmpldif - diff $tmpldif $release_dir/expected-unsorted-links-after-expunge.ldif + diff -u $tmpldif $release_dir/expected-unsorted-links-after-expunge.ldif if [ "$?" != "0" ]; then return 1 fi -- 2.17.1 From 7b9aa9c8b59e9cf287b1bb24279093ecb67b3c9f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 14:52:57 +0100 Subject: [PATCH 03/16] blackbox/dbcheck-links.sh: reproduce lost deleted object problem When a parent object is removed during the tombstone garbage collection before a child object and samba-tool dbcheck runs at the same time, the following can happen: - If the object child had DISALLOW_MOVE_ON_DELETE in systemFlags, samba-tool dbcheck moves the object under the LostAndFound[Config] object (as an originating update!) - The lastKnownParent attribute is removed (as an originating update!) These originating updates cause the object to have an extended time as tombstone. And these changes are replicated to other DCs, which very likely already removed the object completely! This means the destination DC of replication has no chance to handle the object it gets from the source DC with just 2 attributes (name, lastKnownParent). The destination logs something like: No objectClass found in replPropertyMetaData BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5357f591accffbf8c62335c308b985811b66f0b5) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user1.txt | 14 +++ testprogs/blackbox/dbcheck-links.sh | 113 ++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..676281faba58 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt new file mode 100644 index 000000000000..db18b9b188b6 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -0,0 +1,14 @@ +Checking 232 objects +WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' +Remove stale DN link? [YES] +Removed deleted DN on attribute lastKnownParent +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] +Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 7b18e11feb3d..db65dd8db19f 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -238,6 +238,114 @@ dbcheck_missing_link_sid_corruption() { return $? } +add_lost_deleted_user1() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user1.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTIzNC01Njc4LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user1() { + dbcheck "-lost-deleted-user1" "1" "" + return $? +} + +remove_lost_deleted_user1() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -454,6 +562,11 @@ if [ -d $release_dir ]; then testit "missing_link_sid_corruption" missing_link_sid_corruption testit "dbcheck_missing_link_sid_corruption" dbcheck_missing_link_sid_corruption testit "missing_link_sid_clean" dbcheck_clean + testit "add_lost_deleted_user1" add_lost_deleted_user1 + testit "dbcheck_lost_deleted_user1" dbcheck_lost_deleted_user1 + testit "lost_deleted_user1_clean_A" dbcheck_clean + testit "remove_lost_deleted_user1" remove_lost_deleted_user1 + testit "lost_deleted_user1_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From afca069ec3a41993a4b9be0a01c4a870e7e017b8 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:38:38 +0100 Subject: [PATCH 04/16] dsdb:repl_meta_data: allow CONTROL_DBCHECK_FIX_LINK_DN_NAME to by pass rename We need a way to rename an object without updating the replication meta data. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 3e8a435d27da899d0e3dab7cbc0a1c738067eba3) --- source4/dsdb/samdb/ldb_modules/repl_meta_data.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index cfa63af70669..3f00dcb06c94 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -3758,6 +3758,7 @@ static int replmd_rename_callback(struct ldb_request *req, struct ldb_reply *are static int replmd_rename(struct ldb_module *module, struct ldb_request *req) { struct ldb_context *ldb; + struct ldb_control *fix_dn_name_control = NULL; struct replmd_replicated_request *ac; int ret; struct ldb_request *down_req; @@ -3767,6 +3768,12 @@ static int replmd_rename(struct ldb_module *module, struct ldb_request *req) return ldb_next_request(module, req); } + fix_dn_name_control = ldb_request_get_control(req, + DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME); + if (fix_dn_name_control != NULL) { + return ldb_next_request(module, req); + } + ldb = ldb_module_get_ctx(module); ldb_debug(ldb, LDB_DEBUG_TRACE, "replmd_rename\n"); -- 2.17.1 From 2ae62b4d5caa152b94a25ba20424f9d23107aa7a Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 22:45:46 +0100 Subject: [PATCH 05/16] dbcheck: use DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME when renaming deleted objects We should never do originating updates on deleted objects. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 07a8326746f0c444eedf3860b178fc29d84e8d16) --- python/samba/dbchecker.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index bf999ddaab9d..5b8c4f2ebfc4 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -878,7 +878,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: self.samdb.transaction_cancel() - def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val): + def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val, controls): '''handle a wrong dn''' new_rdn = ldb.Dn(self.samdb, str(new_dn)) @@ -895,7 +895,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Not renaming %s to %s" % (obj.dn, new_dn)) return - if self.do_rename(obj.dn, new_rdn, new_parent, ["show_recycled:1", "relax:0"], + if self.do_rename(obj.dn, new_rdn, new_parent, controls, "Failed to rename object %s into %s" % (obj.dn, new_dn)): self.report("Renamed %s into %s" % (obj.dn, new_dn)) @@ -2325,9 +2325,11 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if name_val is not None: parent_dn = None + controls = ["show_recycled:1", "relax:0"] if isDeleted: if not (systemFlags & samba.dsdb.SYSTEM_FLAG_DISALLOW_MOVE_ON_DELETE): parent_dn = deleted_objects_dn + controls += ["local_oid:%s:1" % dsdb.DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME] if parent_dn is None: parent_dn = obj.dn.parent() expected_dn = ldb.Dn(self.samdb, "RDN=RDN,%s" % (parent_dn)) @@ -2338,7 +2340,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if expected_dn != obj.dn: error_count += 1 - self.err_wrong_dn(obj, expected_dn, object_rdn_attr, object_rdn_val, name_val) + self.err_wrong_dn(obj, expected_dn, object_rdn_attr, + object_rdn_val, name_val, controls) elif obj.dn.get_rdn_value() != object_rdn_val: error_count += 1 self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) -- 2.17.1 From 95d2e32e53223a3e0ec39af8d020e7a2a38688a4 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:09:36 +0100 Subject: [PATCH 06/16] dbcheck: do isDeleted, systemFlags and replPropertyMetaData detection first BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 9afcd5331ce567bd80d35175f8e4e21c506e9347) --- python/samba/dbchecker.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 5b8c4f2ebfc4..81c94fbcbd0f 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2088,7 +2088,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count = 0 set_attrs_from_md = set() set_attrs_seen = set() - got_repl_property_meta_data = False got_objectclass = False nc_dn = self.samdb.get_nc_root(obj.dn) @@ -2105,6 +2104,18 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) name_val = None isDeleted = False systemFlags = 0 + repl_meta_data_val = None + + for attrname in obj: + if str(attrname).lower() == 'isdeleted': + if str(obj[attrname][0]) != "FALSE": + isDeleted = True + + if str(attrname).lower() == 'systemflags': + systemFlags = int(obj[attrname][0]) + + if str(attrname).lower() == 'replpropertymetadata': + repl_meta_data_val = obj[attrname][0] for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": @@ -2130,13 +2141,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) else: object_rdn_val = str(obj[attrname][0]) - if str(attrname).lower() == 'isdeleted': - if str(obj[attrname][0]) != "FALSE": - isDeleted = True - - if str(attrname).lower() == 'systemflags': - systemFlags = int(obj[attrname][0]) - if str(attrname).lower() == 'replpropertymetadata': if self.has_replmetadata_zero_invocationid(dn, obj[attrname][0]): error_count += 1 @@ -2166,7 +2170,6 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing incorrect initial attributeID in '%s' on '%s', it should be objectClass" % (attrname, str(dn))) - got_repl_property_meta_data = True continue if str(attrname).lower() == 'ntsecuritydescriptor': @@ -2347,13 +2350,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn))) show_dn = True - if got_repl_property_meta_data: + if repl_meta_data_val: if obj.dn == deleted_objects_dn: isDeletedAttId = 131120 # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container expectedTimeDo = 2650466015990000000 - originating = self.get_originating_time(obj["replPropertyMetaData"][0], isDeletedAttId) + originating = self.get_originating_time(repl_meta_data_val, isDeletedAttId) if originating != expectedTimeDo: if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'): nmsg = ldb.Message() -- 2.17.1 From 457a0a122d89f4bc1619af6e88f84cac0ba371da Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 07/16] dbcheck: don't move already deleted objects to LostAndFound This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6d50ee74920c39cdb18b427bfaaf200775bf2d73) --- python/samba/dbchecker.py | 9 +++++++-- selftest/knownfail.d/dbcheck-list-deleted | 1 - .../expected-dbcheck-link-output-lost-deleted-user1.txt | 8 +++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 81c94fbcbd0f..d6fe261c2b57 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -2391,8 +2391,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) except ldb.LdbError as e11: (enum, estr) = e11.args if enum == ldb.ERR_NO_SUCH_OBJECT: - self.err_missing_parent(obj) - error_count += 1 + if isDeleted: + self.report("WARNING: parent object not found for %s" % (obj.dn)) + self.report("Not moving to LostAndFound " + "(tombstone garbage collection in progress?)") + else: + self.err_missing_parent(obj) + error_count += 1 else: raise diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted index 676281faba58..a8fcb0a223f0 100644 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -1,2 +1 @@ ^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user1_clean_A diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index db18b9b188b6..cfc2644b3cbb 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -7,8 +7,6 @@ Removed deleted DN on attribute lastKnownParent ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -ERROR: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Move object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into LostAndFound? [YES] -Renamed object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into lostAndFound at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Set lastKnownParent on lostAndFound object at CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) -- 2.17.1 From d0d53e54f66188ecef4c347e0db7e6ea721b2eed Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 25 Feb 2019 15:35:22 +0100 Subject: [PATCH 08/16] dbcheck: don't remove dangling one-way links on already deleted objects This would typically happen when the garbage collection removed a parent object before a child object (both with the DISALLOW_MOVE_ON_DELETE bit set in systemFlags), while dbcheck is running at the same time as the garbage collection. In this case the lastKnownParent attributes points a non existing object. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit e388e599495b6d7c38b8b6966332e27f8b958783) --- python/samba/dbchecker.py | 13 +++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 1 - ...ected-dbcheck-link-output-lost-deleted-user1.txt | 7 ++----- 3 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index d6fe261c2b57..31538de7ea6f 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -569,6 +569,19 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) def err_missing_target_dn_or_GUID(self, dn, attrname, val, dsdb_dn): """handle a missing target DN (if specified, GUID form can't be found, and otherwise DN string form can't be found)""" + + # Don't change anything if the object itself is deleted + if str(dn).find('\\0ADEL') != -1: + # We don't bump the error count as Samba produces these + # in normal operation + self.report("WARNING: no target object found for GUID " + "component link %s in deleted object " + "%s - %s" % (attrname, dn, val)) + self.report("Not removing dangling one-way " + "link on deleted object " + "(tombstone garbage collection in progress?)") + return 0 + # check if its a backlink linkID, _ = self.get_attr_linkID_and_reverse_name(attrname) if (linkID & 1 == 0) and str(dsdb_dn).find('\\0ADEL') == -1: diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index a8fcb0a223f0..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_lost_deleted_user1 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt index cfc2644b3cbb..3c55de8fa01f 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user1.txt @@ -1,9 +1,6 @@ Checking 232 objects -WARNING: no target object found for GUID component for DN value lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: target DN is deleted for lastKnownParent in object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Target GUID points at deleted DN ';OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp' -Remove stale DN link? [YES] -Removed deleted DN on attribute lastKnownParent +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1234-5678-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] Rename CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] Renamed CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1234-5678-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -- 2.17.1 From 37746379993c98eb4c0b35d4413b80875438387d Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 28 Feb 2019 18:16:27 +0100 Subject: [PATCH 09/16] dbcheck: add find_repl_attid() helper function BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 598e38d2a5e0832429ba65b4e55bf7127618f894) --- python/samba/dbchecker.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 31538de7ea6f..11fee8ecc73a 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -1491,6 +1491,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) return error_count + def find_repl_attid(self, repl, attid): + for o in repl.ctr.array: + if o.attid == attid: + return o + + return None + def get_originating_time(self, val, attid): '''Read metadata properties and return the originating time for a given attributeId. @@ -1499,11 +1506,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) ''' repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, val) - - for o in repl.ctr.array: - if o.attid == attid: - return o.originating_change_time - + o = self.find_repl_attid(repl, attid) + if o is not None: + return o.originating_change_time return 0 def process_metadata(self, dn, val): -- 2.17.1 From c8ae0d9172d934fdf9a757d279c5c0d1253dcde2 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 11 Mar 2019 23:14:02 +0100 Subject: [PATCH 10/16] blackbox/dbcheck-links.sh: add regression test for lost deleted object repair BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 1ccc21a34d295be3bb2ab481a5918003eae88bf4) --- selftest/knownfail.d/dbcheck-list-deleted | 2 + ...dbcheck-link-output-lost-deleted-user2.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 100 ++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 selftest/knownfail.d/dbcheck-list-deleted create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted new file mode 100644 index 000000000000..670e42b747c6 --- /dev/null +++ b/selftest/knownfail.d/dbcheck-list-deleted @@ -0,0 +1,2 @@ +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean +^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt new file mode 100644 index 000000000000..dfb7422ac0bf --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -0,0 +1,9 @@ +Checking 232 objects +ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (2 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index db65dd8db19f..e14b176693f0 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -346,6 +346,103 @@ remove_lost_deleted_user1() { return 0 } +add_lost_deleted_user2() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user2.ldif + cat > $ldif < Date: Thu, 28 Feb 2019 18:22:18 +0100 Subject: [PATCH 11/16] dbcheck: detect the change after deletion bug Old versions of 'samba-tool dbcheck' could reanimate deleted objects, when running at the same time as the tombstone garbage collection. When the (deleted) parent of a deleted object (with the DISALLOW_MOVE_ON_DELETE bit in systemFlags), is removed before the object itself, dbcheck moved it in the LostAndFound[Config] subtree of the partition as an originating change. That means that the object will be in tombstone state again for 180 days on the local DC. And other DCs fail to replicate the object as it's already removed completely there and the replication only gives the name and lastKnownParent attributes, because all other attributes should already be known to the other DC. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit a1658b306d85452407388b91a745078c9c1f7dc7) --- python/samba/dbchecker.py | 110 ++++++++++++++++++ selftest/knownfail.d/dbcheck-list-deleted | 2 - ...dbcheck-link-output-lost-deleted-user2.txt | 15 ++- 3 files changed, 117 insertions(+), 10 deletions(-) delete mode 100644 selftest/knownfail.d/dbcheck-list-deleted diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 11fee8ecc73a..6538938c664c 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -120,6 +120,7 @@ class dbcheck(object): self.fix_missing_deleted_objects = False self.fix_replica_locations = False self.fix_missing_rid_set_master = False + self.fix_changes_after_deletion_bug = False self.dn_set = set() self.link_id_cache = {} @@ -208,6 +209,14 @@ class dbcheck(object): else: self.rid_set_dn = None + ntds_service_dn = "CN=Directory Service,CN=Windows NT,CN=Services,%s" % \ + self.samdb.get_config_basedn().get_linearized() + res = samdb.search(base=ntds_service_dn, + scope=ldb.SCOPE_BASE, + expression="(objectClass=nTDSService)", + attrs=["tombstoneLifetime"]) + self.tombstoneLifetime = int(res[0]["tombstoneLifetime"][0]) + self.compatibleFeatures = [] self.requiredFeatures = [] @@ -1758,6 +1767,101 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def find_changes_after_deletion(self, repl_val): + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + found = [] + for o in repl.ctr.array: + if o.attid == drsuapi.DRSUAPI_ATTID_isDeleted: + continue + + if o.local_usn <= isDeleted.local_usn: + continue + + if o.originating_change_time <= isDeleted.originating_change_time: + continue + + change_time = samba.nttime2unix(o.originating_change_time) + + delta = change_time - delete_time + if delta <= tombstone_delta: + continue + + # If the modification happened after the tombstone lifetime + # has passed, we have a bug as the object might be deleted + # already on other DCs and won't be able to replicate + # back + found.append(o) + + return found, isDeleted + + def has_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + if len(found) == 0: + return False + + def report_attid(o): + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + + self.report("%s: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + attname, o.attid, o.version, + o.originating_invocation_id, + o.originating_usn, + o.local_usn, + time.ctime(samba.nttime2unix(o.originating_change_time)))) + + self.report("ERROR: object %s, has changes after deletion" % dn) + report_attid(isDeleted) + for o in found: + report_attid(o) + + return True + + def err_changes_after_deletion(self, dn, repl_val): + found, isDeleted = self.find_changes_after_deletion(repl_val) + + in_schema_nc = dn.is_child_of(self.schema_dn) + rdn_attr = dn.get_rdn_name() + rdn_attid = self.samdb_schema.get_attid_from_lDAPDisplayName(rdn_attr, + is_schema_nc=in_schema_nc) + + unexpected = [] + for o in found: + if o.attid == rdn_attid: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_name: + continue + if o.attid == drsuapi.DRSUAPI_ATTID_lastKnownParent: + continue + try: + attname = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid) + except KeyError: + attname = "" % o.attid + unexpected.append(attname) + + if len(unexpected) > 0: + self.report('Unexpeted attributes: %s' % ",".join(unexpected)) + self.report('Not fixing changes after deletion bug') + return + + if not self.confirm_all('Delete broken tombstone object %s deleted %s days ago?' % ( + dn, self.tombstoneLifetime), 'fix_changes_after_deletion_bug'): + self.report('Not fixing changes after deletion bug') + return + + if self.do_delete(dn, ["relax:0"], + "Failed to remove DN %s" % dn): + self.report("Removed DN %s" % dn) + def has_replmetadata_zero_invocationid(self, dn, repl_meta_data): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_meta_data) @@ -2135,6 +2239,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) if str(attrname).lower() == 'replpropertymetadata': repl_meta_data_val = obj[attrname][0] + if isDeleted and repl_meta_data_val: + if self.has_changes_after_deletion(dn, repl_meta_data_val): + error_count += 1 + self.err_changes_after_deletion(dn, repl_meta_data_val) + return error_count + for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": continue diff --git a/selftest/knownfail.d/dbcheck-list-deleted b/selftest/knownfail.d/dbcheck-list-deleted deleted file mode 100644 index 670e42b747c6..000000000000 --- a/selftest/knownfail.d/dbcheck-list-deleted +++ /dev/null @@ -1,2 +0,0 @@ -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.lost_deleted_user2_clean -^samba4.blackbox.dbcheck-links.release-4-5-0-pre1.dbcheck_clean3 diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt index dfb7422ac0bf..9b87ca10c57e 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user2.txt @@ -1,9 +1,8 @@ Checking 232 objects -ERROR: missing GUID component for lastKnownParent in object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -unable to find object for DN OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - (No such Base DN: OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp) -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp - OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-8765-4321-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -Checked 232 objects (2 errors) +ERROR: object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp, has changes after deletion +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +name: attid=0x00090001 version=4 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3772 (local=3772) at Mon Mar 11 13:28:24 2019 +lastKnownParent: attid=0x0009030d version=3 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3773 (local=3773) at Mon Mar 11 13:28:24 2019 +Delete broken tombstone object CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp deleted 180 days ago? [YES] +Removed DN CN=fred\0ADEL:2301a64c-8765-4321-851e-12d4a711cfb4,CN=LostAndFound,DC=release-4-5-0-pre1,DC=samba,DC=corp +Checked 232 objects (1 errors) -- 2.17.1 From f24efc893b3d49ffa5bf2208327e088169d32090 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 10:25:40 +0100 Subject: [PATCH 12/16] python/samba/netcmd: provide SUPPRESS_HELP via Option class BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b61d580fb7dba8ff94e9e98c958e324865cd2f1d) --- python/samba/netcmd/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/samba/netcmd/__init__.py b/python/samba/netcmd/__init__.py index cb22b5dc1b0b..54e9107005a1 100644 --- a/python/samba/netcmd/__init__.py +++ b/python/samba/netcmd/__init__.py @@ -27,6 +27,7 @@ import textwrap class Option(optparse.Option): + SUPPRESS_HELP = optparse.SUPPRESS_HELP pass # This help formatter does text wrapping and preserves newlines -- 2.17.1 From 8f2411f539eeab434b2029b109863b7066320c7a Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:02:18 +0100 Subject: [PATCH 13/16] dbcheck: add --selftest-check-expired-tombstones cmdline option This will be used by dbcheck tests which operate on static/old provision dumps in the following commits. Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 6f9c5ed8de47bb98e21e8064d8e90f963f2f71ca) --- python/samba/netcmd/dbcheck.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index 965288b45573..4912e87c7740 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -74,13 +74,18 @@ class cmd_dbcheck(Command): Option("--reset-well-known-acls", dest="reset_well_known_acls", default=False, action="store_true", help="reset ACLs on objects with well known default ACL values to the default"), Option("-H", "--URL", help="LDB URL for database or target server (defaults to local SAM database)", type=str, metavar="URL", dest="H"), + Option("--selftest-check-expired-tombstones", + dest="selftest_check_expired_tombstones", default=False, action="store_true", + help=Option.SUPPRESS_HELP), # This is only used by tests ] def run(self, DN=None, H=None, verbose=False, fix=False, yes=False, cross_ncs=False, quiet=False, scope="SUB", credopts=None, sambaopts=None, versionopts=None, attrs=None, reindex=False, force_modules=False, - reset_well_known_acls=False, yes_rules=[]): + reset_well_known_acls=False, + selftest_check_expired_tombstones=False, + yes_rules=[]): lp = sambaopts.get_loadparm() -- 2.17.1 From 33bdae7f77c4843ff6b8a54bc5e4f2ccd4897a43 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:04:33 +0100 Subject: [PATCH 14/16] blackbox/dbcheck*.sh: pass --selftest-check-expired-tombstones to dbcheck These tests operate on provision dumps created long ago, they still want to run tests on deleted objects, when the next commits remove processing expired tombstone objects in dbcheck. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit 5fccc4e9044d2e57be33471f5e6b9be7cc37ac3a) --- testprogs/blackbox/dbcheck-links.sh | 18 +++++++++--------- testprogs/blackbox/dbcheck-oldrelease.sh | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index e14b176693f0..24724b902f84 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -42,12 +42,12 @@ dbcheck() { } dbcheck_dangling() { - dbcheck "" "1" "" + dbcheck "" "1" "--selftest-check-expired-tombstones" return $? } dbcheck_one_way() { - dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp" + dbcheck "_one_way" "0" "CN=Configuration,DC=release-4-5-0-pre1,DC=samba,DC=corp --selftest-check-expired-tombstones" return $? } @@ -118,7 +118,7 @@ duplicate_member() { } dbcheck_duplicate_member() { - dbcheck "_duplicate_member" "1" "" + dbcheck "_duplicate_member" "1" "--selftest-check-expired-tombstones" return $? } @@ -234,7 +234,7 @@ EOF } dbcheck_missing_link_sid_corruption() { - dbcheck "-missing-link-sid-corruption" "1" "" + dbcheck "-missing-link-sid-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -332,7 +332,7 @@ EOF } dbcheck_lost_deleted_user1() { - dbcheck "-lost-deleted-user1" "1" "" + dbcheck "-lost-deleted-user1" "1" "--selftest-check-expired-tombstones" return $? } @@ -439,7 +439,7 @@ EOF } dbcheck_lost_deleted_user2() { - dbcheck "-lost-deleted-user2" "1" "" + dbcheck "-lost-deleted-user2" "1" "--selftest-check-expired-tombstones" return $? } @@ -504,7 +504,7 @@ EOF } dbcheck_forward_link_corruption() { - dbcheck "-forward-link-corruption" "1" "" + dbcheck "-forward-link-corruption" "1" "--selftest-check-expired-tombstones" return $? } @@ -565,7 +565,7 @@ EOF } dbcheck_oneway_link_corruption() { - dbcheck "-oneway-link-corruption" "0" "" + dbcheck "-oneway-link-corruption" "0" "--selftest-check-expired-tombstones" return $? } @@ -580,7 +580,7 @@ check_expected_after_dbcheck_oneway_link_corruption() { dbcheck_dangling_multi_valued() { - $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --fix --yes + $PYTHON $BINDIR/samba-tool dbcheck -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --selftest-check-expired-tombstones --fix --yes if [ "$?" != "1" ]; then return 1 fi diff --git a/testprogs/blackbox/dbcheck-oldrelease.sh b/testprogs/blackbox/dbcheck-oldrelease.sh index 67fd6a49b61d..3d0ee2c165ac 100755 --- a/testprogs/blackbox/dbcheck-oldrelease.sh +++ b/testprogs/blackbox/dbcheck-oldrelease.sh @@ -207,7 +207,7 @@ check_expected_before_values() { # This should 'fail', because it returns the number of modified records dbcheck_objectclass() { if [ x$RELEASE = x"release-4-1-6-partial-object" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb --attrs=objectclass $@ else return 1 fi @@ -215,7 +215,7 @@ dbcheck_objectclass() { # This should 'fail', because it returns the number of modified records dbcheck() { - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ } check_expected_after_values() { @@ -285,7 +285,7 @@ check_forced_duplicate_values() { # This should 'fail', because it returns the number of modified records dbcheck_after_dup() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb cn=administrator,cn=users,DC=release-4-1-0rc3,DC=samba,DC=corp $@ else return 1 fi @@ -328,7 +328,7 @@ dbcheck_acl_reset_clean() { # This should 'fail', because it returns the number of modified records dbcheck2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -336,7 +336,7 @@ dbcheck2() { # But having fixed it all up, this should pass dbcheck_clean2() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } @@ -353,7 +353,7 @@ rm_deleted_objects() { # This should 'fail', because it returns the number of modified records dbcheck3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs --fix --yes -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ else exit 1 fi @@ -361,7 +361,7 @@ dbcheck3() { # But having fixed it all up, this should pass dbcheck_clean3() { if [ x$RELEASE = x"release-4-1-0rc3" ]; then - $PYTHON $BINDIR/samba-tool dbcheck --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ + $PYTHON $BINDIR/samba-tool dbcheck --selftest-check-expired-tombstones --cross-ncs -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb $@ fi } -- 2.17.1 From a9e46f80a760e6d26f536893acc9b41c6e9e01bc Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:38:22 +0100 Subject: [PATCH 15/16] blackbox/dbcheck-links.sh: prepare regression test for skipping expired tombstones BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett (cherry picked from commit b096a3117ed9249fd6f65f3221a26c88efbba3b8) --- ...dbcheck-link-output-lost-deleted-user3.txt | 9 ++ testprogs/blackbox/dbcheck-links.sh | 115 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt new file mode 100644 index 000000000000..67ca493c44f7 --- /dev/null +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -0,0 +1,9 @@ +Checking 232 objects +WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) +ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] +Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] +Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp +WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp +Not moving to LostAndFound (tombstone garbage collection in progress?) +Checked 232 objects (1 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 24724b902f84..686f560bb188 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -443,6 +443,116 @@ dbcheck_lost_deleted_user2() { return $? } +add_lost_deleted_user3() { + ldif=$PREFIX_ABS/${RELEASE}/add_lost_deleted_user3.ldif + cat > $ldif <;OU=removed,DC=rel + ease-4-5-0-pre1,DC=samba,DC=corp +isRecycled: TRUE +cn:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +name:: ZnJlZApERUw6MjMwMWE2NGMtMTEyMi01NTY2LTg1MWUtMTJkNGE3MTFjZmI0 +replPropertyMetaData:: AQAAAAAAAAAXAAAAAAAAAAAAAAABAAAAVuGDDQMAAACjlkROuH+XT4o + z0jjbi14tnA4AAAAAAACcDgAAAAAAAAMAAAACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4A + AAAAAACiDgAAAAAAAAEAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAA + AAAAAIAAgABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAADAAAgABAA + AAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAABkBAgABAAAAVuGDDQMAAAC + jlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAAEACQACAAAAV+GDDQMAAACjlkROuH+XT4oz + 0jjbi14tog4AAAAAAACiDgAAAAAAAAgACQADAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tng4AA + AAAAACeDgAAAAAAABAACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAA + AAABkACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAFoACQABAAA + AVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAF4ACQABAAAAVuGDDQMAAACj + lkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAAAAGAACQADAAAAV+GDDQMAAACjlkROuH+XT4oz0 + jjbi14tog4AAAAAAACiDgAAAAAAAGIACQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAA + AAAACiDgAAAAAAAH0ACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnQ4AAAAAAACdDgAAAAA + AAJIACQABAAAAVuGDDQMAAACjlkROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAJ8ACQACAAAA + V+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAN0ACQABAAAAVuGDDQMAAACjl + kROuH+XT4oz0jjbi14tnA4AAAAAAACcDgAAAAAAAC4BCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0j + jbi14tog4AAAAAAACiDgAAAAAAAJACCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAA + AAACiDgAAAAAAAA0DCQABAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAA + AA4DCQACAAAAV+GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAAoICQABAAAAV + +GDDQMAAACjlkROuH+XT4oz0jjbi14tog4AAAAAAACiDgAAAAAAAA== +whenChanged: 20160629043639.0Z +uSNChanged: 3746 +nTSecurityDescriptor:: AQAXjBQAAAAwAAAATAAAAMQAAAABBQAAAAAABRUAAACB/fj4FbukVnK + PlwUAAgAAAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFAAIAAAQAeAACAAAAB1o4ACAAAAADAAAAvjsO + 8/Cf0RG2AwAA+ANnwaV6lr/mDdARooUAqgAwSeIBAQAAAAAAAQAAAAAHWjgAIAAAAAMAAAC/Ow7z8 + J/REbYDAAD4A2fBpXqWv+YN0BGihQCqADBJ4gEBAAAAAAABAAAAAAQA1AcsAAAAAAAkAP8BDwABBQ + AAAAAABRUAAACB/fj4FbukVnKPlwUAAgAAAAAUAP8BDwABAQAAAAAABRIAAAAAABgA/wEPAAECAAA + AAAAFIAAAACQCAAAAABQAlAACAAEBAAAAAAAFCgAAAAUAKAAAAQAAAQAAAFMacqsvHtARmBkAqgBA + UpsBAQAAAAAABQoAAAAFACgAAAEAAAEAAABUGnKrLx7QEZgZAKoAQFKbAQEAAAAAAAUKAAAABQAoA + AABAAABAAAAVhpyqy8e0BGYGQCqAEBSmwEBAAAAAAAFCgAAAAUAKAAwAAAAAQAAAIa4tXdKlNERrr + 0AAPgDZ8EBAQAAAAAABQoAAAAFACgAMAAAAAEAAACylVfkVZTREa69AAD4A2fBAQEAAAAAAAUKAAA + ABQAoADAAAAABAAAAs5VX5FWU0RGuvQAA+ANnwQEBAAAAAAAFCgAAAAUAOAAQAAAAAQAAAPiIcAPh + CtIRtCIAoMlo+TkBBQAAAAAABRUAAACB/fj4FbukVnKPlwUpAgAABQA4ABAAAAABAAAAAEIWTMAg0 + BGnaACqAG4FKQEFAAAAAAAFFQAAAIH9+PgVu6RWco+XBSkCAAAFADgAEAAAAAEAAABAwgq8qXnQEZ + AgAMBPwtTPAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFKQIAAAAAFAAAAAIAAQEAAAAAAAULAAAABQA + oABAAAAABAAAAQi+6WaJ50BGQIADAT8LTzwEBAAAAAAAFCwAAAAUAKAAQAAAAAQAAAIa4tXdKlNER + rr0AAPgDZ8EBAQAAAAAABQsAAAAFACgAEAAAAAEAAACzlVfkVZTREa69AAD4A2fBAQEAAAAAAAULA + AAABQAoABAAAAABAAAAVAGN5Pi80RGHAgDAT7lgUAEBAAAAAAAFCwAAAAUAKAAAAQAAAQAAAFMacq + svHtARmBkAqgBAUpsBAQAAAAAAAQAAAAAFADgAEAAAAAEAAAAQICBfpXnQEZAgAMBPwtTPAQUAAAA + AAAUVAAAAgf34+BW7pFZyj5cFKQIAAAUAOAAwAAAAAQAAAH96lr/mDdARooUAqgAwSeIBBQAAAAAA + BRUAAACB/fj4FbukVnKPlwUFAgAABQAsABAAAAABAAAAHbGpRq5gWkC36P+KWNRW0gECAAAAAAAFI + AAAADACAAAFACwAMAAAAAEAAAAcmrZtIpTREa69AAD4A2fBAQIAAAAAAAUgAAAAMQIAAAUALAAwAA + AAAQAAAGK8BVjJvShEpeKFag9MGF4BAgAAAAAABSAAAAAxAgAABRo8ABAAAAADAAAAAEIWTMAg0BG + naACqAG4FKRTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAAEIWTMAg + 0BGnaACqAG4FKbp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAAAECAgX + 6V50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAADAAAAEC + AgX6V50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAAAADAAA + AQMIKvKl50BGQIADAT8LUzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8ABAAAAAD + AAAAQMIKvKl50BGQIADAT8LUz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo8ABAAA + AADAAAAQi+6WaJ50BGQIADAT8LTzxTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAABRI8AB + AAAAADAAAAQi+6WaJ50BGQIADAT8LTz7p6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqAgAABRo + 8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5ORTMKEg3FLxFmwetbwFeXygBAgAAAAAABSAAAAAqAgAA + BRI8ABAAAAADAAAA+IhwA+EK0hG0IgCgyWj5Obp6lr/mDdARooUAqgAwSeIBAgAAAAAABSAAAAAqA + gAABRo4ABAAAAADAAAAbZ7Gt8cs0hGFTgCgyYP2CIZ6lr/mDdARooUAqgAwSeIBAQAAAAAABQkAAA + AFGjgAEAAAAAMAAABtnsa3xyzSEYVOAKDJg/YInHqWv+YN0BGihQCqADBJ4gEBAAAAAAAFCQAAAAU + SOAAQAAAAAwAAAG2exrfHLNIRhU4AoMmD9gi6epa/5g3QEaKFAKoAMEniAQEAAAAAAAUJAAAABRos + AJQAAgACAAAAFMwoSDcUvEWbB61vAV5fKAECAAAAAAAFIAAAACoCAAAFGiwAlAACAAIAAACcepa/5 + g3QEaKFAKoAMEniAQIAAAAAAAUgAAAAKgIAAAUSLACUAAIAAgAAALp6lr/mDdARooUAqgAwSeIBAg + AAAAAABSAAAAAqAgAABRIoADABAAABAAAA3kfmkW/ZcEuVV9Y/9PPM2AEBAAAAAAAFCgAAAAASJAD + /AQ8AAQUAAAAAAAUVAAAAgf34+BW7pFZyj5cFBwIAAAASGAAEAAAAAQIAAAAAAAUgAAAAKgIAAAAS + GAC9AQ8AAQIAAAAAAAUgAAAAIAIAAA== +EOF + + out=$(TZ=UTC $ldbadd -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb.d/DC%3DRELEASE-4-5-0-PRE1,DC%3DSAMBA,DC%3DCORP.ldb $ldif) + if [ "$?" != "0" ]; then + echo "ldbadd returned:\n$out" + return 1 + fi + + return 0 +} + +dbcheck_lost_deleted_user3() { + # here we don't pass --selftest-check-expired-tombstones + # as we want to test the default + dbcheck "-lost-deleted-user3" "1" "" + return $? +} + +remove_lost_deleted_user3() { + out=$(TZ=UTC $ldbdel -H tdb://$PREFIX_ABS/${RELEASE}/private/sam.ldb "" --show-recycled --relax) + if [ "$?" != "0" ]; then + echo "ldbdel returned:\n$out" + return 1 + fi + + return 0 +} + forward_link_corruption() { # # Step1: add a duplicate forward link from @@ -667,6 +777,11 @@ if [ -d $release_dir ]; then testit "add_lost_deleted_user2" add_lost_deleted_user2 testit "dbcheck_lost_deleted_user2" dbcheck_lost_deleted_user2 testit "lost_deleted_user2_clean" dbcheck_clean + testit "add_lost_deleted_user3" add_lost_deleted_user3 + testit "dbcheck_lost_deleted_user3" dbcheck_lost_deleted_user3 + testit "lost_deleted_user3_clean_A" dbcheck_clean + testit "remove_lost_deleted_user3" remove_lost_deleted_user3 + testit "lost_deleted_user3_clean_B" dbcheck_clean testit "dangling_one_way_dn" dangling_one_way_dn testit "deleted_one_way_dn" deleted_one_way_dn testit "dbcheck_clean3" dbcheck_clean -- 2.17.1 From 86659048d902ec3544f419ac3a13143421c36013 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 12 Mar 2019 11:41:01 +0100 Subject: [PATCH 16/16] dbcheck: don't check expired tombstone objects by default anymore These will be removed anyway and any change on them risks to be an originating update that causes replication problems. BUG: https://bugzilla.samba.org/show_bug.cgi?id=13816 Signed-off-by: Stefan Metzmacher Reviewed-by: Andrew Bartlett Autobuild-User(master): Andrew Bartlett Autobuild-Date(master): Thu Mar 14 03:12:27 UTC 2019 on sn-devel-144 (cherry picked from commit a2c5f8cf41c2dfdc4f122e8427d1dfeabb6ba311) --- python/samba/dbchecker.py | 45 ++++++++++++++++++- python/samba/netcmd/dbcheck.py | 6 ++- ...dbcheck-link-output-lost-deleted-user3.txt | 26 +++++++---- testprogs/blackbox/dbcheck-links.sh | 2 +- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/python/samba/dbchecker.py b/python/samba/dbchecker.py index 6538938c664c..ad2de803f15f 100644 --- a/python/samba/dbchecker.py +++ b/python/samba/dbchecker.py @@ -60,7 +60,8 @@ class dbcheck(object): def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False, yes=False, quiet=False, in_transaction=False, - reset_well_known_acls=False): + reset_well_known_acls=False, + check_expired_tombstones=False): self.samdb = samdb self.dict_oid_name = None self.samdb_schema = (samdb_schema or samdb) @@ -107,6 +108,8 @@ class dbcheck(object): self.fix_doubled_userparameters = False self.fix_sid_rid_set_conflict = False self.reset_well_known_acls = reset_well_known_acls + self.check_expired_tombstones = check_expired_tombstones + self.expired_tombstones = 0 self.reset_all_well_known_acls = False self.in_transaction = in_transaction self.infrastructure_dn = ldb.Dn(samdb, "CN=Infrastructure," + samdb.domain_dn()) @@ -253,6 +256,13 @@ class dbcheck(object): if DN is None: error_count += self.check_rootdse() + if self.expired_tombstones > 0: + self.report("NOTICE: found %d expired tombstones, " + "'samba' will remove them daily, " + "'samba-tool domain tombstones expunge' " + "would do that immediately." % ( + self.expired_tombstones)) + if error_count != 0 and not self.fix: self.report("Please use --fix to fix these errors") @@ -1767,6 +1777,37 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn)) self.samdb.set_session_info(self.system_session_info) + def is_expired_tombstone(self, dn, repl_val): + if self.check_expired_tombstones: + # This is not the default, it's just + # used to keep dbcheck tests work with + # old static provision dumps + return False + + repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) + + isDeleted = self.find_repl_attid(repl, drsuapi.DRSUAPI_ATTID_isDeleted) + + delete_time = samba.nttime2unix(isDeleted.originating_change_time) + current_time = time.time() + + tombstone_delta = self.tombstoneLifetime * (24 * 60 * 60) + + delta = current_time - delete_time + if delta <= tombstone_delta: + return False + + self.report("SKIPING: object %s is an expired tombstone" % dn) + self.report("isDeleted: attid=0x%08x version=%d invocation=%s usn=%s (local=%s) at %s" % ( + isDeleted.attid, + isDeleted.version, + isDeleted.originating_invocation_id, + isDeleted.originating_usn, + isDeleted.local_usn, + time.ctime(samba.nttime2unix(isDeleted.originating_change_time)))) + self.expired_tombstones += 1 + return True + def find_changes_after_deletion(self, repl_val): repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, repl_val) @@ -2244,6 +2285,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base))) error_count += 1 self.err_changes_after_deletion(dn, repl_meta_data_val) return error_count + if self.is_expired_tombstone(dn, repl_meta_data_val): + return error_count for attrname in obj: if attrname == 'dn' or attrname == "distinguishedName": diff --git a/python/samba/netcmd/dbcheck.py b/python/samba/netcmd/dbcheck.py index 4912e87c7740..be251d226dba 100644 --- a/python/samba/netcmd/dbcheck.py +++ b/python/samba/netcmd/dbcheck.py @@ -135,8 +135,10 @@ class cmd_dbcheck(Command): started_transaction = True try: chk = dbcheck(samdb, samdb_schema=samdb_schema, verbose=verbose, - fix=fix, yes=yes, quiet=quiet, in_transaction=started_transaction, - reset_well_known_acls=reset_well_known_acls) + fix=fix, yes=yes, quiet=quiet, + in_transaction=started_transaction, + reset_well_known_acls=reset_well_known_acls, + check_expired_tombstones=selftest_check_expired_tombstones) for option in yes_rules: if hasattr(chk, option): diff --git a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt index 67ca493c44f7..d014bfacae2c 100644 --- a/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt +++ b/source4/selftest/provisions/release-4-5-0-pre1/expected-dbcheck-link-output-lost-deleted-user3.txt @@ -1,9 +1,19 @@ Checking 232 objects -WARNING: no target object found for GUID component link lastKnownParent in deleted object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp - ;OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not removing dangling one-way link on deleted object (tombstone garbage collection in progress?) -ERROR: wrong dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp] cn='fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' name=b'fred\nDEL:2301a64c-1122-5566-851e-12d4a711cfb4' new_dn[CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp] -Rename CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp to CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp? [YES] -Renamed CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp into CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp -WARNING: parent object not found for CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp -Not moving to LostAndFound (tombstone garbage collection in progress?) -Checked 232 objects (1 errors) +SKIPING: object CN=fred\0ADEL:2301a64c-1122-5566-851e-12d4a711cfb4,OU=removed,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=fred\0ADEL:2301a64c-5b42-4ca8-851e-12d4a711cfb4,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3746 (local=3746) at Wed Jun 29 04:36:39 2016 +SKIPING: object CN=dsg\0ADEL:6d66d0ef-cad7-4e5d-b1b6-4a233a21c269,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3734 (local=3734) at Wed Jun 29 04:34:32 2016 +SKIPING: object CN=udg\0ADEL:7cff5537-51b1-4d26-a295-0225dbea8525,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3739 (local=3739) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=usg\0ADEL:d012e8f5-a4bd-40ea-a2a1-68ff2508847d,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3736 (local=3736) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=ddg\0ADEL:fb8c2fe3-5448-43de-99f9-e1d3b9357cfc,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3737 (local=3737) at Wed Jun 29 04:34:34 2016 +SKIPING: object CN=gsg\0ADEL:91aa85cc-fc19-4b8c-9fc7-aaba425439c7,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3735 (local=3735) at Wed Jun 29 04:34:33 2016 +SKIPING: object CN=gdg\0ADEL:e0f581e7-14ee-4fc2-839c-8f46f581c72a,CN=Deleted Objects,DC=release-4-5-0-pre1,DC=samba,DC=corp is an expired tombstone +isDeleted: attid=0x00020030 version=1 invocation=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d usn=3738 (local=3738) at Wed Jun 29 04:34:34 2016 +NOTICE: found 8 expired tombstones, 'samba' will remove them daily, 'samba-tool domain tombstones expunge' would do that immediately. +Checked 232 objects (0 errors) diff --git a/testprogs/blackbox/dbcheck-links.sh b/testprogs/blackbox/dbcheck-links.sh index 686f560bb188..d9d80d47eb38 100755 --- a/testprogs/blackbox/dbcheck-links.sh +++ b/testprogs/blackbox/dbcheck-links.sh @@ -539,7 +539,7 @@ EOF dbcheck_lost_deleted_user3() { # here we don't pass --selftest-check-expired-tombstones # as we want to test the default - dbcheck "-lost-deleted-user3" "1" "" + dbcheck "-lost-deleted-user3" "0" "" return $? } -- 2.17.1