From 8ccb76cb33b2484c8a2e5a56020fa3a8257ca7bc Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 15:31:55 +1000 Subject: [PATCH 1/8] ctdb-tests: Reformat node_has_status() Re-indent and drop non-POSIX left-parenthesis from case labels. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 52227d19735a3305ad633672c70385f443f222f0) --- ctdb/tests/scripts/integration.bash | 94 +++++++++++++++-------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index e0aa312192b..0a400443240 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -315,53 +315,55 @@ wait_until_ready () # This function is becoming nicely overloaded. Soon it will collapse! :-) node_has_status () { - local pnn="$1" - local status="$2" - - local bits fpat mpat rpat - case "$status" in - (unhealthy) bits="?|?|?|1|*" ;; - (healthy) bits="?|?|?|0|*" ;; - (disconnected) bits="1|*" ;; - (connected) bits="0|*" ;; - (banned) bits="?|1|*" ;; - (unbanned) bits="?|0|*" ;; - (disabled) bits="?|?|1|*" ;; - (enabled) bits="?|?|0|*" ;; - (stopped) bits="?|?|?|?|1|*" ;; - (notstopped) bits="?|?|?|?|0|*" ;; - (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; - (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; - (recovered) rpat='^Recovery mode:RECOVERY \(1\)$' ;; - (notlmaster) rpat="^hash:.* lmaster:${pnn}\$" ;; + local pnn="$1" + local status="$2" + + local bits fpat mpat rpat + case "$status" in + unhealthy) bits="?|?|?|1|*" ;; + healthy) bits="?|?|?|0|*" ;; + disconnected) bits="1|*" ;; + connected) bits="0|*" ;; + banned) bits="?|1|*" ;; + unbanned) bits="?|0|*" ;; + disabled) bits="?|?|1|*" ;; + enabled) bits="?|?|0|*" ;; + stopped) bits="?|?|?|?|1|*" ;; + notstopped) bits="?|?|?|?|0|*" ;; + frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; + unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; + recovered) rpat='^Recovery mode:RECOVERY \(1\)$' ;; + notlmaster) rpat="^hash:.* lmaster:${pnn}\$" ;; *) - echo "node_has_status: unknown status \"$status\"" - return 1 - esac - - if [ -n "$bits" ] ; then - local out x line - - out=$($CTDB -X status 2>&1) || return 1 - - { - read x - while read line ; do - # This needs to be done in 2 steps to avoid false matches. - local line_bits="${line#|${pnn}|*|}" - [ "$line_bits" = "$line" ] && continue - [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 - done - return 1 - } <<<"$out" # Yay bash! - elif [ -n "$fpat" ] ; then - $CTDB statistics -n "$pnn" | egrep -q "$fpat" - elif [ -n "$rpat" ] ; then - ! $CTDB status -n "$pnn" | egrep -q "$rpat" - else - echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' - return 1 - fi + echo "node_has_status: unknown status \"$status\"" + return 1 + esac + + if [ -n "$bits" ] ; then + local out x line + + out=$($CTDB -X status 2>&1) || return 1 + + { + read x + while read line ; do + # This needs to be done in 2 steps to + # avoid false matches. + local line_bits="${line#|${pnn}|*|}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && \ + return 0 + done + return 1 + } <<<"$out" # Yay bash! + elif [ -n "$fpat" ] ; then + $CTDB statistics -n "$pnn" | egrep -q "$fpat" + elif [ -n "$rpat" ] ; then + ! $CTDB status -n "$pnn" | egrep -q "$rpat" + else + echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' + return 1 + fi } wait_until_node_has_status () -- 2.23.0.rc1 From 8df372c5c53c8ba3f04525ecb64ae42f58778ab4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 15:40:16 +1000 Subject: [PATCH 2/8] ctdb-tests: Drop unused node statuses frozen/unfrozen Silently drop unused local variable mpat. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 9b09a87326af28877301ad27bcec5bb13744e2b6) --- ctdb/tests/scripts/integration.bash | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 0a400443240..dbf68b05b2e 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -318,7 +318,7 @@ node_has_status () local pnn="$1" local status="$2" - local bits fpat mpat rpat + local bits rpat case "$status" in unhealthy) bits="?|?|?|1|*" ;; healthy) bits="?|?|?|0|*" ;; @@ -330,8 +330,6 @@ node_has_status () enabled) bits="?|?|0|*" ;; stopped) bits="?|?|?|?|1|*" ;; notstopped) bits="?|?|?|?|0|*" ;; - frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; - unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; recovered) rpat='^Recovery mode:RECOVERY \(1\)$' ;; notlmaster) rpat="^hash:.* lmaster:${pnn}\$" ;; *) @@ -356,12 +354,10 @@ node_has_status () done return 1 } <<<"$out" # Yay bash! - elif [ -n "$fpat" ] ; then - $CTDB statistics -n "$pnn" | egrep -q "$fpat" elif [ -n "$rpat" ] ; then ! $CTDB status -n "$pnn" | egrep -q "$rpat" else - echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' + echo 'node_has_status: unknown mode, neither $bits nor $rpat is set' return 1 fi } -- 2.23.0.rc1 From 4aa29528836d0a421217bda8e1c7a295fd638257 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 15:45:41 +1000 Subject: [PATCH 3/8] ctdb-tests: Inline handling of recovered and notlmaster statuses BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit bb59073515ee5f7886b5d9a20d7b2805857c2708) --- ctdb/tests/scripts/integration.bash | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index dbf68b05b2e..712932d7d4b 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -318,7 +318,7 @@ node_has_status () local pnn="$1" local status="$2" - local bits rpat + local bits case "$status" in unhealthy) bits="?|?|?|1|*" ;; healthy) bits="?|?|?|0|*" ;; @@ -330,8 +330,16 @@ node_has_status () enabled) bits="?|?|0|*" ;; stopped) bits="?|?|?|?|1|*" ;; notstopped) bits="?|?|?|?|0|*" ;; - recovered) rpat='^Recovery mode:RECOVERY \(1\)$' ;; - notlmaster) rpat="^hash:.* lmaster:${pnn}\$" ;; + recovered) + ! $CTDB status -n "$pnn" | \ + grep -Eq '^Recovery mode:RECOVERY \(1\)$' + return + ;; + notlmaster) + ! $CTDB status -n "$pnn" | \ + grep -Eq "^hash:.* lmaster:${pnn}\$" + return + ;; *) echo "node_has_status: unknown status \"$status\"" return 1 @@ -354,10 +362,8 @@ node_has_status () done return 1 } <<<"$out" # Yay bash! - elif [ -n "$rpat" ] ; then - ! $CTDB status -n "$pnn" | egrep -q "$rpat" else - echo 'node_has_status: unknown mode, neither $bits nor $rpat is set' + echo 'node_has_status: unknown mode, $bits not set' return 1 fi } -- 2.23.0.rc1 From b82dae2e3d4339b0d83058c564ce89a5333a977a Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 16:43:09 +1000 Subject: [PATCH 4/8] ctdb-tests: Handle special cases first and return All the other cases involve matching bits. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit bff1a3a548a2cace997b767d78bb824438664cb7) --- ctdb/tests/scripts/integration.bash | 59 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 712932d7d4b..c1f57c81eb0 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -318,6 +318,19 @@ node_has_status () local pnn="$1" local status="$2" + case "$status" in + recovered) + ! $CTDB status -n "$pnn" | \ + grep -Eq '^Recovery mode:RECOVERY \(1\)$' + return + ;; + notlmaster) + ! $CTDB status -n "$pnn" | \ + grep -Eq "^hash:.* lmaster:${pnn}\$" + return + ;; + esac + local bits case "$status" in unhealthy) bits="?|?|?|1|*" ;; @@ -330,42 +343,26 @@ node_has_status () enabled) bits="?|?|0|*" ;; stopped) bits="?|?|?|?|1|*" ;; notstopped) bits="?|?|?|?|0|*" ;; - recovered) - ! $CTDB status -n "$pnn" | \ - grep -Eq '^Recovery mode:RECOVERY \(1\)$' - return - ;; - notlmaster) - ! $CTDB status -n "$pnn" | \ - grep -Eq "^hash:.* lmaster:${pnn}\$" - return - ;; *) echo "node_has_status: unknown status \"$status\"" return 1 esac - - if [ -n "$bits" ] ; then - local out x line - - out=$($CTDB -X status 2>&1) || return 1 - - { - read x - while read line ; do - # This needs to be done in 2 steps to - # avoid false matches. - local line_bits="${line#|${pnn}|*|}" - [ "$line_bits" = "$line" ] && continue - [ "${line_bits#${bits}}" != "$line_bits" ] && \ - return 0 - done - return 1 - } <<<"$out" # Yay bash! - else - echo 'node_has_status: unknown mode, $bits not set' + local out x line + + out=$($CTDB -X status 2>&1) || return 1 + + { + read x + while read line ; do + # This needs to be done in 2 steps to + # avoid false matches. + local line_bits="${line#|${pnn}|*|}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && \ + return 0 + done return 1 - fi + } <<<"$out" # Yay bash! } wait_until_node_has_status () -- 2.23.0.rc1 From fb62a22ba2ba4847f75e66b64c5851d8daffc263 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 16:45:07 +1000 Subject: [PATCH 5/8] ctdb-tests: Don't retrieve the VNN map from target node for notlmaster Use the VNN map from the node running node_has_status(). This means that wait_until_node_has_status 1 notlmaster 10 0 will run "ctdb status" on node 0 and check (for up to 10 seconds) if node 1 is in the VNN map. If the LMASTER capability has been dropped on node 1 then the above will wait for the VNN map to be updated on node 0. This will happen as part of the recovery that is triggered by the change of LMASTER capability. The next command will then only be able to attach to $TESTDB after the recovery is complete thus guaranteeing a sane state for the test to continue. This stops simple/79_volatile_db_traverse.sh from going into recovery during the traverse or at some other inconvenient time. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 53daeb2f878af1634a26e05cb86d87e2faf20173) --- ctdb/tests/scripts/integration.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index c1f57c81eb0..cf3a269a344 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -325,8 +325,7 @@ node_has_status () return ;; notlmaster) - ! $CTDB status -n "$pnn" | \ - grep -Eq "^hash:.* lmaster:${pnn}\$" + ! $CTDB status | grep -Eq "^hash:.* lmaster:${pnn}\$" return ;; esac -- 2.23.0.rc1 From 8fd2d65b454c02e56c20f21519d1a65beb6a0d92 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 21 Aug 2019 14:35:09 +1000 Subject: [PATCH 6/8] ctdb-recoverd: Only check for LMASTER nodes in the VNN map BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 5d655ac6f2ff82f8f1c89b06870d600a1a3c7a8a) --- ctdb/server/ctdb_recoverd.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c index 88ef3b1c5a0..41f53fd1fbe 100644 --- a/ctdb/server/ctdb_recoverd.c +++ b/ctdb/server/ctdb_recoverd.c @@ -2982,13 +2982,19 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, return; } - /* verify that all active nodes in the nodemap also exist in - the vnnmap. + /* + * Verify that all active lmaster nodes in the nodemap also + * exist in the vnnmap */ for (j=0; jnum; j++) { if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) { continue; } + if (! ctdb_node_has_capabilities(rec->caps, + ctdb->nodes[j]->pnn, + CTDB_CAP_LMASTER)) { + continue; + } if (nodemap->nodes[j].pnn == pnn) { continue; } @@ -2999,8 +3005,8 @@ static void main_loop(struct ctdb_context *ctdb, struct ctdb_recoverd *rec, } } if (i == vnnmap->size) { - DEBUG(DEBUG_ERR, (__location__ " Node %u is active in the nodemap but did not exist in the vnnmap\n", - nodemap->nodes[j].pnn)); + D_ERR("Active LMASTER node %u is not in the vnnmap\n", + nodemap->nodes[j].pnn); ctdb_set_culprit(rec, nodemap->nodes[j].pnn); do_recovery(rec, mem_ctx, pnn, nodemap, vnnmap); return; -- 2.23.0.rc1 From f02771efaa9438ec032169498112151ac30599d6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 29 Jul 2019 17:22:50 +1000 Subject: [PATCH 7/8] ctdb-tests: Strengthen volatile DB traverse test Check the record count more often, from multiple nodes. Add a case with multiple records. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit ca4df06080709adf0cbebc95b0a70b4090dad5ba) --- ctdb/tests/simple/79_volatile_db_traverse.sh | 67 +++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/ctdb/tests/simple/79_volatile_db_traverse.sh b/ctdb/tests/simple/79_volatile_db_traverse.sh index b73a57a20fe..dbf3d668eb1 100755 --- a/ctdb/tests/simple/79_volatile_db_traverse.sh +++ b/ctdb/tests/simple/79_volatile_db_traverse.sh @@ -45,11 +45,56 @@ try_command_on_node 0 $CTDB writekey "$TESTDB" "foo" "bar0" echo "write foo=bar1 on node 1" try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1" -echo "do traverse on node 0" -try_command_on_node -v 0 $CTDB catdb "$TESTDB" +echo -echo "do traverse on node 1" -try_command_on_node -v 1 $CTDB catdb "$TESTDB" +check_db_num_records () +{ + local node="$1" + local db="$2" + local n="$3" + + echo "Checking on node ${node} to ensure ${db} has ${n} records..." + try_command_on_node "$node" "${CTDB} catdb ${db}" + + num=$(sed -n -e 's|^Dumped \(.*\) records$|\1|p' "$outfile") + if [ "$num" = "$n" ] ; then + echo "OK: Number of records=${num}" + echo + else + echo "BAD: There were ${num} (!= ${n}) records" + cat "$outfile" + exit 1 + fi +} + +check_db_num_records 0 "$TESTDB" 1 +check_db_num_records 1 "$TESTDB" 1 + +cat < Date: Tue, 13 Aug 2019 14:45:33 +1000 Subject: [PATCH 8/8] ctdb-tests: Clear deleted record via recovery instead of vacuuming This test has been flapping because sometimes the record is not vacuumed within the expected time period, perhaps even because the check for the record can interfere with vacuuming. However, instead of waiting for vacuuming the record can be cleared by doing a recovery. This should be much more reliable. BUG: https://bugzilla.samba.org/show_bug.cgi?id=14085 RN: Fix flapping CTDB tests Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs Autobuild-User(master): Martin Schwenke Autobuild-Date(master): Wed Aug 21 13:06:57 UTC 2019 on sn-devel-184 (backported from commit 71ad473ba805abe23bbe6c1a1290612e448e73f3) Signed-off-by: Martin Schwenke --- .../simple/69_recovery_resurrect_deleted.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh index fa8a0dd0bbf..a9d6cd16772 100755 --- a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh +++ b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh @@ -57,17 +57,11 @@ database_has_zero_records () return 0 } -echo "Get vacuum interval" -try_command_on_node -v $second $CTDB getvar VacuumInterval -vacuum_interval="${out#* = }" - -echo "Wait until vacuuming deletes the record on active nodes" -# Why 4? Steps are: -# 1. Original node processes delete queue, asks lmaster to fetch -# 2. lmaster recoverd fetches -# 3. lmaster processes delete queue -# If vacuuming is just missed then need an extra interval. -wait_until $((vacuum_interval * 4)) database_has_zero_records +echo "Trigger a recovery" +try_command_on_node "$second" $CTDB recover + +echo "Checking that database has 0 records" +database_has_zero_records echo "Continue node ${first}" try_command_on_node $first $CTDB continue -- 2.23.0.rc1