From 7456b68069121b4d028a736077b10a843b298418 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Apr 2019 12:09:26 +1000
Subject: [PATCH 01/11] ctdb-tests: Change sanity_check_output() to internally
 use $out

All callers are currently passed $out.  Global variable $out is used
in many other places so use it here to simplify the interface and make
future changes simpler.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 7c3819d1ac264acf998f426e0cef7f6211e0ddee)
---
 ctdb/tests/scripts/integration.bash     | 5 ++---
 ctdb/tests/simple/02_ctdb_tunables.sh   | 3 +--
 ctdb/tests/simple/05_ctdb_listnodes.sh  | 3 +--
 ctdb/tests/simple/09_ctdb_ping.sh       | 6 ++----
 ctdb/tests/simple/12_ctdb_getdebug.sh   | 3 +--
 ctdb/tests/simple/14_ctdb_statistics.sh | 2 +-
 ctdb/tests/simple/24_ctdb_getdbmap.sh   | 4 ++--
 ctdb/tests/simple/25_dumpmemory.sh      | 4 ++--
 ctdb/tests/simple/51_message_ring.sh    | 2 +-
 ctdb/tests/simple/52_fetch_ring.sh      | 2 +-
 ctdb/tests/simple/81_tunnel_ring.sh     | 2 +-
 11 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 6ba531a4a76..7aef0c7ee39 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -108,11 +108,10 @@ sanity_check_output ()
 {
     local min_lines="$1"
     local regexp="$2" # Should be anchored as necessary.
-    local output="$3"
 
     local ret=0
 
-    local num_lines=$(echo "$output" | wc -l)
+    local num_lines=$(echo "$out" | wc -l)
     echo "There are $num_lines lines of output"
     if [ $num_lines -lt $min_lines ] ; then
 	echo "BAD: that's less than the required number (${min_lines})"
@@ -121,7 +120,7 @@ sanity_check_output ()
 
     local status=0
     local unexpected # local doesn't pass through status of command on RHS.
-    unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
+    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
 
     # Note that this is reversed.
     if [ $status -eq 0 ] ; then
diff --git a/ctdb/tests/simple/02_ctdb_tunables.sh b/ctdb/tests/simple/02_ctdb_tunables.sh
index 74163d9b2e6..e205da217e6 100755
--- a/ctdb/tests/simple/02_ctdb_tunables.sh
+++ b/ctdb/tests/simple/02_ctdb_tunables.sh
@@ -19,8 +19,7 @@ try_command_on_node -v 0 "$CTDB listvars"
 
 sanity_check_output \
     5 \
-    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
-    "$out"
+    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$'
 
 echo "Verifying all variable values using \"ctdb getvar\"..."
 
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 7e992c3ae5a..0adb291f758 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -39,8 +39,7 @@ ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
 ipv6_pat='[[:xdigit:]]+:[[:xdigit:]:]+[[:xdigit:]]+'
 sanity_check_output \
     2 \
-    "^${ipv4_pat}|${ipv6_pat}\$" \
-    "$out"
+    "^${ipv4_pat}|${ipv6_pat}\$"
 
 out_0="$out"
 
diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
index 28552b94bfe..b911b896f99 100755
--- a/ctdb/tests/simple/09_ctdb_ping.sh
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@@ -39,8 +39,7 @@ try_command_on_node -v 0 "$CTDB ping -n 1"
 
 sanity_check_output \
     1 \
-    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
-    "$out"
+    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$'
 
 try_command_on_node -v 0 "$CTDB shutdown -n 1"
 
@@ -50,5 +49,4 @@ try_command_on_node -v 0 "! $CTDB ping -n 1"
 
 sanity_check_output \
     1 \
-    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)" \
-    "$out"
+    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)"
diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh
index 1189c1c710d..979392f9a1d 100755
--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
@@ -36,8 +36,7 @@ getdebug_onnode="$out"
 
 sanity_check_output \
     $num_nodes \
-    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$' \
-    "$out"
+    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$'
 
 cmd=""
 n=0
diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh
index 70245ee7e9f..cda6e5f478e 100755
--- a/ctdb/tests/simple/14_ctdb_statistics.sh
+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
@@ -35,4 +35,4 @@ pattern='^(CTDB version 1|Current time of statistics[[:space:]]*:.*|Statistics c
 
 try_command_on_node -v 1 "$CTDB statistics"
 
-sanity_check_output 40 "$pattern" "$out"
+sanity_check_output 40 "$pattern"
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index d8a3c604048..612fec1b49b 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -42,7 +42,7 @@ try_command_on_node -v 0 "$CTDB getdbmap"
 
 db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
 
-sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
 
 num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
 
@@ -51,7 +51,7 @@ for i in $(seq 1 5) ; do
     echo "Creating test database: $f"
     try_command_on_node 0 $CTDB attach "$f"
     try_command_on_node 0 $CTDB getdbmap
-    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
     num=$(echo "$out" | sed -n -e '1s/^.*://p')
     if [ $num = $(($num_db_init + $i)) ] ; then
 	echo "OK: correct number of additional databases"
diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh
index 4f998bf8f3b..3d976c17d0f 100755
--- a/ctdb/tests/simple/25_dumpmemory.sh
+++ b/ctdb/tests/simple/25_dumpmemory.sh
@@ -31,8 +31,8 @@ cluster_is_healthy
 pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$'
 
 try_command_on_node -v 0 "$CTDB dumpmemory"
-sanity_check_output 10 "$pat" "$out"
+sanity_check_output 10 "$pat"
 
 echo
 try_command_on_node -v 0 "$CTDB rddumpmemory"
-sanity_check_output 10 "$pat" "$out"
+sanity_check_output 10 "$pat"
diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
index 8aa8e47fa80..f6f064e43bf 100755
--- a/ctdb/tests/simple/51_message_ring.sh
+++ b/ctdb/tests/simple/51_message_ring.sh
@@ -31,7 +31,7 @@ while read line ; do
 done <<<"$out"
 
 pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\))$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # $prev should look like this:
 #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
index b12b2aa32f0..bdd8186c49e 100755
--- a/ctdb/tests/simple/52_fetch_ring.sh
+++ b/ctdb/tests/simple/52_fetch_ring.sh
@@ -26,7 +26,7 @@ echo "Running fetch_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
 
 pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # Get the last line of output.
 while read line ; do
diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
index 2c5091d3b9b..990039abfcb 100755
--- a/ctdb/tests/simple/81_tunnel_ring.sh
+++ b/ctdb/tests/simple/81_tunnel_ring.sh
@@ -32,7 +32,7 @@ while read line ; do
 done <<<"$out"
 
 pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # $prev should look like this:
 #    pnn[2] count=85400
-- 
2.20.1


From fa55107f8232bf6f35d3bd15d7f8da536496cfae Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Thu, 28 Mar 2019 14:26:52 +1100
Subject: [PATCH 02/11] ctdb-tests: Make try_command_on_node less error-prone

This sometimes fails, apparently due to a cat process in onnode
getting EAGAIN.  The conclusion is that tests that process large
amounts of output should not depend on a sub-shell delivering that
output into a shell variable.

Change try_command_on_node() to leave all of the output in file
$outfile and just put the first 1KB into $out.  $outfile is removed
after each test completes.

Change the implementation of sanity_check_output() to use $outfile
instead of $out.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 9d02452a24625df5f62fd6d45a16effe2fa45fbe)
---
 ctdb/tests/scripts/integration.bash | 30 +++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 7aef0c7ee39..d9afe898ad2 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -72,7 +72,20 @@ ctdb_test_init ()
 
 ########################################
 
-# Sets: $out
+# Sets: $out, $outfile
+# * The first 1KB of output is put into $out
+# * Tests should use $outfile for handling large output
+# * $outfile is removed after each test
+out=""
+outfile="${TEST_VAR_DIR}/try_command_on_node.out"
+
+outfile_cleanup ()
+{
+	rm -f "$outfile"
+}
+
+ctdb_test_exit_hook_add outfile_cleanup
+
 try_command_on_node ()
 {
     local nodespec="$1" ; shift
@@ -91,17 +104,18 @@ try_command_on_node ()
 
     local cmd="$*"
 
-    out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
-
+    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
-	echo "$out"
+	cat "$outfile"
 	return 1
-    }
+    fi
 
     if $verbose ; then
 	echo "Output of \"$cmd\":"
-	echo "$out"
+	cat "$outfile"
     fi
+
+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
 }
 
 sanity_check_output ()
@@ -111,7 +125,7 @@ sanity_check_output ()
 
     local ret=0
 
-    local num_lines=$(echo "$out" | wc -l)
+    local num_lines=$(wc -l <"$outfile")
     echo "There are $num_lines lines of output"
     if [ $num_lines -lt $min_lines ] ; then
 	echo "BAD: that's less than the required number (${min_lines})"
@@ -120,7 +134,7 @@ sanity_check_output ()
 
     local status=0
     local unexpected # local doesn't pass through status of command on RHS.
-    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
+    unexpected=$(grep -Ev "$regexp" "$outfile") || status=$?
 
     # Note that this is reversed.
     if [ $status -eq 0 ] ; then
-- 
2.20.1


From de1f5a6e9bab8a02e316c571e19292ce6a67178c Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Thu, 11 Apr 2019 20:55:20 +1000
Subject: [PATCH 03/11] ctdb-tests: Avoid bulk output in $out, prefer $outfile

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 066cc5b0c561464ed08890d9aa1a1a55b545e9cc)
---
 .../tests/complex/11_ctdb_delip_removes_ip.sh | 10 ++---
 ctdb/tests/complex/18_ctdb_reloadips.sh       |  8 ++--
 ctdb/tests/complex/32_cifs_tickle.sh          |  7 ----
 ctdb/tests/complex/36_smb_reset_server.sh     | 12 +-----
 ctdb/tests/complex/37_nfs_reset_server.sh     |  4 +-
 ctdb/tests/complex/60_rogueip_releaseip.sh    |  2 +-
 ctdb/tests/complex/scripts/local.bash         |  5 +--
 ctdb/tests/scripts/integration.bash           | 40 +++++--------------
 ctdb/tests/simple/02_ctdb_tunables.sh         |  3 +-
 ctdb/tests/simple/05_ctdb_listnodes.sh        |  2 +-
 ctdb/tests/simple/08_ctdb_isnotrecmaster.sh   | 10 ++---
 ctdb/tests/simple/11_ctdb_ip.sh               | 14 ++++---
 ctdb/tests/simple/15_ctdb_statisticsreset.sh  | 21 +++++-----
 ctdb/tests/simple/19_ip_takeover_noop.sh      |  4 +-
 ctdb/tests/simple/20_delip_iface_gc.sh        | 10 ++---
 ctdb/tests/simple/21_ctdb_attach.sh           |  7 ++--
 ctdb/tests/simple/23_ctdb_moveip.sh           | 25 +++++++++++-
 ctdb/tests/simple/24_ctdb_getdbmap.sh         |  6 +--
 ..._config_check_error_on_unreachable_ctdb.sh |  6 +--
 ctdb/tests/simple/27_ctdb_detach.sh           | 21 +++++-----
 ctdb/tests/simple/35_ctdb_getreclock.sh       |  2 +-
 ctdb/tests/simple/51_message_ring.sh          | 12 +++---
 ctdb/tests/simple/52_fetch_ring.sh            | 12 +++---
 ctdb/tests/simple/53_transaction_loop.sh      |  4 +-
 .../simple/54_transaction_loop_recovery.sh    |  4 +-
 ctdb/tests/simple/55_ctdb_ptrans.sh           | 12 +++---
 .../56_replicated_transaction_recovery.sh     |  4 +-
 ctdb/tests/simple/58_ctdb_restoredb.sh        |  8 ++--
 .../simple/69_recovery_resurrect_deleted.sh   | 10 ++---
 ctdb/tests/simple/70_recoverpdbbyseqnum.sh    |  4 +-
 ctdb/tests/simple/71_ctdb_wipedb.sh           |  4 +-
 .../simple/72_update_record_persistent.sh     |  4 +-
 .../tests/simple/75_readonly_records_basic.sh | 24 +++++------
 ctdb/tests/simple/77_ctdb_db_recovery.sh      |  6 +--
 ctdb/tests/simple/79_volatile_db_traverse.sh  |  4 +-
 ctdb/tests/simple/80_ctdb_traverse.sh         |  2 +-
 ctdb/tests/simple/81_tunnel_ring.sh           | 12 +++---
 ctdb/tests/simple/90_debug_hung_script.sh     |  6 +--
 38 files changed, 161 insertions(+), 190 deletions(-)

diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
index b5c8866d67a..543472c0f22 100755
--- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
+++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
@@ -22,8 +22,8 @@ cluster_is_healthy
 select_test_node_and_ips
 get_test_ip_mask_and_iface
 
-echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..."
-try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet6?[[:space:]]*${test_ip}/'"
+echo "Checking that node ${test_node} hosts ${test_ip}..."
+try_command_on_node $test_node "ip addr show to ${test_ip} | grep -q ."
 
 echo "Attempting to remove ${test_ip} from node ${test_node}."
 try_command_on_node $test_node $CTDB delip $test_ip
@@ -33,10 +33,10 @@ wait_until_ips_are_on_node '!' $test_node $test_ip
 timeout=60
 increment=5
 count=0
-echo "Waiting for ${test_ip} to disappear from ${iface}..."
+echo "Waiting for ${test_ip} to disappear from node ${test_node}..."
 while : ; do
-    try_command_on_node -v $test_node "ip addr show dev $iface"
-    if echo "$out" | grep -E 'inet6?[[:space:]]*${test_ip}/'; then
+    try_command_on_node -v $test_node "ip addr show to ${test_node}"
+    if -n "$out" ; then
 	echo "Still there..."
 	if [ $(($count * $increment)) -ge $timeout ] ; then
 	    echo "BAD: Timed out waiting..."
diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
index 2beff771874..4ba1b26a8e8 100755
--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
@@ -48,12 +48,12 @@ select_test_node_and_ips
 
 echo "Getting public IP information from CTDB..."
 try_command_on_node any "$CTDB ip -X -v all"
-ctdb_ip_info=$(echo "$out" | awk -F'|' 'NR > 1 { print $2, $3, $5 }')
+ctdb_ip_info=$(awk -F'|' 'NR > 1 { print $2, $3, $5 }' "$outfile")
 
 echo "Getting IP information from interfaces..."
 try_command_on_node all "ip addr show"
-ip_addr_info=$(echo "$out" | \
-    awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }')
+ip_addr_info=$(awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }' \
+		   "$outfile")
 
 prefix=""
 for b in $(seq 0 255) ; do
@@ -168,7 +168,7 @@ check_ips ()
 
     try_command_on_node $test_node "ip addr show dev ${iface}"
     local ip_addrs_file=$(mktemp)
-    echo "$out" | \
+    cat "$outfile" | \
 	sed -n -e "s@.*inet * \(${prefix//./\.}\.[0-9]*\)/.*@\1@p" | \
 	sort >"$ip_addrs_file"
 
diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh
index 4f2cdadbdfc..bfe3df4e82f 100755
--- a/ctdb/tests/complex/32_cifs_tickle.sh
+++ b/ctdb/tests/complex/32_cifs_tickle.sh
@@ -61,13 +61,6 @@ echo "Source socket is $src_socket"
 # we sometimes beat the registration.
 echo "Checking if CIFS connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    die "BAD: Socket not tracked by CTDB."
-fi
 
 # This is almost immediate.  However, it is sent between nodes
 # asynchonously, so it is worth checking...
diff --git a/ctdb/tests/complex/36_smb_reset_server.sh b/ctdb/tests/complex/36_smb_reset_server.sh
index 0de77722fc3..870b80661aa 100755
--- a/ctdb/tests/complex/36_smb_reset_server.sh
+++ b/ctdb/tests/complex/36_smb_reset_server.sh
@@ -59,16 +59,8 @@ echo "Source socket is $src_socket"
 
 # This should happen as soon as connection is up... but unless we wait
 # we sometimes beat the registration.
-echo "Checking if CIFS connection is tracked by CTDB on test node..."
+echo "Waiting until SMB connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    echo "BAD: Socket not tracked by CTDB."
-    exit 1
-fi
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -80,7 +72,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node
diff --git a/ctdb/tests/complex/37_nfs_reset_server.sh b/ctdb/tests/complex/37_nfs_reset_server.sh
index 7190af0f552..32ff9295cc6 100755
--- a/ctdb/tests/complex/37_nfs_reset_server.sh
+++ b/ctdb/tests/complex/37_nfs_reset_server.sh
@@ -60,7 +60,7 @@ echo "Source socket is $src_socket"
 echo "Wait until NFS connection is tracked by CTDB on test node ..."
 wait_until $((monitor_interval * 2)) \
 	   check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
+cat "$outfile"
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -72,7 +72,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node
diff --git a/ctdb/tests/complex/60_rogueip_releaseip.sh b/ctdb/tests/complex/60_rogueip_releaseip.sh
index 2fddc06f867..88e4e554c34 100755
--- a/ctdb/tests/complex/60_rogueip_releaseip.sh
+++ b/ctdb/tests/complex/60_rogueip_releaseip.sh
@@ -31,7 +31,7 @@ for i in $all_pnns ; do
 		continue
 	fi
 	try_command_on_node $i "$CTDB ip"
-	n=$(awk -v ip="$test_ip" '$1 == ip { print }' <<<"$out")
+	n=$(awk -v ip="$test_ip" '$1 == ip { print }' "$outfile")
 	if [ -n "$n" ] ; then
 		other_node="$i"
 		break
diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash
index 7787de8f111..787f597edcc 100644
--- a/ctdb/tests/complex/scripts/local.bash
+++ b/ctdb/tests/complex/scripts/local.bash
@@ -67,7 +67,7 @@ check_tickles ()
     local src_socket="$4"
     try_command_on_node $node ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    [ "${out/SRC: ${src_socket} /}" != "$out" ]
+    grep -Fq "SRC: ${src_socket} " "$outfile"
 }
 
 check_tickles_all ()
@@ -79,8 +79,7 @@ check_tickles_all ()
 
     try_command_on_node all ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    local t="${src_socket//./\\.}"
-    local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
+    local count=$(grep -Fc "SRC: ${src_socket} " "$outfile" || true)
     [ $count -eq $numnodes ]
 }
 
diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index d9afe898ad2..ce5bd576b24 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -148,30 +148,7 @@ sanity_check_output ()
     return $ret
 }
 
-sanity_check_ips ()
-{
-    local ips="$1" # list of "ip node" lines
-
-    echo "Sanity checking IPs..."
-
-    local x ipp prev
-    prev=""
-    while read x ipp ; do
-	[ "$ipp" = "-1" ] && break
-	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
-	    echo "OK"
-	    return 0
-	fi
-	prev="$ipp"
-    done <<<"$ips"
-
-    echo "BAD: a node was -1 or IPs are only assigned to one node:"
-    echo "$ips"
-    echo "Are you running an old version of CTDB?"
-    return 1
-}
-
-# This returns a list of "ip node" lines in $out
+# This returns a list of "ip node" lines in $outfile
 all_ips_on_node()
 {
     local node="$1"
@@ -192,9 +169,9 @@ _select_test_node_and_ips ()
 	    test_node="$pnn"
 	fi
 	if [ "$pnn" = "$test_node" ] ; then
-            test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+	    test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
     test_ip="${test_node_ips%% *}"
@@ -264,7 +241,7 @@ delete_ip_from_all_nodes ()
 	    if [ "$_ip" = "$_i" ] ; then
 		_nodes="${_nodes}${_nodes:+,}${_pnn}"
 	    fi
-	done <<<"$out" # bashism
+	done <"$outfile"
     done
 
     try_command_on_node -pq "$_nodes" "$CTDB delip $_ip"
@@ -435,7 +412,7 @@ ips_are_on_node ()
 	    if $negating ; then
 		ips="${ips/${check}}"
 	    fi
-	done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+	done <"$outfile"
     done
 
     ips="${ips// }" # Remove any spaces.
@@ -477,7 +454,7 @@ node_has_some_ips ()
 	if [ "$node" = "$pnn" ] ; then
 	    return 0
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     return 1
 }
@@ -614,7 +591,8 @@ wait_for_monitor_event ()
 	return 1
     }
 
-    local ctdb_scriptstatus_original="$out"
+    mv "$outfile" "${outfile}.orig"
+
     wait_until 120 _ctdb_scriptstatus_changed
 }
 
@@ -625,7 +603,7 @@ _ctdb_scriptstatus_changed ()
 	return 1
     }
 
-    [ "$out" != "$ctdb_scriptstatus_original" ]
+    ! diff "$outfile" "${outfile}.orig" >/dev/null
 }
 
 #######################################
diff --git a/ctdb/tests/simple/02_ctdb_tunables.sh b/ctdb/tests/simple/02_ctdb_tunables.sh
index e205da217e6..2e7a08b6f6f 100755
--- a/ctdb/tests/simple/02_ctdb_tunables.sh
+++ b/ctdb/tests/simple/02_ctdb_tunables.sh
@@ -23,7 +23,6 @@ sanity_check_output \
 
 echo "Verifying all variable values using \"ctdb getvar\"..."
 
-echo "$out" |
 while read var x val ; do
     try_command_on_node 0 "$CTDB getvar $var"
 
@@ -33,7 +32,7 @@ while read var x val ; do
 	echo "MISMATCH on $var: $val != $val2"
 	exit 1
     fi
-done
+done <"$outfile"
 
 echo "GOOD: all tunables match"
 
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 0adb291f758..722c9d2a035 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -32,7 +32,7 @@ cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listnodes"
 
-num_nodes=$(echo "$out" | wc -l)
+num_nodes=$(wc -l <"$outfile")
 
 # Each line should look like an IP address.
 ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
index 12c8164b223..68e22879ba3 100755
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@@ -33,13 +33,11 @@ set -e
 cluster_is_healthy
 
 cmd="$CTDB isnotrecmaster || true"
-try_command_on_node all "$cmd"
-echo "Output of \"$cmd\":"
-echo "$out"
+try_command_on_node -v all "$cmd"
 
-num_all_lines=$(echo "$out" |  wc -l)
-num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
-num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
+num_all_lines=$(wc -l <"$outfile")
+num_rm_lines=$(grep -Fc 'this node is the recmaster' "$outfile") || true
+num_not_rm_lines=$(grep -Fc 'this node is not the recmaster' "$outfile") || true
 
 if [ $num_rm_lines -eq 1 ] ; then
     echo "OK, there is only 1 recmaster"
diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
index 4b11729a1e1..d9fdd28f775 100755
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@@ -33,18 +33,20 @@ cluster_is_healthy
 
 echo "Getting list of public IPs..."
 try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
-ips=$(echo "$out" | sed \
+ips=$(sed \
 	-e 's@ node\[@ @' \
-	-e 's@\].*$@@')
-machineout=$(echo "$out" | sed -r \
+	-e 's@\].*$@@' \
+	"$outfile")
+machineout=$(sed -r \
 	-e 's@^| |$@\|@g' \
 	-e 's@[[:alpha:]]+\[@@g' \
-	-e 's@\]@@g')
+	-e 's@\]@@g' \
+	"$outfile")
 
 if [ -z "$TEST_LOCAL_DAEMONS" ]; then
 	while read ip pnn ; do
-		try_command_on_node $pnn "ip addr show"
-		if [ "${out/inet* ${ip}\/}" != "$out" ] ; then
+		try_command_on_node $pnn "ip addr show to ${ip}"
+		if [ -n "$out" ] ; then
 			echo "GOOD: node $pnn appears to have $ip assigned"
 		else
 			die "BAD: node $pnn does not appear to have $ip assigned"
diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
index 8843a4dd9f7..1dce7b39965 100755
--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
@@ -37,10 +37,11 @@ num_nodes="$out"
 
 get_stat ()
 {
-    local label="$1"
-    local out="$2"
+	local label="$1"
 
-    echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
+	cat "$outfile" |
+	sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" |
+	head -1
 }
 
 check_reduced ()
@@ -59,20 +60,20 @@ check_reduced ()
 n=0
 while [ $n -lt $num_nodes ] ; do
     echo "Getting initial statistics for node ${n}..."
-    
+
     try_command_on_node -v $n $CTDB statistics
 
-    before_req_control=$(get_stat "req_control" "$out")
-    before_reply_control=$(get_stat "reply_control" "$out")
-    before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    before_req_control=$(get_stat "req_control")
+    before_reply_control=$(get_stat "reply_control")
+    before_node_packets_recv=$(get_stat "node_packets_recv")
 
     try_command_on_node $n $CTDB statisticsreset
 
     try_command_on_node -v $n $CTDB statistics
 
-    after_req_control=$(get_stat "req_control" "$out")
-    after_reply_control=$(get_stat "reply_control" "$out")
-    after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    after_req_control=$(get_stat "req_control")
+    after_reply_control=$(get_stat "reply_control")
+    after_node_packets_recv=$(get_stat "node_packets_recv")
 
     check_reduced "req_control" "$before_req_control" "$after_req_control"
     check_reduced "reply_control" "$before_reply_control" "$after_reply_control"
diff --git a/ctdb/tests/simple/19_ip_takeover_noop.sh b/ctdb/tests/simple/19_ip_takeover_noop.sh
index 8aedb339d94..93caf668c0b 100755
--- a/ctdb/tests/simple/19_ip_takeover_noop.sh
+++ b/ctdb/tests/simple/19_ip_takeover_noop.sh
@@ -34,9 +34,7 @@ while read ip pnn ; do
 	if [ "$pnn" != "-1" ] ; then
 		die "BAD: IP address ${ip} is assigned to node ${pnn}"
 	fi
-done <<EOF
-$out
-EOF
+done <"$outfile"
 
 echo "GOOD: All IP addresses are unassigned"
 
diff --git a/ctdb/tests/simple/20_delip_iface_gc.sh b/ctdb/tests/simple/20_delip_iface_gc.sh
index 6cc39580c5e..2756f64ceb1 100755
--- a/ctdb/tests/simple/20_delip_iface_gc.sh
+++ b/ctdb/tests/simple/20_delip_iface_gc.sh
@@ -19,14 +19,14 @@ select_test_node_and_ips
 
 # Find interfaces on test node
 try_command_on_node $test_node "$CTDB ifaces -X"
-ifaces=$(awk -F'|' 'NR > 1 { print $2 }' <<<"$out")
+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
 echo "Node ${test_node} has interfaces: ${ifaces}"
 
 # Delete all IPs on each interface...  deleting IPs from one interface
 # can cause other interfaces to disappear, so we need to be careful...
 for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
 
     if [ -z "$info" ] ; then
 	echo "Interface ${i} missing... assuming already deleted!"
@@ -38,7 +38,7 @@ for i in $ifaces ; do
 
     try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
     awk -F'|' -v i="$i" \
-	'$6 == i { print $2 }' <<<"$out" |
+	'$6 == i { print $2 }' "$outfile" |
     while read ip ; do
 	echo "  $ip"
 	try_command_on_node $test_node "$CTDB delip $ip"
@@ -46,8 +46,8 @@ for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ipreallocate"
 
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
-    
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
     if [ -z "$info" ] ; then
 	echo "GOOD: Interface ${i} has been garbage collected"
     else
diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
index 88f91e69acd..7965ced6052 100755
--- a/ctdb/tests/simple/21_ctdb_attach.sh
+++ b/ctdb/tests/simple/21_ctdb_attach.sh
@@ -36,9 +36,8 @@ cluster_is_healthy
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 lastnode=$(( numnodes - 1 ))
 
 ######################################################################
@@ -55,7 +54,7 @@ check_db ()
 	echo "$out"
 	exit 1
     else
-	local flags=$(awk '{print $4}' <<<"$out") || true
+	local flags=$(awk '{print $4}' "$outfile") || true
 	if [ "$flags" = "$flag" ]; then
 	    echo "GOOD: database $db is attached on node $node with flag $flag"
 	else
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
index 4220d1268e4..699ccc3d2ee 100755
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -23,7 +23,28 @@ cluster_is_healthy
 
 select_test_node_and_ips
 
-sanity_check_ips "$out"
+sanity_check_ips ()
+{
+    echo "Sanity checking IPs..."
+
+    local x ipp prev
+    prev=""
+    while read x ipp ; do
+	[ "$ipp" = "-1" ] && break
+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+	    echo "OK"
+	    return 0
+	fi
+	prev="$ipp"
+    done <"$outfile"
+
+    echo "BAD: a node was -1 or IPs are only assigned to one node:"
+    cat "$outfile"
+    echo "Are you running an old version of CTDB?"
+    return 1
+}
+
+sanity_check_ips
 
 # Find a target node - it must be willing to host $test_ip
 try_command_on_node any "$CTDB listnodes | wc -l"
@@ -37,7 +58,7 @@ for i in $(seq 0 $(($num_nodes - 1)) ) ; do
 	    to_node="$i"
 	    break 2
 	fi
-    done <<<"$out"
+    done <"$outfile"
 done
 
 if [ -z "$to_node" ] ; then
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 612fec1b49b..32d6d26a4ff 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -44,7 +44,7 @@ db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[
 
 sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
 
-num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+num_db_init=$(sed -n -e '1s/.*://p' "$outfile")
 
 for i in $(seq 1 5) ; do
     f=$(make_temp_db_filename)
@@ -52,14 +52,14 @@ for i in $(seq 1 5) ; do
     try_command_on_node 0 $CTDB attach "$f"
     try_command_on_node 0 $CTDB getdbmap
     sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
-    num=$(echo "$out" | sed -n -e '1s/^.*://p')
+    num=$(sed -n -e '1s/^.*://p' "$outfile")
     if [ $num = $(($num_db_init + $i)) ] ; then
 	echo "OK: correct number of additional databases"
     else
 	echo "BAD: no additional database"
 	exit 1
     fi
-    if [ "${out/name:${f} /}" != "$out" ] ; then
+    if awk '{print $2}' "$outfile" | grep -Fqx "name:$f" ; then
 	echo "OK: getdbmap knows about \"$f\""
     else
 	echo "BAD: getdbmap does not know about \"$f\""
diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index f356c820260..d7c0c0f9100 100755
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -42,8 +42,8 @@ cluster_is_healthy
 
 test_node=1
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 echo "There are $num_nodes nodes."
 
 echo "Shutting down node ${test_node}..."
@@ -58,7 +58,7 @@ pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error:
 for i in ip disable enable "ban 0" unban listvars ; do
     try_command_on_node -v 0 ! $CTDB $i -n $test_node
 
-    if egrep -q "$pat" <<<"$out" ; then
+    if egrep -q "$pat" "$outfile" ; then
 	echo "OK: \"ctdb ${i}\" fails with expected \"disconnected node\" message"
     else
 	echo "BAD: \"ctdb ${i}\" does not fail with expected \"disconnected node\" message"
diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
index e8ca2382e88..f7f7a7e8c40 100755
--- a/ctdb/tests/simple/27_ctdb_detach.sh
+++ b/ctdb/tests/simple/27_ctdb_detach.sh
@@ -32,9 +32,8 @@ cluster_is_healthy
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 
 ######################################################################
 
@@ -43,12 +42,12 @@ check_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq $numnodes ]; then
 	echo "GOOD: database $db is attached on all nodes"
     else
 	echo "BAD: database $db is not attached on all nodes"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -58,12 +57,12 @@ check_no_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq 0 ]; then
 	echo "GOOD: database $db is not attached any more"
     else
 	echo "BAD: database $db is still attached"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -134,12 +133,12 @@ echo
 echo "Write a key to database"
 try_command_on_node 0 $CTDB writekey $testdb1 foo bar
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 1 ]; then
     echo "GOOD: Key added to database"
 else
     echo "BAD: Key did not get added to database"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi
 
@@ -158,11 +157,11 @@ check_db "$testdb1"
 echo
 echo "Check if the database is empty"
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 0 ]; then
     echo "GOOD: Database $testdb1 is empty"
 else
     echo "BAD: Database $testdb1 is not empty"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi
diff --git a/ctdb/tests/simple/35_ctdb_getreclock.sh b/ctdb/tests/simple/35_ctdb_getreclock.sh
index 0a2f3d5b92e..8cedd34045b 100755
--- a/ctdb/tests/simple/35_ctdb_getreclock.sh
+++ b/ctdb/tests/simple/35_ctdb_getreclock.sh
@@ -25,7 +25,7 @@ if [ -z "$out" ] ; then
     exit 0
 fi
 
-n=$(echo "$out" | sort -u | wc -l)
+n=$(sort -u "$outfile" | wc -l)
 if [ "$n" = 1 ] ; then
 	echo "GOOD: All nodes have the same recovery lock setting"
 else
diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
index f6f064e43bf..6f90c8fd5bb 100755
--- a/ctdb/tests/simple/51_message_ring.sh
+++ b/ctdb/tests/simple/51_message_ring.sh
@@ -19,23 +19,21 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running message_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND message_ring -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\))$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
-stuff="${prev##Ring\[*\]: }"
+stuff="${last##Ring\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
index bdd8186c49e..05632024cb5 100755
--- a/ctdb/tests/simple/52_fetch_ring.sh
+++ b/ctdb/tests/simple/52_fetch_ring.sh
@@ -19,8 +19,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running fetch_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
@@ -29,13 +29,11 @@ pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)?
 sanity_check_output 1 "$pat"
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
-# $prev should look like this:
+# $last should look like this:
 #    Fetch[1]: 10670.93 msgs/sec
-stuff="${prev##*Fetch\[*\]: }"
+stuff="${last##*Fetch\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/53_transaction_loop.sh b/ctdb/tests/simple/53_transaction_loop.sh
index 2102e401384..d47761216ce 100755
--- a/ctdb/tests/simple/53_transaction_loop.sh
+++ b/ctdb/tests/simple/53_transaction_loop.sh
@@ -24,8 +24,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/54_transaction_loop_recovery.sh b/ctdb/tests/simple/54_transaction_loop_recovery.sh
index 61ac29ca178..7beacb9c913 100755
--- a/ctdb/tests/simple/54_transaction_loop_recovery.sh
+++ b/ctdb/tests/simple/54_transaction_loop_recovery.sh
@@ -43,8 +43,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/55_ctdb_ptrans.sh b/ctdb/tests/simple/55_ctdb_ptrans.sh
index e7db3aea456..72b18348398 100755
--- a/ctdb/tests/simple/55_ctdb_ptrans.sh
+++ b/ctdb/tests/simple/55_ctdb_ptrans.sh
@@ -51,11 +51,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records were inserted"
@@ -75,11 +75,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records found"
@@ -116,11 +116,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 0 ] ; then
     echo "BAD: expected 0 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 0 records found"
diff --git a/ctdb/tests/simple/56_replicated_transaction_recovery.sh b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
index f09ed550c82..4d5c1297fa9 100755
--- a/ctdb/tests/simple/56_replicated_transaction_recovery.sh
+++ b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
@@ -44,8 +44,8 @@ TESTDB="replicated_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB replicated"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/58_ctdb_restoredb.sh b/ctdb/tests/simple/58_ctdb_restoredb.sh
index eacbdfc5465..579233e0e80 100755
--- a/ctdb/tests/simple/58_ctdb_restoredb.sh
+++ b/ctdb/tests/simple/58_ctdb_restoredb.sh
@@ -36,10 +36,10 @@ set -e
 cluster_is_healthy
 
 try_command_on_node 0 $CTDB status
-generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="restoredb_test.tdb"
@@ -105,7 +105,7 @@ fi
 wait_until_ready
 
 try_command_on_node 0 $CTDB status
-new_generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+new_generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
 echo "Old generation = $generation"
 echo "New generation = $new_generation"
diff --git a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
index 2c8fc12edef..8126c49b83c 100755
--- a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
+++ b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
@@ -20,9 +20,9 @@ testdb="rec_test.tdb"
 echo "Getting list of nodes..."
 try_command_on_node -v any "onnode -pq all ctdb pnn | grep '^[0-9][0-9]*$'"
 
-first=$(echo "$out" | sed -n -e '1p')
-second=$(echo "$out" | sed -n -e '2p')
-notfirst=$(echo "$out" | tail -n +2)
+first=$(sed -n -e '1p' "$outfile")
+second=$(sed -n -e '2p' "$outfile")
+notfirst=$(tail -n +2 "$outfile")
 
 echo "Create/wipe test database ${testdb}"
 try_command_on_node $first $CTDB attach "$testdb"
@@ -46,7 +46,7 @@ database_has_zero_records ()
 	local n
 	for n in $notfirst ; do
 		try_command_on_node $n $CTDB cattdb "$testdb"
-		if echo "$out" | grep -q '^key(' ; then
+		if grep -q '^key(' "$outfile" ; then
 			return 1
 		fi
 	done
@@ -74,7 +74,7 @@ wait_until_node_has_status $first notstopped
 echo "Get database contents"
 try_command_on_node -v $first $CTDB catdb "$testdb"
 
-if echo "$out" | grep -q '^key(' ; then
+if grep -q '^key(' "$outfile" ; then
 	echo "BAD: Deleted record has been resurrected"
 	exit 1
 fi
diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
index 1722288b562..84e0ec15c43 100755
--- a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
+++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
@@ -35,8 +35,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 add_record_per_node ()
 {
diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh
index 6fae93b0a43..9305339d1e3 100755
--- a/ctdb/tests/simple/71_ctdb_wipedb.sh
+++ b/ctdb/tests/simple/71_ctdb_wipedb.sh
@@ -32,8 +32,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="persistent_test.tdb"
diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh
index 4ab69d81b61..90b5ee06151 100755
--- a/ctdb/tests/simple/72_update_record_persistent.sh
+++ b/ctdb/tests/simple/72_update_record_persistent.sh
@@ -34,8 +34,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 test_db="persistent_test.tdb"
 
diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh
index c5662f571c6..a83b4d31530 100755
--- a/ctdb/tests/simple/75_readonly_records_basic.sh
+++ b/ctdb/tests/simple/75_readonly_records_basic.sh
@@ -49,12 +49,12 @@ check_no_readonly ()
 {
     try_command_on_node all $CTDB cattdb $testdb
     local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS"
-    local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true
+    local numreadonly=$(grep -c -E "$ro_flags" "$outfile") || true
     if [ $numreadonly -eq 0 ] ; then
 	echo "GOOD: no read-only delegations"
     else
 	echo "BAD: there are read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -72,36 +72,36 @@ check_readonly ()
     local count
 
     try_command_on_node $dmaster $CTDB cattdb $testdb
-    count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
     if [ $count -eq 1 ] ; then
 	echo "GOOD: dmaster ${dmaster} has read-only delegations"
     else
 	echo "BAD: dmaster ${dmaster} has no read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
-    count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
     if [ $count -ne 0 ] ; then
 	echo "BAD: dmaster ${dmaster} has a read-only copy"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 
     local o
     for o in $others ; do
 	try_command_on_node $o $CTDB cattdb $testdb
-	count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
 	if [ $count -eq 1 ] ; then
 	    echo "GOOD: node ${o} has a read-only copy"
 	else
 	    echo "BAD: node ${o} has no read-only copy"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
-	count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
 	if [ $count -ne 0 ] ; then
 	    echo "BAD: other node ${o} has read-only delegations"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
     done
@@ -111,7 +111,7 @@ check_readonly ()
 
 echo "Get list of nodes..."
 try_command_on_node any $CTDB -X listnodes
-all_nodes=$(awk -F'|' '{print $2}' <<<"$out")
+all_nodes=$(awk -F'|' '{print $2}' "$outfile")
 
 ######################################################################
 
@@ -138,7 +138,7 @@ try_command_on_node all $CTDB setdbreadonly $testdb
 
 # Database should be tagged as READONLY
 try_command_on_node 0 $CTDB getdbmap
-db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out")
+db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' "$outfile")
 if grep -q "READONLY" <<<"$db_details" ; then
     echo "GOOD: read-only record support is enabled"
 else
diff --git a/ctdb/tests/simple/77_ctdb_db_recovery.sh b/ctdb/tests/simple/77_ctdb_db_recovery.sh
index b2703eedbc4..a8bc2d95ef6 100755
--- a/ctdb/tests/simple/77_ctdb_db_recovery.sh
+++ b/ctdb/tests/simple/77_ctdb_db_recovery.sh
@@ -93,7 +93,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value1
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -108,7 +108,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value2
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -119,7 +119,7 @@ wait_until_node_has_status $recmaster recovered
 # Verify record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 if [ "$out" = "Data: size:6 ptr:[value2]" ]; then
 	echo "GOOD: Recovery did not corrupt database"
 else
diff --git a/ctdb/tests/simple/79_volatile_db_traverse.sh b/ctdb/tests/simple/79_volatile_db_traverse.sh
index b98be4d23f9..af7e962f579 100755
--- a/ctdb/tests/simple/79_volatile_db_traverse.sh
+++ b/ctdb/tests/simple/79_volatile_db_traverse.sh
@@ -75,7 +75,7 @@ try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1"
 echo "do traverse on node 0"
 try_command_on_node -v 0 $CTDB catdb "$TESTDB"
 
-num=$(echo "$out" | sed -n -e 's|^Dumped \(.*\) records$|\1|p')
+num=$(sed -n -e 's|^Dumped \(.*\) records$|\1|p' "$outfile")
 if [ "$num" = 1 ] ; then
 	echo "OK: There was 1 record"
 else
@@ -83,7 +83,7 @@ else
 	exit 1
 fi
 
-if echo "$out" | grep -q "^data(4) = \"bar1\"\$" ; then
+if grep -q "^data(4) = \"bar1\"\$" "$outfile" ; then
 	echo "OK: Data from node 1 was returned"
 else
 	echo "BAD: Data from node 1 was not returned"
diff --git a/ctdb/tests/simple/80_ctdb_traverse.sh b/ctdb/tests/simple/80_ctdb_traverse.sh
index 544d8932cd9..d28f9c2814e 100755
--- a/ctdb/tests/simple/80_ctdb_traverse.sh
+++ b/ctdb/tests/simple/80_ctdb_traverse.sh
@@ -58,7 +58,7 @@ done
 echo "Start a traverse and collect records"
 try_command_on_node 0 $CTDB catdb $TESTDB
 
-num_read=$(echo "$out" | tail -n 1 | cut -d\  -f2)
+num_read=$(tail -n 1 "$outfile" | cut -d\  -f2)
 if [ $num_read -eq $num_records ]; then
 	echo "GOOD: All $num_records records retrieved"
 	status=0
diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
index 990039abfcb..c78c4209ea6 100755
--- a/ctdb/tests/simple/81_tunnel_ring.sh
+++ b/ctdb/tests/simple/81_tunnel_ring.sh
@@ -19,24 +19,22 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running tunnel_test on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND \
 	tunnel_test -t 30 -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    pnn[2] count=85400
-stuff="${prev##pnn\[*\] }"
+stuff="${last##pnn\[*\] }"
 mps="${stuff% msgs/sec}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/90_debug_hung_script.sh b/ctdb/tests/simple/90_debug_hung_script.sh
index 9e2072c98b6..e8cdefec734 100755
--- a/ctdb/tests/simple/90_debug_hung_script.sh
+++ b/ctdb/tests/simple/90_debug_hung_script.sh
@@ -54,11 +54,9 @@ wait_for_monitor_event $test_node
 
 echo "Waiting for debugging output to appear..."
 # Use test -s because the file is created above using mktemp
-wait_until 60 onnode $test_node test -s "$debug_output"
+wait_until 60 test -s "$debug_output"
 
 echo "Checking output of hung script debugging..."
-try_command_on_node -v $test_node cat "$debug_output"
-hung_script_output="$out"
 
 # Can we actually read kernel stacks
 if try_command_on_node $test_node "cat /proc/$$/stack >/dev/null 2>&1" ; then
@@ -72,7 +70,7 @@ fi
 
 while IFS="" read pattern ; do
     [ -n "$pattern" ] || continue
-    if grep -- "^${pattern}\$" <<<"$hung_script_output" >/dev/null ; then
+    if grep -q -- "^${pattern}\$" "$debug_output" ; then
 	printf 'GOOD: output contains "%s"\n' "$pattern"
     else
 	printf 'BAD: output does not contain "%s"\n' "$pattern"
-- 
2.20.1


From 83a6e7b2ce9eec529898758f86650f2c2bed782a Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Sat, 27 Apr 2019 14:54:09 +1000
Subject: [PATCH 04/11] ctdb-tests: Wait to allow database attach/detach to
 take effect

Sometimes the detach test fails:

  Check detaching single test database detach_test1.tdb
  BAD: database detach_test1.tdb is still attached
  Number of databases:4
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.0/db/volatile/detach_test4.tdb.0
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.0/db/volatile/detach_test3.tdb.0
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.0/db/volatile/detach_test2.tdb.0
  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.0/db/volatile/detach_test1.tdb.0
  Number of databases:3
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.1/db/volatile/detach_test4.tdb.1
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.1/db/volatile/detach_test3.tdb.1
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.1/db/volatile/detach_test2.tdb.1
  Number of databases:4
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.2/db/volatile/detach_test4.tdb.2
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.2/db/volatile/detach_test3.tdb.2
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.2/db/volatile/detach_test2.tdb.2
  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.2/db/volatile/detach_test1.tdb.2
  *** TEST COMPLETED (RC=1) AT 2019-04-27 03:35:40, CLEANING UP...

When issued from a client, the detach control re-broadcasts itself
asynchronously to all nodes and then returns success.  The controls to
some nodes to do the actual detach may still be in flight when success
is returned to the client.  Therefore, the test should wait for a few
seconds to allow the asynchronous controls to complete.

The same is true for the attach control, so workaround the problem in
the attach test too.

An alternative is to make the attach and detach controls synchronous
by avoiding the broadcast and waiting for the results of the
individual controls sent to the nodes.  However, a simple
implementation would involve adding new nested event loops.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 3cb53a7a05409925024d6a67bcfaeb962d896e0b)
---
 ctdb/tests/simple/21_ctdb_attach.sh | 44 ++++++++++++++--------
 ctdb/tests/simple/27_ctdb_detach.sh | 58 +++++++++++++++++++----------
 2 files changed, 66 insertions(+), 36 deletions(-)

diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
index 7965ced6052..b9ae9481d91 100755
--- a/ctdb/tests/simple/21_ctdb_attach.sh
+++ b/ctdb/tests/simple/21_ctdb_attach.sh
@@ -42,27 +42,39 @@ lastnode=$(( numnodes - 1 ))
 
 ######################################################################
 
-# Confirm that the database is attached
+# Confirm that the database is attached with appropriate flags
+check_db_once ()
+{
+	local pnn="$1"
+	local db="$2"
+
+	try_command_on_node "$pnn" $CTDB getdbmap
+	if grep -qF "name:${db}" "$outfile" >/dev/null ; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_db ()
 {
-    pnn="$1"
-    db="$2"
-    flag="$3"
-    try_command_on_node $pnn "$CTDB getdbmap | grep $db"
-    if [ -z "$out" ] ; then
-	echo "BAD: database $db is not attached on node $node"
-	echo "$out"
-	exit 1
-    else
-	local flags=$(awk '{print $4}' "$outfile") || true
+	local pnn="$1"
+	local db="$2"
+	local flag="$3"
+
+	local flags
+
+	echo "Waiting until database ${db} is attached on node ${pnn}"
+	wait_until 10 check_db_once "$pnn" "$db"
+
+	flags=$(awk -v db="$db" '$2 == "name:" db {print $4}' "$outfile")
 	if [ "$flags" = "$flag" ]; then
-	    echo "GOOD: database $db is attached on node $node with flag $flag"
+		echo "GOOD: db ${db} attached on node ${pnn} with flag $flag"
 	else
-	    echo "BAD: database $db is attached on node $node with wrong flag"
-	    echo "$out"
-	    exit 1
+		echo "BAD: db ${db} attached on node ${pnn} with wrong flag"
+		cat "$outfile"
+		exit 1
 	fi
-    fi
 }
 
 ######################################################################
diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
index f7f7a7e8c40..5d1e12328c6 100755
--- a/ctdb/tests/simple/27_ctdb_detach.sh
+++ b/ctdb/tests/simple/27_ctdb_detach.sh
@@ -38,33 +38,51 @@ numnodes="$out"
 ######################################################################
 
 # Confirm that the database is attached
+check_db_once ()
+{
+	local db="$1"
+
+	local num_db
+
+	try_command_on_node all "$CTDB getdbmap"
+	num_db=$(grep -cF "name:${db}" "$outfile") || true
+	if [ "$num_db" -eq "$numnodes" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_db ()
 {
-    db="$1"
-    try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" "$outfile") || true
-    if [ $num_db -eq $numnodes ]; then
-	echo "GOOD: database $db is attached on all nodes"
-    else
-	echo "BAD: database $db is not attached on all nodes"
-	cat "$outfile"
-	exit 1
-    fi
+	local db="$1"
+
+	echo "Waiting until database ${db} is attached on all nodes"
+	wait_until 10 check_db_once "$db"
 }
 
 # Confirm that no nodes have databases attached
+check_no_db_once ()
+{
+	local db="$1"
+
+	local num_db
+
+	try_command_on_node all "$CTDB getdbmap"
+	num_db=$(grep -cF "name:${db}" "$outfile") || true
+	if [ "$num_db" -eq 0 ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_no_db ()
 {
-    db="$1"
-    try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" "$outfile") || true
-    if [ $num_db -eq 0 ]; then
-	echo "GOOD: database $db is not attached any more"
-    else
-	echo "BAD: database $db is still attached"
-	cat "$outfile"
-	exit 1
-    fi
+	local db="$1"
+
+	echo "Waiting until database ${db} is detached on all nodes"
+	wait_until 10 check_no_db_once "$db"
 }
 
 ######################################################################
-- 
2.20.1


From 8ec2fb0cb856bba4e7dbfc14d502ebc3f4f337ed Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 1 May 2019 15:10:28 +1000
Subject: [PATCH 05/11] ctdb-tests: Fix usage message

Since commit 0e9ead8f28fced3ebfa888786a1dc5bb59e734a3 daemons have
been shut down after each test, so this option no longer has anything
to do with killing daemons.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit a2ab6485e027ebb13871c7d83b7626ac5c9b98c0)
---
 ctdb/tests/run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh
index f5be01a8d66..a9c68464883 100755
--- a/ctdb/tests/run_tests.sh
+++ b/ctdb/tests/run_tests.sh
@@ -7,7 +7,7 @@ Usage: $0 [OPTIONS] [TESTS]
 Options:
   -A		Use "cat -A" to print test output (only some tests)
   -c		Run integration tests on a cluster
-  -C		Clean up - kill daemons and remove TEST_VAR_DIR when done
+  -C		Remove TEST_VAR_DIR when done
   -d		Print descriptions of tests instead of filenames (dodgy!)
   -D		Show diff between failed/expected test output (some tests only)
   -e		Exit on the first test failure
-- 
2.20.1


From 48a32612ea081aaef323efe13acf9144f96846a3 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 1 May 2019 15:17:14 +1000
Subject: [PATCH 06/11] ctdb-tests: Don't clean up test var directory in
 autotest target

If the directory is always cleaned up then it is not possible to look
at daemon logs to debug test failures.

This target is only really used by autobuild.py, which (optionally)
cleans up the parent directory anyway.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Tue May  7 06:56:01 UTC 2019 on sn-devel-184

(cherry picked from commit 5a9e338330fe136908a3a17a5df81c054c5cc5b0)
---
 ctdb/wscript | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/wscript b/ctdb/wscript
index 4adf60046c3..dd319a3ce7b 100644
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -1132,7 +1132,7 @@ def test(ctx):
 
 def autotest(ctx):
     env = samba_utils.LOAD_ENVIRONMENT()
-    cmd = 'tests/run_tests.sh -e -S %s -C' % env.SOCKET_WRAPPER_SO_PATH
+    cmd = 'tests/run_tests.sh -e -S %s' % env.SOCKET_WRAPPER_SO_PATH
     ret = samba_utils.RUN_COMMAND(cmd)
     if ret != 0:
         print('autotest exited with exit status %d' % ret)
-- 
2.20.1


From dd9c6417461de892855fed45bd54d39af33db78a Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Fri, 10 May 2019 19:22:16 +1000
Subject: [PATCH 07/11] ctdb-tests: Actually restart if cluster doesn't become
 healthy

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 97ad353a67ce0232d7ca5637f1bf8886e2df1aca)
---
 ctdb/tests/scripts/integration.bash | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index ce5bd576b24..32a729d0249 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -520,6 +520,7 @@ ctdb_init ()
 {
     local i
     for i in $(seq 1 5) ; do
+	ctdb_stop_all >/dev/null 2>&1 || :
 	ctdb_start_all || {
 	    echo "Start failed.  Trying again in a few seconds..."
 	    sleep_for 5
-- 
2.20.1


From 91ab60558062398240438fd144369e68ecf45c9e Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Sun, 12 May 2019 07:52:13 +1000
Subject: [PATCH 08/11] ctdb-tests: Remove old socket wrapper state directory
 during setup

Otherwise, when looping tests for a long time, nodes are unable to
connect to each other.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Mon May 13 08:42:44 UTC 2019 on sn-devel-184

(cherry picked from commit c75fbeaa96108cd4dc193ef5f4170977804e5104)
---
 ctdb/tests/local_daemons.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ctdb/tests/local_daemons.sh b/ctdb/tests/local_daemons.sh
index 3c3897feb50..5a92a24e9e7 100755
--- a/ctdb/tests/local_daemons.sh
+++ b/ctdb/tests/local_daemons.sh
@@ -119,6 +119,7 @@ setup_socket_wrapper ()
 	ln -s "$_socket_wrapper_so" "$_so"
 
 	_d="${directory}/sw"
+	rm -rf "$_d"
 	mkdir -p "$_d"
 }
 
-- 
2.20.1


From 0da384c06edc3695430a25a136b1725897a9bcb0 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 13 May 2019 17:40:15 +1000
Subject: [PATCH 09/11] ctdb-tests: Capture output in $out on failure as well

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit cf00db40355b49443263187f9d97934f91287e51)
---
 ctdb/tests/scripts/integration.bash | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 32a729d0249..30725c48e53 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -104,7 +104,11 @@ try_command_on_node ()
 
     local cmd="$*"
 
-    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
+    local status=0
+    onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 || status=$?
+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
+
+    if [ $status -ne 0 ] ; then
 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
 	cat "$outfile"
 	return 1
@@ -114,8 +118,6 @@ try_command_on_node ()
 	echo "Output of \"$cmd\":"
 	cat "$outfile"
     fi
-
-    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
 }
 
 sanity_check_output ()
-- 
2.20.1


From d3b33ba0fe149172101f75b4598ce4d078b2d93c Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 7 May 2019 15:29:19 +1000
Subject: [PATCH 10/11] ctdb-tests: Make ctdb reloadips tests more reliable

ctdb reloadips will fail if it can't disable takover runs.  The most
likely reason for this is that there is already a takeover run in
progress.  We can't predict when this will happen, so retry if this
occurs.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 8be4ee1a28d5c037955832b6f827d40f28f02796)
---
 ctdb/tests/complex/18_ctdb_reloadips.sh | 35 ++++++++++++++++++++++---
 ctdb/tests/simple/18_ctdb_reloadips.sh  | 33 ++++++++++++++++++++---
 2 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
index 4ba1b26a8e8..26782130430 100755
--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
@@ -184,6 +184,33 @@ check_ips ()
     fi
 }
 
+# ctdb reloadips will fail if it can't disable takover runs.  The most
+# likely reason for this is that there is already a takeover run in
+# progress.  We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+	local retry_max=10
+	local retry_count=0
+	while : ; do
+		if try_command_on_node "$test_node" "$CTDB reloadips" ; then
+			return 0
+		fi
+
+		if [ "$out" != "Failed to disable takeover runs" ] ; then
+			return 1
+		fi
+
+		if [ $retry_count -ge $retry_max ] ; then
+			return 1
+		fi
+
+		retry_count=$((retry_count + 1))
+		echo "Retrying..."
+		sleep_for 1
+	done
+}
+
 ####################
 
 new_ip_max=100
@@ -193,7 +220,7 @@ new_ip_max=100
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" 1 $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 1 $new_ip_max
 
@@ -207,7 +234,7 @@ echo "Using 'ctdb reloadips' to remove the 1st address just added..."
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" 2 $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 2 $new_ip_max
 
@@ -222,7 +249,7 @@ echo "Updating to include only about 1/2 of the new IPs..."
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" $start $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" $start $new_ip_max
 
@@ -234,6 +261,6 @@ try_command_on_node any $CTDB sync
 echo "Restoring original IP configuration..."
 restore_public_addresses
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 0
diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
index 9c60a14a93e..61c3a561da9 100755
--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
@@ -42,9 +42,36 @@ restore_public_addresses ()
 }
 ctdb_test_exit_hook_add restore_public_addresses
 
+# ctdb reloadips will fail if it can't disable takover runs.  The most
+# likely reason for this is that there is already a takeover run in
+# progress.  We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+	local retry_max=10
+	local retry_count=0
+	while : ; do
+		if try_command_on_node any "$CTDB reloadips all" ; then
+			return 0
+		fi
+
+		if [ "$out" != "Failed to disable takeover runs" ] ; then
+			return 1
+		fi
+
+		if [ $retry_count -ge $retry_max ] ; then
+			return 1
+		fi
+
+		retry_count=$((retry_count + 1))
+		echo "Retrying..."
+		sleep_for 1
+	done
+}
+
 try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 echo "Getting list of public IPs on node $test_node"
 try_command_on_node $test_node "$CTDB ip | tail -n +2"
@@ -64,7 +91,7 @@ try_command_on_node any $CTDB sync
 echo "Restoring addresses"
 restore_public_addresses
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 echo "Getting list of public IPs on node $test_node"
 try_command_on_node $test_node "$CTDB ip | tail -n +2"
@@ -85,7 +112,7 @@ echo "Removing IP $test_ip from node $test_node"
 
 try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 try_command_on_node $test_node $CTDB ip
 
-- 
2.20.1


From 7d6c86b69d772ecbc8b1fd1e5f37061d81ea4af8 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 7 May 2019 15:42:49 +1000
Subject: [PATCH 11/11] ctdb-tests: Fix logic error in simple ctdb reloadips
 test

There is a chance that restoring IP addresses to the test node will
result in different IP addresses being assigned to that node.
Removing a single IP address may then fail (or be a no-op) if it is
done after the restore.

So, swap the single IP address removal to happen first, then restore,
then remove all IP addresses.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit dc89db8ca6aadd4a9f7e8a85843c53709d04587c)
---
 ctdb/tests/simple/18_ctdb_reloadips.sh | 37 ++++++++++++++------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
index 61c3a561da9..451fca3a866 100755
--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
@@ -29,8 +29,6 @@ cluster_is_healthy
 
 select_test_node_and_ips
 
-echo "Emptying public addresses file on $test_node"
-
 try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show
 addresses="${out}/public_addresses"
 echo "Public addresses file on node $test_node is \"$addresses\""
@@ -69,25 +67,31 @@ do_ctdb_reloadips ()
 	done
 }
 
-try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
+
+echo "Removing IP $test_ip from node $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
 
 do_ctdb_reloadips
 
-echo "Getting list of public IPs on node $test_node"
-try_command_on_node $test_node "$CTDB ip | tail -n +2"
+try_command_on_node $test_node $CTDB ip
 
-if [ -n "$out" ] ; then
+if grep "^${test_ip} " <<<"$out" ; then
     cat <<EOF
-BAD: node $test_node still has ips:
+BAD: node $test_node can still host IP $test_ip:
 $out
 EOF
     exit 1
 fi
 
-echo "GOOD: no IPs left on node $test_node"
+cat <<EOF
+GOOD: node $test_node is no longer hosting IP $test_ip:
+$out
+EOF
 
 try_command_on_node any $CTDB sync
 
+
 echo "Restoring addresses"
 restore_public_addresses
 
@@ -108,23 +112,22 @@ EOF
 
 try_command_on_node any $CTDB sync
 
-echo "Removing IP $test_ip from node $test_node"
 
-try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
+echo "Emptying public addresses file on $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
 
 do_ctdb_reloadips
 
-try_command_on_node $test_node $CTDB ip
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
 
-if grep "^${test_ip} " <<<"$out" ; then
+if [ -n "$out" ] ; then
     cat <<EOF
-BAD: node $test_node can still host IP $test_ip:
+BAD: node $test_node still has ips:
 $out
 EOF
     exit 1
 fi
 
-cat <<EOF
-GOOD: node $test_node is no longer hosting IP $test_ip:
-$out
-EOF
+echo "GOOD: no IPs left on node $test_node"
-- 
2.20.1