From fe124cc7fac3f046270f03442a27024df2af5f50 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 30 Apr 2019 12:09:26 +1000
Subject: [PATCH 1/9] ctdb-tests: Change sanity_check_output() to internally
 use $out

All callers are currently passed $out.  Global variable $out is used
in many other places so use it here to simplify the interface and make
future changes simpler.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 7c3819d1ac264acf998f426e0cef7f6211e0ddee)
---
 ctdb/tests/scripts/integration.bash     | 5 ++---
 ctdb/tests/simple/02_ctdb_listvars.sh   | 3 +--
 ctdb/tests/simple/05_ctdb_listnodes.sh  | 3 +--
 ctdb/tests/simple/09_ctdb_ping.sh       | 6 ++----
 ctdb/tests/simple/12_ctdb_getdebug.sh   | 3 +--
 ctdb/tests/simple/14_ctdb_statistics.sh | 2 +-
 ctdb/tests/simple/24_ctdb_getdbmap.sh   | 4 ++--
 ctdb/tests/simple/25_dumpmemory.sh      | 4 ++--
 ctdb/tests/simple/51_message_ring.sh    | 2 +-
 ctdb/tests/simple/52_fetch_ring.sh      | 2 +-
 ctdb/tests/simple/81_tunnel_ring.sh     | 2 +-
 11 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 3750c442eba..bc4d2b429ff 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -113,11 +113,10 @@ sanity_check_output ()
 {
     local min_lines="$1"
     local regexp="$2" # Should be anchored as necessary.
-    local output="$3"
 
     local ret=0
 
-    local num_lines=$(echo "$output" | wc -l)
+    local num_lines=$(echo "$out" | wc -l)
     echo "There are $num_lines lines of output"
     if [ $num_lines -lt $min_lines ] ; then
 	echo "BAD: that's less than the required number (${min_lines})"
@@ -126,7 +125,7 @@ sanity_check_output ()
 
     local status=0
     local unexpected # local doesn't pass through status of command on RHS.
-    unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
+    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
 
     # Note that this is reversed.
     if [ $status -eq 0 ] ; then
diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh
index 2f709a8cc0d..9041b6cd216 100755
--- a/ctdb/tests/simple/02_ctdb_listvars.sh
+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
@@ -37,5 +37,4 @@ try_command_on_node -v 0 "$CTDB listvars"
 
 sanity_check_output \
     5 \
-    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
-    "$out"
+    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$'
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 9e48a68e3c0..87aa776d038 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -39,8 +39,7 @@ ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
 ipv6_pat='[[:xdigit:]]+:[[:xdigit:]:]+[[:xdigit:]]+'
 sanity_check_output \
     2 \
-    "^${ipv4_pat}|${ipv6_pat}\$" \
-    "$out"
+    "^${ipv4_pat}|${ipv6_pat}\$"
 
 out_0="$out"
 
diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
index ab6ba146df6..40006a221d0 100755
--- a/ctdb/tests/simple/09_ctdb_ping.sh
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@@ -42,8 +42,7 @@ try_command_on_node -v 0 "$CTDB ping -n 1"
 
 sanity_check_output \
     1 \
-    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
-    "$out"
+    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$'
 
 try_command_on_node -v 0 "$CTDB shutdown -n 1"
 
@@ -53,5 +52,4 @@ try_command_on_node -v 0 "! $CTDB ping -n 1"
 
 sanity_check_output \
     1 \
-    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)" \
-    "$out"
+    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)"
diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh
index 226184813c0..7e6177a3ad4 100755
--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
@@ -36,8 +36,7 @@ getdebug_onnode="$out"
 
 sanity_check_output \
     $num_nodes \
-    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$' \
-    "$out"
+    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$'
 
 cmd=""
 n=0
diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh
index 5ff22d7f64c..eda2dc54749 100755
--- a/ctdb/tests/simple/14_ctdb_statistics.sh
+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
@@ -35,4 +35,4 @@ pattern='^(CTDB version 1|Current time of statistics[[:space:]]*:.*|Statistics c
 
 try_command_on_node -v 1 "$CTDB statistics"
 
-sanity_check_output 40 "$pattern" "$out"
+sanity_check_output 40 "$pattern"
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 9bed5903914..60458c2c8a7 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -45,7 +45,7 @@ try_command_on_node -v 0 "$CTDB getdbmap"
 
 db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
 
-sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
 
 num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
 
@@ -54,7 +54,7 @@ for i in $(seq 1 5) ; do
     echo "Creating test database: $f"
     try_command_on_node 0 $CTDB attach "$f"
     try_command_on_node 0 $CTDB getdbmap
-    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
+    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
     num=$(echo "$out" | sed -n -e '1s/^.*://p')
     if [ $num = $(($num_db_init + $i)) ] ; then
 	echo "OK: correct number of additional databases"
diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh
index 36fecfed016..6aa9300383f 100755
--- a/ctdb/tests/simple/25_dumpmemory.sh
+++ b/ctdb/tests/simple/25_dumpmemory.sh
@@ -31,8 +31,8 @@ cluster_is_healthy
 pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$'
 
 try_command_on_node -v 0 "$CTDB dumpmemory"
-sanity_check_output 10 "$pat" "$out"
+sanity_check_output 10 "$pat"
 
 echo
 try_command_on_node -v 0 "$CTDB rddumpmemory"
-sanity_check_output 10 "$pat" "$out"
+sanity_check_output 10 "$pat"
diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
index e1bdab383c7..cecea6c81bb 100755
--- a/ctdb/tests/simple/51_message_ring.sh
+++ b/ctdb/tests/simple/51_message_ring.sh
@@ -31,7 +31,7 @@ while read line ; do
 done <<<"$out"
 
 pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\))$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # $prev should look like this:
 #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
index 81e6a7a6bb0..078adcbe30f 100755
--- a/ctdb/tests/simple/52_fetch_ring.sh
+++ b/ctdb/tests/simple/52_fetch_ring.sh
@@ -26,7 +26,7 @@ echo "Running fetch_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
 
 pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # Get the last line of output.
 while read line ; do
diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
index c98b139e9cc..31ede40bafd 100755
--- a/ctdb/tests/simple/81_tunnel_ring.sh
+++ b/ctdb/tests/simple/81_tunnel_ring.sh
@@ -32,7 +32,7 @@ while read line ; do
 done <<<"$out"
 
 pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
-sanity_check_output 1 "$pat" "$out"
+sanity_check_output 1 "$pat"
 
 # $prev should look like this:
 #    pnn[2] count=85400
-- 
2.20.1


From 4abc8264315d7e1afc32d950b6a5281b0b091cd8 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Thu, 28 Mar 2019 14:26:52 +1100
Subject: [PATCH 2/9] ctdb-tests: Make try_command_on_node less error-prone

This sometimes fails, apparently due to a cat process in onnode
getting EAGAIN.  The conclusion is that tests that process large
amounts of output should not depend on a sub-shell delivering that
output into a shell variable.

Change try_command_on_node() to leave all of the output in file
$outfile and just put the first 1KB into $out.  $outfile is removed
after each test completes.

Change the implementation of sanity_check_output() to use $outfile
instead of $out.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 9d02452a24625df5f62fd6d45a16effe2fa45fbe)
---
 ctdb/tests/scripts/integration.bash | 30 +++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index bc4d2b429ff..9185b1fb8e3 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -77,7 +77,20 @@ ctdb_test_init ()
 
 ########################################
 
-# Sets: $out
+# Sets: $out, $outfile
+# * The first 1KB of output is put into $out
+# * Tests should use $outfile for handling large output
+# * $outfile is removed after each test
+out=""
+outfile="${TEST_VAR_DIR}/try_command_on_node.out"
+
+outfile_cleanup ()
+{
+	rm -f "$outfile"
+}
+
+ctdb_test_exit_hook_add outfile_cleanup
+
 try_command_on_node ()
 {
     local nodespec="$1" ; shift
@@ -96,17 +109,18 @@ try_command_on_node ()
 
     local cmd="$*"
 
-    out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
-
+    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
-	echo "$out"
+	cat "$outfile"
 	return 1
-    }
+    fi
 
     if $verbose ; then
 	echo "Output of \"$cmd\":"
-	echo "$out"
+	cat "$outfile"
     fi
+
+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
 }
 
 sanity_check_output ()
@@ -116,7 +130,7 @@ sanity_check_output ()
 
     local ret=0
 
-    local num_lines=$(echo "$out" | wc -l)
+    local num_lines=$(wc -l <"$outfile")
     echo "There are $num_lines lines of output"
     if [ $num_lines -lt $min_lines ] ; then
 	echo "BAD: that's less than the required number (${min_lines})"
@@ -125,7 +139,7 @@ sanity_check_output ()
 
     local status=0
     local unexpected # local doesn't pass through status of command on RHS.
-    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
+    unexpected=$(grep -Ev "$regexp" "$outfile") || status=$?
 
     # Note that this is reversed.
     if [ $status -eq 0 ] ; then
-- 
2.20.1


From ed82783c65d3d4e6b0cb8b0d2cbecb79faf24733 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Thu, 11 Apr 2019 20:55:20 +1000
Subject: [PATCH 3/9] ctdb-tests: Avoid bulk output in $out, prefer $outfile

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 066cc5b0c561464ed08890d9aa1a1a55b545e9cc)
---
 .../tests/complex/11_ctdb_delip_removes_ip.sh | 10 ++---
 ctdb/tests/complex/18_ctdb_reloadips.sh       |  8 ++--
 ctdb/tests/complex/32_cifs_tickle.sh          |  8 ----
 ctdb/tests/complex/36_smb_reset_server.sh     | 12 +-----
 ctdb/tests/complex/37_nfs_reset_server.sh     |  4 +-
 ctdb/tests/complex/60_rogueip_releaseip.sh    |  2 +-
 ctdb/tests/complex/scripts/local.bash         |  5 +--
 ctdb/tests/scripts/integration.bash           | 40 +++++--------------
 ctdb/tests/simple/03_ctdb_getvar.sh           |  3 +-
 ctdb/tests/simple/05_ctdb_listnodes.sh        |  2 +-
 ctdb/tests/simple/08_ctdb_isnotrecmaster.sh   | 10 ++---
 ctdb/tests/simple/11_ctdb_ip.sh               | 27 +++++++------
 ctdb/tests/simple/15_ctdb_statisticsreset.sh  | 21 +++++-----
 ctdb/tests/simple/19_ip_takeover_noop.sh      |  4 +-
 ctdb/tests/simple/20_delip_iface_gc.sh        | 10 ++---
 ctdb/tests/simple/21_ctdb_attach.sh           |  7 ++--
 ctdb/tests/simple/23_ctdb_moveip.sh           | 25 +++++++++++-
 ctdb/tests/simple/24_ctdb_getdbmap.sh         |  6 +--
 ..._config_check_error_on_unreachable_ctdb.sh |  6 +--
 ctdb/tests/simple/27_ctdb_detach.sh           | 21 +++++-----
 ctdb/tests/simple/35_ctdb_getreclock.sh       |  2 +-
 ctdb/tests/simple/51_message_ring.sh          | 12 +++---
 ctdb/tests/simple/52_fetch_ring.sh            | 12 +++---
 ctdb/tests/simple/53_transaction_loop.sh      |  4 +-
 .../simple/54_transaction_loop_recovery.sh    |  4 +-
 ctdb/tests/simple/55_ctdb_ptrans.sh           | 12 +++---
 .../56_replicated_transaction_recovery.sh     |  4 +-
 ctdb/tests/simple/58_ctdb_restoredb.sh        |  8 ++--
 .../simple/69_recovery_resurrect_deleted.sh   | 10 ++---
 ctdb/tests/simple/70_recoverpdbbyseqnum.sh    |  4 +-
 ctdb/tests/simple/71_ctdb_wipedb.sh           |  4 +-
 .../simple/72_update_record_persistent.sh     |  4 +-
 .../tests/simple/75_readonly_records_basic.sh | 24 +++++------
 ctdb/tests/simple/77_ctdb_db_recovery.sh      |  6 +--
 ctdb/tests/simple/79_volatile_db_traverse.sh  |  4 +-
 ctdb/tests/simple/80_ctdb_traverse.sh         |  2 +-
 ctdb/tests/simple/81_tunnel_ring.sh           | 12 +++---
 ctdb/tests/simple/90_debug_hung_script.sh     |  6 +--
 38 files changed, 167 insertions(+), 198 deletions(-)

diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
index d6a3b3c04ff..a41be09eab1 100755
--- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
+++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
@@ -25,8 +25,8 @@ ctdb_restart_when_done
 select_test_node_and_ips
 get_test_ip_mask_and_iface
 
-echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..."
-try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet6?[[:space:]]*${test_ip}/'"
+echo "Checking that node ${test_node} hosts ${test_ip}..."
+try_command_on_node $test_node "ip addr show to ${test_ip} | grep -q ."
 
 echo "Attempting to remove ${test_ip} from node ${test_node}."
 try_command_on_node $test_node $CTDB delip $test_ip
@@ -36,10 +36,10 @@ wait_until_ips_are_on_node '!' $test_node $test_ip
 timeout=60
 increment=5
 count=0
-echo "Waiting for ${test_ip} to disappear from ${iface}..."
+echo "Waiting for ${test_ip} to disappear from node ${test_node}..."
 while : ; do
-    try_command_on_node -v $test_node "ip addr show dev $iface"
-    if echo "$out" | grep -E 'inet6?[[:space:]]*${test_ip}/'; then
+    try_command_on_node -v $test_node "ip addr show to ${test_node}"
+    if -n "$out" ; then
 	echo "Still there..."
 	if [ $(($count * $increment)) -ge $timeout ] ; then
 	    echo "BAD: Timed out waiting..."
diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
index e0632f30564..1dded769741 100755
--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
@@ -51,12 +51,12 @@ select_test_node_and_ips
 
 echo "Getting public IP information from CTDB..."
 try_command_on_node any "$CTDB ip -X -v all"
-ctdb_ip_info=$(echo "$out" | awk -F'|' 'NR > 1 { print $2, $3, $5 }')
+ctdb_ip_info=$(awk -F'|' 'NR > 1 { print $2, $3, $5 }' "$outfile")
 
 echo "Getting IP information from interfaces..."
 try_command_on_node all "ip addr show"
-ip_addr_info=$(echo "$out" | \
-    awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }')
+ip_addr_info=$(awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }' \
+		   "$outfile")
 
 prefix=""
 for b in $(seq 0 255) ; do
@@ -171,7 +171,7 @@ check_ips ()
 
     try_command_on_node $test_node "ip addr show dev ${iface}"
     local ip_addrs_file=$(mktemp)
-    echo "$out" | \
+    cat "$outfile" | \
 	sed -n -e "s@.*inet * \(${prefix//./\.}\.[0-9]*\)/.*@\1@p" | \
 	sort >"$ip_addrs_file"
 
diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh
index 366824501da..ac00d449293 100755
--- a/ctdb/tests/complex/32_cifs_tickle.sh
+++ b/ctdb/tests/complex/32_cifs_tickle.sh
@@ -64,14 +64,6 @@ echo "Source socket is $src_socket"
 # we sometimes beat the registration.
 echo "Checking if CIFS connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    echo "BAD: Socket not tracked by CTDB."
-    testfailures=1
-fi
 
 # This is almost immediate.  However, it is sent between nodes
 # asynchonously, so it is worth checking...
diff --git a/ctdb/tests/complex/36_smb_reset_server.sh b/ctdb/tests/complex/36_smb_reset_server.sh
index beff1a22014..c55cb9f251e 100755
--- a/ctdb/tests/complex/36_smb_reset_server.sh
+++ b/ctdb/tests/complex/36_smb_reset_server.sh
@@ -62,16 +62,8 @@ echo "Source socket is $src_socket"
 
 # This should happen as soon as connection is up... but unless we wait
 # we sometimes beat the registration.
-echo "Checking if CIFS connection is tracked by CTDB on test node..."
+echo "Waiting until SMB connection is tracked by CTDB on test node..."
 wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
-
-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
-    echo "GOOD: CIFS connection tracked OK by CTDB."
-else
-    echo "BAD: Socket not tracked by CTDB."
-    exit 1
-fi
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -83,7 +75,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node
diff --git a/ctdb/tests/complex/37_nfs_reset_server.sh b/ctdb/tests/complex/37_nfs_reset_server.sh
index 07bd034d6c7..e38d4510d49 100755
--- a/ctdb/tests/complex/37_nfs_reset_server.sh
+++ b/ctdb/tests/complex/37_nfs_reset_server.sh
@@ -63,7 +63,7 @@ echo "Source socket is $src_socket"
 echo "Wait until NFS connection is tracked by CTDB on test node ..."
 wait_until $((monitor_interval * 2)) \
 	   check_tickles $test_node $test_ip $test_port $src_socket
-echo "$out"
+cat "$outfile"
 
 # It would be nice if ss consistently used local/peer instead of src/dst
 ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
@@ -75,7 +75,7 @@ if [ -z "$out" ] ; then
 	exit 1
 fi
 echo "GOOD: ss lists the socket:"
-echo "$out"
+cat "$outfile"
 
 echo "Disabling node $test_node"
 try_command_on_node 1 $CTDB disable -n $test_node
diff --git a/ctdb/tests/complex/60_rogueip_releaseip.sh b/ctdb/tests/complex/60_rogueip_releaseip.sh
index 0daf289c83a..7e2f89a1e3b 100755
--- a/ctdb/tests/complex/60_rogueip_releaseip.sh
+++ b/ctdb/tests/complex/60_rogueip_releaseip.sh
@@ -34,7 +34,7 @@ for i in $all_pnns ; do
 		continue
 	fi
 	try_command_on_node $i "$CTDB ip"
-	n=$(awk -v ip="$test_ip" '$1 == ip { print }' <<<"$out")
+	n=$(awk -v ip="$test_ip" '$1 == ip { print }' "$outfile")
 	if [ -n "$n" ] ; then
 		other_node="$i"
 		break
diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash
index 7787de8f111..787f597edcc 100644
--- a/ctdb/tests/complex/scripts/local.bash
+++ b/ctdb/tests/complex/scripts/local.bash
@@ -67,7 +67,7 @@ check_tickles ()
     local src_socket="$4"
     try_command_on_node $node ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    [ "${out/SRC: ${src_socket} /}" != "$out" ]
+    grep -Fq "SRC: ${src_socket} " "$outfile"
 }
 
 check_tickles_all ()
@@ -79,8 +79,7 @@ check_tickles_all ()
 
     try_command_on_node all ctdb gettickles $test_ip $test_port
     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
-    local t="${src_socket//./\\.}"
-    local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
+    local count=$(grep -Fc "SRC: ${src_socket} " "$outfile" || true)
     [ $count -eq $numnodes ]
 }
 
diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 9185b1fb8e3..0fbd39daec8 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -153,30 +153,7 @@ sanity_check_output ()
     return $ret
 }
 
-sanity_check_ips ()
-{
-    local ips="$1" # list of "ip node" lines
-
-    echo "Sanity checking IPs..."
-
-    local x ipp prev
-    prev=""
-    while read x ipp ; do
-	[ "$ipp" = "-1" ] && break
-	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
-	    echo "OK"
-	    return 0
-	fi
-	prev="$ipp"
-    done <<<"$ips"
-
-    echo "BAD: a node was -1 or IPs are only assigned to one node:"
-    echo "$ips"
-    echo "Are you running an old version of CTDB?"
-    return 1
-}
-
-# This returns a list of "ip node" lines in $out
+# This returns a list of "ip node" lines in $outfile
 all_ips_on_node()
 {
     local node="$1"
@@ -197,9 +174,9 @@ _select_test_node_and_ips ()
 	    test_node="$pnn"
 	fi
 	if [ "$pnn" = "$test_node" ] ; then
-            test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
+	    test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
     test_ip="${test_node_ips%% *}"
@@ -269,7 +246,7 @@ delete_ip_from_all_nodes ()
 	    if [ "$_ip" = "$_i" ] ; then
 		_nodes="${_nodes}${_nodes:+,}${_pnn}"
 	    fi
-	done <<<"$out" # bashism
+	done <"$outfile"
     done
 
     try_command_on_node -pq "$_nodes" "$CTDB delip $_ip"
@@ -439,7 +416,7 @@ ips_are_on_node ()
 	    if $negating ; then
 		ips="${ips/${check}}"
 	    fi
-	done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+	done <"$outfile"
     done
 
     ips="${ips// }" # Remove any spaces.
@@ -481,7 +458,7 @@ node_has_some_ips ()
 	if [ "$node" = "$pnn" ] ; then
 	    return 0
 	fi
-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
+    done <"$outfile"
 
     return 1
 }
@@ -623,7 +600,8 @@ wait_for_monitor_event ()
 	return 1
     }
 
-    local ctdb_scriptstatus_original="$out"
+    mv "$outfile" "${outfile}.orig"
+
     wait_until 120 _ctdb_scriptstatus_changed
 }
 
@@ -634,7 +612,7 @@ _ctdb_scriptstatus_changed ()
 	return 1
     }
 
-    [ "$out" != "$ctdb_scriptstatus_original" ]
+    ! diff "$outfile" "${outfile}.orig" >/dev/null
 }
 
 #######################################
diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh
index cae3dd14159..7d795598d0a 100755
--- a/ctdb/tests/simple/03_ctdb_getvar.sh
+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
@@ -38,7 +38,6 @@ try_command_on_node -v 0 "$CTDB listvars"
 
 echo "Verifying all variable values using \"ctdb getvar\"..."
 
-echo "$out" |
 while read var x val ; do
     try_command_on_node 0 "$CTDB getvar $var"
 
@@ -48,4 +47,4 @@ while read var x val ; do
 	echo "MISMATCH on $var: $val != $val2"
 	exit 1
     fi
-done
+done <"$outfile"
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 87aa776d038..5e64032fd5d 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -32,7 +32,7 @@ cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listnodes"
 
-num_nodes=$(echo "$out" | wc -l)
+num_nodes=$(wc -l <"$outfile")
 
 # Each line should look like an IP address.
 ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
index 138f59c6e79..b6e7287e776 100755
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@@ -33,13 +33,11 @@ set -e
 cluster_is_healthy
 
 cmd="$CTDB isnotrecmaster || true"
-try_command_on_node all "$cmd"
-echo "Output of \"$cmd\":"
-echo "$out"
+try_command_on_node -v all "$cmd"
 
-num_all_lines=$(echo "$out" |  wc -l)
-num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
-num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
+num_all_lines=$(wc -l <"$outfile")
+num_rm_lines=$(grep -Fc 'this node is the recmaster' "$outfile") || true
+num_not_rm_lines=$(grep -Fc 'this node is not the recmaster' "$outfile") || true
 
 if [ $num_rm_lines -eq 1 ] ; then
     echo "OK, there is only 1 recmaster"
diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
index 8bca2c608c5..3e22d5d5b29 100755
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@@ -33,24 +33,25 @@ cluster_is_healthy
 
 echo "Getting list of public IPs..."
 try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
-ips=$(echo "$out" | sed \
+ips=$(sed \
 	-e 's@ node\[@ @' \
-	-e 's@\].*$@@')
-machineout=$(echo "$out" | sed -r \
+	-e 's@\].*$@@' \
+	"$outfile")
+machineout=$(sed -r \
 	-e 's@^| |$@\|@g' \
 	-e 's@[[:alpha:]]+\[@@g' \
-	-e 's@\]@@g')
+	-e 's@\]@@g' \
+	"$outfile")
 
 if [ -z "$TEST_LOCAL_DAEMONS" ]; then
-    while read ip pnn ; do
-        try_command_on_node $pnn "ip addr show"
-        if [ "${out/inet* ${ip}\/}" != "$out" ] ; then
-            echo "GOOD: node $pnn appears to have $ip assigned"
-        else
-            echo "BAD:  node $pnn does not appear to have $ip assigned"
-            testfailures=1
-        fi
-    done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
+	while read ip pnn ; do
+		try_command_on_node $pnn "ip addr show to ${ip}"
+		if [ -n "$out" ] ; then
+			echo "GOOD: node $pnn appears to have $ip assigned"
+		else
+			die "BAD: node $pnn does not appear to have $ip assigned"
+		fi
+	done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
 fi
 
 [ "$testfailures" != 1 ] && echo "Looks good!"
diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
index eaa60d69205..4f0c2bd9d01 100755
--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
@@ -37,10 +37,11 @@ num_nodes="$out"
 
 get_stat ()
 {
-    local label="$1"
-    local out="$2"
+	local label="$1"
 
-    echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
+	cat "$outfile" |
+	sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" |
+	head -1
 }
 
 check_reduced ()
@@ -60,20 +61,20 @@ check_reduced ()
 n=0
 while [ $n -lt $num_nodes ] ; do
     echo "Getting initial statistics for node ${n}..."
-    
+
     try_command_on_node -v $n $CTDB statistics
 
-    before_req_control=$(get_stat "req_control" "$out")
-    before_reply_control=$(get_stat "reply_control" "$out")
-    before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    before_req_control=$(get_stat "req_control")
+    before_reply_control=$(get_stat "reply_control")
+    before_node_packets_recv=$(get_stat "node_packets_recv")
 
     try_command_on_node $n $CTDB statisticsreset
 
     try_command_on_node -v $n $CTDB statistics
 
-    after_req_control=$(get_stat "req_control" "$out")
-    after_reply_control=$(get_stat "reply_control" "$out")
-    after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
+    after_req_control=$(get_stat "req_control")
+    after_reply_control=$(get_stat "reply_control")
+    after_node_packets_recv=$(get_stat "node_packets_recv")
 
     check_reduced "req_control" "$before_req_control" "$after_req_control"
     check_reduced "reply_control" "$before_reply_control" "$after_reply_control"
diff --git a/ctdb/tests/simple/19_ip_takeover_noop.sh b/ctdb/tests/simple/19_ip_takeover_noop.sh
index 0b86241db2a..48236a72031 100755
--- a/ctdb/tests/simple/19_ip_takeover_noop.sh
+++ b/ctdb/tests/simple/19_ip_takeover_noop.sh
@@ -46,9 +46,7 @@ while read ip pnn ; do
 	if [ "$pnn" != "-1" ] ; then
 		die "BAD: IP address ${ip} is assigned to node ${pnn}"
 	fi
-done <<EOF
-$out
-EOF
+done <"$outfile"
 
 echo "GOOD: All IP addresses are unassigned"
 
diff --git a/ctdb/tests/simple/20_delip_iface_gc.sh b/ctdb/tests/simple/20_delip_iface_gc.sh
index e73eed20cc0..9a4442d6ebe 100755
--- a/ctdb/tests/simple/20_delip_iface_gc.sh
+++ b/ctdb/tests/simple/20_delip_iface_gc.sh
@@ -22,14 +22,14 @@ select_test_node_and_ips
 
 # Find interfaces on test node
 try_command_on_node $test_node "$CTDB ifaces -X"
-ifaces=$(awk -F'|' 'NR > 1 { print $2 }' <<<"$out")
+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
 echo "Node ${test_node} has interfaces: ${ifaces}"
 
 # Delete all IPs on each interface...  deleting IPs from one interface
 # can cause other interfaces to disappear, so we need to be careful...
 for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
 
     if [ -z "$info" ] ; then
 	echo "Interface ${i} missing... assuming already deleted!"
@@ -41,7 +41,7 @@ for i in $ifaces ; do
 
     try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
     awk -F'|' -v i="$i" \
-	'$6 == i { print $2 }' <<<"$out" |
+	'$6 == i { print $2 }' "$outfile" |
     while read ip ; do
 	echo "  $ip"
 	try_command_on_node $test_node "$CTDB delip $ip"
@@ -49,8 +49,8 @@ for i in $ifaces ; do
     try_command_on_node $test_node "$CTDB ipreallocate"
 
     try_command_on_node $test_node "$CTDB ifaces -X"
-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
-    
+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
     if [ -z "$info" ] ; then
 	echo "GOOD: Interface ${i} has been garbage collected"
     else
diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
index 11b600800a4..d9dc697d934 100755
--- a/ctdb/tests/simple/21_ctdb_attach.sh
+++ b/ctdb/tests/simple/21_ctdb_attach.sh
@@ -39,9 +39,8 @@ ctdb_restart_when_done
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 lastnode=$(( numnodes - 1 ))
 
 ######################################################################
@@ -58,7 +57,7 @@ check_db ()
 	echo "$out"
 	exit 1
     else
-	local flags=$(awk '{print $4}' <<<"$out") || true
+	local flags=$(awk '{print $4}' "$outfile") || true
 	if [ "$flags" = "$flag" ]; then
 	    echo "GOOD: database $db is attached on node $node with flag $flag"
 	else
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
index c6e70bf4e94..77c23eb621b 100755
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -26,7 +26,28 @@ ctdb_restart_when_done
 
 select_test_node_and_ips
 
-sanity_check_ips "$out"
+sanity_check_ips ()
+{
+    echo "Sanity checking IPs..."
+
+    local x ipp prev
+    prev=""
+    while read x ipp ; do
+	[ "$ipp" = "-1" ] && break
+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+	    echo "OK"
+	    return 0
+	fi
+	prev="$ipp"
+    done <"$outfile"
+
+    echo "BAD: a node was -1 or IPs are only assigned to one node:"
+    cat "$outfile"
+    echo "Are you running an old version of CTDB?"
+    return 1
+}
+
+sanity_check_ips
 
 # Find a target node - it must be willing to host $test_ip
 try_command_on_node any "$CTDB listnodes | wc -l"
@@ -40,7 +61,7 @@ for i in $(seq 0 $(($num_nodes - 1)) ) ; do
 	    to_node="$i"
 	    break 2
 	fi
-    done <<<"$out"
+    done <"$outfile"
 done
 
 if [ -z "$to_node" ] ; then
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 60458c2c8a7..a92262a177b 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -47,7 +47,7 @@ db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[
 
 sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
 
-num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+num_db_init=$(sed -n -e '1s/.*://p' "$outfile")
 
 for i in $(seq 1 5) ; do
     f=$(make_temp_db_filename)
@@ -55,14 +55,14 @@ for i in $(seq 1 5) ; do
     try_command_on_node 0 $CTDB attach "$f"
     try_command_on_node 0 $CTDB getdbmap
     sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
-    num=$(echo "$out" | sed -n -e '1s/^.*://p')
+    num=$(sed -n -e '1s/^.*://p' "$outfile")
     if [ $num = $(($num_db_init + $i)) ] ; then
 	echo "OK: correct number of additional databases"
     else
 	echo "BAD: no additional database"
 	exit 1
     fi
-    if [ "${out/name:${f} /}" != "$out" ] ; then
+    if awk '{print $2}' "$outfile" | grep -Fqx "name:$f" ; then
 	echo "OK: getdbmap knows about \"$f\""
     else
 	echo "BAD: getdbmap does not know about \"$f\""
diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index 6642b1759f9..7ec6f489a57 100755
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -45,8 +45,8 @@ ctdb_restart_when_done
 
 test_node=1
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 echo "There are $num_nodes nodes."
 
 echo "Shutting down node ${test_node}..."
@@ -61,7 +61,7 @@ pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error:
 for i in ip disable enable "ban 0" unban listvars ; do
     try_command_on_node -v 0 ! $CTDB $i -n $test_node
 
-    if egrep -q "$pat" <<<"$out" ; then
+    if egrep -q "$pat" "$outfile" ; then
 	echo "OK: \"ctdb ${i}\" fails with expected \"disconnected node\" message"
     else
 	echo "BAD: \"ctdb ${i}\" does not fail with expected \"disconnected node\" message"
diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
index 42ec635fb65..29f3f691dd5 100755
--- a/ctdb/tests/simple/27_ctdb_detach.sh
+++ b/ctdb/tests/simple/27_ctdb_detach.sh
@@ -35,9 +35,8 @@ ctdb_restart_when_done
 
 ######################################################################
 
-try_command_on_node 0 "$CTDB listnodes -X"
-listnodes_output="$out"
-numnodes=$(wc -l <<<"$listnodes_output")
+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
+numnodes="$out"
 
 ######################################################################
 
@@ -46,12 +45,12 @@ check_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq $numnodes ]; then
 	echo "GOOD: database $db is attached on all nodes"
     else
 	echo "BAD: database $db is not attached on all nodes"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -61,12 +60,12 @@ check_no_db ()
 {
     db="$1"
     try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" <<<"$out") || true
+    local num_db=$(grep -cF "$db" "$outfile") || true
     if [ $num_db -eq 0 ]; then
 	echo "GOOD: database $db is not attached any more"
     else
 	echo "BAD: database $db is still attached"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -137,12 +136,12 @@ echo
 echo "Write a key to database"
 try_command_on_node 0 $CTDB writekey $testdb1 foo bar
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 1 ]; then
     echo "GOOD: Key added to database"
 else
     echo "BAD: Key did not get added to database"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi
 
@@ -161,11 +160,11 @@ check_db "$testdb1"
 echo
 echo "Check if the database is empty"
 try_command_on_node 0 $CTDB catdb $testdb1
-num_keys=$(echo "$out" | sed -n -e 's/Dumped \([0-9]*\) records/\1/p') || true
+num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true
 if [ -n "$num_keys" -a $num_keys -eq 0 ]; then
     echo "GOOD: Database $testdb1 is empty"
 else
     echo "BAD: Database $testdb1 is not empty"
-    echo "$out"
+    cat "$outfile"
     exit 1
 fi
diff --git a/ctdb/tests/simple/35_ctdb_getreclock.sh b/ctdb/tests/simple/35_ctdb_getreclock.sh
index 5d24e2dd145..304e6c3df12 100755
--- a/ctdb/tests/simple/35_ctdb_getreclock.sh
+++ b/ctdb/tests/simple/35_ctdb_getreclock.sh
@@ -25,7 +25,7 @@ if [ -z "$out" ] ; then
     exit 0
 fi
 
-n=$(echo "$out" | sort -u | wc -l)
+n=$(sort -u "$outfile" | wc -l)
 if [ "$n" = 1 ] ; then
 	echo "GOOD: All nodes have the same recovery lock setting"
 else
diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
index cecea6c81bb..9b8403f2a7e 100755
--- a/ctdb/tests/simple/51_message_ring.sh
+++ b/ctdb/tests/simple/51_message_ring.sh
@@ -19,23 +19,21 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running message_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND message_ring -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\))$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
-stuff="${prev##Ring\[*\]: }"
+stuff="${last##Ring\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
index 078adcbe30f..8f2ac3bb451 100755
--- a/ctdb/tests/simple/52_fetch_ring.sh
+++ b/ctdb/tests/simple/52_fetch_ring.sh
@@ -19,8 +19,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running fetch_ring on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
@@ -29,13 +29,11 @@ pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)?
 sanity_check_output 1 "$pat"
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
-# $prev should look like this:
+# $last should look like this:
 #    Fetch[1]: 10670.93 msgs/sec
-stuff="${prev##*Fetch\[*\]: }"
+stuff="${last##*Fetch\[*\]: }"
 mps="${stuff% msgs/sec*}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/53_transaction_loop.sh b/ctdb/tests/simple/53_transaction_loop.sh
index 8291a8d55b1..d1db2a9a32b 100755
--- a/ctdb/tests/simple/53_transaction_loop.sh
+++ b/ctdb/tests/simple/53_transaction_loop.sh
@@ -24,8 +24,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/54_transaction_loop_recovery.sh b/ctdb/tests/simple/54_transaction_loop_recovery.sh
index a5d94ffbf89..cb2c1033acd 100755
--- a/ctdb/tests/simple/54_transaction_loop_recovery.sh
+++ b/ctdb/tests/simple/54_transaction_loop_recovery.sh
@@ -45,8 +45,8 @@ TESTDB="persistent_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB persistent"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/55_ctdb_ptrans.sh b/ctdb/tests/simple/55_ctdb_ptrans.sh
index ba92899a613..ff007816d05 100755
--- a/ctdb/tests/simple/55_ctdb_ptrans.sh
+++ b/ctdb/tests/simple/55_ctdb_ptrans.sh
@@ -51,11 +51,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records were inserted"
@@ -75,11 +75,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 3 ] ; then
     echo "BAD: expected 3 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 3 records found"
@@ -116,11 +116,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
 
 try_command_on_node 0 $CTDB catdb "$TESTDB"
 
-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
 
 if [ $n -ne 0 ] ; then
     echo "BAD: expected 0 keys in..."
-    echo "$out"
+    cat "$outfile"
     exit 1
 else
     echo "GOOD: 0 records found"
diff --git a/ctdb/tests/simple/56_replicated_transaction_recovery.sh b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
index 61827145bc2..980173a35fd 100755
--- a/ctdb/tests/simple/56_replicated_transaction_recovery.sh
+++ b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
@@ -46,8 +46,8 @@ TESTDB="replicated_trans.tdb"
 try_command_on_node 0 "$CTDB attach $TESTDB replicated"
 try_command_on_node 0 "$CTDB wipedb $TESTDB"
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
     CTDB_TEST_TIMELIMIT=30
diff --git a/ctdb/tests/simple/58_ctdb_restoredb.sh b/ctdb/tests/simple/58_ctdb_restoredb.sh
index 31b81a6dcb3..3a9d4d24557 100755
--- a/ctdb/tests/simple/58_ctdb_restoredb.sh
+++ b/ctdb/tests/simple/58_ctdb_restoredb.sh
@@ -36,10 +36,10 @@ set -e
 cluster_is_healthy
 
 try_command_on_node 0 $CTDB status
-generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="restoredb_test.tdb"
@@ -105,7 +105,7 @@ fi
 wait_until_ready
 
 try_command_on_node 0 $CTDB status
-new_generation=$(echo "$out" | sed -n -e 's/^Generation:\([0-9]*\)/\1/p')
+new_generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile")
 
 echo "Old generation = $generation"
 echo "New generation = $new_generation"
diff --git a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
index 95e79fdd491..fa8a0dd0bbf 100755
--- a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
+++ b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
@@ -23,9 +23,9 @@ testdb="rec_test.tdb"
 echo "Getting list of nodes..."
 try_command_on_node -v any "onnode -pq all ctdb pnn | grep '^[0-9][0-9]*$'"
 
-first=$(echo "$out" | sed -n -e '1p')
-second=$(echo "$out" | sed -n -e '2p')
-notfirst=$(echo "$out" | tail -n +2)
+first=$(sed -n -e '1p' "$outfile")
+second=$(sed -n -e '2p' "$outfile")
+notfirst=$(tail -n +2 "$outfile")
 
 echo "Create/wipe test database ${testdb}"
 try_command_on_node $first $CTDB attach "$testdb"
@@ -49,7 +49,7 @@ database_has_zero_records ()
 	local n
 	for n in $notfirst ; do
 		try_command_on_node $n $CTDB cattdb "$testdb"
-		if echo "$out" | grep -q '^key(' ; then
+		if grep -q '^key(' "$outfile" ; then
 			return 1
 		fi
 	done
@@ -76,7 +76,7 @@ wait_until_node_has_status $first notstopped
 echo "Get database contents"
 try_command_on_node -v $first $CTDB catdb "$testdb"
 
-if echo "$out" | grep -q '^key(' ; then
+if grep -q '^key(' "$outfile" ; then
 	echo "BAD: Deleted record has been resurrected"
 	exit 1
 fi
diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
index 255d58d2156..0099d08359d 100755
--- a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
+++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
@@ -38,8 +38,8 @@ cluster_is_healthy
 # Reset configuration
 ctdb_restart_when_done
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 add_record_per_node ()
 {
diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh
index 1919ea70a96..e5e3182f68e 100755
--- a/ctdb/tests/simple/71_ctdb_wipedb.sh
+++ b/ctdb/tests/simple/71_ctdb_wipedb.sh
@@ -32,8 +32,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 # 2.
 test_db="persistent_test.tdb"
diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh
index 8f028d0d0f1..5e7edabc365 100755
--- a/ctdb/tests/simple/72_update_record_persistent.sh
+++ b/ctdb/tests/simple/72_update_record_persistent.sh
@@ -34,8 +34,8 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 test_db="persistent_test.tdb"
 
diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh
index 48e16dfed8c..c4664ba3bcd 100755
--- a/ctdb/tests/simple/75_readonly_records_basic.sh
+++ b/ctdb/tests/simple/75_readonly_records_basic.sh
@@ -52,12 +52,12 @@ check_no_readonly ()
 {
     try_command_on_node all $CTDB cattdb $testdb
     local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS"
-    local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true
+    local numreadonly=$(grep -c -E "$ro_flags" "$outfile") || true
     if [ $numreadonly -eq 0 ] ; then
 	echo "GOOD: no read-only delegations"
     else
 	echo "BAD: there are read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 }
@@ -75,36 +75,36 @@ check_readonly ()
     local count
 
     try_command_on_node $dmaster $CTDB cattdb $testdb
-    count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
     if [ $count -eq 1 ] ; then
 	echo "GOOD: dmaster ${dmaster} has read-only delegations"
     else
 	echo "BAD: dmaster ${dmaster} has no read-only delegations"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
-    count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+    count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
     if [ $count -ne 0 ] ; then
 	echo "BAD: dmaster ${dmaster} has a read-only copy"
-	echo "$out"
+	cat "$outfile"
 	exit 1
     fi
 
     local o
     for o in $others ; do
 	try_command_on_node $o $CTDB cattdb $testdb
-	count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
 	if [ $count -eq 1 ] ; then
 	    echo "GOOD: node ${o} has a read-only copy"
 	else
 	    echo "BAD: node ${o} has no read-only copy"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
-	count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
+	count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
 	if [ $count -ne 0 ] ; then
 	    echo "BAD: other node ${o} has read-only delegations"
-	    echo "$out"
+	    cat "$outfile"
 	    exit 1
 	fi
     done
@@ -114,7 +114,7 @@ check_readonly ()
 
 echo "Get list of nodes..."
 try_command_on_node any $CTDB -X listnodes
-all_nodes=$(awk -F'|' '{print $2}' <<<"$out")
+all_nodes=$(awk -F'|' '{print $2}' "$outfile")
 
 ######################################################################
 
@@ -141,7 +141,7 @@ try_command_on_node all $CTDB setdbreadonly $testdb
 
 # Database should be tagged as READONLY
 try_command_on_node 0 $CTDB getdbmap
-db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out")
+db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' "$outfile")
 if grep -q "READONLY" <<<"$db_details" ; then
     echo "GOOD: read-only record support is enabled"
 else
diff --git a/ctdb/tests/simple/77_ctdb_db_recovery.sh b/ctdb/tests/simple/77_ctdb_db_recovery.sh
index 6dbc0cf639c..00102083124 100755
--- a/ctdb/tests/simple/77_ctdb_db_recovery.sh
+++ b/ctdb/tests/simple/77_ctdb_db_recovery.sh
@@ -96,7 +96,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value1
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -111,7 +111,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value2
 # Fetch a record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 
 # Do a recovery
 echo "force recovery"
@@ -122,7 +122,7 @@ wait_until_node_has_status $recmaster recovered
 # Verify record   key=test1
 echo "read key(test1)"
 try_command_on_node $recmaster $CTDB readkey $TESTDB test1
-echo "$out"
+cat "$outfile"
 if [ "$out" = "Data: size:6 ptr:[value2]" ]; then
 	echo "GOOD: Recovery did not corrupt database"
 else
diff --git a/ctdb/tests/simple/79_volatile_db_traverse.sh b/ctdb/tests/simple/79_volatile_db_traverse.sh
index 50732cab330..b73a57a20fe 100755
--- a/ctdb/tests/simple/79_volatile_db_traverse.sh
+++ b/ctdb/tests/simple/79_volatile_db_traverse.sh
@@ -78,7 +78,7 @@ try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1"
 echo "do traverse on node 0"
 try_command_on_node -v 0 $CTDB catdb "$TESTDB"
 
-num=$(echo "$out" | sed -n -e 's|^Dumped \(.*\) records$|\1|p')
+num=$(sed -n -e 's|^Dumped \(.*\) records$|\1|p' "$outfile")
 if [ "$num" = 1 ] ; then
 	echo "OK: There was 1 record"
 else
@@ -86,7 +86,7 @@ else
 	exit 1
 fi
 
-if echo "$out" | grep -q "^data(4) = \"bar1\"\$" ; then
+if grep -q "^data(4) = \"bar1\"\$" "$outfile" ; then
 	echo "OK: Data from node 1 was returned"
 else
 	echo "BAD: Data from node 1 was not returned"
diff --git a/ctdb/tests/simple/80_ctdb_traverse.sh b/ctdb/tests/simple/80_ctdb_traverse.sh
index 334c684d17c..6470b179029 100755
--- a/ctdb/tests/simple/80_ctdb_traverse.sh
+++ b/ctdb/tests/simple/80_ctdb_traverse.sh
@@ -61,7 +61,7 @@ done
 echo "Start a traverse and collect records"
 try_command_on_node 0 $CTDB catdb $TESTDB
 
-num_read=$(echo "$out" | tail -n 1 | cut -d\  -f2)
+num_read=$(tail -n 1 "$outfile" | cut -d\  -f2)
 if [ $num_read -eq $num_records ]; then
 	echo "GOOD: All $num_records records retrieved"
 	status=0
diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
index 31ede40bafd..10c6d2b9a6c 100755
--- a/ctdb/tests/simple/81_tunnel_ring.sh
+++ b/ctdb/tests/simple/81_tunnel_ring.sh
@@ -19,24 +19,22 @@ set -e
 
 cluster_is_healthy
 
-try_command_on_node 0 "$CTDB listnodes"
-num_nodes=$(echo "$out" | wc -l)
+try_command_on_node 0 "$CTDB listnodes | wc -l"
+num_nodes="$out"
 
 echo "Running tunnel_test on all $num_nodes nodes."
 try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND \
 	tunnel_test -t 30 -n $num_nodes
 
 # Get the last line of output.
-while read line ; do
-    prev=$line
-done <<<"$out"
+last=$(tail -n 1 "$outfile")
 
 pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
 sanity_check_output 1 "$pat"
 
-# $prev should look like this:
+# $last should look like this:
 #    pnn[2] count=85400
-stuff="${prev##pnn\[*\] }"
+stuff="${last##pnn\[*\] }"
 mps="${stuff% msgs/sec}"
 
 if [ ${mps%.*} -ge 10 ] ; then
diff --git a/ctdb/tests/simple/90_debug_hung_script.sh b/ctdb/tests/simple/90_debug_hung_script.sh
index 8b8e22b3239..da0535fe7fc 100755
--- a/ctdb/tests/simple/90_debug_hung_script.sh
+++ b/ctdb/tests/simple/90_debug_hung_script.sh
@@ -57,11 +57,9 @@ wait_for_monitor_event $test_node
 
 echo "Waiting for debugging output to appear..."
 # Use test -s because the file is created above using mktemp
-wait_until 60 onnode $test_node test -s "$debug_output"
+wait_until 60 test -s "$debug_output"
 
 echo "Checking output of hung script debugging..."
-try_command_on_node -v $test_node cat "$debug_output"
-hung_script_output="$out"
 
 # Can we actually read kernel stacks
 if try_command_on_node $test_node "cat /proc/$$/stack >/dev/null 2>&1" ; then
@@ -75,7 +73,7 @@ fi
 
 while IFS="" read pattern ; do
     [ -n "$pattern" ] || continue
-    if grep -- "^${pattern}\$" <<<"$hung_script_output" >/dev/null ; then
+    if grep -q -- "^${pattern}\$" "$debug_output" ; then
 	printf 'GOOD: output contains "%s"\n' "$pattern"
     else
 	printf 'BAD: output does not contain "%s"\n' "$pattern"
-- 
2.20.1


From fbe6507fba037816779f9fa7919b068fb3c786f2 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Sat, 27 Apr 2019 14:54:09 +1000
Subject: [PATCH 4/9] ctdb-tests: Wait to allow database attach/detach to take
 effect

Sometimes the detach test fails:

  Check detaching single test database detach_test1.tdb
  BAD: database detach_test1.tdb is still attached
  Number of databases:4
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.0/db/volatile/detach_test4.tdb.0
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.0/db/volatile/detach_test3.tdb.0
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.0/db/volatile/detach_test2.tdb.0
  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.0/db/volatile/detach_test1.tdb.0
  Number of databases:3
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.1/db/volatile/detach_test4.tdb.1
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.1/db/volatile/detach_test3.tdb.1
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.1/db/volatile/detach_test2.tdb.1
  Number of databases:4
  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.2/db/volatile/detach_test4.tdb.2
  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.2/db/volatile/detach_test3.tdb.2
  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.2/db/volatile/detach_test2.tdb.2
  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.2/db/volatile/detach_test1.tdb.2
  *** TEST COMPLETED (RC=1) AT 2019-04-27 03:35:40, CLEANING UP...

When issued from a client, the detach control re-broadcasts itself
asynchronously to all nodes and then returns success.  The controls to
some nodes to do the actual detach may still be in flight when success
is returned to the client.  Therefore, the test should wait for a few
seconds to allow the asynchronous controls to complete.

The same is true for the attach control, so workaround the problem in
the attach test too.

An alternative is to make the attach and detach controls synchronous
by avoiding the broadcast and waiting for the results of the
individual controls sent to the nodes.  However, a simple
implementation would involve adding new nested event loops.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 3cb53a7a05409925024d6a67bcfaeb962d896e0b)
---
 ctdb/tests/simple/21_ctdb_attach.sh | 44 ++++++++++++++--------
 ctdb/tests/simple/27_ctdb_detach.sh | 58 +++++++++++++++++++----------
 2 files changed, 66 insertions(+), 36 deletions(-)

diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
index d9dc697d934..9bd85759475 100755
--- a/ctdb/tests/simple/21_ctdb_attach.sh
+++ b/ctdb/tests/simple/21_ctdb_attach.sh
@@ -45,27 +45,39 @@ lastnode=$(( numnodes - 1 ))
 
 ######################################################################
 
-# Confirm that the database is attached
+# Confirm that the database is attached with appropriate flags
+check_db_once ()
+{
+	local pnn="$1"
+	local db="$2"
+
+	try_command_on_node "$pnn" $CTDB getdbmap
+	if grep -qF "name:${db}" "$outfile" >/dev/null ; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_db ()
 {
-    pnn="$1"
-    db="$2"
-    flag="$3"
-    try_command_on_node $pnn "$CTDB getdbmap | grep $db"
-    if [ -z "$out" ] ; then
-	echo "BAD: database $db is not attached on node $node"
-	echo "$out"
-	exit 1
-    else
-	local flags=$(awk '{print $4}' "$outfile") || true
+	local pnn="$1"
+	local db="$2"
+	local flag="$3"
+
+	local flags
+
+	echo "Waiting until database ${db} is attached on node ${pnn}"
+	wait_until 10 check_db_once "$pnn" "$db"
+
+	flags=$(awk -v db="$db" '$2 == "name:" db {print $4}' "$outfile")
 	if [ "$flags" = "$flag" ]; then
-	    echo "GOOD: database $db is attached on node $node with flag $flag"
+		echo "GOOD: db ${db} attached on node ${pnn} with flag $flag"
 	else
-	    echo "BAD: database $db is attached on node $node with wrong flag"
-	    echo "$out"
-	    exit 1
+		echo "BAD: db ${db} attached on node ${pnn} with wrong flag"
+		cat "$outfile"
+		exit 1
 	fi
-    fi
 }
 
 ######################################################################
diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
index 29f3f691dd5..ee0207ae909 100755
--- a/ctdb/tests/simple/27_ctdb_detach.sh
+++ b/ctdb/tests/simple/27_ctdb_detach.sh
@@ -41,33 +41,51 @@ numnodes="$out"
 ######################################################################
 
 # Confirm that the database is attached
+check_db_once ()
+{
+	local db="$1"
+
+	local num_db
+
+	try_command_on_node all "$CTDB getdbmap"
+	num_db=$(grep -cF "name:${db}" "$outfile") || true
+	if [ "$num_db" -eq "$numnodes" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_db ()
 {
-    db="$1"
-    try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" "$outfile") || true
-    if [ $num_db -eq $numnodes ]; then
-	echo "GOOD: database $db is attached on all nodes"
-    else
-	echo "BAD: database $db is not attached on all nodes"
-	cat "$outfile"
-	exit 1
-    fi
+	local db="$1"
+
+	echo "Waiting until database ${db} is attached on all nodes"
+	wait_until 10 check_db_once "$db"
 }
 
 # Confirm that no nodes have databases attached
+check_no_db_once ()
+{
+	local db="$1"
+
+	local num_db
+
+	try_command_on_node all "$CTDB getdbmap"
+	num_db=$(grep -cF "name:${db}" "$outfile") || true
+	if [ "$num_db" -eq 0 ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 check_no_db ()
 {
-    db="$1"
-    try_command_on_node all $CTDB getdbmap
-    local num_db=$(grep -cF "$db" "$outfile") || true
-    if [ $num_db -eq 0 ]; then
-	echo "GOOD: database $db is not attached any more"
-    else
-	echo "BAD: database $db is still attached"
-	cat "$outfile"
-	exit 1
-    fi
+	local db="$1"
+
+	echo "Waiting until database ${db} is detached on all nodes"
+	wait_until 10 check_no_db_once "$db"
 }
 
 ######################################################################
-- 
2.20.1


From 4ad4a8b8cdba6742718042d092195c7fd4510e61 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 1 May 2019 15:10:28 +1000
Subject: [PATCH 5/9] ctdb-tests: Fix usage message

Since commit 0e9ead8f28fced3ebfa888786a1dc5bb59e734a3 daemons have
been shut down after each test, so this option no longer has anything
to do with killing daemons.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit a2ab6485e027ebb13871c7d83b7626ac5c9b98c0)
---
 ctdb/tests/run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh
index 8eead527b33..3c7e794d251 100755
--- a/ctdb/tests/run_tests.sh
+++ b/ctdb/tests/run_tests.sh
@@ -7,7 +7,7 @@ Usage: $0 [OPTIONS] [TESTS]
 Options:
   -A		Use "cat -A" to print test output (only some tests)
   -c		Run integration tests on a cluster
-  -C		Clean up - kill daemons and remove TEST_VAR_DIR when done
+  -C		Remove TEST_VAR_DIR when done
   -d		Print descriptions of tests instead of filenames (dodgy!)
   -D		Show diff between failed/expected test output (some tests only)
   -e		Exit on the first test failure
-- 
2.20.1


From 7203a346d62c6e598463686e3a07e33d00d67dac Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Wed, 1 May 2019 15:17:14 +1000
Subject: [PATCH 6/9] ctdb-tests: Don't clean up test var directory in autotest
 target

If the directory is always cleaned up then it is not possible to look
at daemon logs to debug test failures.

This target is only really used by autobuild.py, which (optionally)
cleans up the parent directory anyway.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Tue May  7 06:56:01 UTC 2019 on sn-devel-184

(cherry picked from commit 5a9e338330fe136908a3a17a5df81c054c5cc5b0)
---
 ctdb/wscript | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ctdb/wscript b/ctdb/wscript
index 323da8cf12a..cbb8bd0fd19 100644
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -1117,7 +1117,7 @@ def test(ctx):
 
 def autotest(ctx):
     env = samba_utils.LOAD_ENVIRONMENT()
-    cmd = 'tests/run_tests.sh -e -S %s -C' % env.SOCKET_WRAPPER_SO_PATH
+    cmd = 'tests/run_tests.sh -e -S %s' % env.SOCKET_WRAPPER_SO_PATH
     ret = samba_utils.RUN_COMMAND(cmd)
     if ret != 0:
         print('autotest exited with exit status %d' % ret)
-- 
2.20.1


From 6fdeb29db1eff042974ab771004775296dcf482e Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Mon, 13 May 2019 17:40:15 +1000
Subject: [PATCH 7/9] ctdb-tests: Capture output in $out on failure as well

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit cf00db40355b49443263187f9d97934f91287e51)
---
 ctdb/tests/scripts/integration.bash | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
index 0fbd39daec8..e0aa312192b 100644
--- a/ctdb/tests/scripts/integration.bash
+++ b/ctdb/tests/scripts/integration.bash
@@ -109,7 +109,11 @@ try_command_on_node ()
 
     local cmd="$*"
 
-    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
+    local status=0
+    onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 || status=$?
+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
+
+    if [ $status -ne 0 ] ; then
 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
 	cat "$outfile"
 	return 1
@@ -119,8 +123,6 @@ try_command_on_node ()
 	echo "Output of \"$cmd\":"
 	cat "$outfile"
     fi
-
-    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
 }
 
 sanity_check_output ()
-- 
2.20.1


From 6dbd56440c125caea3bfe53f8b06105ca21665c6 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 7 May 2019 15:29:19 +1000
Subject: [PATCH 8/9] ctdb-tests: Make ctdb reloadips tests more reliable

ctdb reloadips will fail if it can't disable takover runs.  The most
likely reason for this is that there is already a takeover run in
progress.  We can't predict when this will happen, so retry if this
occurs.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit 8be4ee1a28d5c037955832b6f827d40f28f02796)
---
 ctdb/tests/complex/18_ctdb_reloadips.sh | 35 ++++++++++++++++++++++---
 ctdb/tests/simple/18_ctdb_reloadips.sh  | 33 ++++++++++++++++++++---
 2 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
index 1dded769741..d6efc6bf631 100755
--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
@@ -187,6 +187,33 @@ check_ips ()
     fi
 }
 
+# ctdb reloadips will fail if it can't disable takover runs.  The most
+# likely reason for this is that there is already a takeover run in
+# progress.  We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+	local retry_max=10
+	local retry_count=0
+	while : ; do
+		if try_command_on_node "$test_node" "$CTDB reloadips" ; then
+			return 0
+		fi
+
+		if [ "$out" != "Failed to disable takeover runs" ] ; then
+			return 1
+		fi
+
+		if [ $retry_count -ge $retry_max ] ; then
+			return 1
+		fi
+
+		retry_count=$((retry_count + 1))
+		echo "Retrying..."
+		sleep_for 1
+	done
+}
+
 ####################
 
 new_ip_max=100
@@ -196,7 +223,7 @@ new_ip_max=100
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" 1 $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 1 $new_ip_max
 
@@ -210,7 +237,7 @@ echo "Using 'ctdb reloadips' to remove the 1st address just added..."
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" 2 $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 2 $new_ip_max
 
@@ -225,7 +252,7 @@ echo "Updating to include only about 1/2 of the new IPs..."
 add_ips_to_original_config \
     $test_node "$addresses" "$iface" "$prefix" $start $new_ip_max
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" $start $new_ip_max
 
@@ -237,6 +264,6 @@ try_command_on_node any $CTDB sync
 echo "Restoring original IP configuration..."
 restore_public_addresses
 
-try_command_on_node $test_node "$CTDB reloadips"
+do_ctdb_reloadips
 
 check_ips $test_node "$iface" "$prefix" 0
diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
index 95b819fd82b..6c55c676ca5 100755
--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
@@ -45,9 +45,36 @@ restore_public_addresses ()
 }
 ctdb_test_exit_hook_add restore_public_addresses
 
+# ctdb reloadips will fail if it can't disable takover runs.  The most
+# likely reason for this is that there is already a takeover run in
+# progress.  We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+	local retry_max=10
+	local retry_count=0
+	while : ; do
+		if try_command_on_node any "$CTDB reloadips all" ; then
+			return 0
+		fi
+
+		if [ "$out" != "Failed to disable takeover runs" ] ; then
+			return 1
+		fi
+
+		if [ $retry_count -ge $retry_max ] ; then
+			return 1
+		fi
+
+		retry_count=$((retry_count + 1))
+		echo "Retrying..."
+		sleep_for 1
+	done
+}
+
 try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 echo "Getting list of public IPs on node $test_node"
 try_command_on_node $test_node "$CTDB ip | tail -n +2"
@@ -67,7 +94,7 @@ try_command_on_node any $CTDB sync
 echo "Restoring addresses"
 restore_public_addresses
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 echo "Getting list of public IPs on node $test_node"
 try_command_on_node $test_node "$CTDB ip | tail -n +2"
@@ -88,7 +115,7 @@ echo "Removing IP $test_ip from node $test_node"
 
 try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
 
-try_command_on_node any $CTDB reloadips all
+do_ctdb_reloadips
 
 try_command_on_node $test_node $CTDB ip
 
-- 
2.20.1


From e6436f94bdd8fa3570034e9674bb235778340334 Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 7 May 2019 15:42:49 +1000
Subject: [PATCH 9/9] ctdb-tests: Fix logic error in simple ctdb reloadips test

There is a chance that restoring IP addresses to the test node will
result in different IP addresses being assigned to that node.
Removing a single IP address may then fail (or be a no-op) if it is
done after the restore.

So, swap the single IP address removal to happen first, then restore,
then remove all IP addresses.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit dc89db8ca6aadd4a9f7e8a85843c53709d04587c)
---
 ctdb/tests/simple/18_ctdb_reloadips.sh | 37 ++++++++++++++------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
index 6c55c676ca5..7a18f66b9fa 100755
--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
@@ -32,8 +32,6 @@ ctdb_restart_when_done
 
 select_test_node_and_ips
 
-echo "Emptying public addresses file on $test_node"
-
 try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show
 addresses="${out}/public_addresses"
 echo "Public addresses file on node $test_node is \"$addresses\""
@@ -72,25 +70,31 @@ do_ctdb_reloadips ()
 	done
 }
 
-try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
+
+echo "Removing IP $test_ip from node $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
 
 do_ctdb_reloadips
 
-echo "Getting list of public IPs on node $test_node"
-try_command_on_node $test_node "$CTDB ip | tail -n +2"
+try_command_on_node $test_node $CTDB ip
 
-if [ -n "$out" ] ; then
+if grep "^${test_ip} " <<<"$out" ; then
     cat <<EOF
-BAD: node $test_node still has ips:
+BAD: node $test_node can still host IP $test_ip:
 $out
 EOF
     exit 1
 fi
 
-echo "GOOD: no IPs left on node $test_node"
+cat <<EOF
+GOOD: node $test_node is no longer hosting IP $test_ip:
+$out
+EOF
 
 try_command_on_node any $CTDB sync
 
+
 echo "Restoring addresses"
 restore_public_addresses
 
@@ -111,23 +115,22 @@ EOF
 
 try_command_on_node any $CTDB sync
 
-echo "Removing IP $test_ip from node $test_node"
 
-try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
+echo "Emptying public addresses file on $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
 
 do_ctdb_reloadips
 
-try_command_on_node $test_node $CTDB ip
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
 
-if grep "^${test_ip} " <<<"$out" ; then
+if [ -n "$out" ] ; then
     cat <<EOF
-BAD: node $test_node can still host IP $test_ip:
+BAD: node $test_node still has ips:
 $out
 EOF
     exit 1
 fi
 
-cat <<EOF
-GOOD: node $test_node is no longer hosting IP $test_ip:
-$out
-EOF
+echo "GOOD: no IPs left on node $test_node"
-- 
2.20.1