Attachment 15155 Details for Bug 13924 – Patch for 4.9

[patch] Patch for 4.9

BZ13924-v4-9.patch (text/plain), 70.39 KB, created by Martin Schwenke on 2019-05-15 02:13:13 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Martin Schwenke

Created: 2019-05-15 02:13:13 UTC

Size: 70.39 KB

patch

obsolete

>From fe124cc7fac3f046270f03442a27024df2af5f50 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Tue, 30 Apr 2019 12:09:26 +1000
>Subject: [PATCH 1/9] ctdb-tests: Change sanity_check_output() to internally
> use $out
>
>All callers are currently passed $out.  Global variable $out is used
>in many other places so use it here to simplify the interface and make
>future changes simpler.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit 7c3819d1ac264acf998f426e0cef7f6211e0ddee)
>---
> ctdb/tests/scripts/integration.bash     | 5 ++---
> ctdb/tests/simple/02_ctdb_listvars.sh   | 3 +--
> ctdb/tests/simple/05_ctdb_listnodes.sh  | 3 +--
> ctdb/tests/simple/09_ctdb_ping.sh       | 6 ++----
> ctdb/tests/simple/12_ctdb_getdebug.sh   | 3 +--
> ctdb/tests/simple/14_ctdb_statistics.sh | 2 +-
> ctdb/tests/simple/24_ctdb_getdbmap.sh   | 4 ++--
> ctdb/tests/simple/25_dumpmemory.sh      | 4 ++--
> ctdb/tests/simple/51_message_ring.sh    | 2 +-
> ctdb/tests/simple/52_fetch_ring.sh      | 2 +-
> ctdb/tests/simple/81_tunnel_ring.sh     | 2 +-
> 11 files changed, 15 insertions(+), 21 deletions(-)
>
>diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
>index 3750c442eba..bc4d2b429ff 100644
>--- a/ctdb/tests/scripts/integration.bash
>+++ b/ctdb/tests/scripts/integration.bash
>@@ -113,11 +113,10 @@ sanity_check_output ()
> {
>     local min_lines="$1"
>     local regexp="$2" # Should be anchored as necessary.
>-    local output="$3"
> 
>     local ret=0
> 
>-    local num_lines=$(echo "$output" | wc -l)
>+    local num_lines=$(echo "$out" | wc -l)
>     echo "There are $num_lines lines of output"
>     if [ $num_lines -lt $min_lines ] ; then
> 	echo "BAD: that's less than the required number (${min_lines})"
>@@ -126,7 +125,7 @@ sanity_check_output ()
> 
>     local status=0
>     local unexpected # local doesn't pass through status of command on RHS.
>-    unexpected=$(echo "$output" | egrep -v "$regexp") || status=$?
>+    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
> 
>     # Note that this is reversed.
>     if [ $status -eq 0 ] ; then
>diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh
>index 2f709a8cc0d..9041b6cd216 100755
>--- a/ctdb/tests/simple/02_ctdb_listvars.sh
>+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
>@@ -37,5 +37,4 @@ try_command_on_node -v 0 "$CTDB listvars"
> 
> sanity_check_output \
>     5 \
>-    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \
>-    "$out"
>+    '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$'
>diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
>index 9e48a68e3c0..87aa776d038 100755
>--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
>+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
>@@ -39,8 +39,7 @@ ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
> ipv6_pat='[[:xdigit:]]+:[[:xdigit:]:]+[[:xdigit:]]+'
> sanity_check_output \
>     2 \
>-    "^${ipv4_pat}|${ipv6_pat}\$" \
>-    "$out"
>+    "^${ipv4_pat}|${ipv6_pat}\$"
> 
> out_0="$out"
> 
>diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
>index ab6ba146df6..40006a221d0 100755
>--- a/ctdb/tests/simple/09_ctdb_ping.sh
>+++ b/ctdb/tests/simple/09_ctdb_ping.sh
>@@ -42,8 +42,7 @@ try_command_on_node -v 0 "$CTDB ping -n 1"
> 
> sanity_check_output \
>     1 \
>-    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+$[[:digit:]]+ clients$$' \
>-    "$out"
>+    '^response from 1 time=-?[.0-9]+ sec[[:space:]]+$[[:digit:]]+ clients$$'
> 
> try_command_on_node -v 0 "$CTDB shutdown -n 1"
> 
>@@ -53,5 +52,4 @@ try_command_on_node -v 0 "! $CTDB ping -n 1"
> 
> sanity_check_output \
>     1 \
>-    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)" \
>-    "$out"
>+    "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)"
>diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh
>index 226184813c0..7e6177a3ad4 100755
>--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
>+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
>@@ -36,8 +36,7 @@ getdebug_onnode="$out"
> 
> sanity_check_output \
>     $num_nodes \
>-    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$' \
>-    "$out"
>+    '^(ERROR|WARNING|NOTICE|INFO|DEBUG)$'
> 
> cmd=""
> n=0
>diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh
>index 5ff22d7f64c..eda2dc54749 100755
>--- a/ctdb/tests/simple/14_ctdb_statistics.sh
>+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
>@@ -35,4 +35,4 @@ pattern='^(CTDB version 1|Current time of statistics[[:space:]]*:.*|Statistics c
> 
> try_command_on_node -v 1 "$CTDB statistics"
> 
>-sanity_check_output 40 "$pattern" "$out"
>+sanity_check_output 40 "$pattern"
>diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
>index 9bed5903914..60458c2c8a7 100755
>--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
>+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
>@@ -45,7 +45,7 @@ try_command_on_node -v 0 "$CTDB getdbmap"
> 
> db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
> 
>-sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
>+sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
> 
> num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
> 
>@@ -54,7 +54,7 @@ for i in $(seq 1 5) ; do
>     echo "Creating test database: $f"
>     try_command_on_node 0 $CTDB attach "$f"
>     try_command_on_node 0 $CTDB getdbmap
>-    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
>+    sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
>     num=$(echo "$out" | sed -n -e '1s/^.*://p')
>     if [ $num = $(($num_db_init + $i)) ] ; then
> 	echo "OK: correct number of additional databases"
>diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh
>index 36fecfed016..6aa9300383f 100755
>--- a/ctdb/tests/simple/25_dumpmemory.sh
>+++ b/ctdb/tests/simple/25_dumpmemory.sh
>@@ -31,8 +31,8 @@ cluster_is_healthy
> pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks $ref [[:digit:]]+$[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ $total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks$)$'
> 
> try_command_on_node -v 0 "$CTDB dumpmemory"
>-sanity_check_output 10 "$pat" "$out"
>+sanity_check_output 10 "$pat"
> 
> echo
> try_command_on_node -v 0 "$CTDB rddumpmemory"
>-sanity_check_output 10 "$pat" "$out"
>+sanity_check_output 10 "$pat"
>diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
>index e1bdab383c7..cecea6c81bb 100755
>--- a/ctdb/tests/simple/51_message_ring.sh
>+++ b/ctdb/tests/simple/51_message_ring.sh
>@@ -31,7 +31,7 @@ while read line ; do
> done <<<"$out"
> 
> pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec $\+ve=[[:digit:]]+ -ve=[[:digit:]]+$)$'
>-sanity_check_output 1 "$pat" "$out"
>+sanity_check_output 1 "$pat"
> 
> # $prev should look like this:
> #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
>diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
>index 81e6a7a6bb0..078adcbe30f 100755
>--- a/ctdb/tests/simple/52_fetch_ring.sh
>+++ b/ctdb/tests/simple/52_fetch_ring.sh
>@@ -26,7 +26,7 @@ echo "Running fetch_ring on all $num_nodes nodes."
> try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
> 
> pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
>-sanity_check_output 1 "$pat" "$out"
>+sanity_check_output 1 "$pat"
> 
> # Get the last line of output.
> while read line ; do
>diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
>index c98b139e9cc..31ede40bafd 100755
>--- a/ctdb/tests/simple/81_tunnel_ring.sh
>+++ b/ctdb/tests/simple/81_tunnel_ring.sh
>@@ -32,7 +32,7 @@ while read line ; do
> done <<<"$out"
> 
> pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
>-sanity_check_output 1 "$pat" "$out"
>+sanity_check_output 1 "$pat"
> 
> # $prev should look like this:
> #    pnn[2] count=85400
>-- 
>2.20.1
>
>
>From 4abc8264315d7e1afc32d950b6a5281b0b091cd8 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Thu, 28 Mar 2019 14:26:52 +1100
>Subject: [PATCH 2/9] ctdb-tests: Make try_command_on_node less error-prone
>
>This sometimes fails, apparently due to a cat process in onnode
>getting EAGAIN.  The conclusion is that tests that process large
>amounts of output should not depend on a sub-shell delivering that
>output into a shell variable.
>
>Change try_command_on_node() to leave all of the output in file
>$outfile and just put the first 1KB into $out.  $outfile is removed
>after each test completes.
>
>Change the implementation of sanity_check_output() to use $outfile
>instead of $out.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit 9d02452a24625df5f62fd6d45a16effe2fa45fbe)
>---
> ctdb/tests/scripts/integration.bash | 30 +++++++++++++++++++++--------
> 1 file changed, 22 insertions(+), 8 deletions(-)
>
>diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
>index bc4d2b429ff..9185b1fb8e3 100644
>--- a/ctdb/tests/scripts/integration.bash
>+++ b/ctdb/tests/scripts/integration.bash
>@@ -77,7 +77,20 @@ ctdb_test_init ()
> 
> ########################################
> 
>-# Sets: $out
>+# Sets: $out, $outfile
>+# * The first 1KB of output is put into $out
>+# * Tests should use $outfile for handling large output
>+# * $outfile is removed after each test
>+out=""
>+outfile="${TEST_VAR_DIR}/try_command_on_node.out"
>+
>+outfile_cleanup ()
>+{
>+	rm -f "$outfile"
>+}
>+
>+ctdb_test_exit_hook_add outfile_cleanup
>+
> try_command_on_node ()
> {
>     local nodespec="$1" ; shift
>@@ -96,17 +109,18 @@ try_command_on_node ()
> 
>     local cmd="$*"
> 
>-    out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || {
>-
>+    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
> 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
>-	echo "$out"
>+	cat "$outfile"
> 	return 1
>-    }
>+    fi
> 
>     if $verbose ; then
> 	echo "Output of \"$cmd\":"
>-	echo "$out"
>+	cat "$outfile"
>     fi
>+
>+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
> }
> 
> sanity_check_output ()
>@@ -116,7 +130,7 @@ sanity_check_output ()
> 
>     local ret=0
> 
>-    local num_lines=$(echo "$out" | wc -l)
>+    local num_lines=$(wc -l <"$outfile")
>     echo "There are $num_lines lines of output"
>     if [ $num_lines -lt $min_lines ] ; then
> 	echo "BAD: that's less than the required number (${min_lines})"
>@@ -125,7 +139,7 @@ sanity_check_output ()
> 
>     local status=0
>     local unexpected # local doesn't pass through status of command on RHS.
>-    unexpected=$(echo "$out" | egrep -v "$regexp") || status=$?
>+    unexpected=$(grep -Ev "$regexp" "$outfile") || status=$?
> 
>     # Note that this is reversed.
>     if [ $status -eq 0 ] ; then
>-- 
>2.20.1
>
>
>From ed82783c65d3d4e6b0cb8b0d2cbecb79faf24733 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Thu, 11 Apr 2019 20:55:20 +1000
>Subject: [PATCH 3/9] ctdb-tests: Avoid bulk output in $out, prefer $outfile
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit 066cc5b0c561464ed08890d9aa1a1a55b545e9cc)
>---
> .../tests/complex/11_ctdb_delip_removes_ip.sh | 10 ++---
> ctdb/tests/complex/18_ctdb_reloadips.sh       |  8 ++--
> ctdb/tests/complex/32_cifs_tickle.sh          |  8 ----
> ctdb/tests/complex/36_smb_reset_server.sh     | 12 +-----
> ctdb/tests/complex/37_nfs_reset_server.sh     |  4 +-
> ctdb/tests/complex/60_rogueip_releaseip.sh    |  2 +-
> ctdb/tests/complex/scripts/local.bash         |  5 +--
> ctdb/tests/scripts/integration.bash           | 40 +++++--------------
> ctdb/tests/simple/03_ctdb_getvar.sh           |  3 +-
> ctdb/tests/simple/05_ctdb_listnodes.sh        |  2 +-
> ctdb/tests/simple/08_ctdb_isnotrecmaster.sh   | 10 ++---
> ctdb/tests/simple/11_ctdb_ip.sh               | 27 +++++++------
> ctdb/tests/simple/15_ctdb_statisticsreset.sh  | 21 +++++-----
> ctdb/tests/simple/19_ip_takeover_noop.sh      |  4 +-
> ctdb/tests/simple/20_delip_iface_gc.sh        | 10 ++---
> ctdb/tests/simple/21_ctdb_attach.sh           |  7 ++--
> ctdb/tests/simple/23_ctdb_moveip.sh           | 25 +++++++++++-
> ctdb/tests/simple/24_ctdb_getdbmap.sh         |  6 +--
> ..._config_check_error_on_unreachable_ctdb.sh |  6 +--
> ctdb/tests/simple/27_ctdb_detach.sh           | 21 +++++-----
> ctdb/tests/simple/35_ctdb_getreclock.sh       |  2 +-
> ctdb/tests/simple/51_message_ring.sh          | 12 +++---
> ctdb/tests/simple/52_fetch_ring.sh            | 12 +++---
> ctdb/tests/simple/53_transaction_loop.sh      |  4 +-
> .../simple/54_transaction_loop_recovery.sh    |  4 +-
> ctdb/tests/simple/55_ctdb_ptrans.sh           | 12 +++---
> .../56_replicated_transaction_recovery.sh     |  4 +-
> ctdb/tests/simple/58_ctdb_restoredb.sh        |  8 ++--
> .../simple/69_recovery_resurrect_deleted.sh   | 10 ++---
> ctdb/tests/simple/70_recoverpdbbyseqnum.sh    |  4 +-
> ctdb/tests/simple/71_ctdb_wipedb.sh           |  4 +-
> .../simple/72_update_record_persistent.sh     |  4 +-
> .../tests/simple/75_readonly_records_basic.sh | 24 +++++------
> ctdb/tests/simple/77_ctdb_db_recovery.sh      |  6 +--
> ctdb/tests/simple/79_volatile_db_traverse.sh  |  4 +-
> ctdb/tests/simple/80_ctdb_traverse.sh         |  2 +-
> ctdb/tests/simple/81_tunnel_ring.sh           | 12 +++---
> ctdb/tests/simple/90_debug_hung_script.sh     |  6 +--
> 38 files changed, 167 insertions(+), 198 deletions(-)
>
>diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
>index d6a3b3c04ff..a41be09eab1 100755
>--- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
>+++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh
>@@ -25,8 +25,8 @@ ctdb_restart_when_done
> select_test_node_and_ips
> get_test_ip_mask_and_iface
> 
>-echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..."
>-try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet6?[[:space:]]*${test_ip}/'"
>+echo "Checking that node ${test_node} hosts ${test_ip}..."
>+try_command_on_node $test_node "ip addr show to ${test_ip} | grep -q ."
> 
> echo "Attempting to remove ${test_ip} from node ${test_node}."
> try_command_on_node $test_node $CTDB delip $test_ip
>@@ -36,10 +36,10 @@ wait_until_ips_are_on_node '!' $test_node $test_ip
> timeout=60
> increment=5
> count=0
>-echo "Waiting for ${test_ip} to disappear from ${iface}..."
>+echo "Waiting for ${test_ip} to disappear from node ${test_node}..."
> while : ; do
>-    try_command_on_node -v $test_node "ip addr show dev $iface"
>-    if echo "$out" | grep -E 'inet6?[[:space:]]*${test_ip}/'; then
>+    try_command_on_node -v $test_node "ip addr show to ${test_node}"
>+    if -n "$out" ; then
> 	echo "Still there..."
> 	if [ $(($count * $increment)) -ge $timeout ] ; then
> 	    echo "BAD: Timed out waiting..."
>diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
>index e0632f30564..1dded769741 100755
>--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
>+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
>@@ -51,12 +51,12 @@ select_test_node_and_ips
> 
> echo "Getting public IP information from CTDB..."
> try_command_on_node any "$CTDB ip -X -v all"
>-ctdb_ip_info=$(echo "$out" | awk -F'|' 'NR > 1 { print $2, $3, $5 }')
>+ctdb_ip_info=$(awk -F'|' 'NR > 1 { print $2, $3, $5 }' "$outfile")
> 
> echo "Getting IP information from interfaces..."
> try_command_on_node all "ip addr show"
>-ip_addr_info=$(echo "$out" | \
>-    awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }')
>+ip_addr_info=$(awk '$1 == "inet" { ip = $2; sub(/\/.*/, "", ip); print ip }' \
>+		   "$outfile")
> 
> prefix=""
> for b in $(seq 0 255) ; do
>@@ -171,7 +171,7 @@ check_ips ()
> 
>     try_command_on_node $test_node "ip addr show dev ${iface}"
>     local ip_addrs_file=$(mktemp)
>-    echo "$out" | \
>+    cat "$outfile" | \
> 	sed -n -e "s@.*inet * $${prefix//./\.}\.[0-9]*$/.*@\1@p" | \
> 	sort >"$ip_addrs_file"
> 
>diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh
>index 366824501da..ac00d449293 100755
>--- a/ctdb/tests/complex/32_cifs_tickle.sh
>+++ b/ctdb/tests/complex/32_cifs_tickle.sh
>@@ -64,14 +64,6 @@ echo "Source socket is $src_socket"
> # we sometimes beat the registration.
> echo "Checking if CIFS connection is tracked by CTDB on test node..."
> wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
>-echo "$out"
>-
>-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
>-    echo "GOOD: CIFS connection tracked OK by CTDB."
>-else
>-    echo "BAD: Socket not tracked by CTDB."
>-    testfailures=1
>-fi
> 
> # This is almost immediate.  However, it is sent between nodes
> # asynchonously, so it is worth checking...
>diff --git a/ctdb/tests/complex/36_smb_reset_server.sh b/ctdb/tests/complex/36_smb_reset_server.sh
>index beff1a22014..c55cb9f251e 100755
>--- a/ctdb/tests/complex/36_smb_reset_server.sh
>+++ b/ctdb/tests/complex/36_smb_reset_server.sh
>@@ -62,16 +62,8 @@ echo "Source socket is $src_socket"
> 
> # This should happen as soon as connection is up... but unless we wait
> # we sometimes beat the registration.
>-echo "Checking if CIFS connection is tracked by CTDB on test node..."
>+echo "Waiting until SMB connection is tracked by CTDB on test node..."
> wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket
>-echo "$out"
>-
>-if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then
>-    echo "GOOD: CIFS connection tracked OK by CTDB."
>-else
>-    echo "BAD: Socket not tracked by CTDB."
>-    exit 1
>-fi
> 
> # It would be nice if ss consistently used local/peer instead of src/dst
> ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
>@@ -83,7 +75,7 @@ if [ -z "$out" ] ; then
> 	exit 1
> fi
> echo "GOOD: ss lists the socket:"
>-echo "$out"
>+cat "$outfile"
> 
> echo "Disabling node $test_node"
> try_command_on_node 1 $CTDB disable -n $test_node
>diff --git a/ctdb/tests/complex/37_nfs_reset_server.sh b/ctdb/tests/complex/37_nfs_reset_server.sh
>index 07bd034d6c7..e38d4510d49 100755
>--- a/ctdb/tests/complex/37_nfs_reset_server.sh
>+++ b/ctdb/tests/complex/37_nfs_reset_server.sh
>@@ -63,7 +63,7 @@ echo "Source socket is $src_socket"
> echo "Wait until NFS connection is tracked by CTDB on test node ..."
> wait_until $((monitor_interval * 2)) \
> 	   check_tickles $test_node $test_ip $test_port $src_socket
>-echo "$out"
>+cat "$outfile"
> 
> # It would be nice if ss consistently used local/peer instead of src/dst
> ss_filter="src ${test_ip}:${test_port} dst ${src_socket}"
>@@ -75,7 +75,7 @@ if [ -z "$out" ] ; then
> 	exit 1
> fi
> echo "GOOD: ss lists the socket:"
>-echo "$out"
>+cat "$outfile"
> 
> echo "Disabling node $test_node"
> try_command_on_node 1 $CTDB disable -n $test_node
>diff --git a/ctdb/tests/complex/60_rogueip_releaseip.sh b/ctdb/tests/complex/60_rogueip_releaseip.sh
>index 0daf289c83a..7e2f89a1e3b 100755
>--- a/ctdb/tests/complex/60_rogueip_releaseip.sh
>+++ b/ctdb/tests/complex/60_rogueip_releaseip.sh
>@@ -34,7 +34,7 @@ for i in $all_pnns ; do
> 		continue
> 	fi
> 	try_command_on_node $i "$CTDB ip"
>-	n=$(awk -v ip="$test_ip" '$1 == ip { print }' <<<"$out")
>+	n=$(awk -v ip="$test_ip" '$1 == ip { print }' "$outfile")
> 	if [ -n "$n" ] ; then
> 		other_node="$i"
> 		break
>diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash
>index 7787de8f111..787f597edcc 100644
>--- a/ctdb/tests/complex/scripts/local.bash
>+++ b/ctdb/tests/complex/scripts/local.bash
>@@ -67,7 +67,7 @@ check_tickles ()
>     local src_socket="$4"
>     try_command_on_node $node ctdb gettickles $test_ip $test_port
>     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
>-    [ "${out/SRC: ${src_socket} /}" != "$out" ]
>+    grep -Fq "SRC: ${src_socket} " "$outfile"
> }
> 
> check_tickles_all ()
>@@ -79,8 +79,7 @@ check_tickles_all ()
> 
>     try_command_on_node all ctdb gettickles $test_ip $test_port
>     # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
>-    local t="${src_socket//./\\.}"
>-    local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
>+    local count=$(grep -Fc "SRC: ${src_socket} " "$outfile" || true)
>     [ $count -eq $numnodes ]
> }
> 
>diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
>index 9185b1fb8e3..0fbd39daec8 100644
>--- a/ctdb/tests/scripts/integration.bash
>+++ b/ctdb/tests/scripts/integration.bash
>@@ -153,30 +153,7 @@ sanity_check_output ()
>     return $ret
> }
> 
>-sanity_check_ips ()
>-{
>-    local ips="$1" # list of "ip node" lines
>-
>-    echo "Sanity checking IPs..."
>-
>-    local x ipp prev
>-    prev=""
>-    while read x ipp ; do
>-	[ "$ipp" = "-1" ] && break
>-	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
>-	    echo "OK"
>-	    return 0
>-	fi
>-	prev="$ipp"
>-    done <<<"$ips"
>-
>-    echo "BAD: a node was -1 or IPs are only assigned to one node:"
>-    echo "$ips"
>-    echo "Are you running an old version of CTDB?"
>-    return 1
>-}
>-
>-# This returns a list of "ip node" lines in $out
>+# This returns a list of "ip node" lines in $outfile
> all_ips_on_node()
> {
>     local node="$1"
>@@ -197,9 +174,9 @@ _select_test_node_and_ips ()
> 	    test_node="$pnn"
> 	fi
> 	if [ "$pnn" = "$test_node" ] ; then
>-            test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
>+	    test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}"
> 	fi
>-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
>+    done <"$outfile"
> 
>     echo "Selected node ${test_node} with IPs: ${test_node_ips}."
>     test_ip="${test_node_ips%% *}"
>@@ -269,7 +246,7 @@ delete_ip_from_all_nodes ()
> 	    if [ "$_ip" = "$_i" ] ; then
> 		_nodes="${_nodes}${_nodes:+,}${_pnn}"
> 	    fi
>-	done <<<"$out" # bashism
>+	done <"$outfile"
>     done
> 
>     try_command_on_node -pq "$_nodes" "$CTDB delip $_ip"
>@@ -439,7 +416,7 @@ ips_are_on_node ()
> 	    if $negating ; then
> 		ips="${ips/${check}}"
> 	    fi
>-	done <<<"$out" # bashism to avoid problem setting variable in pipeline.
>+	done <"$outfile"
>     done
> 
>     ips="${ips// }" # Remove any spaces.
>@@ -481,7 +458,7 @@ node_has_some_ips ()
> 	if [ "$node" = "$pnn" ] ; then
> 	    return 0
> 	fi
>-    done <<<"$out" # bashism to avoid problem setting variable in pipeline.
>+    done <"$outfile"
> 
>     return 1
> }
>@@ -623,7 +600,8 @@ wait_for_monitor_event ()
> 	return 1
>     }
> 
>-    local ctdb_scriptstatus_original="$out"
>+    mv "$outfile" "${outfile}.orig"
>+
>     wait_until 120 _ctdb_scriptstatus_changed
> }
> 
>@@ -634,7 +612,7 @@ _ctdb_scriptstatus_changed ()
> 	return 1
>     }
> 
>-    [ "$out" != "$ctdb_scriptstatus_original" ]
>+    ! diff "$outfile" "${outfile}.orig" >/dev/null
> }
> 
> #######################################
>diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh
>index cae3dd14159..7d795598d0a 100755
>--- a/ctdb/tests/simple/03_ctdb_getvar.sh
>+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
>@@ -38,7 +38,6 @@ try_command_on_node -v 0 "$CTDB listvars"
> 
> echo "Verifying all variable values using \"ctdb getvar\"..."
> 
>-echo "$out" |
> while read var x val ; do
>     try_command_on_node 0 "$CTDB getvar $var"
> 
>@@ -48,4 +47,4 @@ while read var x val ; do
> 	echo "MISMATCH on $var: $val != $val2"
> 	exit 1
>     fi
>-done
>+done <"$outfile"
>diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
>index 87aa776d038..5e64032fd5d 100755
>--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
>+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
>@@ -32,7 +32,7 @@ cluster_is_healthy
> 
> try_command_on_node -v 0 "$CTDB listnodes"
> 
>-num_nodes=$(echo "$out" | wc -l)
>+num_nodes=$(wc -l <"$outfile")
> 
> # Each line should look like an IP address.
> ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
>diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
>index 138f59c6e79..b6e7287e776 100755
>--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
>+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
>@@ -33,13 +33,11 @@ set -e
> cluster_is_healthy
> 
> cmd="$CTDB isnotrecmaster || true"
>-try_command_on_node all "$cmd"
>-echo "Output of \"$cmd\":"
>-echo "$out"
>+try_command_on_node -v all "$cmd"
> 
>-num_all_lines=$(echo "$out" |  wc -l)
>-num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true
>-num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true
>+num_all_lines=$(wc -l <"$outfile")
>+num_rm_lines=$(grep -Fc 'this node is the recmaster' "$outfile") || true
>+num_not_rm_lines=$(grep -Fc 'this node is not the recmaster' "$outfile") || true
> 
> if [ $num_rm_lines -eq 1 ] ; then
>     echo "OK, there is only 1 recmaster"
>diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
>index 8bca2c608c5..3e22d5d5b29 100755
>--- a/ctdb/tests/simple/11_ctdb_ip.sh
>+++ b/ctdb/tests/simple/11_ctdb_ip.sh
>@@ -33,24 +33,25 @@ cluster_is_healthy
> 
> echo "Getting list of public IPs..."
> try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
>-ips=$(echo "$out" | sed \
>+ips=$(sed \
> 	-e 's@ node\[@ @' \
>-	-e 's@\].*$@@')
>-machineout=$(echo "$out" | sed -r \
>+	-e 's@\].*$@@' \
>+	"$outfile")
>+machineout=$(sed -r \
> 	-e 's@^| |$@\|@g' \
> 	-e 's@[[:alpha:]]+\[@@g' \
>-	-e 's@\]@@g')
>+	-e 's@\]@@g' \
>+	"$outfile")
> 
> if [ -z "$TEST_LOCAL_DAEMONS" ]; then
>-    while read ip pnn ; do
>-        try_command_on_node $pnn "ip addr show"
>-        if [ "${out/inet* ${ip}\/}" != "$out" ] ; then
>-            echo "GOOD: node $pnn appears to have $ip assigned"
>-        else
>-            echo "BAD:  node $pnn does not appear to have $ip assigned"
>-            testfailures=1
>-        fi
>-    done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
>+	while read ip pnn ; do
>+		try_command_on_node $pnn "ip addr show to ${ip}"
>+		if [ -n "$out" ] ; then
>+			echo "GOOD: node $pnn appears to have $ip assigned"
>+		else
>+			die "BAD: node $pnn does not appear to have $ip assigned"
>+		fi
>+	done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
> fi
> 
> [ "$testfailures" != 1 ] && echo "Looks good!"
>diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
>index eaa60d69205..4f0c2bd9d01 100755
>--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
>+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
>@@ -37,10 +37,11 @@ num_nodes="$out"
> 
> get_stat ()
> {
>-    local label="$1"
>-    local out="$2"
>+	local label="$1"
> 
>-    echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1
>+	cat "$outfile" |
>+	sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" |
>+	head -1
> }
> 
> check_reduced ()
>@@ -60,20 +61,20 @@ check_reduced ()
> n=0
> while [ $n -lt $num_nodes ] ; do
>     echo "Getting initial statistics for node ${n}..."
>-    
>+
>     try_command_on_node -v $n $CTDB statistics
> 
>-    before_req_control=$(get_stat "req_control" "$out")
>-    before_reply_control=$(get_stat "reply_control" "$out")
>-    before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
>+    before_req_control=$(get_stat "req_control")
>+    before_reply_control=$(get_stat "reply_control")
>+    before_node_packets_recv=$(get_stat "node_packets_recv")
> 
>     try_command_on_node $n $CTDB statisticsreset
> 
>     try_command_on_node -v $n $CTDB statistics
> 
>-    after_req_control=$(get_stat "req_control" "$out")
>-    after_reply_control=$(get_stat "reply_control" "$out")
>-    after_node_packets_recv=$(get_stat "node_packets_recv" "$out")
>+    after_req_control=$(get_stat "req_control")
>+    after_reply_control=$(get_stat "reply_control")
>+    after_node_packets_recv=$(get_stat "node_packets_recv")
> 
>     check_reduced "req_control" "$before_req_control" "$after_req_control"
>     check_reduced "reply_control" "$before_reply_control" "$after_reply_control"
>diff --git a/ctdb/tests/simple/19_ip_takeover_noop.sh b/ctdb/tests/simple/19_ip_takeover_noop.sh
>index 0b86241db2a..48236a72031 100755
>--- a/ctdb/tests/simple/19_ip_takeover_noop.sh
>+++ b/ctdb/tests/simple/19_ip_takeover_noop.sh
>@@ -46,9 +46,7 @@ while read ip pnn ; do
> 	if [ "$pnn" != "-1" ] ; then
> 		die "BAD: IP address ${ip} is assigned to node ${pnn}"
> 	fi
>-done <<EOF
>-$out
>-EOF
>+done <"$outfile"
> 
> echo "GOOD: All IP addresses are unassigned"
> 
>diff --git a/ctdb/tests/simple/20_delip_iface_gc.sh b/ctdb/tests/simple/20_delip_iface_gc.sh
>index e73eed20cc0..9a4442d6ebe 100755
>--- a/ctdb/tests/simple/20_delip_iface_gc.sh
>+++ b/ctdb/tests/simple/20_delip_iface_gc.sh
>@@ -22,14 +22,14 @@ select_test_node_and_ips
> 
> # Find interfaces on test node
> try_command_on_node $test_node "$CTDB ifaces -X"
>-ifaces=$(awk -F'|' 'NR > 1 { print $2 }' <<<"$out")
>+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
> echo "Node ${test_node} has interfaces: ${ifaces}"
> 
> # Delete all IPs on each interface...  deleting IPs from one interface
> # can cause other interfaces to disappear, so we need to be careful...
> for i in $ifaces ; do
>     try_command_on_node $test_node "$CTDB ifaces -X"
>-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
>+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
> 
>     if [ -z "$info" ] ; then
> 	echo "Interface ${i} missing... assuming already deleted!"
>@@ -41,7 +41,7 @@ for i in $ifaces ; do
> 
>     try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
>     awk -F'|' -v i="$i" \
>-	'$6 == i { print $2 }' <<<"$out" |
>+	'$6 == i { print $2 }' "$outfile" |
>     while read ip ; do
> 	echo "  $ip"
> 	try_command_on_node $test_node "$CTDB delip $ip"
>@@ -49,8 +49,8 @@ for i in $ifaces ; do
>     try_command_on_node $test_node "$CTDB ipreallocate"
> 
>     try_command_on_node $test_node "$CTDB ifaces -X"
>-    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out")
>-    
>+    info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
>+
>     if [ -z "$info" ] ; then
> 	echo "GOOD: Interface ${i} has been garbage collected"
>     else
>diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
>index 11b600800a4..d9dc697d934 100755
>--- a/ctdb/tests/simple/21_ctdb_attach.sh
>+++ b/ctdb/tests/simple/21_ctdb_attach.sh
>@@ -39,9 +39,8 @@ ctdb_restart_when_done
> 
> ######################################################################
> 
>-try_command_on_node 0 "$CTDB listnodes -X"
>-listnodes_output="$out"
>-numnodes=$(wc -l <<<"$listnodes_output")
>+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
>+numnodes="$out"
> lastnode=$(( numnodes - 1 ))
> 
> ######################################################################
>@@ -58,7 +57,7 @@ check_db ()
> 	echo "$out"
> 	exit 1
>     else
>-	local flags=$(awk '{print $4}' <<<"$out") || true
>+	local flags=$(awk '{print $4}' "$outfile") || true
> 	if [ "$flags" = "$flag" ]; then
> 	    echo "GOOD: database $db is attached on node $node with flag $flag"
> 	else
>diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
>index c6e70bf4e94..77c23eb621b 100755
>--- a/ctdb/tests/simple/23_ctdb_moveip.sh
>+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
>@@ -26,7 +26,28 @@ ctdb_restart_when_done
> 
> select_test_node_and_ips
> 
>-sanity_check_ips "$out"
>+sanity_check_ips ()
>+{
>+    echo "Sanity checking IPs..."
>+
>+    local x ipp prev
>+    prev=""
>+    while read x ipp ; do
>+	[ "$ipp" = "-1" ] && break
>+	if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
>+	    echo "OK"
>+	    return 0
>+	fi
>+	prev="$ipp"
>+    done <"$outfile"
>+
>+    echo "BAD: a node was -1 or IPs are only assigned to one node:"
>+    cat "$outfile"
>+    echo "Are you running an old version of CTDB?"
>+    return 1
>+}
>+
>+sanity_check_ips
> 
> # Find a target node - it must be willing to host $test_ip
> try_command_on_node any "$CTDB listnodes | wc -l"
>@@ -40,7 +61,7 @@ for i in $(seq 0 $(($num_nodes - 1)) ) ; do
> 	    to_node="$i"
> 	    break 2
> 	fi
>-    done <<<"$out"
>+    done <"$outfile"
> done
> 
> if [ -z "$to_node" ] ; then
>diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
>index 60458c2c8a7..a92262a177b 100755
>--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
>+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
>@@ -47,7 +47,7 @@ db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[
> 
> sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
> 
>-num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
>+num_db_init=$(sed -n -e '1s/.*://p' "$outfile")
> 
> for i in $(seq 1 5) ; do
>     f=$(make_temp_db_filename)
>@@ -55,14 +55,14 @@ for i in $(seq 1 5) ; do
>     try_command_on_node 0 $CTDB attach "$f"
>     try_command_on_node 0 $CTDB getdbmap
>     sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern"
>-    num=$(echo "$out" | sed -n -e '1s/^.*://p')
>+    num=$(sed -n -e '1s/^.*://p' "$outfile")
>     if [ $num = $(($num_db_init + $i)) ] ; then
> 	echo "OK: correct number of additional databases"
>     else
> 	echo "BAD: no additional database"
> 	exit 1
>     fi
>-    if [ "${out/name:${f} /}" != "$out" ] ; then
>+    if awk '{print $2}' "$outfile" | grep -Fqx "name:$f" ; then
> 	echo "OK: getdbmap knows about \"$f\""
>     else
> 	echo "BAD: getdbmap does not know about \"$f\""
>diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
>index 6642b1759f9..7ec6f489a57 100755
>--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
>+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
>@@ -45,8 +45,8 @@ ctdb_restart_when_done
> 
> test_node=1
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> echo "There are $num_nodes nodes."
> 
> echo "Shutting down node ${test_node}..."
>@@ -61,7 +61,7 @@ pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error:
> for i in ip disable enable "ban 0" unban listvars ; do
>     try_command_on_node -v 0 ! $CTDB $i -n $test_node
> 
>-    if egrep -q "$pat" <<<"$out" ; then
>+    if egrep -q "$pat" "$outfile" ; then
> 	echo "OK: \"ctdb ${i}\" fails with expected \"disconnected node\" message"
>     else
> 	echo "BAD: \"ctdb ${i}\" does not fail with expected \"disconnected node\" message"
>diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
>index 42ec635fb65..29f3f691dd5 100755
>--- a/ctdb/tests/simple/27_ctdb_detach.sh
>+++ b/ctdb/tests/simple/27_ctdb_detach.sh
>@@ -35,9 +35,8 @@ ctdb_restart_when_done
> 
> ######################################################################
> 
>-try_command_on_node 0 "$CTDB listnodes -X"
>-listnodes_output="$out"
>-numnodes=$(wc -l <<<"$listnodes_output")
>+try_command_on_node 0 "$CTDB listnodes -X | wc -l"
>+numnodes="$out"
> 
> ######################################################################
> 
>@@ -46,12 +45,12 @@ check_db ()
> {
>     db="$1"
>     try_command_on_node all $CTDB getdbmap
>-    local num_db=$(grep -cF "$db" <<<"$out") || true
>+    local num_db=$(grep -cF "$db" "$outfile") || true
>     if [ $num_db -eq $numnodes ]; then
> 	echo "GOOD: database $db is attached on all nodes"
>     else
> 	echo "BAD: database $db is not attached on all nodes"
>-	echo "$out"
>+	cat "$outfile"
> 	exit 1
>     fi
> }
>@@ -61,12 +60,12 @@ check_no_db ()
> {
>     db="$1"
>     try_command_on_node all $CTDB getdbmap
>-    local num_db=$(grep -cF "$db" <<<"$out") || true
>+    local num_db=$(grep -cF "$db" "$outfile") || true
>     if [ $num_db -eq 0 ]; then
> 	echo "GOOD: database $db is not attached any more"
>     else
> 	echo "BAD: database $db is still attached"
>-	echo "$out"
>+	cat "$outfile"
> 	exit 1
>     fi
> }
>@@ -137,12 +136,12 @@ echo
> echo "Write a key to database"
> try_command_on_node 0 $CTDB writekey $testdb1 foo bar
> try_command_on_node 0 $CTDB catdb $testdb1
>-num_keys=$(echo "$out" | sed -n -e 's/Dumped $[0-9]*$ records/\1/p') || true
>+num_keys=$(sed -n -e 's/Dumped $[0-9]*$ records/\1/p' "$outfile") || true
> if [ -n "$num_keys" -a $num_keys -eq 1 ]; then
>     echo "GOOD: Key added to database"
> else
>     echo "BAD: Key did not get added to database"
>-    echo "$out"
>+    cat "$outfile"
>     exit 1
> fi
> 
>@@ -161,11 +160,11 @@ check_db "$testdb1"
> echo
> echo "Check if the database is empty"
> try_command_on_node 0 $CTDB catdb $testdb1
>-num_keys=$(echo "$out" | sed -n -e 's/Dumped $[0-9]*$ records/\1/p') || true
>+num_keys=$(sed -n -e 's/Dumped $[0-9]*$ records/\1/p' "$outfile") || true
> if [ -n "$num_keys" -a $num_keys -eq 0 ]; then
>     echo "GOOD: Database $testdb1 is empty"
> else
>     echo "BAD: Database $testdb1 is not empty"
>-    echo "$out"
>+    cat "$outfile"
>     exit 1
> fi
>diff --git a/ctdb/tests/simple/35_ctdb_getreclock.sh b/ctdb/tests/simple/35_ctdb_getreclock.sh
>index 5d24e2dd145..304e6c3df12 100755
>--- a/ctdb/tests/simple/35_ctdb_getreclock.sh
>+++ b/ctdb/tests/simple/35_ctdb_getreclock.sh
>@@ -25,7 +25,7 @@ if [ -z "$out" ] ; then
>     exit 0
> fi
> 
>-n=$(echo "$out" | sort -u | wc -l)
>+n=$(sort -u "$outfile" | wc -l)
> if [ "$n" = 1 ] ; then
> 	echo "GOOD: All nodes have the same recovery lock setting"
> else
>diff --git a/ctdb/tests/simple/51_message_ring.sh b/ctdb/tests/simple/51_message_ring.sh
>index cecea6c81bb..9b8403f2a7e 100755
>--- a/ctdb/tests/simple/51_message_ring.sh
>+++ b/ctdb/tests/simple/51_message_ring.sh
>@@ -19,23 +19,21 @@ set -e
> 
> cluster_is_healthy
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> echo "Running message_ring on all $num_nodes nodes."
> try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND message_ring -n $num_nodes
> 
> # Get the last line of output.
>-while read line ; do
>-    prev=$line
>-done <<<"$out"
>+last=$(tail -n 1 "$outfile")
> 
> pat='^(Waiting for cluster|Ring\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec $\+ve=[[:digit:]]+ -ve=[[:digit:]]+$)$'
> sanity_check_output 1 "$pat"
> 
>-# $prev should look like this:
>+# $last should look like this:
> #    Ring[1]: 10670.93 msgs/sec (+ve=53391 -ve=53373)
>-stuff="${prev##Ring\[*\]: }"
>+stuff="${last##Ring\[*\]: }"
> mps="${stuff% msgs/sec*}"
> 
> if [ ${mps%.*} -ge 10 ] ; then
>diff --git a/ctdb/tests/simple/52_fetch_ring.sh b/ctdb/tests/simple/52_fetch_ring.sh
>index 078adcbe30f..8f2ac3bb451 100755
>--- a/ctdb/tests/simple/52_fetch_ring.sh
>+++ b/ctdb/tests/simple/52_fetch_ring.sh
>@@ -19,8 +19,8 @@ set -e
> 
> cluster_is_healthy
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> echo "Running fetch_ring on all $num_nodes nodes."
> try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND fetch_ring -n $num_nodes
>@@ -29,13 +29,11 @@ pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)?
> sanity_check_output 1 "$pat"
> 
> # Get the last line of output.
>-while read line ; do
>-    prev=$line
>-done <<<"$out"
>+last=$(tail -n 1 "$outfile")
> 
>-# $prev should look like this:
>+# $last should look like this:
> #    Fetch[1]: 10670.93 msgs/sec
>-stuff="${prev##*Fetch\[*\]: }"
>+stuff="${last##*Fetch\[*\]: }"
> mps="${stuff% msgs/sec*}"
> 
> if [ ${mps%.*} -ge 10 ] ; then
>diff --git a/ctdb/tests/simple/53_transaction_loop.sh b/ctdb/tests/simple/53_transaction_loop.sh
>index 8291a8d55b1..d1db2a9a32b 100755
>--- a/ctdb/tests/simple/53_transaction_loop.sh
>+++ b/ctdb/tests/simple/53_transaction_loop.sh
>@@ -24,8 +24,8 @@ TESTDB="persistent_trans.tdb"
> try_command_on_node 0 "$CTDB attach $TESTDB persistent"
> try_command_on_node 0 "$CTDB wipedb $TESTDB"
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
>     CTDB_TEST_TIMELIMIT=30
>diff --git a/ctdb/tests/simple/54_transaction_loop_recovery.sh b/ctdb/tests/simple/54_transaction_loop_recovery.sh
>index a5d94ffbf89..cb2c1033acd 100755
>--- a/ctdb/tests/simple/54_transaction_loop_recovery.sh
>+++ b/ctdb/tests/simple/54_transaction_loop_recovery.sh
>@@ -45,8 +45,8 @@ TESTDB="persistent_trans.tdb"
> try_command_on_node 0 "$CTDB attach $TESTDB persistent"
> try_command_on_node 0 "$CTDB wipedb $TESTDB"
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
>     CTDB_TEST_TIMELIMIT=30
>diff --git a/ctdb/tests/simple/55_ctdb_ptrans.sh b/ctdb/tests/simple/55_ctdb_ptrans.sh
>index ba92899a613..ff007816d05 100755
>--- a/ctdb/tests/simple/55_ctdb_ptrans.sh
>+++ b/ctdb/tests/simple/55_ctdb_ptrans.sh
>@@ -51,11 +51,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
> 
> try_command_on_node 0 $CTDB catdb "$TESTDB"
> 
>-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
>+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
> 
> if [ $n -ne 3 ] ; then
>     echo "BAD: expected 3 keys in..."
>-    echo "$out"
>+    cat "$outfile"
>     exit 1
> else
>     echo "GOOD: 3 records were inserted"
>@@ -75,11 +75,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
> 
> try_command_on_node 0 $CTDB catdb "$TESTDB"
> 
>-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
>+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
> 
> if [ $n -ne 3 ] ; then
>     echo "BAD: expected 3 keys in..."
>-    echo "$out"
>+    cat "$outfile"
>     exit 1
> else
>     echo "GOOD: 3 records found"
>@@ -116,11 +116,11 @@ echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB"
> 
> try_command_on_node 0 $CTDB catdb "$TESTDB"
> 
>-n=$(echo "$out" | grep -c '^key.*= "key.*"' || true)
>+n=$(grep -c '^key.*= "key.*"' "$outfile" || true)
> 
> if [ $n -ne 0 ] ; then
>     echo "BAD: expected 0 keys in..."
>-    echo "$out"
>+    cat "$outfile"
>     exit 1
> else
>     echo "GOOD: 0 records found"
>diff --git a/ctdb/tests/simple/56_replicated_transaction_recovery.sh b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
>index 61827145bc2..980173a35fd 100755
>--- a/ctdb/tests/simple/56_replicated_transaction_recovery.sh
>+++ b/ctdb/tests/simple/56_replicated_transaction_recovery.sh
>@@ -46,8 +46,8 @@ TESTDB="replicated_trans.tdb"
> try_command_on_node 0 "$CTDB attach $TESTDB replicated"
> try_command_on_node 0 "$CTDB wipedb $TESTDB"
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then
>     CTDB_TEST_TIMELIMIT=30
>diff --git a/ctdb/tests/simple/58_ctdb_restoredb.sh b/ctdb/tests/simple/58_ctdb_restoredb.sh
>index 31b81a6dcb3..3a9d4d24557 100755
>--- a/ctdb/tests/simple/58_ctdb_restoredb.sh
>+++ b/ctdb/tests/simple/58_ctdb_restoredb.sh
>@@ -36,10 +36,10 @@ set -e
> cluster_is_healthy
> 
> try_command_on_node 0 $CTDB status
>-generation=$(echo "$out" | sed -n -e 's/^Generation:$[0-9]*$/\1/p')
>+generation=$(sed -n -e 's/^Generation:$[0-9]*$/\1/p' "$outfile")
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> # 2.
> test_db="restoredb_test.tdb"
>@@ -105,7 +105,7 @@ fi
> wait_until_ready
> 
> try_command_on_node 0 $CTDB status
>-new_generation=$(echo "$out" | sed -n -e 's/^Generation:$[0-9]*$/\1/p')
>+new_generation=$(sed -n -e 's/^Generation:$[0-9]*$/\1/p' "$outfile")
> 
> echo "Old generation = $generation"
> echo "New generation = $new_generation"
>diff --git a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
>index 95e79fdd491..fa8a0dd0bbf 100755
>--- a/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
>+++ b/ctdb/tests/simple/69_recovery_resurrect_deleted.sh
>@@ -23,9 +23,9 @@ testdb="rec_test.tdb"
> echo "Getting list of nodes..."
> try_command_on_node -v any "onnode -pq all ctdb pnn | grep '^[0-9][0-9]*$'"
> 
>-first=$(echo "$out" | sed -n -e '1p')
>-second=$(echo "$out" | sed -n -e '2p')
>-notfirst=$(echo "$out" | tail -n +2)
>+first=$(sed -n -e '1p' "$outfile")
>+second=$(sed -n -e '2p' "$outfile")
>+notfirst=$(tail -n +2 "$outfile")
> 
> echo "Create/wipe test database ${testdb}"
> try_command_on_node $first $CTDB attach "$testdb"
>@@ -49,7 +49,7 @@ database_has_zero_records ()
> 	local n
> 	for n in $notfirst ; do
> 		try_command_on_node $n $CTDB cattdb "$testdb"
>-		if echo "$out" | grep -q '^key(' ; then
>+		if grep -q '^key(' "$outfile" ; then
> 			return 1
> 		fi
> 	done
>@@ -76,7 +76,7 @@ wait_until_node_has_status $first notstopped
> echo "Get database contents"
> try_command_on_node -v $first $CTDB catdb "$testdb"
> 
>-if echo "$out" | grep -q '^key(' ; then
>+if grep -q '^key(' "$outfile" ; then
> 	echo "BAD: Deleted record has been resurrected"
> 	exit 1
> fi
>diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
>index 255d58d2156..0099d08359d 100755
>--- a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
>+++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh
>@@ -38,8 +38,8 @@ cluster_is_healthy
> # Reset configuration
> ctdb_restart_when_done
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> add_record_per_node ()
> {
>diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh
>index 1919ea70a96..e5e3182f68e 100755
>--- a/ctdb/tests/simple/71_ctdb_wipedb.sh
>+++ b/ctdb/tests/simple/71_ctdb_wipedb.sh
>@@ -32,8 +32,8 @@ set -e
> 
> cluster_is_healthy
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> # 2.
> test_db="persistent_test.tdb"
>diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh
>index 8f028d0d0f1..5e7edabc365 100755
>--- a/ctdb/tests/simple/72_update_record_persistent.sh
>+++ b/ctdb/tests/simple/72_update_record_persistent.sh
>@@ -34,8 +34,8 @@ set -e
> 
> cluster_is_healthy
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> test_db="persistent_test.tdb"
> 
>diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh
>index 48e16dfed8c..c4664ba3bcd 100755
>--- a/ctdb/tests/simple/75_readonly_records_basic.sh
>+++ b/ctdb/tests/simple/75_readonly_records_basic.sh
>@@ -52,12 +52,12 @@ check_no_readonly ()
> {
>     try_command_on_node all $CTDB cattdb $testdb
>     local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS"
>-    local numreadonly=$(grep -c -E "$ro_flags" <<<"$out") || true
>+    local numreadonly=$(grep -c -E "$ro_flags" "$outfile") || true
>     if [ $numreadonly -eq 0 ] ; then
> 	echo "GOOD: no read-only delegations"
>     else
> 	echo "BAD: there are read-only delegations"
>-	echo "$out"
>+	cat "$outfile"
> 	exit 1
>     fi
> }
>@@ -75,36 +75,36 @@ check_readonly ()
>     local count
> 
>     try_command_on_node $dmaster $CTDB cattdb $testdb
>-    count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
>+    count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
>     if [ $count -eq 1 ] ; then
> 	echo "GOOD: dmaster ${dmaster} has read-only delegations"
>     else
> 	echo "BAD: dmaster ${dmaster} has no read-only delegations"
>-	echo "$out"
>+	cat "$outfile"
> 	exit 1
>     fi
>-    count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
>+    count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
>     if [ $count -ne 0 ] ; then
> 	echo "BAD: dmaster ${dmaster} has a read-only copy"
>-	echo "$out"
>+	cat "$outfile"
> 	exit 1
>     fi
> 
>     local o
>     for o in $others ; do
> 	try_command_on_node $o $CTDB cattdb $testdb
>-	count=$(grep -c -E "RO_HAVE_READONLY" <<<"$out") || true
>+	count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true
> 	if [ $count -eq 1 ] ; then
> 	    echo "GOOD: node ${o} has a read-only copy"
> 	else
> 	    echo "BAD: node ${o} has no read-only copy"
>-	    echo "$out"
>+	    cat "$outfile"
> 	    exit 1
> 	fi
>-	count=$(grep -c -E "RO_HAVE_DELEGATIONS" <<<"$out") || true
>+	count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true
> 	if [ $count -ne 0 ] ; then
> 	    echo "BAD: other node ${o} has read-only delegations"
>-	    echo "$out"
>+	    cat "$outfile"
> 	    exit 1
> 	fi
>     done
>@@ -114,7 +114,7 @@ check_readonly ()
> 
> echo "Get list of nodes..."
> try_command_on_node any $CTDB -X listnodes
>-all_nodes=$(awk -F'|' '{print $2}' <<<"$out")
>+all_nodes=$(awk -F'|' '{print $2}' "$outfile")
> 
> ######################################################################
> 
>@@ -141,7 +141,7 @@ try_command_on_node all $CTDB setdbreadonly $testdb
> 
> # Database should be tagged as READONLY
> try_command_on_node 0 $CTDB getdbmap
>-db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' <<<"$out")
>+db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' "$outfile")
> if grep -q "READONLY" <<<"$db_details" ; then
>     echo "GOOD: read-only record support is enabled"
> else
>diff --git a/ctdb/tests/simple/77_ctdb_db_recovery.sh b/ctdb/tests/simple/77_ctdb_db_recovery.sh
>index 6dbc0cf639c..00102083124 100755
>--- a/ctdb/tests/simple/77_ctdb_db_recovery.sh
>+++ b/ctdb/tests/simple/77_ctdb_db_recovery.sh
>@@ -96,7 +96,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value1
> # Fetch a record   key=test1
> echo "read key(test1)"
> try_command_on_node $recmaster $CTDB readkey $TESTDB test1
>-echo "$out"
>+cat "$outfile"
> 
> # Do a recovery
> echo "force recovery"
>@@ -111,7 +111,7 @@ try_command_on_node $recmaster $CTDB writekey $TESTDB test1 value2
> # Fetch a record   key=test1
> echo "read key(test1)"
> try_command_on_node $recmaster $CTDB readkey $TESTDB test1
>-echo "$out"
>+cat "$outfile"
> 
> # Do a recovery
> echo "force recovery"
>@@ -122,7 +122,7 @@ wait_until_node_has_status $recmaster recovered
> # Verify record   key=test1
> echo "read key(test1)"
> try_command_on_node $recmaster $CTDB readkey $TESTDB test1
>-echo "$out"
>+cat "$outfile"
> if [ "$out" = "Data: size:6 ptr:[value2]" ]; then
> 	echo "GOOD: Recovery did not corrupt database"
> else
>diff --git a/ctdb/tests/simple/79_volatile_db_traverse.sh b/ctdb/tests/simple/79_volatile_db_traverse.sh
>index 50732cab330..b73a57a20fe 100755
>--- a/ctdb/tests/simple/79_volatile_db_traverse.sh
>+++ b/ctdb/tests/simple/79_volatile_db_traverse.sh
>@@ -78,7 +78,7 @@ try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1"
> echo "do traverse on node 0"
> try_command_on_node -v 0 $CTDB catdb "$TESTDB"
> 
>-num=$(echo "$out" | sed -n -e 's|^Dumped $.*$ records$|\1|p')
>+num=$(sed -n -e 's|^Dumped $.*$ records$|\1|p' "$outfile")
> if [ "$num" = 1 ] ; then
> 	echo "OK: There was 1 record"
> else
>@@ -86,7 +86,7 @@ else
> 	exit 1
> fi
> 
>-if echo "$out" | grep -q "^data(4) = \"bar1\"\$" ; then
>+if grep -q "^data(4) = \"bar1\"\$" "$outfile" ; then
> 	echo "OK: Data from node 1 was returned"
> else
> 	echo "BAD: Data from node 1 was not returned"
>diff --git a/ctdb/tests/simple/80_ctdb_traverse.sh b/ctdb/tests/simple/80_ctdb_traverse.sh
>index 334c684d17c..6470b179029 100755
>--- a/ctdb/tests/simple/80_ctdb_traverse.sh
>+++ b/ctdb/tests/simple/80_ctdb_traverse.sh
>@@ -61,7 +61,7 @@ done
> echo "Start a traverse and collect records"
> try_command_on_node 0 $CTDB catdb $TESTDB
> 
>-num_read=$(echo "$out" | tail -n 1 | cut -d\  -f2)
>+num_read=$(tail -n 1 "$outfile" | cut -d\  -f2)
> if [ $num_read -eq $num_records ]; then
> 	echo "GOOD: All $num_records records retrieved"
> 	status=0
>diff --git a/ctdb/tests/simple/81_tunnel_ring.sh b/ctdb/tests/simple/81_tunnel_ring.sh
>index 31ede40bafd..10c6d2b9a6c 100755
>--- a/ctdb/tests/simple/81_tunnel_ring.sh
>+++ b/ctdb/tests/simple/81_tunnel_ring.sh
>@@ -19,24 +19,22 @@ set -e
> 
> cluster_is_healthy
> 
>-try_command_on_node 0 "$CTDB listnodes"
>-num_nodes=$(echo "$out" | wc -l)
>+try_command_on_node 0 "$CTDB listnodes | wc -l"
>+num_nodes="$out"
> 
> echo "Running tunnel_test on all $num_nodes nodes."
> try_command_on_node -v -p all $CTDB_TEST_WRAPPER $VALGRIND \
> 	tunnel_test -t 30 -n $num_nodes
> 
> # Get the last line of output.
>-while read line ; do
>-    prev=$line
>-done <<<"$out"
>+last=$(tail -n 1 "$outfile")
> 
> pat='^(Waiting for cluster|pnn\[[[:digit:]]+\] [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
> sanity_check_output 1 "$pat"
> 
>-# $prev should look like this:
>+# $last should look like this:
> #    pnn[2] count=85400
>-stuff="${prev##pnn\[*\] }"
>+stuff="${last##pnn\[*\] }"
> mps="${stuff% msgs/sec}"
> 
> if [ ${mps%.*} -ge 10 ] ; then
>diff --git a/ctdb/tests/simple/90_debug_hung_script.sh b/ctdb/tests/simple/90_debug_hung_script.sh
>index 8b8e22b3239..da0535fe7fc 100755
>--- a/ctdb/tests/simple/90_debug_hung_script.sh
>+++ b/ctdb/tests/simple/90_debug_hung_script.sh
>@@ -57,11 +57,9 @@ wait_for_monitor_event $test_node
> 
> echo "Waiting for debugging output to appear..."
> # Use test -s because the file is created above using mktemp
>-wait_until 60 onnode $test_node test -s "$debug_output"
>+wait_until 60 test -s "$debug_output"
> 
> echo "Checking output of hung script debugging..."
>-try_command_on_node -v $test_node cat "$debug_output"
>-hung_script_output="$out"
> 
> # Can we actually read kernel stacks
> if try_command_on_node $test_node "cat /proc/$$/stack >/dev/null 2>&1" ; then
>@@ -75,7 +73,7 @@ fi
> 
> while IFS="" read pattern ; do
>     [ -n "$pattern" ] || continue
>-    if grep -- "^${pattern}\$" <<<"$hung_script_output" >/dev/null ; then
>+    if grep -q -- "^${pattern}\$" "$debug_output" ; then
> 	printf 'GOOD: output contains "%s"\n' "$pattern"
>     else
> 	printf 'BAD: output does not contain "%s"\n' "$pattern"
>-- 
>2.20.1
>
>
>From fbe6507fba037816779f9fa7919b068fb3c786f2 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Sat, 27 Apr 2019 14:54:09 +1000
>Subject: [PATCH 4/9] ctdb-tests: Wait to allow database attach/detach to take
> effect
>
>Sometimes the detach test fails:
>
>  Check detaching single test database detach_test1.tdb
>  BAD: database detach_test1.tdb is still attached
>  Number of databases:4
>  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.0/db/volatile/detach_test4.tdb.0
>  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.0/db/volatile/detach_test3.tdb.0
>  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.0/db/volatile/detach_test2.tdb.0
>  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.0/db/volatile/detach_test1.tdb.0
>  Number of databases:3
>  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.1/db/volatile/detach_test4.tdb.1
>  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.1/db/volatile/detach_test3.tdb.1
>  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.1/db/volatile/detach_test2.tdb.1
>  Number of databases:4
>  dbid:0x5ae995ee name:detach_test4.tdb path:tests/var/simple/node.2/db/volatile/detach_test4.tdb.2
>  dbid:0xd84cc13c name:detach_test3.tdb path:tests/var/simple/node.2/db/volatile/detach_test3.tdb.2
>  dbid:0x8e8e8cef name:detach_test2.tdb path:tests/var/simple/node.2/db/volatile/detach_test2.tdb.2
>  dbid:0xc62491f4 name:detach_test1.tdb path:tests/var/simple/node.2/db/volatile/detach_test1.tdb.2
>  *** TEST COMPLETED (RC=1) AT 2019-04-27 03:35:40, CLEANING UP...
>
>When issued from a client, the detach control re-broadcasts itself
>asynchronously to all nodes and then returns success.  The controls to
>some nodes to do the actual detach may still be in flight when success
>is returned to the client.  Therefore, the test should wait for a few
>seconds to allow the asynchronous controls to complete.
>
>The same is true for the attach control, so workaround the problem in
>the attach test too.
>
>An alternative is to make the attach and detach controls synchronous
>by avoiding the broadcast and waiting for the results of the
>individual controls sent to the nodes.  However, a simple
>implementation would involve adding new nested event loops.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit 3cb53a7a05409925024d6a67bcfaeb962d896e0b)
>---
> ctdb/tests/simple/21_ctdb_attach.sh | 44 ++++++++++++++--------
> ctdb/tests/simple/27_ctdb_detach.sh | 58 +++++++++++++++++++----------
> 2 files changed, 66 insertions(+), 36 deletions(-)
>
>diff --git a/ctdb/tests/simple/21_ctdb_attach.sh b/ctdb/tests/simple/21_ctdb_attach.sh
>index d9dc697d934..9bd85759475 100755
>--- a/ctdb/tests/simple/21_ctdb_attach.sh
>+++ b/ctdb/tests/simple/21_ctdb_attach.sh
>@@ -45,27 +45,39 @@ lastnode=$(( numnodes - 1 ))
> 
> ######################################################################
> 
>-# Confirm that the database is attached
>+# Confirm that the database is attached with appropriate flags
>+check_db_once ()
>+{
>+	local pnn="$1"
>+	local db="$2"
>+
>+	try_command_on_node "$pnn" $CTDB getdbmap
>+	if grep -qF "name:${db}" "$outfile" >/dev/null ; then
>+		return 0
>+	else
>+		return 1
>+	fi
>+}
>+
> check_db ()
> {
>-    pnn="$1"
>-    db="$2"
>-    flag="$3"
>-    try_command_on_node $pnn "$CTDB getdbmap | grep $db"
>-    if [ -z "$out" ] ; then
>-	echo "BAD: database $db is not attached on node $node"
>-	echo "$out"
>-	exit 1
>-    else
>-	local flags=$(awk '{print $4}' "$outfile") || true
>+	local pnn="$1"
>+	local db="$2"
>+	local flag="$3"
>+
>+	local flags
>+
>+	echo "Waiting until database ${db} is attached on node ${pnn}"
>+	wait_until 10 check_db_once "$pnn" "$db"
>+
>+	flags=$(awk -v db="$db" '$2 == "name:" db {print $4}' "$outfile")
> 	if [ "$flags" = "$flag" ]; then
>-	    echo "GOOD: database $db is attached on node $node with flag $flag"
>+		echo "GOOD: db ${db} attached on node ${pnn} with flag $flag"
> 	else
>-	    echo "BAD: database $db is attached on node $node with wrong flag"
>-	    echo "$out"
>-	    exit 1
>+		echo "BAD: db ${db} attached on node ${pnn} with wrong flag"
>+		cat "$outfile"
>+		exit 1
> 	fi
>-    fi
> }
> 
> ######################################################################
>diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh
>index 29f3f691dd5..ee0207ae909 100755
>--- a/ctdb/tests/simple/27_ctdb_detach.sh
>+++ b/ctdb/tests/simple/27_ctdb_detach.sh
>@@ -41,33 +41,51 @@ numnodes="$out"
> ######################################################################
> 
> # Confirm that the database is attached
>+check_db_once ()
>+{
>+	local db="$1"
>+
>+	local num_db
>+
>+	try_command_on_node all "$CTDB getdbmap"
>+	num_db=$(grep -cF "name:${db}" "$outfile") || true
>+	if [ "$num_db" -eq "$numnodes" ]; then
>+		return 0
>+	else
>+		return 1
>+	fi
>+}
>+
> check_db ()
> {
>-    db="$1"
>-    try_command_on_node all $CTDB getdbmap
>-    local num_db=$(grep -cF "$db" "$outfile") || true
>-    if [ $num_db -eq $numnodes ]; then
>-	echo "GOOD: database $db is attached on all nodes"
>-    else
>-	echo "BAD: database $db is not attached on all nodes"
>-	cat "$outfile"
>-	exit 1
>-    fi
>+	local db="$1"
>+
>+	echo "Waiting until database ${db} is attached on all nodes"
>+	wait_until 10 check_db_once "$db"
> }
> 
> # Confirm that no nodes have databases attached
>+check_no_db_once ()
>+{
>+	local db="$1"
>+
>+	local num_db
>+
>+	try_command_on_node all "$CTDB getdbmap"
>+	num_db=$(grep -cF "name:${db}" "$outfile") || true
>+	if [ "$num_db" -eq 0 ]; then
>+		return 0
>+	else
>+		return 1
>+	fi
>+}
>+
> check_no_db ()
> {
>-    db="$1"
>-    try_command_on_node all $CTDB getdbmap
>-    local num_db=$(grep -cF "$db" "$outfile") || true
>-    if [ $num_db -eq 0 ]; then
>-	echo "GOOD: database $db is not attached any more"
>-    else
>-	echo "BAD: database $db is still attached"
>-	cat "$outfile"
>-	exit 1
>-    fi
>+	local db="$1"
>+
>+	echo "Waiting until database ${db} is detached on all nodes"
>+	wait_until 10 check_no_db_once "$db"
> }
> 
> ######################################################################
>-- 
>2.20.1
>
>
>From 4ad4a8b8cdba6742718042d092195c7fd4510e61 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Wed, 1 May 2019 15:10:28 +1000
>Subject: [PATCH 5/9] ctdb-tests: Fix usage message
>
>Since commit 0e9ead8f28fced3ebfa888786a1dc5bb59e734a3 daemons have
>been shut down after each test, so this option no longer has anything
>to do with killing daemons.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit a2ab6485e027ebb13871c7d83b7626ac5c9b98c0)
>---
> ctdb/tests/run_tests.sh | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh
>index 8eead527b33..3c7e794d251 100755
>--- a/ctdb/tests/run_tests.sh
>+++ b/ctdb/tests/run_tests.sh
>@@ -7,7 +7,7 @@ Usage: $0 [OPTIONS] [TESTS]
> Options:
>   -A		Use "cat -A" to print test output (only some tests)
>   -c		Run integration tests on a cluster
>-  -C		Clean up - kill daemons and remove TEST_VAR_DIR when done
>+  -C		Remove TEST_VAR_DIR when done
>   -d		Print descriptions of tests instead of filenames (dodgy!)
>   -D		Show diff between failed/expected test output (some tests only)
>   -e		Exit on the first test failure
>-- 
>2.20.1
>
>
>From 7203a346d62c6e598463686e3a07e33d00d67dac Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Wed, 1 May 2019 15:17:14 +1000
>Subject: [PATCH 6/9] ctdb-tests: Don't clean up test var directory in autotest
> target
>
>If the directory is always cleaned up then it is not possible to look
>at daemon logs to debug test failures.
>
>This target is only really used by autobuild.py, which (optionally)
>cleans up the parent directory anyway.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>
>Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
>Autobuild-Date(master): Tue May  7 06:56:01 UTC 2019 on sn-devel-184
>
>(cherry picked from commit 5a9e338330fe136908a3a17a5df81c054c5cc5b0)
>---
> ctdb/wscript | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/ctdb/wscript b/ctdb/wscript
>index 323da8cf12a..cbb8bd0fd19 100644
>--- a/ctdb/wscript
>+++ b/ctdb/wscript
>@@ -1117,7 +1117,7 @@ def test(ctx):
> 
> def autotest(ctx):
>     env = samba_utils.LOAD_ENVIRONMENT()
>-    cmd = 'tests/run_tests.sh -e -S %s -C' % env.SOCKET_WRAPPER_SO_PATH
>+    cmd = 'tests/run_tests.sh -e -S %s' % env.SOCKET_WRAPPER_SO_PATH
>     ret = samba_utils.RUN_COMMAND(cmd)
>     if ret != 0:
>         print('autotest exited with exit status %d' % ret)
>-- 
>2.20.1
>
>
>From 6fdeb29db1eff042974ab771004775296dcf482e Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Mon, 13 May 2019 17:40:15 +1000
>Subject: [PATCH 7/9] ctdb-tests: Capture output in $out on failure as well
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit cf00db40355b49443263187f9d97934f91287e51)
>---
> ctdb/tests/scripts/integration.bash | 8 +++++---
> 1 file changed, 5 insertions(+), 3 deletions(-)
>
>diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash
>index 0fbd39daec8..e0aa312192b 100644
>--- a/ctdb/tests/scripts/integration.bash
>+++ b/ctdb/tests/scripts/integration.bash
>@@ -109,7 +109,11 @@ try_command_on_node ()
> 
>     local cmd="$*"
> 
>-    if ! onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 ; then
>+    local status=0
>+    onnode -q $onnode_opts "$nodespec" "$cmd" >"$outfile" 2>&1 || status=$?
>+    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
>+
>+    if [ $status -ne 0 ] ; then
> 	echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\""
> 	cat "$outfile"
> 	return 1
>@@ -119,8 +123,6 @@ try_command_on_node ()
> 	echo "Output of \"$cmd\":"
> 	cat "$outfile"
>     fi
>-
>-    out=$(dd if="$outfile" bs=1k count=1 2>/dev/null)
> }
> 
> sanity_check_output ()
>-- 
>2.20.1
>
>
>From 6dbd56440c125caea3bfe53f8b06105ca21665c6 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Tue, 7 May 2019 15:29:19 +1000
>Subject: [PATCH 8/9] ctdb-tests: Make ctdb reloadips tests more reliable
>
>ctdb reloadips will fail if it can't disable takover runs.  The most
>likely reason for this is that there is already a takeover run in
>progress.  We can't predict when this will happen, so retry if this
>occurs.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit 8be4ee1a28d5c037955832b6f827d40f28f02796)
>---
> ctdb/tests/complex/18_ctdb_reloadips.sh | 35 ++++++++++++++++++++++---
> ctdb/tests/simple/18_ctdb_reloadips.sh  | 33 ++++++++++++++++++++---
> 2 files changed, 61 insertions(+), 7 deletions(-)
>
>diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh
>index 1dded769741..d6efc6bf631 100755
>--- a/ctdb/tests/complex/18_ctdb_reloadips.sh
>+++ b/ctdb/tests/complex/18_ctdb_reloadips.sh
>@@ -187,6 +187,33 @@ check_ips ()
>     fi
> }
> 
>+# ctdb reloadips will fail if it can't disable takover runs.  The most
>+# likely reason for this is that there is already a takeover run in
>+# progress.  We can't predict when this will happen, so retry if this
>+# occurs.
>+do_ctdb_reloadips ()
>+{
>+	local retry_max=10
>+	local retry_count=0
>+	while : ; do
>+		if try_command_on_node "$test_node" "$CTDB reloadips" ; then
>+			return 0
>+		fi
>+
>+		if [ "$out" != "Failed to disable takeover runs" ] ; then
>+			return 1
>+		fi
>+
>+		if [ $retry_count -ge $retry_max ] ; then
>+			return 1
>+		fi
>+
>+		retry_count=$((retry_count + 1))
>+		echo "Retrying..."
>+		sleep_for 1
>+	done
>+}
>+
> ####################
> 
> new_ip_max=100
>@@ -196,7 +223,7 @@ new_ip_max=100
> add_ips_to_original_config \
>     $test_node "$addresses" "$iface" "$prefix" 1 $new_ip_max
> 
>-try_command_on_node $test_node "$CTDB reloadips"
>+do_ctdb_reloadips
> 
> check_ips $test_node "$iface" "$prefix" 1 $new_ip_max
> 
>@@ -210,7 +237,7 @@ echo "Using 'ctdb reloadips' to remove the 1st address just added..."
> add_ips_to_original_config \
>     $test_node "$addresses" "$iface" "$prefix" 2 $new_ip_max
> 
>-try_command_on_node $test_node "$CTDB reloadips"
>+do_ctdb_reloadips
> 
> check_ips $test_node "$iface" "$prefix" 2 $new_ip_max
> 
>@@ -225,7 +252,7 @@ echo "Updating to include only about 1/2 of the new IPs..."
> add_ips_to_original_config \
>     $test_node "$addresses" "$iface" "$prefix" $start $new_ip_max
> 
>-try_command_on_node $test_node "$CTDB reloadips"
>+do_ctdb_reloadips
> 
> check_ips $test_node "$iface" "$prefix" $start $new_ip_max
> 
>@@ -237,6 +264,6 @@ try_command_on_node any $CTDB sync
> echo "Restoring original IP configuration..."
> restore_public_addresses
> 
>-try_command_on_node $test_node "$CTDB reloadips"
>+do_ctdb_reloadips
> 
> check_ips $test_node "$iface" "$prefix" 0
>diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
>index 95b819fd82b..6c55c676ca5 100755
>--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
>+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
>@@ -45,9 +45,36 @@ restore_public_addresses ()
> }
> ctdb_test_exit_hook_add restore_public_addresses
> 
>+# ctdb reloadips will fail if it can't disable takover runs.  The most
>+# likely reason for this is that there is already a takeover run in
>+# progress.  We can't predict when this will happen, so retry if this
>+# occurs.
>+do_ctdb_reloadips ()
>+{
>+	local retry_max=10
>+	local retry_count=0
>+	while : ; do
>+		if try_command_on_node any "$CTDB reloadips all" ; then
>+			return 0
>+		fi
>+
>+		if [ "$out" != "Failed to disable takeover runs" ] ; then
>+			return 1
>+		fi
>+
>+		if [ $retry_count -ge $retry_max ] ; then
>+			return 1
>+		fi
>+
>+		retry_count=$((retry_count + 1))
>+		echo "Retrying..."
>+		sleep_for 1
>+	done
>+}
>+
> try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
> 
>-try_command_on_node any $CTDB reloadips all
>+do_ctdb_reloadips
> 
> echo "Getting list of public IPs on node $test_node"
> try_command_on_node $test_node "$CTDB ip | tail -n +2"
>@@ -67,7 +94,7 @@ try_command_on_node any $CTDB sync
> echo "Restoring addresses"
> restore_public_addresses
> 
>-try_command_on_node any $CTDB reloadips all
>+do_ctdb_reloadips
> 
> echo "Getting list of public IPs on node $test_node"
> try_command_on_node $test_node "$CTDB ip | tail -n +2"
>@@ -88,7 +115,7 @@ echo "Removing IP $test_ip from node $test_node"
> 
> try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
> 
>-try_command_on_node any $CTDB reloadips all
>+do_ctdb_reloadips
> 
> try_command_on_node $test_node $CTDB ip
> 
>-- 
>2.20.1
>
>
>From e6436f94bdd8fa3570034e9674bb235778340334 Mon Sep 17 00:00:00 2001
>From: Martin Schwenke <martin@meltin.net>
>Date: Tue, 7 May 2019 15:42:49 +1000
>Subject: [PATCH 9/9] ctdb-tests: Fix logic error in simple ctdb reloadips test
>
>There is a chance that restoring IP addresses to the test node will
>result in different IP addresses being assigned to that node.
>Removing a single IP address may then fail (or be a no-op) if it is
>done after the restore.
>
>So, swap the single IP address removal to happen first, then restore,
>then remove all IP addresses.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=13924
>
>Signed-off-by: Martin Schwenke <martin@meltin.net>
>Reviewed-by: Amitay Isaacs <amitay@gmail.com>
>(cherry picked from commit dc89db8ca6aadd4a9f7e8a85843c53709d04587c)
>---
> ctdb/tests/simple/18_ctdb_reloadips.sh | 37 ++++++++++++++------------
> 1 file changed, 20 insertions(+), 17 deletions(-)
>
>diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh
>index 6c55c676ca5..7a18f66b9fa 100755
>--- a/ctdb/tests/simple/18_ctdb_reloadips.sh
>+++ b/ctdb/tests/simple/18_ctdb_reloadips.sh
>@@ -32,8 +32,6 @@ ctdb_restart_when_done
> 
> select_test_node_and_ips
> 
>-echo "Emptying public addresses file on $test_node"
>-
> try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show
> addresses="${out}/public_addresses"
> echo "Public addresses file on node $test_node is \"$addresses\""
>@@ -72,25 +70,31 @@ do_ctdb_reloadips ()
> 	done
> }
> 
>-try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
>+
>+echo "Removing IP $test_ip from node $test_node"
>+
>+try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
> 
> do_ctdb_reloadips
> 
>-echo "Getting list of public IPs on node $test_node"
>-try_command_on_node $test_node "$CTDB ip | tail -n +2"
>+try_command_on_node $test_node $CTDB ip
> 
>-if [ -n "$out" ] ; then
>+if grep "^${test_ip} " <<<"$out" ; then
>     cat <<EOF
>-BAD: node $test_node still has ips:
>+BAD: node $test_node can still host IP $test_ip:
> $out
> EOF
>     exit 1
> fi
> 
>-echo "GOOD: no IPs left on node $test_node"
>+cat <<EOF
>+GOOD: node $test_node is no longer hosting IP $test_ip:
>+$out
>+EOF
> 
> try_command_on_node any $CTDB sync
> 
>+
> echo "Restoring addresses"
> restore_public_addresses
> 
>@@ -111,23 +115,22 @@ EOF
> 
> try_command_on_node any $CTDB sync
> 
>-echo "Removing IP $test_ip from node $test_node"
> 
>-try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
>+echo "Emptying public addresses file on $test_node"
>+
>+try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
> 
> do_ctdb_reloadips
> 
>-try_command_on_node $test_node $CTDB ip
>+echo "Getting list of public IPs on node $test_node"
>+try_command_on_node $test_node "$CTDB ip | tail -n +2"
> 
>-if grep "^${test_ip} " <<<"$out" ; then
>+if [ -n "$out" ] ; then
>     cat <<EOF
>-BAD: node $test_node can still host IP $test_ip:
>+BAD: node $test_node still has ips:
> $out
> EOF
>     exit 1
> fi
> 
>-cat <<EOF
>-GOOD: node $test_node is no longer hosting IP $test_ip:
>-$out
>-EOF
>+echo "GOOD: no IPs left on node $test_node"
>-- 
>2.20.1
>

Flags:

martins: review? (amitay)

Actions: View

Attachments on bug 13924: 15155 | 15156