From 8d11e8676f4d78d72eb759368f82359662a47d4d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 27 Feb 2015 15:13:23 +1100 Subject: [PATCH 01/10] ctdb-tests: Extend ctdb stub to support "ptrans", "pdelete", "catdb" Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit d057ca04a9eec0f2aa3d792da0a4648e3716685a) Conflicts: ctdb/tests/eventscripts/stubs/ctdb --- ctdb/tests/eventscripts/stubs/ctdb | 102 ++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 14 deletions(-) diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb index c444196..382d1d2 100755 --- a/ctdb/tests/eventscripts/stubs/ctdb +++ b/ctdb/tests/eventscripts/stubs/ctdb @@ -66,12 +66,6 @@ ctdb_killtcp () done } -setup_pstore () -{ - pstore_dir="$CTDB_VARDIR/fake-ctdb/pstore/$1" - mkdir -p "$pstore_dir" -} - parse_nodespec () { if [ "$nodespec" = "all" ] ; then @@ -304,6 +298,89 @@ ctdb_natgwlist () ###################################################################### +_t_setup () +{ + _t_dir="$CTDB_VARDIR/fake-ctdb/fake-tdb/$1" + mkdir -p "$_t_dir" +} + +_t_put () +{ + echo "$2" >"${_t_dir}/$1" +} + +_t_get () +{ + cat "${_t_dir}/$1" +} + +_t_del () +{ + rm -f "${_t_dir}/$1" +} + +ctdb_pstore () +{ + _t_setup "$2" + _t_put "$3" "$4" +} + +ctdb_pdelete () +{ + _t_setup "$2" + _t_del "$3" +} + +ctdb_pfetch () +{ + _t_setup "$2" + _t_get "$3" >"$4" 2>/dev/null +} + +ctdb_ptrans () +{ + _t_setup "$2" + + while IFS="" read _line ; do + _k=$(echo "$_line" | sed -n -e 's@^"\([^"]*\)" "[^"]*"$@\1@p') + _v=$(echo "$_line" | sed -e 's@^"[^"]*" "\([^"]*\)"$@\1@') + [ -n "$_k" ] || die "ctdb ptrans: bad line \"${line}\"" + if [ -n "$_v" ] ; then + _t_put "$_k" "$_v" + else + _t_del "$_k" + fi + done +} + +ctdb_catdb () +{ + _t_setup "$2" + + # This will break on keys with spaces but we don't have any of + # those yet. + _count=0 + for _i in "${_t_dir}/"* ; do + [ -r "$_i" ] || continue + _k="${_i##*/}" # basename + _v=$(_t_get "$_k") + _kn=$(echo -n "$_k" | wc -c) + _vn=$(echo -n "$_v" | wc -c) + cat <"$tickles_file" ;; - pstore) - setup_pstore "$2" - cat "$4" >"${pstore_dir}/$3" - ;; - pfetch) - setup_pstore "$2" - cat "${pstore_dir}/$3" >"$4" 2>/dev/null - ;; + pstore) ctdb_pstore "$@" ;; + pdelete) ctdb_pdelete "$@" ;; + pfetch) ctdb_pfetch "$@" ;; + ptrans) ctdb_ptrans "$@" ;; + catdb) ctdb_catdb "$@" ;; ifaces) # Assume -Y. echo "|Name|LinkStatus|References|" -- 2.1.4 From c152b7248f5b0360e7f24dca08a602b8a373b03d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 27 Feb 2015 15:15:18 +1100 Subject: [PATCH 02/10] ctdb-tests: Extend ctdb stub to support "ip" with and without -X Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 2aeb518637af29da03984470d874b94dfb18e34e) --- ctdb/tests/eventscripts/stubs/ctdb | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb index 382d1d2..7745dd9 100755 --- a/ctdb/tests/eventscripts/stubs/ctdb +++ b/ctdb/tests/eventscripts/stubs/ctdb @@ -146,14 +146,18 @@ ctdb_ip () # If nobody has done any IP-fu then generate a layout. [ -f "$FAKE_CTDB_IP_LAYOUT" ] || ip_reallocate - if $verbose ; then - echo ":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:" + _mypnn=$(ctdb_pnn | sed -e 's@PNN:@@') + + if $machine_readable ; then + if $verbose ; then + echo "|Public IP|Node|ActiveInterface|AvailableInterfaces|ConfiguredInterfaces|" + else + echo "|Public IP|Node|" + fi else - echo ":Public IP:Node:" + echo "Public IPs on node ${_mypnn}" fi - _mypnn=$(ctdb_pnn | sed -e 's@PNN:@@') - # Join public addresses file with $FAKE_CTDB_IP_LAYOUT, and # process output line by line... _pa="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" @@ -167,9 +171,17 @@ ctdb_ip () if [ "$_pnn" = "$_mypnn" ]; then _my_iface="$_first_iface" fi - echo "|${_ip}|${_pnn}|${_my_iface}|${_first_iface}|${_ifaces}|" + if $machine_readable ; then + echo "|${_ip}|${_pnn}|${_my_iface}|${_first_iface}|${_ifaces}|" + else + echo "${_ip} node[${_pnn}] active[${_my_iface}] available[${_first_iface}] configured[[${_ifaces}]" + fi else - echo "|${_ip}|${_pnn}|" + if $machine_readable ; then + echo "|${_ip}|${_pnn}|" + else + echo "${_ip} ${_pnn}" + fi fi done } @@ -445,7 +457,7 @@ case "$1" in exit 0 ;; scriptstatus) - $machine_readable || not_implemented "$1, without -Y" + $machine_readable || not_implemented "$1, without -X" [ "$2" != "all" ] || not_implemented "scriptstatus all" # For now just assume everything is good. echo "|Type|Name|Code|Status|Start|End|Error Output...|" -- 2.1.4 From 6d5908427eff52d6befc773c7136f3ab7d2d3d15 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 27 Feb 2015 15:17:30 +1100 Subject: [PATCH 03/10] ctdb-tests: Support testing scripts that change directory Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 9317d82c19a0eb51ff6293d00328a5c36b063a2c) --- ctdb/tests/eventscripts/scripts/local.sh | 4 ++++ ctdb/tests/scripts/common.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index c4e86d9..ebde664 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -9,6 +9,10 @@ EVENTSCRIPTS_PATH="" if [ -d "${TEST_SUBDIR}/stubs" ] ; then EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs" + case "$EVENTSCRIPTS_PATH" in + /*) : ;; + *) EVENTSCRIPTS_PATH="${PWD}/${EVENTSCRIPTS_PATH}" ;; + esac fi export EVENTSCRIPTS_PATH diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 754ea2e..287fb71 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -30,6 +30,10 @@ if [ -f "${_test_dir}/run_tests.sh" ] ; then fi _test_bin_dir="${TEST_BIN_DIR:-${ctdb_dir}/bin}" +case "$_test_bin_dir" in + /*) : ;; + *) _test_bin_dir="${PWD}/${_test_bin_dir}" ;; +esac if [ -d "$_test_bin_dir" ] ; then PATH="${_test_bin_dir}:$PATH" fi -- 2.1.4 From e75188462afe4d9a8d003beba1f27855656916ee Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 27 Feb 2015 15:19:04 +1100 Subject: [PATCH 04/10] ctdb-tests: Extend eventscript unit test infrastructure for other scripts There's so much infrastructure here that it would be a shame not to use it for testing things like statd-callout. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 7e7c24ca7a422f2258962216b0184eda8d49827f) --- ctdb/tests/eventscripts/scripts/local.sh | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index ebde664..8873062 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -973,19 +973,32 @@ define_test () # Remaining format should be NN.service.event.NNN or NN.service.NNN: _num="${_f##*.}" _f="${_f%.*}" + case "$_f" in - *.*.*) + [0-9][0-9].*.*) script="${_f%.*}" event="${_f##*.}" + script_dir="${CTDB_BASE}/events.d" ;; - *.*) + [0-9][0-9].*) script="$_f" unset event + script_dir="${CTDB_BASE}/events.d" + ;; + *.*) + script="${_f%.*}" + event="${_f##*.}" + script_dir="${CTDB_BASE}" ;; *) - die "Internal error - unknown testcase filename format" + script="${_f%.*}" + unset event + script_dir="${CTDB_BASE}" esac + [ -x "${script_dir}/${script}" ] || \ + die "Internal error - unable to find script \"${script_dir}/${script}\"" + printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" } @@ -1010,14 +1023,14 @@ simple_test () _extra_header=$(_extra_header) - echo "Running eventscript \"$script $event${1:+ }$*\"" + echo "Running script \"$script $event${1:+ }$*\"" _shell="" if $TEST_COMMAND_TRACE ; then _shell="sh -x" else _shell="sh" fi - _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1) + _out=$($_shell "${script_dir}/${script}" "$event" "$@" 2>&1) result_check "$_extra_header" } @@ -1116,7 +1129,7 @@ iterate_test () else _shell="sh" fi - _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) + _out=$($_shell "${script_dir}/${script}" "$event" $args 2>&1) _rc=$? _fout=$(echo "$_out" | result_filter) -- 2.1.4 From 44c83baa5e884f6f921736e775520d379aca9133 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 27 Feb 2015 15:20:56 +1100 Subject: [PATCH 05/10] ctdb-tests: Make setup of public addresses more obvious Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit d98c7ba382189161c5b8cbbebbdfbe36f1456572) --- ctdb/tests/eventscripts/scripts/local.sh | 39 +++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index 8873062..111a027 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -275,6 +275,33 @@ ctdb_set_output () eventscripts_test_add_cleanup "rm -f $_out $_rc" } +# For now this creates the same public addresses each time. However, +# it could be made more flexible. +setup_public_addresses () +{ + if [ -f "$CTDB_PUBLIC_ADDRESSES" -a \ + "${CTDB_PUBLIC_ADDRESSES%/*}" = "$EVENTSCRIPTS_TESTS_VAR_DIR" ] ; then + rm "$CTDB_PUBLIC_ADDRESSES" + fi + + export CTDB_PUBLIC_ADDRESSES=$(mktemp \ + --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR" \ + "public-addresses-XXXXXXXX") + + echo "Setting up CTDB_PUBLIC_ADDRESSES=${CTDB_PUBLIC_ADDRESSES}" + cat >"$CTDB_PUBLIC_ADDRESSES" <>"$CTDB_PUBLIC_ADDRESSES" - done - eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" - fi + setup_public_addresses export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb" -- 2.1.4 From d51e6797d7d4df7a9433da5c5668074311b39ca0 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 4 Mar 2015 11:51:20 +1100 Subject: [PATCH 06/10] ctdb-tests: Unit tests for statd-callout With improvements to unit test infrastructure to support. This includes linking the real statd-callout into etc-ctdb/ in place of the placeholder script. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 22602f76bc1ec91e807a8f1cd45ba6fb4c05e622) --- ctdb/tests/eventscripts/etc-ctdb/statd-callout | 6 +- ctdb/tests/eventscripts/scripts/local.sh | 80 +++++++++++++++++++++++--- ctdb/tests/eventscripts/statd-callout.001.sh | 14 +++++ ctdb/tests/eventscripts/statd-callout.002.sh | 15 +++++ ctdb/tests/eventscripts/statd-callout.003.sh | 15 +++++ ctdb/tests/eventscripts/statd-callout.004.sh | 18 ++++++ ctdb/tests/eventscripts/statd-callout.005.sh | 25 ++++++++ ctdb/tests/eventscripts/statd-callout.006.sh | 27 +++++++++ ctdb/tests/eventscripts/stubs/id | 3 + ctdb/tests/eventscripts/stubs/smnotify | 3 + ctdb/wscript | 3 +- 11 files changed, 196 insertions(+), 13 deletions(-) mode change 100755 => 120000 ctdb/tests/eventscripts/etc-ctdb/statd-callout create mode 100755 ctdb/tests/eventscripts/statd-callout.001.sh create mode 100755 ctdb/tests/eventscripts/statd-callout.002.sh create mode 100755 ctdb/tests/eventscripts/statd-callout.003.sh create mode 100755 ctdb/tests/eventscripts/statd-callout.004.sh create mode 100755 ctdb/tests/eventscripts/statd-callout.005.sh create mode 100755 ctdb/tests/eventscripts/statd-callout.006.sh create mode 100755 ctdb/tests/eventscripts/stubs/id create mode 100755 ctdb/tests/eventscripts/stubs/smnotify diff --git a/ctdb/tests/eventscripts/etc-ctdb/statd-callout b/ctdb/tests/eventscripts/etc-ctdb/statd-callout deleted file mode 100755 index 51779bd..0000000 --- a/ctdb/tests/eventscripts/etc-ctdb/statd-callout +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# For now, always succeed. - -exit 0 diff --git a/ctdb/tests/eventscripts/etc-ctdb/statd-callout b/ctdb/tests/eventscripts/etc-ctdb/statd-callout new file mode 120000 index 0000000..dbd3c98 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/statd-callout @@ -0,0 +1 @@ +../../../config/statd-callout \ No newline at end of file diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index 111a027..c00fda1 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -526,6 +526,65 @@ EOF ###################################################################### +ctdb_catdb_format_pairs () +{ + _count=0 + + while read _k _v ; do + _kn=$(echo -n "$_k" | wc -c) + _vn=$(echo -n "$_v" | wc -c) + cat < Date: Thu, 26 Feb 2015 15:34:51 +1100 Subject: [PATCH 07/10] ctdb-scripts: Fix a regression in statd-callout Commit 4638010abb116aed0c180207aaa11475277aecb7 changed from using gensub() to gsub() in awk. However, it didn't halve the number of backslashes in the target strings. This is necessary because backslash is used in gensub() target strings to allow substitution of text matching parenthesised subexpressions. This is not the case with gsub(). So, halve the number of backslashes in the target string where gsub() is used in statd-callout. This is the only target string broken by changes made by the above commit Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 032441d9a2974584cde455e4dbd5cc33fe6a23c2) --- ctdb/config/statd-callout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/config/statd-callout b/ctdb/config/statd-callout index e2a955e..b929425 100755 --- a/ctdb/config/statd-callout +++ b/ctdb/config/statd-callout @@ -146,7 +146,7 @@ case "$1" in # but only for the server-IPs that are hosted on this node. sed_expr=$(ctdb ip | tail -n +2 | awk -v pnn=$pnn 'pnn == $2 { \ - ip = $1; gsub(/\./, "\\\\.", ip); \ + ip = $1; gsub(/\./, "\\.", ip); \ printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }') statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort) -- 2.1.4 From 3d8b581fac1cbf735cd24b13603749f00988ee0e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 13 Feb 2015 20:55:43 +1100 Subject: [PATCH 08/10] ctdb-scripts: Change statd-callout to be more scalable Updating ctdb.tdb on each add-client, del-client and each delete during notify was too ambitious. Persistent transactions do not perform well enough to do this. Revert to having add-client and del-client create touch files. Each monitor event calls "statd-callout update" to convert touch files into ctdb.tdb records. Update testcases to do the "update" and add an extra test. Signed-off-by: Martin Schwenke Pair-programmed-with: Amitay Isaacs Reviewed-by: Amitay Isaacs (cherry picked from commit 500c6e194babe06b6aead7a053a9442c94db6e38) --- ctdb/config/events.d/60.ganesha | 1 + ctdb/config/events.d/60.nfs | 1 + ctdb/config/functions | 10 ++ ctdb/config/statd-callout | 140 ++++++++++++++++----------- ctdb/tests/eventscripts/statd-callout.001.sh | 1 + ctdb/tests/eventscripts/statd-callout.002.sh | 3 +- ctdb/tests/eventscripts/statd-callout.003.sh | 5 +- ctdb/tests/eventscripts/statd-callout.004.sh | 1 + ctdb/tests/eventscripts/statd-callout.005.sh | 2 + ctdb/tests/eventscripts/statd-callout.006.sh | 2 + ctdb/tests/eventscripts/statd-callout.007.sh | 16 +++ 11 files changed, 123 insertions(+), 59 deletions(-) create mode 100755 ctdb/tests/eventscripts/statd-callout.007.sh diff --git a/ctdb/config/events.d/60.ganesha b/ctdb/config/events.d/60.ganesha index df0912d..46edb0a 100755 --- a/ctdb/config/events.d/60.ganesha +++ b/ctdb/config/events.d/60.ganesha @@ -221,6 +221,7 @@ case "$1" in } || exit $? update_tickles 2049 + nfs_update_lock_info # check that statd responds to rpc requests # if statd is not running we try to restart it diff --git a/ctdb/config/events.d/60.nfs b/ctdb/config/events.d/60.nfs index c4e798e..d570cd7 100755 --- a/ctdb/config/events.d/60.nfs +++ b/ctdb/config/events.d/60.nfs @@ -91,6 +91,7 @@ case "$1" in } || exit $? update_tickles 2049 + nfs_update_lock_info nfs_check_rpc_services diff --git a/ctdb/config/functions b/ctdb/config/functions index 02db675..36cb0c1 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -654,6 +654,16 @@ get_tcp_connections_for_ip () {print $4" "$5}' } +################################################################## +# use statd-callout to update NFS lock info +################################################################## +nfs_update_lock_info () +{ + if [ -x "$CTDB_BASE/statd-callout" ] ; then + "$CTDB_BASE/statd-callout" update + fi +} + ######################################################## # start/stop the Ganesha nfs service ######################################################## diff --git a/ctdb/config/statd-callout b/ctdb/config/statd-callout index b929425..4a331ac 100755 --- a/ctdb/config/statd-callout +++ b/ctdb/config/statd-callout @@ -30,7 +30,19 @@ loadconfig nfs nl=" " +ctdb_setup_service_state_dir "statd-callout" + +cd "$service_state_dir" || \ + die "Failed to change directory to \"${service_state_dir}\"" + case "$1" in + # Keep a single file to keep track of the last "add-client" or + # "del-client'. These get pushed to ctdb.tdb during "update", + # which will generally be run once each "monitor" cycle. In this + # way we avoid scalability problems with flood of persistent + # transactions after a "notify" when all the clients re-take their + # locks. + add-client) # statd does not tell us to which IP the client connected so # we must add it to all the IPs that we serve @@ -38,42 +50,47 @@ case "$1" in pnn=$(ctdb xpnn | sed -e 's/.*://') date=$(date '+%s') ctdb ip -X | - tail -n +2 | { - # This all needs to be in the end of the pipe so it - # doesn't get lost - items="" - while IFS="|" read x sip node x ; do - [ "$node" = "$pnn" ] || continue # not us - key="statd-state@${sip}@${cip}" - item="\"${key}\" \"${date}\"" - items="${items}${items:+${nl}}${item}" - done - if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then - die "Failed to add clients" - fi - } + tail -n +2 | + while IFS="|" read x sip node x ; do + [ "$node" = "$pnn" ] || continue # not us + key="statd-state@${sip}@${cip}" + echo "\"${key}\" \"${date}\"" >"$key" + done ;; - del-client) + + del-client) # statd does not tell us from which IP the client disconnected # so we must add it to all the IPs that we serve cip="$2" pnn=$(ctdb xpnn | sed -e 's/.*://') ctdb ip -X | - tail -n +2 | { - # This all needs to be in the end of the pipe so it - # doesn't get lost - items="" - while IFS="|" read x sip node x ; do - [ "$node" = "$pnn" ] || continue # not us - key="statd-state@${sip}@${cip}" - item="\"${key}\" \"\"" - items="${items}${items:+${nl}}${item}" - done - if ! echo "$items" | ctdb ptrans "ctdb.tdb" ; then - die "Failed to delete clients" - fi - } + tail -n +2 | + while IFS="|" read x sip node x ; do + [ "$node" = "$pnn" ] || continue # not us + key="statd-state@${sip}@${cip}" + echo "\"${key}\" \"\"" >"$key" + done ;; + + update) + files=$(echo statd-state@*) + if [ "$files" = "statd-state@*" ] ; then + # No files! + exit 0 + fi + # Filter out lines for any IP addresses that are not currently + # hosted public IP addresses. + pnn=$(ctdb xpnn | sed -e 's/.*://') + ctdb_ips=$(ctdb ip | tail -n +2) + sed_expr=$(echo "$ctdb_ips" | + awk -v pnn=$pnn 'pnn == $2 { \ + ip = $1; gsub(/\./, "\\.", ip); \ + printf "/statd-state@%s@/p\n", ip }') + if cat $files | sed -n "$sed_expr" | ctdb ptrans "ctdb.tdb" ; then + rm $files + fi + ;; + notify) # we must restart the lockmanager (on all nodes) so that we get # a clusterwide grace period (so other clients dont take out @@ -144,7 +161,8 @@ case "$1" in # Construct a sed expression to take catdb output and produce pairs of: # server-IP client-IP # but only for the server-IPs that are hosted on this node. - sed_expr=$(ctdb ip | tail -n +2 | + ctdb_all_ips=$(ctdb ip -n all | tail -n +2) + sed_expr=$(echo "$ctdb_all_ips" | awk -v pnn=$pnn 'pnn == $2 { \ ip = $1; gsub(/\./, "\\.", ip); \ printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }') @@ -152,34 +170,42 @@ case "$1" in statd_state=$(ctdb catdb ctdb.tdb | sed -n "$sed_expr" | sort) [ -n "$statd_state" ] || exit 0 - # The following is dangerous if this script times out before - # all of the smnotify commands are run. Revert to individual - # pdelete commands for now and consider optimising smnotify to - # read all the data from stdin and then run it in the - # background. - # - # Delete all the items from the TDB - #if ! echo "$statd_state" | \ - # awk '{ printf "\"statd-state@%s@%s\" \"\"\n", $1, $2 }') | \ - # ctdb ptrans ctdb.tdb ; then + prev="" + echo "$statd_state" | { + # This all needs to be in the same command group at the + # end of the pipe so it doesn't get lost when the loop + # completes. + items="" + while read sip cip ; do + # Collect item to delete from the DB + key="statd-state@${sip}@${cip}" + item="\"${key}\" \"\"" + items="${items}${items:+${nl}}${item}" - # die "Yikes!" - #fi + # NOTE: Consider optimising smnotify to read all the + # data from stdin and then run it in the background. + + # Reset stateval for each serverip + [ "$sip" = "$prev" ] || stateval="$state_even" + # Send notifies for server shutdown + smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval + smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval + # Send notifies for server startup + stateval=$(($stateval + 1)) + smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval + smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval + done - prev="" - echo "$statd_state" | - while read sip cip ; do - # Delete the entry from the DB - ctdb pdelete ctdb.tdb "statd-state@${sip}@${cip}" - # Reset stateval for each serverip - [ "$sip" = "$prev" ] || stateval="$state_even" - # Send notifies for server shutdown - smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval - smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval - # Send notifies for server startup - stateval=$(($stateval + 1)) - smnotify --client=$cip --ip=$sip --server=$sip --stateval=$stateval - smnotify --client=$cip --ip=$sip --server=$NFS_HOSTNAME --stateval=$stateval - done + echo "$items" | ctdb ptrans "ctdb.tdb" + } + + # Remove any stale touch files (i.e. for IPs not currently + # hosted on this node and created since the last "update"). + # There's nothing else we can do with them at this stage. + echo "$ctdb_all_ips" | + awk -v pnn=$pnn 'pnn != $2 { print $1 }' | + while read sip ; do + rm -f "statd-state@${sip}@"* + done ;; esac diff --git a/ctdb/tests/eventscripts/statd-callout.001.sh b/ctdb/tests/eventscripts/statd-callout.001.sh index 5f7b7e2..29b9fbc 100755 --- a/ctdb/tests/eventscripts/statd-callout.001.sh +++ b/ctdb/tests/eventscripts/statd-callout.001.sh @@ -10,5 +10,6 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" +simple_test_event "update" check_ctdb_tdb_statd_state "192.168.123.45" diff --git a/ctdb/tests/eventscripts/statd-callout.002.sh b/ctdb/tests/eventscripts/statd-callout.002.sh index f8778f7..009da1b 100755 --- a/ctdb/tests/eventscripts/statd-callout.002.sh +++ b/ctdb/tests/eventscripts/statd-callout.002.sh @@ -2,7 +2,7 @@ . "${TEST_SCRIPTS_DIR}/unit.sh" -define_test "2 x add-client" +define_test "2 x add-client, update" setup_ctdb @@ -11,5 +11,6 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" simple_test_event "add-client" "192.168.123.46" +simple_test_event "update" check_ctdb_tdb_statd_state "192.168.123.45" "192.168.123.46" diff --git a/ctdb/tests/eventscripts/statd-callout.003.sh b/ctdb/tests/eventscripts/statd-callout.003.sh index 1319ee4..ed28de6 100755 --- a/ctdb/tests/eventscripts/statd-callout.003.sh +++ b/ctdb/tests/eventscripts/statd-callout.003.sh @@ -2,7 +2,7 @@ . "${TEST_SCRIPTS_DIR}/unit.sh" -define_test "add-client, del-client" +define_test "add-client, update, del-client, update" setup_ctdb @@ -10,6 +10,9 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" +simple_test_event "update" + simple_test_event "del-client" "192.168.123.45" +simple_test_event "update" check_ctdb_tdb_statd_state diff --git a/ctdb/tests/eventscripts/statd-callout.004.sh b/ctdb/tests/eventscripts/statd-callout.004.sh index 5702b85..011ced9 100755 --- a/ctdb/tests/eventscripts/statd-callout.004.sh +++ b/ctdb/tests/eventscripts/statd-callout.004.sh @@ -10,6 +10,7 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" +simple_test_event "update" check_ctdb_tdb_statd_state "192.168.123.45" diff --git a/ctdb/tests/eventscripts/statd-callout.005.sh b/ctdb/tests/eventscripts/statd-callout.005.sh index 65e291d..ceb4445 100755 --- a/ctdb/tests/eventscripts/statd-callout.005.sh +++ b/ctdb/tests/eventscripts/statd-callout.005.sh @@ -10,11 +10,13 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" +simple_test_event "update" FAKE_CTDB_PNN=1 ok_null simple_test_event "add-client" "192.168.123.46" +simple_test_event "update" FAKE_CTDB_PNN=0 diff --git a/ctdb/tests/eventscripts/statd-callout.006.sh b/ctdb/tests/eventscripts/statd-callout.006.sh index df8af88..0db86fe 100755 --- a/ctdb/tests/eventscripts/statd-callout.006.sh +++ b/ctdb/tests/eventscripts/statd-callout.006.sh @@ -10,11 +10,13 @@ FAKE_DATE_OUTPUT="1234565789" ok_null simple_test_event "add-client" "192.168.123.45" +simple_test_event "update" FAKE_CTDB_PNN=1 ok_null simple_test_event "add-client" "192.168.123.46" +simple_test_event "update" FAKE_CTDB_PNN=0 diff --git a/ctdb/tests/eventscripts/statd-callout.007.sh b/ctdb/tests/eventscripts/statd-callout.007.sh new file mode 100755 index 0000000..32339cd --- /dev/null +++ b/ctdb/tests/eventscripts/statd-callout.007.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "add-client, del-client, update" + +setup_ctdb + +FAKE_DATE_OUTPUT="1234565789" + +ok_null +simple_test_event "add-client" "192.168.123.45" +simple_test_event "del-client" "192.168.123.45" +simple_test_event "update" + +check_ctdb_tdb_statd_state -- 2.1.4 From fde45b73e2bd2eb0b361c9154e809d2cafdefc8b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 13 Feb 2015 15:42:20 +1100 Subject: [PATCH 09/10] ctdb-scripts: Remove unused function nfs_statd_update() Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 50ddc2c35643389c2f249c6ad4496ab73a1bfc99) --- ctdb/config/functions | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ctdb/config/functions b/ctdb/config/functions index 36cb0c1..7a3607f 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -815,23 +815,6 @@ startstop_nfslock() { esac } -# Periodically update the statd database -nfs_statd_update () -{ - _update_period="$1" - - _statd_update_trigger="$service_state_dir/update-trigger" - [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger" - - _last_update=$(stat --printf="%Y" "$_statd_update_trigger") - _current_time=$(date +"%s") - if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then - touch "$_statd_update_trigger" - $CTDB_BASE/statd-callout updatelocal & - $CTDB_BASE/statd-callout updateremote & - fi -} - ######################################################## add_ip_to_iface () -- 2.1.4 From 6e73ceb6530dd3bcc10fe5f607ebe4731a8adbf8 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 5 Mar 2015 13:10:32 +1100 Subject: [PATCH 10/10] ctdb-scripts: Add a 'rm' stub so statd-callout tests work correctly statd-callout tries to remove global files from /var/lib/nfs/statd and this causes errors in tests. Add an rm stub that ignores attempts to remove these files but invokes /bin/rm for anything else. Signed-off-by: Amitay Isaacs Pair-programmed-by: Martin Schwenke Reviewed-by: Martin Schwenke (cherry picked from commit 956e51707d7ddcff060352f54d11ff42bdcc51ef) --- ctdb/tests/eventscripts/stubs/rm | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 ctdb/tests/eventscripts/stubs/rm diff --git a/ctdb/tests/eventscripts/stubs/rm b/ctdb/tests/eventscripts/stubs/rm new file mode 100755 index 0000000..64b4d18 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rm @@ -0,0 +1,6 @@ +#!/bin/sh +# Make statd-callout happy +case "$*" in + */var/lib/nfs/statd/sm*) : ;; + *) exec /bin/rm "$@" ;; +esac -- 2.1.4