From c2cf90e6ff3e92420d285bc1bc510c30abacd41f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 19 Nov 2014 16:35:35 +1100 Subject: [PATCH 01/29] ctdb-tools: Produce machine readable output with new function printm() printm() is a printf(3) replacement and must be used to printing any machine readable output. It currently just calls vprintf(3). Later it will change the field delimiter. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit fbacbb9c7868e22c04980af3602bae59dd5fe34d) Conflicts: ctdb/tools/ctdb.c --- ctdb/tools/ctdb.c | 188 +++++++++++++++++++++++++++++------------------------- 1 file changed, 102 insertions(+), 86 deletions(-) diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c index 1ba2be1..a45ff20 100644 --- a/ctdb/tools/ctdb.c +++ b/ctdb/tools/ctdb.c @@ -69,6 +69,20 @@ static int control_version(struct ctdb_context *ctdb, int argc, const char **arg return 0; } +/* Like printf(3) but substitute for separator in format */ +static int printm(const char *format, ...) PRINTF_ATTRIBUTE(1,2); +static int printm(const char *format, ...) +{ + va_list ap; + int ret; + + va_start(ap, format); + ret = vprintf(format, ap); + va_end(ap); + + return ret; +} + #define CTDB_NOMEM_ABORT(p) do { if (!(p)) { \ DEBUG(DEBUG_ALERT,("ctdb fatal error: %s\n", \ "Out of memory in " __location__ )); \ @@ -452,64 +466,64 @@ static void show_statistics(struct ctdb_statistics *s, int show_header) if (options.machinereadable){ if (show_header) { - printf("CTDB version:"); - printf("Current time of statistics:"); - printf("Statistics collected since:"); + printm("CTDB version:"); + printm("Current time of statistics:"); + printm("Statistics collected since:"); for (i=0;istatistics_current_time.tv_sec); - printf("%d:", (int)s->statistics_start_time.tv_sec); + printm("num_reclock_ctdbd_latency:"); + printm("min_reclock_ctdbd_latency:"); + printm("avg_reclock_ctdbd_latency:"); + printm("max_reclock_ctdbd_latency:"); + + printm("num_reclock_recd_latency:"); + printm("min_reclock_recd_latency:"); + printm("avg_reclock_recd_latency:"); + printm("max_reclock_recd_latency:"); + + printm("num_call_latency:"); + printm("min_call_latency:"); + printm("avg_call_latency:"); + printm("max_call_latency:"); + + printm("num_lockwait_latency:"); + printm("min_lockwait_latency:"); + printm("avg_lockwait_latency:"); + printm("max_lockwait_latency:"); + + printm("num_childwrite_latency:"); + printm("min_childwrite_latency:"); + printm("avg_childwrite_latency:"); + printm("max_childwrite_latency:"); + printm("\n"); + } + printm("%d:", CTDB_PROTOCOL); + printm("%d:", (int)s->statistics_current_time.tv_sec); + printm("%d:", (int)s->statistics_start_time.tv_sec); for (i=0;ireclock.ctdbd.num); - printf("%.6f:", s->reclock.ctdbd.min); - printf("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0); - printf("%.6f:", s->reclock.ctdbd.max); - - printf("%d:", s->reclock.recd.num); - printf("%.6f:", s->reclock.recd.min); - printf("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0); - printf("%.6f:", s->reclock.recd.max); - - printf("%d:", s->call_latency.num); - printf("%.6f:", s->call_latency.min); - printf("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0); - printf("%.6f:", s->call_latency.max); - - printf("%d:", s->childwrite_latency.num); - printf("%.6f:", s->childwrite_latency.min); - printf("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0); - printf("%.6f:", s->childwrite_latency.max); - printf("\n"); + printm("%d:", *(uint32_t *)(fields[i].offset+(uint8_t *)s)); + } + printm("%d:", s->reclock.ctdbd.num); + printm("%.6f:", s->reclock.ctdbd.min); + printm("%.6f:", s->reclock.ctdbd.num?s->reclock.ctdbd.total/s->reclock.ctdbd.num:0.0); + printm("%.6f:", s->reclock.ctdbd.max); + + printm("%d:", s->reclock.recd.num); + printm("%.6f:", s->reclock.recd.min); + printm("%.6f:", s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0); + printm("%.6f:", s->reclock.recd.max); + + printm("%d:", s->call_latency.num); + printm("%.6f:", s->call_latency.min); + printm("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0); + printm("%.6f:", s->call_latency.max); + + printm("%d:", s->childwrite_latency.num); + printm("%.6f:", s->childwrite_latency.min); + printm("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0); + printm("%.6f:", s->childwrite_latency.max); + printm("\n"); } else { printf("CTDB version %u\n", CTDB_PROTOCOL); printf("Current time of statistics : %s", ctime(&s->statistics_current_time.tv_sec)); @@ -763,8 +777,8 @@ static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv } if (options.machinereadable){ - printf(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n"); - printf(":%u:%u:%u:%lf\n", + printm(":Current Node Time:Ctdb Start Time:Last Recovery/Failover Time:Last Recovery/IPFailover Duration:\n"); + printm(":%u:%u:%u:%lf\n", (unsigned int)uptime->current_time.tv_sec, (unsigned int)uptime->ctdbd_start_time.tv_sec, (unsigned int)uptime->last_recovery_finished.tv_sec, @@ -960,14 +974,14 @@ static bool is_partially_online(struct ctdb_context *ctdb, struct ctdb_node_and_ static void control_status_header_machine(void) { - printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped" + printm(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped" ":Inactive:PartiallyOnline:ThisNode:\n"); } static int control_status_1_machine(struct ctdb_context *ctdb, int mypnn, struct ctdb_node_and_flags *node) { - printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn, + printm(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%c:\n", node->pnn, ctdb_addr_to_str(&node->addr), !!(node->flags&NODE_FLAGS_DISCONNECTED), !!(node->flags&NODE_FLAGS_BANNED), @@ -1353,8 +1367,8 @@ static int control_natgwlist(struct ctdb_context *ctdb, int argc, const char **a } if (options.machinereadable) { - printf(":Node:IP:\n"); - printf(":%d:%s:\n", pnn, ip); + printm(":Node:IP:\n"); + printm(":%d:%s:\n", pnn, ip); } else { printf("%d %s\n", pnn, ip); } @@ -1435,7 +1449,7 @@ static int control_one_scriptstatus(struct ctdb_context *ctdb, break; } if (options.machinereadable) { - printf(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n", + printm(":%s:%s:%i:%s:%lu.%06lu:%lu.%06lu:%s:\n", ctdb_eventscript_call_names[type], script_status->scripts[i].name, script_status->scripts[i].status, @@ -1511,7 +1525,7 @@ static int control_scriptstatus(struct ctdb_context *ctdb, } if (options.machinereadable) { - printf(":Type:Name:Code:Status:Start:End:Error Output...:\n"); + printm(":Type:Name:Code:Status:Start:End:Error Output...:\n"); } for (type = min; type < max; type++) { @@ -1692,13 +1706,13 @@ static int control_get_tickles(struct ctdb_context *ctdb, int argc, const char * } if (options.machinereadable){ - printf(":source ip:port:destination ip:port:\n"); + printm(":source ip:port:destination ip:port:\n"); for (i=0;itickles.num;i++) { if (port && port != ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)) { continue; } - printf(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port)); - printf(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)); + printm(":%s:%u", ctdb_addr_to_str(&list->tickles.connections[i].src_addr), ntohs(list->tickles.connections[i].src_addr.ip.sin_port)); + printm(":%s:%u:\n", ctdb_addr_to_str(&list->tickles.connections[i].dst_addr), ntohs(list->tickles.connections[i].dst_addr.ip.sin_port)); } } else { printf("Tickles for ip:%s\n", ctdb_addr_to_str(&list->addr)); @@ -2943,11 +2957,11 @@ static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv) } if (options.machinereadable){ - printf(":Public IP:Node:"); + printm(":Public IP:Node:"); if (options.verbose){ - printf("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:"); + printm("ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:"); } - printf("\n"); + printm("\n"); } else { if (options.pnn == CTDB_BROADCAST_ALL) { printf("Public IPs on ALL nodes\n"); @@ -3007,11 +3021,11 @@ static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv) } if (options.machinereadable){ - printf(":%s:%d:", + printm(":%s:%d:", ctdb_addr_to_str(&ips->ips[ips->num-i].addr), ips->ips[ips->num-i].pnn); if (options.verbose){ - printf("%s:%s:%s:", + printm("%s:%s:%s:", aciface?aciface:"", avifaces?avifaces:"", cifaces?cifaces:""); @@ -3110,14 +3124,14 @@ static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv } if (options.machinereadable){ - printf(":Name:LinkStatus:References:\n"); + printm(":Name:LinkStatus:References:\n"); } else { printf("Interfaces on node %u\n", options.pnn); } for (i=0; inum; i++) { if (options.machinereadable){ - printf(":%s:%s:%u\n", + printm(":%s:%s:%u\n", ifaces->ifaces[i].name, ifaces->ifaces[i].link_state?"1":"0", (unsigned int)ifaces->ifaces[i].references); @@ -3520,8 +3534,8 @@ static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char ** if (!options.machinereadable){ printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode); } else { - printf(":mode:\n"); - printf(":%d:\n",monmode); + printm(":mode:\n"); + printm(":%d:\n",monmode); } return 0; } @@ -3547,8 +3561,8 @@ static int control_getcapabilities(struct ctdb_context *ctdb, int argc, const ch printf("LVS: %s\n", (capabilities&CTDB_CAP_LVS)?"YES":"NO"); printf("NATGW: %s\n", (capabilities&CTDB_CAP_NATGW)?"YES":"NO"); } else { - printf(":RECMASTER:LMASTER:LVS:NATGW:\n"); - printf(":%d:%d:%d:%d:\n", + printm(":RECMASTER:LMASTER:LVS:NATGW:\n"); + printm(":%d:%d:%d:%d:\n", !!(capabilities&CTDB_CAP_RECMASTER), !!(capabilities&CTDB_CAP_LMASTER), !!(capabilities&CTDB_CAP_LVS), @@ -3658,9 +3672,11 @@ static int control_lvsmaster(struct ctdb_context *ctdb, int argc, const char **a } if (n->num > 0) { ret = 0; - printf(options.machinereadable ? - "%d\n" : "Node %d is LVS master\n", - n->nodes[0].pnn); + if (options.machinereadable) { + printm("%d\n", n->nodes[0].pnn); + } else { + printf("Node %d is LVS master\n", n->nodes[0].pnn); + } goto done; } } @@ -4582,7 +4598,7 @@ static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **ar } if(options.machinereadable){ - printf(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n"); + printm(":ID:Name:Path:Persistent:Sticky:Unhealthy:ReadOnly:\n"); for(i=0;inum;i++){ const char *path; const char *name; @@ -4600,7 +4616,7 @@ static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **ar persistent = dbmap->dbs[i].flags & CTDB_DB_FLAGS_PERSISTENT; readonly = dbmap->dbs[i].flags & CTDB_DB_FLAGS_READONLY; sticky = dbmap->dbs[i].flags & CTDB_DB_FLAGS_STICKY; - printf(":0x%08X:%s:%s:%d:%d:%d:%d:\n", + printm(":0x%08X:%s:%s:%d:%d:%d:%d:\n", dbmap->dbs[i].dbid, name, path, !!(persistent), !!(sticky), !!(health), !!(readonly)); @@ -4842,8 +4858,8 @@ static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **ar return ret; } else { if (options.machinereadable){ - printf(":Name:Level:\n"); - printf(":%s:%d:\n",get_debug_by_level(level),level); + printm(":Name:Level:\n"); + printm(":%s:%d:\n",get_debug_by_level(level),level); } else { printf("Node %u is at debug level %s (%d)\n", options.pnn, get_debug_by_level(level), level); } @@ -4866,7 +4882,7 @@ static int control_getreclock(struct ctdb_context *ctdb, int argc, const char ** } else { if (options.machinereadable){ if (reclock != NULL) { - printf("%s", reclock); + printm("%s", reclock); } } else { if (reclock == NULL) { @@ -6142,7 +6158,7 @@ static int control_listnodes(struct ctdb_context *ctdb, int argc, const char **a for(pnn_node=pnn_nodes;pnn_node;pnn_node=pnn_node->next) { const char *addr = ctdb_addr_to_str(&pnn_node->addr); if (options.machinereadable){ - printf(":%d:%s:\n", pnn_node->pnn, addr); + printm(":%d:%s:\n", pnn_node->pnn, addr); } else { printf("%s\n", addr); } -- 2.1.3 From 035824e87c7e4e874ce5ad13753599ff43d62168 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 19 Nov 2014 17:15:21 +1100 Subject: [PATCH 02/29] ctdb-tools: Add -x option to specify delimiter for machine readable output To support this, update printm() to replace ':' in format string with options.machineseparator, which is a string but must contain a single character. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 3b90e45bae555cc4a47fe9958b86628d41084868) --- ctdb/doc/ctdb.1.xml | 12 +++++++++++- ctdb/tools/ctdb.c | 32 +++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml index efa5d85..08ed564 100644 --- a/ctdb/doc/ctdb.1.xml +++ b/ctdb/doc/ctdb.1.xml @@ -137,7 +137,17 @@ Produce output in machine readable form for easier parsing - by scripts. Not all commands support this option. + by scripts. This uses a field delimiter of ':'. Not all + commands support this option. + + + + + -x SEPARATOR + + + Use SEPARATOR to delimit fields in machine readable output. + This implies -Y. diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c index a45ff20..d533b15 100644 --- a/ctdb/tools/ctdb.c +++ b/ctdb/tools/ctdb.c @@ -43,6 +43,7 @@ static struct { uint32_t pnn; uint32_t *nodes; int machinereadable; + const char *machineseparator; int verbose; int maxruntime; int printemptyrecords; @@ -75,9 +76,18 @@ static int printm(const char *format, ...) { va_list ap; int ret; + size_t len = strlen(format); + char new_format[len+1]; + + strcpy(new_format, format); + + if (options.machineseparator[0] != ':') { + all_string_sub(new_format, + ":", options.machineseparator, len + 1); + } va_start(ap, format); - ret = vprintf(format, ap); + ret = vprintf(new_format, ap); va_end(ap); return ret; @@ -6338,7 +6348,8 @@ static void usage(void) "Usage: ctdb [options] \n" \ "Options:\n" \ " -n choose node number, or 'all' (defaults to local node)\n" -" -Y generate machinereadable output\n" +" -Y generate machine readable output\n" +" -x specify delimiter for machine readable output\n" " -v generate verbose output\n" " -t set timelimit for control in seconds (default %u)\n", options.timelimit); printf("Controls:\n"); @@ -6370,7 +6381,8 @@ int main(int argc, const char *argv[]) POPT_CTDB_CMDLINE { "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" }, { "node", 'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" }, - { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL }, + { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machine readable output", NULL }, + { NULL, 'x', POPT_ARG_STRING, &options.machineseparator, 0, "specify separator for machine readable output", "char" }, { "verbose", 'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL }, { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" }, { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL }, @@ -6428,6 +6440,20 @@ int main(int argc, const char *argv[]) } } + if (options.machineseparator != NULL) { + if (strlen(options.machineseparator) != 1) { + printf("Invalid separator \"%s\" - " + "must be single character\n", + options.machineseparator); + exit(1); + } + + /* -x implies -Y */ + options.machinereadable = true; + } else if (options.machinereadable) { + options.machineseparator = ":"; + } + signal(SIGALRM, ctdb_alarm); alarm(options.maxruntime); -- 2.1.3 From 3367107e895b245b7815af1fc337d3ea2fc17b3c Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 19 Nov 2014 18:19:50 +1100 Subject: [PATCH 03/29] ctdb-tools: Add -X option for machine parsable output with separator '|' Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 7428f809a76044fcbc98fd5f718e331ce183941d) --- ctdb/doc/ctdb.1.xml | 14 ++++++++++++++ ctdb/tools/ctdb.c | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml index 08ed564..15bcabf 100644 --- a/ctdb/doc/ctdb.1.xml +++ b/ctdb/doc/ctdb.1.xml @@ -152,6 +152,20 @@ + -X + + + Produce output in machine readable form for easier parsing + by scripts. This uses a field delimiter of '|'. Not all + commands support this option. + + + This is equivalent to "-x|" and avoids some shell quoting + issues. + + + + -t TIMEOUT diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c index d533b15..072832d 100644 --- a/ctdb/tools/ctdb.c +++ b/ctdb/tools/ctdb.c @@ -6376,6 +6376,7 @@ int main(int argc, const char *argv[]) { struct ctdb_context *ctdb; char *nodestring = NULL; + int machineparsable = 0; struct poptOption popt_options[] = { POPT_AUTOHELP POPT_CTDB_CMDLINE @@ -6383,6 +6384,7 @@ int main(int argc, const char *argv[]) { "node", 'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" }, { "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machine readable output", NULL }, { NULL, 'x', POPT_ARG_STRING, &options.machineseparator, 0, "specify separator for machine readable output", "char" }, + { NULL, 'X', POPT_ARG_NONE, &machineparsable, 0, "enable machine parsable output with separator |", NULL }, { "verbose", 'v', POPT_ARG_NONE, &options.verbose, 0, "enable verbose output", NULL }, { "maxruntime", 'T', POPT_ARG_INT, &options.maxruntime, 0, "die if runtime exceeds this limit (in seconds)", "integer" }, { "print-emptyrecords", 0, POPT_ARG_NONE, &options.printemptyrecords, 0, "print the empty records when dumping databases (catdb, cattdb, dumpdbbackup)", NULL }, @@ -6440,6 +6442,9 @@ int main(int argc, const char *argv[]) } } + if (machineparsable) { + options.machineseparator = "|"; + } if (options.machineseparator != NULL) { if (strlen(options.machineseparator) != 1) { printf("Invalid separator \"%s\" - " -- 2.1.3 From 808a3a94ed9e626dab614140de6b2582458ec672 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 20 Nov 2014 14:32:46 +1100 Subject: [PATCH 04/29] ctdb-scripts: Update eventscripts to use ctdb -X instead of ctdb -Y Also update associated eventscript unit tests and ctdb stub. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 4940f191d37b5deadc8b1edf0cd516674e5d5d64) --- ctdb/config/events.d/10.interface | 2 +- ctdb/config/events.d/13.per_ip_routing | 6 +++--- ctdb/config/events.d/62.cnfs | 2 +- ctdb/config/events.d/70.iscsi | 2 +- ctdb/config/functions | 20 ++++++++++---------- ctdb/config/statd-callout | 8 ++++---- ctdb/tests/eventscripts/scripts/local.sh | 6 +++--- ctdb/tests/eventscripts/stubs/ctdb | 22 +++++++++++----------- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ctdb/config/events.d/10.interface b/ctdb/config/events.d/10.interface index 018f767..8207fd3 100755 --- a/ctdb/config/events.d/10.interface +++ b/ctdb/config/events.d/10.interface @@ -46,7 +46,7 @@ get_all_interfaces () # Get the interfaces for which CTDB has public IPs configured. # That is, for all but the 1st line, get the 1st field. - ctdb_ifaces=$(ctdb -Y ifaces | sed -e '1d' -e 's@^:@@' -e 's@:.*@@') + ctdb_ifaces=$(ctdb -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@') # Add $ctdb_interfaces and uniquify all_interfaces=$(echo $all_interfaces $ctdb_ifaces | tr ' ' '\n' | sort -u) diff --git a/ctdb/config/events.d/13.per_ip_routing b/ctdb/config/events.d/13.per_ip_routing index ee83632..cd0020e 100755 --- a/ctdb/config/events.d/13.per_ip_routing +++ b/ctdb/config/events.d/13.per_ip_routing @@ -290,7 +290,7 @@ flush_rules_and_routes () # routes. add_missing_routes () { - ctdb ip -v -Y | { + ctdb ip -v -X | { read _x # skip header line # Read the rest of the lines. We're only interested in the @@ -299,7 +299,7 @@ add_missing_routes () # non-local addresses. For each IP local address we check if # the relevant routing table is populated and populate it if # not. - while IFS=":" read _x _ip _x _iface _x ; do + while IFS="|" read _x _ip _x _iface _x ; do [ -n "$_iface" ] || continue _table_id="${table_id_prefix}${_ip}" @@ -317,7 +317,7 @@ add_missing_routes () remove_bogus_routes () { # Get a IPs current hosted by this node, each anchored with '@'. - _ips=$(ctdb ip -v -Y | awk -F: 'NR > 1 && $4 != "" {printf "@%s@\n", $2}') + _ips=$(ctdb ip -v -X | awk -F'|' 'NR > 1 && $4 != "" {printf "@%s@\n", $2}') ip rule show | while read _p _x _i _x _t ; do diff --git a/ctdb/config/events.d/62.cnfs b/ctdb/config/events.d/62.cnfs index da02acc..a6ca0c4 100755 --- a/ctdb/config/events.d/62.cnfs +++ b/ctdb/config/events.d/62.cnfs @@ -53,7 +53,7 @@ case "$1" in # Wait until we no longer serve any ip addresses at all PNN=`ctdb pnn | cut -d: -f2` - while `ctdb -Y ip | cut -d: -f3 | egrep "^$PNN$" >/dev/null`; do + while `ctdb -X ip | cut -d'|' -f3 | egrep "^$PNN$" >/dev/null`; do sleep 1 done ;; diff --git a/ctdb/config/events.d/70.iscsi b/ctdb/config/events.d/70.iscsi index cedaf40..4627822 100755 --- a/ctdb/config/events.d/70.iscsi +++ b/ctdb/config/events.d/70.iscsi @@ -36,7 +36,7 @@ case "$1" in # start the iscsi daemon tgtd >/dev/null 2>/dev/null - ips=$(ctdb -Y ip | awk -F: -v pnn=$this_node '$3 == pnn {print $2}') + ips=$(ctdb -X ip | awk -F'|' -v pnn=$this_node '$3 == pnn {print $2}') for ip in $ips ; do script="${CTDB_START_ISCSI_SCRIPTS}/${ip}.sh" if [ -x "$script" ] ; then diff --git a/ctdb/config/functions b/ctdb/config/functions index 3f2ccee..05bcd38 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -1095,19 +1095,19 @@ ctdb_reconfigure_release_lock () ctdb_replay_monitor_status () { echo "Replaying previous status for this script due to reconfigure..." - # Leading colon (':') is missing in some versions... - _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:") + # Leading separator ('|') is missing in some versions... + _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|") # Output looks like this: - # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar: + # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar| # This is the cheapest way of getting fields in the middle. - set -- $(IFS=":" ; echo $_out) + set -- $(IFS="|" ; echo $_out) _code="$3" _status="$4" # The error output field can include colons so we'll try to # preserve them. The weak checking at the beginning tries to make - # this work for both broken (no leading ':') and fixed output. - _out="${_out%:}" - _err_out="${_out#*monitor:${script_name}:*:*:*:*:}" + # this work for both broken (no leading '|') and fixed output. + _out="${_out%|}" + _err_out="${_out#*monitor|${script_name}|*|*|*|*|}" case "$_status" in OK) : ;; # Do nothing special. TIMEDOUT) @@ -1370,7 +1370,7 @@ update_tickles () _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:} # What public IPs do I hold? - _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}') + _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}') # IPs as a regexp choice _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))" @@ -1387,8 +1387,8 @@ update_tickles () _my_tickles="${tickledir}/${_port}.tickles" rm -f "$_my_tickles" for _i in $_ips ; do - ctdb -Y gettickles $_i $_port | - awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }' + ctdb -X gettickles $_i $_port | + awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }' done | sort >"$_my_tickles" diff --git a/ctdb/config/statd-callout b/ctdb/config/statd-callout index 53b408d..5e8eb0e 100755 --- a/ctdb/config/statd-callout +++ b/ctdb/config/statd-callout @@ -37,12 +37,12 @@ case "$1" in cip="$2" pnn=$(ctdb xpnn | sed -e 's/.*://') date=$(date '+%s') - ctdb ip -Y | + ctdb ip -X | tail -n +2 | { # This all needs to be in the end of the pipe so it # doesn't get lost items="" - while IFS=":" read x sip node x ; do + while IFS="|" read x sip node x ; do [ "$node" = "$pnn" ] || continue # not us key="statd-state@${sip}@${cip}" item="\"${key}\" \"${date}\"" @@ -58,12 +58,12 @@ case "$1" in # so we must add it to all the IPs that we serve cip="$2" pnn=$(ctdb xpnn | sed -e 's/.*://') - ctdb ip -Y | + ctdb ip -X | tail -n +2 | { # This all needs to be in the end of the pipe so it # doesn't get lost items="" - while IFS=":" read x sip node x ; do + while IFS="|" read x sip node x ; do [ "$node" = "$pnn" ] || continue # not us key="statd-state@${sip}@${cip}" item="\"${key}\" \"\"" diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index deaab60..c4e86d9 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -354,7 +354,7 @@ Swap: 5719 246 5473" ctdb_get_interfaces () { # The echo/subshell forces all the output onto 1 line. - echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') + echo $(ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}') } ctdb_get_1_interface () @@ -377,10 +377,10 @@ ctdb_get_all_public_addresses () # Each line is suitable for passing to takeip/releaseip ctdb_get_my_public_addresses () { - ctdb ip -v -Y | { + ctdb ip -v -X | { read _x # skip header line - while IFS=":" read _x _ip _x _iface _x ; do + while IFS="|" read _x _ip _x _iface _x ; do [ -n "$_iface" ] || continue while IFS="/$IFS" read _i _maskbits _x ; do if [ "$_ip" = "$_i" ] ; then diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb index b8b3e67..c444196 100755 --- a/ctdb/tests/eventscripts/stubs/ctdb +++ b/ctdb/tests/eventscripts/stubs/ctdb @@ -13,7 +13,7 @@ not_implemented_exit_code=1 usage () { cat >&2 < Date: Thu, 20 Nov 2014 14:39:59 +1100 Subject: [PATCH 05/29] ctdb-tools: Update onnode and ctdb-diagnostics to use ctdb -X Also update onnode unit tests. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 55df9c86c19e261a2a384ffc4b77c596c84e53a0) --- ctdb/tests/onnode/0070.sh | 10 +++++----- ctdb/tests/onnode/0071.sh | 10 +++++----- ctdb/tests/onnode/0072.sh | 10 +++++----- ctdb/tests/onnode/0075.sh | 10 +++++----- ctdb/tests/onnode/stubs/onnode-buggy-001 | 12 ++++++------ ctdb/tools/ctdb_diagnostics | 2 +- ctdb/tools/onnode | 6 +++--- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ctdb/tests/onnode/0070.sh b/ctdb/tests/onnode/0070.sh index b071e80..d649f82 100755 --- a/ctdb/tests/onnode/0070.sh +++ b/ctdb/tests/onnode/0070.sh @@ -7,11 +7,11 @@ cmd="$ONNODE ok hostname" define_test "$cmd" "all nodes OK" ctdb_set_output </dev/null) + ctdb_status_output=$(ctdb -X status 2>/dev/null) if [ $? -ne 0 ] ; then echo "${prog}: unable to get status of CTDB nodes" >&2 exit 1 @@ -164,14 +164,14 @@ get_nodes_with_status () local i for i in $ctdb_status_output ; do # Try removing bits from end. - local t="${i%:${bits}:}" + local t="${i%|${bits}|}" if [ "$t" != "$i" ] ; then # Succeeded. Get address. NOTE: this is an optimisation. # It might be better to get the node number and then get # the nth node to get the address. This would make things # more consistent if $ctdb_base/nodes actually contained # hostnames. - nodes="${nodes} ${t#:*:}" + nodes="${nodes} ${t#|*|}" fi done @@ -186,7 +186,7 @@ get_node_with_property () local prop_node="" if [ "${ctdb_props##:${prop}:}" = "$ctdb_props" ] ; then - prop_node=$(ctdb "$prop" -Y 2>/dev/null) + prop_node=$(ctdb "$prop" -X 2>/dev/null) # We only want the first line. local nl=" " diff --git a/ctdb/tools/ctdb_diagnostics b/ctdb/tools/ctdb_diagnostics index 2a51e1b..3f2fa63 100755 --- a/ctdb/tools/ctdb_diagnostics +++ b/ctdb/tools/ctdb_diagnostics @@ -17,7 +17,7 @@ EOF } -nodes=$(ctdb listnodes -Y | cut -d: -f2) +nodes=$(ctdb listnodes -X | cut -d'|' -f2) bad_nodes="" diff_opts= no_ads=false diff --git a/ctdb/tools/onnode b/ctdb/tools/onnode index 33d0e20..96d569a 100755 --- a/ctdb/tools/onnode +++ b/ctdb/tools/onnode @@ -145,7 +145,7 @@ get_nodes_with_status () local status="$2" if [ -z "$ctdb_status_output" ] ; then - ctdb_status_output=$(ctdb -Y status 2>&1) + ctdb_status_output=$(ctdb -X status 2>&1) if [ $? -ne 0 ] ; then echo "${prog}: unable to get status of CTDB nodes" >&2 echo "$ctdb_status_output" >&2 @@ -158,7 +158,7 @@ get_nodes_with_status () ( local i - IFS="${IFS}:" + IFS="${IFS}|" while IFS="" read i ; do set -- $i # split line on colons @@ -196,7 +196,7 @@ get_node_with_property () local prop_node="" if [ "${ctdb_props##:${prop}:}" = "$ctdb_props" ] ; then # Not in cache. - prop_node=$(ctdb "$prop" -Y 2>/dev/null) + prop_node=$(ctdb "$prop" -X 2>/dev/null) if [ $? -eq 0 ] ; then if [ "$prop" = "natgwlist" ] ; then prop_node="${prop_node%% *}" # 1st word -- 2.1.3 From 80d5812c75e0d08ff8fb19fdac36bd9e1c1adede Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 20 Nov 2014 15:03:25 +1100 Subject: [PATCH 06/29] ctdb-tests: Update integration tests to use ctdb -X Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 2c4de75d8754616891e97222cfb2ec58fdd8eac2) --- ctdb/tests/complex/18_ctdb_reloadips.sh | 4 +-- ctdb/tests/complex/34_nfs_tickle_restart.sh | 4 +-- ctdb/tests/events.d/00.test | 4 +-- ctdb/tests/scripts/integration.bash | 34 +++++++++++++------------- ctdb/tests/simple/11_ctdb_ip.sh | 8 +++--- ctdb/tests/simple/12_ctdb_getdebug.sh | 12 ++++----- ctdb/tests/simple/20_delip_iface_gc.sh | 18 +++++++------- ctdb/tests/simple/27_ctdb_detach.sh | 2 +- ctdb/tests/simple/75_readonly_records_basic.sh | 4 +-- 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/ctdb/tests/complex/18_ctdb_reloadips.sh b/ctdb/tests/complex/18_ctdb_reloadips.sh index 042683c..13f7c21 100755 --- a/ctdb/tests/complex/18_ctdb_reloadips.sh +++ b/ctdb/tests/complex/18_ctdb_reloadips.sh @@ -50,8 +50,8 @@ select_test_node_and_ips # the provided prefix. Note that this is an IPv4-specific test. echo "Getting public IP information from CTDB..." -try_command_on_node any "$CTDB ip -Y -v -n all" -ctdb_ip_info=$(echo "$out" | awk -F: 'NR > 1 { print $2, $3, $5 }') +try_command_on_node any "$CTDB ip -X -v -n all" +ctdb_ip_info=$(echo "$out" | awk -F'|' 'NR > 1 { print $2, $3, $5 }') echo "Getting IP information from interfaces..." try_command_on_node all "ip addr show" diff --git a/ctdb/tests/complex/34_nfs_tickle_restart.sh b/ctdb/tests/complex/34_nfs_tickle_restart.sh index b7eea4c..365a017 100755 --- a/ctdb/tests/complex/34_nfs_tickle_restart.sh +++ b/ctdb/tests/complex/34_nfs_tickle_restart.sh @@ -45,7 +45,7 @@ monitor_interval="${out#*= }" #echo "Monitor interval on node $test_node is $monitor_interval seconds." select_test_node_and_ips -try_command_on_node $test_node "$CTDB listnodes -Y" +try_command_on_node $test_node "$CTDB listnodes -X" listnodes_output="$out" numnodes=$(wc -l <<<"$listnodes_output") @@ -67,7 +67,7 @@ echo "Wait until NFS connection is tracked by CTDB on test node ..." wait_until 10 check_tickles $test_node $test_ip $test_port $src_socket echo "Select a node to restart ctdbd" -rn=$(awk -F: -v test_node=$test_node \ +rn=$(awk -F'|' -v test_node=$test_node \ '$2 != test_node { print $2 ; exit }' <<<"$listnodes_output") echo "Restarting CTDB on node ${rn}" diff --git a/ctdb/tests/events.d/00.test b/ctdb/tests/events.d/00.test index e3e15eb..0ae0987 100755 --- a/ctdb/tests/events.d/00.test +++ b/ctdb/tests/events.d/00.test @@ -26,9 +26,9 @@ case $cmd in ;; startup) echo "ctdb startup event" - IFACES=`ctdb ifaces -Y | grep -v '^:Name:LinkStatus:References:'` + IFACES=`ctdb ifaces -X | grep -v '^|Name|LinkStatus|References|'` for I in $IFACES; do - IFACE=`echo -n "$I" | cut -d ':' -f2` + IFACE=`echo -n "$I" | cut -d '|' -f2` ctdb setifacelink $IFACE up done exit 0; diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 1835949..b75e8f9 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -163,13 +163,13 @@ all_ips_on_node() { local node="$1" try_command_on_node $node \ - "$CTDB ip -Y | awk -F: 'NR > 1 { print \$2, \$3 }'" + "$CTDB ip -X | awk -F'|' 'NR > 1 { print \$2, \$3 }'" } _select_test_node_and_ips () { try_command_on_node any \ - "$CTDB ip -Y -n all | awk -F: 'NR > 1 { print \$2, \$3 }'" + "$CTDB ip -X -n all | awk -F'|' 'NR > 1 { print \$2, \$3 }'" test_node="" # this matches no PNN test_node_ips="" @@ -209,7 +209,7 @@ select_test_node_and_ips () get_test_ip_mask_and_iface () { # Find the interface - try_command_on_node $test_node "$CTDB ip -v -Y | awk -F: -v ip=$test_ip '\$2 == ip { print \$4 }'" + try_command_on_node $test_node "$CTDB ip -v -X | awk -F'|' -v ip=$test_ip '\$2 == ip { print \$4 }'" iface="$out" if [ -z "$TEST_LOCAL_DAEMONS" ] ; then @@ -334,16 +334,16 @@ node_has_status () local bits fpat mpat rpat case "$status" in - (unhealthy) bits="?:?:?:1:*" ;; - (healthy) bits="?:?:?:0:*" ;; - (disconnected) bits="1:*" ;; - (connected) bits="0:*" ;; - (banned) bits="?:1:*" ;; - (unbanned) bits="?:0:*" ;; - (disabled) bits="?:?:1:*" ;; - (enabled) bits="?:?:0:*" ;; - (stopped) bits="?:?:?:?:1:*" ;; - (notstopped) bits="?:?:?:?:0:*" ;; + (unhealthy) bits="?|?|?|1|*" ;; + (healthy) bits="?|?|?|0|*" ;; + (disconnected) bits="1|*" ;; + (connected) bits="0|*" ;; + (banned) bits="?|1|*" ;; + (unbanned) bits="?|0|*" ;; + (disabled) bits="?|?|1|*" ;; + (enabled) bits="?|?|0|*" ;; + (stopped) bits="?|?|?|?|1|*" ;; + (notstopped) bits="?|?|?|?|0|*" ;; (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; @@ -357,13 +357,13 @@ node_has_status () if [ -n "$bits" ] ; then local out x line - out=$($CTDB -Y status 2>&1) || return 1 + out=$($CTDB -X status 2>&1) || return 1 { read x while read line ; do # This needs to be done in 2 steps to avoid false matches. - local line_bits="${line#:${pnn}:*:}" + local line_bits="${line#|${pnn}|*|}" [ "$line_bits" = "$line" ] && continue [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 done @@ -566,7 +566,7 @@ restart_ctdb () # Cluster is still healthy. Good, we're done! if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then echo "Cluster became UNHEALTHY again [$(date)]" - onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb status -X 2>&1 onnode -p all ctdb scriptstatus 2>&1 echo "Restarting..." continue @@ -580,7 +580,7 @@ restart_ctdb () done echo "Cluster UNHEALTHY... too many attempts..." - onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb status -X 2>&1 onnode -p all ctdb scriptstatus 2>&1 # Try to make the calling test fail diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh index c1aec0e..0e02b5e 100755 --- a/ctdb/tests/simple/11_ctdb_ip.sh +++ b/ctdb/tests/simple/11_ctdb_ip.sh @@ -15,7 +15,7 @@ Steps: 2. Run 'ctdb ip' on one of the nodes and verify the list of IP addresses displayed (cross check the result with the output of 'ip addr show' on the node). -3. Verify that colon-separated output is generated with the -Y option. +3. Verify that pipe-separated output is generated with the -X option. Expected results: @@ -37,14 +37,14 @@ ips=$(echo "$out" | sed \ -e 's@ node\[@ @' \ -e 's@\].*$@@') machineout=$(echo "$out" | sed -r \ - -e 's@^| |$@:@g' \ + -e 's@^| |$@\|@g' \ -e 's@[[:alpha:]]+\[@@g' \ -e 's@\]@@g') if [ -z "$TEST_LOCAL_DAEMONS" ]; then while read ip pnn ; do try_command_on_node $pnn "ip addr show" - if [ "${out/inet ${ip}\/}" != "$out" ] ; then + if [ "${out/inet* ${ip}\/}" != "$out" ] ; then echo "GOOD: node $pnn appears to have $ip assigned" else echo "BAD: node $pnn does not appear to have $ip assigned" @@ -55,7 +55,7 @@ fi [ "$testfailures" != 1 ] && echo "Looks good!" -cmd="$CTDB -Y ip -n all | tail -n +2" +cmd="$CTDB -X ip -n all | tail -n +2" echo "Checking that \"$cmd\" produces expected output..." try_command_on_node 1 "$cmd" diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh index 4a4926d..cdd9e34 100755 --- a/ctdb/tests/simple/12_ctdb_getdebug.sh +++ b/ctdb/tests/simple/12_ctdb_getdebug.sh @@ -13,7 +13,7 @@ Steps: 1. Verify that the status on all of the ctdb nodes is 'OK'. 2. Get the current debug level on a node, using 'ctdb getdebug -n '. -3. Verify that colon-separated output is generated with the -Y option. +3. Verify that pipe-separated output is generated with the -X option. 4. Verify that the '-n all' option shows the debug level on all nodes. Expected results: @@ -61,19 +61,19 @@ else testfailures=1 fi -colons="" +seps="" nl=" " while read line ; do - t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]]+)\)$@:\1:\2:@') - colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}" + t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]]+)\)$@\|\1\|\2|@') + seps="${seps}${seps:+${nl}}|Name|Level|${nl}${t}" done <<<"$getdebug_onnode" -cmd="$CTDB -Y getdebug -n all" +cmd="$CTDB -X getdebug -n all" echo "Checking that \"$cmd\" produces expected output..." try_command_on_node 1 "$cmd" -if [ "$out" = "$colons" ] ; then +if [ "$out" = "$seps" ] ; then echo "Yep, looks good!" else echo "Nope, it looks like this:" diff --git a/ctdb/tests/simple/20_delip_iface_gc.sh b/ctdb/tests/simple/20_delip_iface_gc.sh index bc43567..83de495 100755 --- a/ctdb/tests/simple/20_delip_iface_gc.sh +++ b/ctdb/tests/simple/20_delip_iface_gc.sh @@ -19,19 +19,19 @@ cluster_is_healthy ctdb_restart_when_done echo "Getting public IPs information..." -try_command_on_node -v any "$CTDB ip -v -n all -Y | tail -n +2" +try_command_on_node -v any "$CTDB ip -v -n all -X | tail -n +2" ip_info="$out" # Select the first node and find out its interfaces -test_node=$(awk -F: 'NR == 1 { print $3}' <<<"$ip_info") -ifaces=$(awk -F: -v tn=$test_node '$3 == tn { print $6 }' <<<"$ip_info" | sed 's@, @ @g' | xargs -n 1 | sort -u) +test_node=$(awk -F'|' 'NR == 1 { print $3}' <<<"$ip_info") +ifaces=$(awk -F'|' -v tn=$test_node '$3 == tn { print $6 }' <<<"$ip_info" | sed 's@, @ @g' | xargs -n 1 | sort -u) echo "Selected test node ${test_node} with interfaces: ${ifaces}" # Delete all IPs on each interface... deleting IPs from one interface # can cause other interfaces to disappear, so we need to be careful... for i in $ifaces ; do - try_command_on_node $test_node "$CTDB ifaces -Y" - info=$(awk -F: -v iface="$i" '$2 == iface { print $0 }' <<<"$out") + try_command_on_node $test_node "$CTDB ifaces -X" + info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out") if [ -z "$info" ] ; then echo "Interface ${i} missing... assuming already deleted!" @@ -41,16 +41,16 @@ for i in $ifaces ; do echo "Deleting IPs on interface ${i}, with this information:" echo " $info" - try_command_on_node $test_node "$CTDB ip -v -Y | tail -n +2" - awk -F: -v i="$i" \ + try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2" + awk -F'|' -v i="$i" \ '$6 == i { print $2 }' <<<"$out" | while read ip ; do echo " $ip" try_command_on_node $test_node "$CTDB delip $ip" done - try_command_on_node $test_node "$CTDB ifaces -Y" - info=$(awk -F: -v iface="$i" '$2 == iface { print $0 }' <<<"$out") + try_command_on_node $test_node "$CTDB ifaces -X" + info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' <<<"$out") if [ -z "$info" ] ; then echo "GOOD: Interface ${i} has been garbage collected" diff --git a/ctdb/tests/simple/27_ctdb_detach.sh b/ctdb/tests/simple/27_ctdb_detach.sh index 108a270..4b7b3b5 100755 --- a/ctdb/tests/simple/27_ctdb_detach.sh +++ b/ctdb/tests/simple/27_ctdb_detach.sh @@ -35,7 +35,7 @@ ctdb_restart_when_done ###################################################################### -try_command_on_node 0 "$CTDB listnodes -Y" +try_command_on_node 0 "$CTDB listnodes -X" listnodes_output="$out" numnodes=$(wc -l <<<"$listnodes_output") diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh index 6cd2cce..e4d5708 100755 --- a/ctdb/tests/simple/75_readonly_records_basic.sh +++ b/ctdb/tests/simple/75_readonly_records_basic.sh @@ -113,8 +113,8 @@ check_readonly () ###################################################################### echo "Get list of nodes..." -try_command_on_node any $CTDB -Y listnodes -all_nodes=$(awk -F: '{print $2}' <<<"$out") +try_command_on_node any $CTDB -X listnodes +all_nodes=$(awk -F'|' '{print $2}' <<<"$out") ###################################################################### -- 2.1.3 From c9796d2909cf3c255dc5edc655e34cd296e332c1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 10:45:57 +1100 Subject: [PATCH 07/29] ctdb-tool: Fix "ctdb -Y ifaces" output to have trailing delimiters In the CTDB CLI tool source code and the documentation example. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 550fb8ce877cf980b4fd3be4f964449b369cf61e) --- ctdb/doc/ctdb.1.xml | 8 ++++---- ctdb/tools/ctdb.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml index 15bcabf..a3b0762 100644 --- a/ctdb/doc/ctdb.1.xml +++ b/ctdb/doc/ctdb.1.xml @@ -537,10 +537,10 @@ name:eth2 link:up references:1 # ctdb ifaces -Y :Name:LinkStatus:References: -:eth5:1:2 -:eth4:0:0 -:eth3:1:1 -:eth2:1:1 +:eth5:1:2: +:eth4:0:0: +:eth3:1:1: +:eth2:1:1: diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c index 072832d..44775c7 100644 --- a/ctdb/tools/ctdb.c +++ b/ctdb/tools/ctdb.c @@ -3141,7 +3141,7 @@ static int control_ifaces(struct ctdb_context *ctdb, int argc, const char **argv for (i=0; inum; i++) { if (options.machinereadable){ - printm(":%s:%s:%u\n", + printm(":%s:%s:%u:\n", ifaces->ifaces[i].name, ifaces->ifaces[i].link_state?"1":"0", (unsigned int)ifaces->ifaces[i].references); -- 2.1.3 From 010e4ced260698a91375ba7fcf4b0ec2bb1d9eed Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 10:47:22 +1100 Subject: [PATCH 08/29] ctdb-doc: Update examples to use ctdb -X Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 0d61b6137306ee0eab2981e3c213023c332da550) --- ctdb/doc/ctdb.1.xml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml index a3b0762..087227e 100644 --- a/ctdb/doc/ctdb.1.xml +++ b/ctdb/doc/ctdb.1.xml @@ -535,12 +535,12 @@ name:eth4 link:down references:0 name:eth3 link:up references:1 name:eth2 link:up references:1 -# ctdb ifaces -Y -:Name:LinkStatus:References: -:eth5:1:2: -:eth4:0:0: -:eth3:1:1: -:eth2:1:1: +# ctdb -X ifaces +|Name|LinkStatus|References| +|eth5|1|2| +|eth4|0|0| +|eth3|1|1| +|eth2|1|1| @@ -553,7 +553,7 @@ name:eth2 link:up references:1 Example -# ctdb ip +# ctdb ip -v Public IPs on node 0 172.31.91.82 node[1] active[] available[eth2,eth3] configured[eth2,eth3] 172.31.91.83 node[0] active[eth3] available[eth2,eth3] configured[eth2,eth3] @@ -564,16 +564,16 @@ Public IPs on node 0 172.31.92.84 node[1] active[] available[eth5] configured[eth4,eth5] 172.31.92.85 node[0] active[eth5] available[eth5] configured[eth4,eth5] -# ctdb ip -Y -:Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces: -:172.31.91.82:1::eth2,eth3:eth2,eth3: -:172.31.91.83:0:eth3:eth2,eth3:eth2,eth3: -:172.31.91.84:1::eth2,eth3:eth2,eth3: -:172.31.91.85:0:eth2:eth2,eth3:eth2,eth3: -:172.31.92.82:1::eth5:eth4,eth5: -:172.31.92.83:0:eth5:eth5:eth4,eth5: -:172.31.92.84:1::eth5:eth4,eth5: -:172.31.92.85:0:eth5:eth5:eth4,eth5: +# ctdb -X ip -v +|Public IP|Node|ActiveInterface|AvailableInterfaces|ConfiguredInterfaces| +|172.31.91.82|1||eth2,eth3|eth2,eth3| +|172.31.91.83|0|eth3|eth2,eth3|eth2,eth3| +|172.31.91.84|1||eth2,eth3|eth2,eth3| +|172.31.91.85|0|eth2|eth2,eth3|eth2,eth3| +|172.31.92.82|1||eth5|eth4,eth5| +|172.31.92.83|0|eth5|eth5|eth4,eth5| +|172.31.92.84|1||eth5|eth4,eth5| +|172.31.92.85|0|eth5|eth5|eth4,eth5| @@ -1172,9 +1172,9 @@ dbid:0x7bbbd26c name:passdb.tdb path:/var/ctdb/persistent/passdb.tdb.0 PERSISTEN Number of databases:1 dbid:0xb775fff6 name:secrets.tdb path:/var/ctdb/persistent/secrets.tdb.0 PERSISTENT UNHEALTHY -# ctdb -Y getdbmap -:ID:Name:Path:Persistent:Unhealthy: -:0x7bbbd26c:passdb.tdb:/var/ctdb/persistent/passdb.tdb.0:1:0: +# ctdb -X getdbmap +|ID|Name|Path|Persistent|Unhealthy| +|0x7bbbd26c|passdb.tdb|/var/ctdb/persistent/passdb.tdb.0|1|0| -- 2.1.3 From 8b0d0db7c60ee9b34ee2f3e3323c9e855cec6079 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 10:48:25 +1100 Subject: [PATCH 09/29] ctdb-utils: Update Nagios code to use ctdb -X Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit b41c1bdaa1df92ee6c510ae6749d0524b88ef828) --- ctdb/utils/nagios/check_ctdb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 ctdb/utils/nagios/check_ctdb diff --git a/ctdb/utils/nagios/check_ctdb b/ctdb/utils/nagios/check_ctdb old mode 100644 new mode 100755 index 837a0a4..7803f9a --- a/ctdb/utils/nagios/check_ctdb +++ b/ctdb/utils/nagios/check_ctdb @@ -177,7 +177,7 @@ sub safe_close_command { if ($info eq "scriptstatus") { $result = OK; - safe_open_command('ctdb', '-Y', 'scriptstatus'); + safe_open_command('ctdb', '-X', 'scriptstatus'); if ($result == OK) { my $script_count = 0; my $ok_script_count = 0; @@ -187,7 +187,7 @@ if ($info eq "scriptstatus") { next if $. == 1; # Header $script_count++; chop; - my ($col0, $type, $name, $code, $status, $start, $end, @error) = split(":"); + my ($col0, $type, $name, $code, $status, $start, $end, @error) = split("|"); if ($col0 ne '') { # Old version, before 30 Aug 2011 and commit a779d83a6213 ($type, $name, $code, $status, $start, $end, @error) = ($col0, $type, $name, $code, $status, $start, $end, @error); -- 2.1.3 From 241cb1027a6d034b3eb288ef28db7f21bf1dd393 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 14:37:54 +1100 Subject: [PATCH 10/29] ctdb-scripts: Add IPv6 addresses support in ip_maskbits_iface() It also prints a third word, the address family. This is either "inet" or "inet6". Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit ed029ae0a1faa56bf882a71d10828e2a90ab0bc7) --- ctdb/config/functions | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ctdb/config/functions b/ctdb/config/functions index 05bcd38..eca767e 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -864,8 +864,15 @@ ip_maskbits_iface () { _addr="$1" - ip addr show to "${_addr}/32" 2>/dev/null | \ - awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }' + case "$_addr" in + *:*) _family="inet6" ; _bits=128 ;; + *) _family="inet" ; _bits=32 ;; + esac + + ip addr show to "${_addr}/${_bits}" 2>/dev/null | \ + awk -v family="${_family}" \ + 'NR == 1 { iface = gensub(":$", "", 1, $2) } \ + $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }' } drop_ip () -- 2.1.3 From bc2637995ec62a91b48e57356ae7113904fd1118 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 14:39:43 +1100 Subject: [PATCH 11/29] ctdb-scripts: New functions ip6tables() and iptables_wrapper() ip6tables() uses the same lock as iptables(). This is done on suspicion. iptables_wrapper() takes 1st argument "inet" or "inet6", and the rest is passed to the correct iptables variant. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit c314ae0b2af4a902cdd003ec6d663fe5b62b003b) --- ctdb/config/functions | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ctdb/config/functions b/ctdb/config/functions index eca767e..82afefc 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -1331,10 +1331,23 @@ ctdb_standard_event_handler () } # iptables doesn't like being re-entered, so flock-wrap it. -iptables() +iptables () { flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@" } +ip6tables () +{ + flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/ip6tables "$@" +} +iptables_wrapper () +{ + _family="$1" ; shift + if [ "$_family" = "inet6" ] ; then + ip6tables "$@" + else + iptables "$@" + fi +} # AIX (and perhaps others?) doesn't have mktemp if ! which mktemp >/dev/null 2>&1 ; then -- 2.1.3 From 943d738b867ed43382d1f013833641f5d5cbfdfe Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 14:46:00 +1100 Subject: [PATCH 12/29] ctdb-scripts: Make 10.interface IPv6-safe Add checking to "releaseip" and "updateip" to ensure that the given IP address is really on the given interface with the given netmask. If reality doesn't match the given arguments then believe reality. Use new function iptables_wrapper() instead of calling iptables() directly. Use new function flush_route_cache() instead of doing IPv4-specific /proc magic. Remove setting of otherwise unused variable "failed". Fix a test for which the error message has changed. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 6471541d6d2bc9f2af0ff92b280abbd1d933cf88) --- ctdb/config/events.d/10.interface | 85 +++++++++++++++------- ctdb/config/functions | 6 ++ .../eventscripts/10.interface.releaseip.002.sh | 5 +- 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/ctdb/config/events.d/10.interface b/ctdb/config/events.d/10.interface index 8207fd3..51d1b97 100755 --- a/ctdb/config/events.d/10.interface +++ b/ctdb/config/events.d/10.interface @@ -137,6 +137,34 @@ monitor_interfaces() return 1 } +# Sets: iface, ip, maskbits, family +get_iface_ip_maskbits_family () +{ + _iface_in="$1" + ip="$2" + _maskbits_in="$3" + + set -- $(ip_maskbits_iface "$ip") + if [ -n "$1" ] ; then + maskbits="$1" + iface="$2" + family="$3" + + if [ "$iface" != "$_iface_in" ] ; then + printf \ + 'WARNING: Public IP %s hosted on interface %s but VNN says %s\n' \ + "$ip" "$iface" "$_iface_in" + fi + if [ "$maskbits" != "$_maskbits_in" ] ; then + printf \ + 'WARNING: Public IP %s has %s bit netmask but VNN says %s\n' \ + "$ip" "$maskbits" "$_maskbits_in" + fi + else + die "ERROR: Unable to determine interface for IP ${ip}" + fi +} + ctdb_check_args "$@" case "$1" in @@ -174,10 +202,13 @@ case "$1" in } # cope with the script being killed while we have the interface blocked - iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null + case "$ip" in + *:*) family="inet6" ;; + *) family="inet" ;; + esac + iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null - # flush our route cache - set_proc sys/net/ipv4/route/flush 1 + flush_route_cache ;; @@ -194,25 +225,23 @@ case "$1" in # 2) use netstat -tn to find existing connections, and kill them # 3) remove the IP from the interface # 4) remove the firewall rule - iface=$2 - ip=$3 - maskbits=$4 + shift + get_iface_ip_maskbits_family "$@" - failed=0 # we do an extra delete to cope with the script being killed - iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null - iptables -I INPUT -i $iface -d $ip -j DROP + iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null + iptables_wrapper $family -I INPUT -i $iface -d $ip -j DROP kill_tcp_connections $ip delete_ip_from_iface $iface $ip $maskbits || { - iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null - exit 1; + iptables_wrapper $family \ + -D INPUT -i $iface -d $ip -j DROP 2> /dev/null + exit 1 } - iptables -D INPUT -i $iface -d $ip -j DROP 2> /dev/null + iptables_wrapper $family -D INPUT -i $iface -d $ip -j DROP 2> /dev/null - # flush our route cache - set_proc sys/net/ipv4/route/flush 1 + flush_route_cache ;; ################################################## @@ -224,34 +253,36 @@ case "$1" in # we finally remove it from the old interface. # # 1) firewall this IP, so no new external packets arrive for it - # 2) add the IP to the new interface - # 3) remove the IP from the old interface + # 2) remove the IP from the old interface (and new interface, to be sure) + # 3) add the IP to the new interface # 4) remove the firewall rule # 5) use ctdb gratiousarp to propagate the new mac address # 6) use netstat -tn to find existing connections, and tickle them - oiface=$2 + _oiface=$2 niface=$3 - ip=$4 - maskbits=$5 + _ip=$4 + _maskbits=$5 + + get_iface_ip_maskbits_family "$_oiface" "$ip" "$maskbits" + oiface="$iface" - failed=0 # we do an extra delete to cope with the script being killed - iptables -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null - iptables -I INPUT -i $oiface -d $ip -j DROP + iptables_wrapper $family -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null + iptables_wrapper $family -I INPUT -i $oiface -d $ip -j DROP delete_ip_from_iface $oiface $ip $maskbits 2>/dev/null delete_ip_from_iface $niface $ip $maskbits 2>/dev/null add_ip_to_iface $niface $ip $maskbits || { - iptables -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null - exit 1; + iptables_wrapper $family \ + -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null + exit 1 } # cope with the script being killed while we have the interface blocked - iptables -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null + iptables_wrapper $family -D INPUT -i $oiface -d $ip -j DROP 2> /dev/null - # flush our route cache - set_proc sys/net/ipv4/route/flush 1 + flush_route_cache # propagate the new mac address ctdb gratiousarp $ip $niface diff --git a/ctdb/config/functions b/ctdb/config/functions index 82afefc..190618b 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -895,6 +895,12 @@ drop_all_public_ips () done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}" } +flush_route_cache () +{ + set_proc sys/net/ipv4/route/flush 1 + set_proc sys/net/ipv6/route/flush 1 +} + ######################################################## # Simple counters _ctdb_counter_common () { diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.002.sh b/ctdb/tests/eventscripts/10.interface.releaseip.002.sh index 9bcb7f1..4b9726c 100755 --- a/ctdb/tests/eventscripts/10.interface.releaseip.002.sh +++ b/ctdb/tests/eventscripts/10.interface.releaseip.002.sh @@ -9,9 +9,6 @@ setup_ctdb public_address=$(ctdb_get_1_public_address) ip="${public_address% *}" ; ip="${ip#* }" -required_result 1 < Date: Fri, 21 Nov 2014 14:52:47 +1100 Subject: [PATCH 13/29] ctdb-daemon: Trust vnn->interface for an IP when releasing it ctdb_sys_find_ifname() doesn't work for IPv6 addresses so don't use it. Trust the eventscript to do sanity checking on the interface. Current warnings are replaced with equivalents generated by the eventscript. The unlikely message: Public IP %s is hosted on interface %s but we have no VNN will be replaced by: WARNING: Public IP %s hosted on interface %s but VNN says __none__ which is clear enough. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 108b1be0ee62af7ecb3c775f45c540dd07a527bf) --- ctdb/server/ctdb_takeover.c | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/ctdb/server/ctdb_takeover.c b/ctdb/server/ctdb_takeover.c index 7603fbc..f8369da 100644 --- a/ctdb/server/ctdb_takeover.c +++ b/ctdb/server/ctdb_takeover.c @@ -996,30 +996,7 @@ int32_t ctdb_control_release_ip(struct ctdb_context *ctdb, return -1; } - if (ctdb->do_checkpublicip) { - iface = ctdb_sys_find_ifname(&pip->addr); - if (iface == NULL) { - DEBUG(DEBUG_ERR, ("Could not find which interface the ip address is hosted on. can not release it\n")); - return 0; - } - if (vnn->iface == NULL) { - DEBUG(DEBUG_WARNING, - ("Public IP %s is hosted on interface %s but we have no VNN\n", - ctdb_addr_to_str(&pip->addr), - iface)); - } else if (strcmp(iface, ctdb_vnn_iface_string(vnn)) != 0) { - DEBUG(DEBUG_WARNING, - ("Public IP %s is hosted on inteterface %s but VNN says %s\n", - ctdb_addr_to_str(&pip->addr), - iface, - ctdb_vnn_iface_string(vnn))); - /* Should we fix vnn->iface? If we do, what - * happens to reference counts? - */ - } - } else { - iface = strdup(ctdb_vnn_iface_string(vnn)); - } + iface = strdup(ctdb_vnn_iface_string(vnn)); DEBUG(DEBUG_NOTICE,("Release of IP %s/%u on interface %s node:%d\n", ctdb_addr_to_str(&pip->addr), -- 2.1.3 From 1fa808918e5a46f973e1e9f8ce4d4200691da618 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 20 Nov 2014 21:58:31 +1100 Subject: [PATCH 14/29] ctdb-eventscripts: Specify broadcast optionally to ip addr add Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit d4212bd6a533b4b54b56e376a9246f2396cba253) --- ctdb/config/functions | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ctdb/config/functions b/ctdb/config/functions index 190618b..e50e98a 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -834,7 +834,13 @@ add_ip_to_iface () ip link set "$_iface" up || \ die "Failed to bringup interface $_iface" - ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || { + # Only need to define broadcast for IPv4 + case "$ip" in + *:*) _bcast="" ;; + *) _bcast="brd +" ;; + esac + + ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || { echo "Failed to add $_ip/$_maskbits on dev $_iface" return 1 } -- 2.1.3 From 1331a1d454d43d1b6b66aaaa219d337abd3ba93c Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 21 Nov 2014 17:33:21 +1100 Subject: [PATCH 15/29] ctdb-scripts: Wait until IPv6 addresses are not "tentative" There are a few potential failure modes when adding an IPv6 address. It takes a little while of duplicate address detection to complete, so wait for a while. After a timeout, also need to check to see if duplicate address detection failed - if it did then actually drop the IP address. This really needs some careful thinking. If CTDB disappears on a node but the node's IP addresses are still on interfaces then the above failure mode could cause the takeover nodes to become banned. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit d0b2375c3d754da3cd68e34617ab3edd36e9317b) --- ctdb/config/functions | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ctdb/config/functions b/ctdb/config/functions index e50e98a..1583bfc 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -844,6 +844,29 @@ add_ip_to_iface () echo "Failed to add $_ip/$_maskbits on dev $_iface" return 1 } + + # Wait 5 seconds for IPv6 addresses to stop being tentative... + if [ -z "$_bcast" ] ; then + for _x in $(seq 1 10) ; do + ip addr show to "${_ip}/128" | grep -q "tentative" || break + sleep 0.5 + done + + # If the address was a duplicate then it won't be on the + # interface so flag an error. + _t=$(ip addr show to "${_ip}/128") + case "$_t" in + "") + echo "Failed to add $_ip/$_maskbits on dev $_iface" + return 1 + ;; + *tentative*|*dadfailed*) + echo "Failed to add $_ip/$_maskbits on dev $_iface" + ip addr del "$_ip/$_maskbits" dev "$_iface" + return 1 + ;; + esac + fi } delete_ip_from_iface() -- 2.1.3 From 9950b9bf022a4b797bf042e6f0dc8d9b0aa61e05 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 25 Nov 2014 12:38:23 +1100 Subject: [PATCH 16/29] ctdb-daemon: Fix IP address comparisons for IPv6 addresses Before storing node IP address, convert into the correct abbreviated string form for IPv6 addresses. Signed-off-by: Amitay Isaacs Reviewed-by: Martin Schwenke (cherry picked from commit e3c59d83d0ace9d7421d40d33fe917fb82bb38d8) --- ctdb/common/ctdb_util.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ctdb/common/ctdb_util.c b/ctdb/common/ctdb_util.c index bdff425..137e0a8 100644 --- a/ctdb/common/ctdb_util.c +++ b/ctdb/common/ctdb_util.c @@ -100,12 +100,20 @@ int ctdb_parse_address(struct ctdb_context *ctdb, struct ctdb_address *address) { struct servent *se; + ctdb_sock_addr addr; setservent(0); se = getservbyname("ctdb", "tcp"); endservent(); - - address->address = talloc_strdup(mem_ctx, str); + + /* Parse IP address and re-convert to string. This ensure correct + * string form for IPv6 addresses. + */ + if (! parse_ip(str, NULL, 0, &addr)) { + return -1; + } + + address->address = talloc_strdup(mem_ctx, ctdb_addr_to_str(&addr)); CTDB_NO_MEMORY(ctdb, address->address); if (se == NULL) { -- 2.1.3 From 35100fb97fe1eb495cf4a220191ab9eb68ee5cc1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 28 Nov 2014 21:49:08 +1100 Subject: [PATCH 17/29] ctdb-tools: Bracket IP addresses in onnode (for IPv6) Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 2d480792e7d0b3f6628186dc77f54b708e315dd1) --- ctdb/tools/onnode | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctdb/tools/onnode b/ctdb/tools/onnode index 96d569a..2ca3113 100755 --- a/ctdb/tools/onnode +++ b/ctdb/tools/onnode @@ -320,8 +320,8 @@ push() for f in $files ; do $verbose && echo "Pushing $f" case "$f" in - /*) rsync "$f" "${host}:${f}" ;; - *) rsync "${PWD}/${f}" "${host}:${PWD}/${f}" ;; + /*) rsync "$f" "[${host}]:${f}" ;; + *) rsync "${PWD}/${f}" "[${host}]:${PWD}/${f}" ;; esac done } -- 2.1.3 From ff11a71e3a49f436ea90b4c9b91a3980333e125b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 26 Nov 2014 21:31:42 +1100 Subject: [PATCH 18/29] ctdb-tests: Extend regexp to match IPv6 addresses Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 2a83b74597a3366536e1935e2e8ff23493503117) --- ctdb/tests/simple/05_ctdb_listnodes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh index a84e4af..9e48a68 100755 --- a/ctdb/tests/simple/05_ctdb_listnodes.sh +++ b/ctdb/tests/simple/05_ctdb_listnodes.sh @@ -35,9 +35,11 @@ try_command_on_node -v 0 "$CTDB listnodes" num_nodes=$(echo "$out" | wc -l) # Each line should look like an IP address. +ipv4_pat='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+' +ipv6_pat='[[:xdigit:]]+:[[:xdigit:]:]+[[:xdigit:]]+' sanity_check_output \ 2 \ - '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' \ + "^${ipv4_pat}|${ipv6_pat}\$" \ "$out" out_0="$out" -- 2.1.3 From 6cdd23ecf27fcd5834a02e978fcce1e15b8333a7 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 26 Nov 2014 21:32:22 +1100 Subject: [PATCH 19/29] ctdb-tests: Try to handle IPv6 addresses for local daemons If CTDB_USE_IPV6 is set then use IPv6 addresses for nodes and public IPs. This can be useful for some simple tests. However, the node address actually needs to be on lo so that ctdbd can bind to the port on that address, so they actually need to be added as root before running tests, like this: for i in $(seq 1 10) ; do ip addr add "fc00:10::${i}/64" dev lo ; done IPv4 127.0.0.0/8 addresses are somehow magic and only one needs to be on lo so that many can be bound to. Also change the IPv4 node addresses to be (slightly) more exotic. For both IPv4 and IPv6, choose addresses that are compatible with socket wrapper. Signed-off-by: Martin Schwenke Signed-off-by: Amitay Isaacs (socket wrapper fixes) Reviewed-by: Amitay Isaacs Reviewed-by: Martin Schwenke (socket wrapper fixes) (cherry picked from commit d9d07fff34143d251b4987cdb1cff1e8b3384e64) --- ctdb/tests/simple/scripts/local_daemons.bash | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ctdb/tests/simple/scripts/local_daemons.bash b/ctdb/tests/simple/scripts/local_daemons.bash index a227a5d..9b5cf04 100644 --- a/ctdb/tests/simple/scripts/local_daemons.bash +++ b/ctdb/tests/simple/scripts/local_daemons.bash @@ -67,10 +67,18 @@ setup_ctdb () local i for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do if [ "${CTDB_USE_IPV6}x" != "x" ]; then - echo ::$i >>"$CTDB_NODES" - ip addr add ::$i/128 dev lo + j=$((printf "%02x" $i)) + echo "fd00::5357:5f${j}" >>"$CTDB_NODES" + # FIXME: need to add addresses to lo as root before running :-( + # ip addr add "fc00:10::${i}/64" dev lo + # 2 public addresses on most nodes, just to make things interesting. + if [ $(($i - 1)) -ne $no_public_ips ] ; then + echo "fc00:10::1:${i}/64 lo" >>"$public_addresses_all" + echo "fc00:10::1:$(($i + $TEST_LOCAL_DAEMONS))/64 lo" >>"$public_addresses_all" + fi else - echo 127.0.0.$i >>"$CTDB_NODES" + j=$(( $i + 10)) + echo 127.0.0.$j >>"$CTDB_NODES" # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then echo "192.168.234.$i/24 lo" >>"$public_addresses_all" -- 2.1.3 From f567d5a395ca2baab6c71876de9b57ff824ed8ed Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2014 15:57:35 +1100 Subject: [PATCH 20/29] ctdb-tests: Bracket IP addresses in NFS mounts and scp command (for IPv6) Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 62164ec52fd1082863bf5017a5170f74f18a07c3) --- ctdb/tests/complex/44_failover_nfs_oneway.sh | 2 +- ctdb/tests/scripts/integration.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ctdb/tests/complex/44_failover_nfs_oneway.sh b/ctdb/tests/complex/44_failover_nfs_oneway.sh index aaec2ed..1bc09f5 100755 --- a/ctdb/tests/complex/44_failover_nfs_oneway.sh +++ b/ctdb/tests/complex/44_failover_nfs_oneway.sh @@ -59,7 +59,7 @@ ctdb_test_exit_hook_add rm -f "$local_f" dd if=/dev/urandom of=$local_f bs=1k count=1 local_sum=$(sum $local_f) -scp -p "$local_f" "${test_ip}:${nfs_remote_file}" +scp -p "$local_f" "[${test_ip}]:${nfs_remote_file}" try_command_on_node $test_node "chmod 644 $nfs_remote_file" nfs_sum=$(sum $nfs_local_file) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index b75e8f9..2ae0342 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -669,7 +669,7 @@ nfs_test_setup () echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..." mount -o timeo=1,hard,intr,vers=3 \ - ${test_ip}:${nfs_first_export} ${nfs_mnt_d} + "[${test_ip}]:${nfs_first_export}" ${nfs_mnt_d} } nfs_test_cleanup () -- 2.1.3 From 6d8cb4c9ef316d06116449f8b1435e751b70019c Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2014 15:58:20 +1100 Subject: [PATCH 21/29] ctdb-tests: Extend regexps to handle IPv6 address matching Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 62acf5986f52a3118ed4e3638c5ac8b1f9c0adf8) --- ctdb/tests/complex/11_ctdb_delip_removes_ip.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh index d67cb07..d023e98 100755 --- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh +++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh @@ -26,7 +26,7 @@ select_test_node_and_ips get_test_ip_mask_and_iface echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..." -try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${test_ip}/'" +try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet6?[[:space:]]*${test_ip}/'" echo "Attempting to remove ${test_ip} from node ${test_node}." try_command_on_node $test_node $CTDB delip $test_ip @@ -38,7 +38,7 @@ count=0 echo "Waiting for ${test_ip} to disappear from ${iface}..." while : ; do try_command_on_node -v $test_node "ip addr show dev $iface" - if echo "$out" | grep -E 'inet[[:space:]]*${test_ip}/'; then + if echo "$out" | grep -E 'inet6?[[:space:]]*${test_ip}/'; then echo "Still there..." if [ $(($count * $increment)) -ge $timeout ] ; then echo "BAD: Timed out waiting..." -- 2.1.3 From 21bf983cf53e90b2486e0edfdd2ba43bc38ead8c Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 1 Dec 2014 13:50:42 +1100 Subject: [PATCH 22/29] ctdb-tests: Use ping_wrapper to do relevant ping or ping6 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 255705c030135bd54a1f7a7dc40cbf00c2fb39c9) --- ctdb/tests/complex/33_gratuitous_arp.sh | 2 +- ctdb/tests/complex/41_failover_ping_discrete.sh | 4 ++-- ctdb/tests/complex/scripts/local.bash | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ctdb/tests/complex/33_gratuitous_arp.sh b/ctdb/tests/complex/33_gratuitous_arp.sh index 721b0f2..a51aa0c 100755 --- a/ctdb/tests/complex/33_gratuitous_arp.sh +++ b/ctdb/tests/complex/33_gratuitous_arp.sh @@ -57,7 +57,7 @@ echo "Removing ${test_ip} from the local ARP table..." arp -d $test_ip >/dev/null 2>&1 || true echo "Pinging ${test_ip}..." -ping -q -n -c 1 $test_ip +ping_wrapper -q -n -c 1 $test_ip echo "Getting MAC address associated with ${test_ip}..." original_mac=$(arp -n $test_ip | awk '$2 == "ether" {print $3}') diff --git a/ctdb/tests/complex/41_failover_ping_discrete.sh b/ctdb/tests/complex/41_failover_ping_discrete.sh index 88b2013..1fe7c1f 100755 --- a/ctdb/tests/complex/41_failover_ping_discrete.sh +++ b/ctdb/tests/complex/41_failover_ping_discrete.sh @@ -51,7 +51,7 @@ echo "Removing ${test_ip} from the local ARP table..." arp -d $test_ip >/dev/null 2>&1 || true echo "Pinging ${test_ip}..." -ping -q -n -c 1 $test_ip +ping_wrapper -q -n -c 1 $test_ip gratarp_sniff_start @@ -65,4 +65,4 @@ echo "Removing ${test_ip} from the local ARP table again..." arp -d $test_ip >/dev/null 2>&1 || true echo "Pinging ${test_ip} again..." -ping -q -n -c 1 $test_ip +ping_wrapper -q -n -c 1 $test_ip diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash index 6fbc1ae..fcd919b 100644 --- a/ctdb/tests/complex/scripts/local.bash +++ b/ctdb/tests/complex/scripts/local.bash @@ -170,3 +170,10 @@ ctdb_test_check_real_cluster () done } +ping_wrapper () +{ + case "$*" in + *:*) ping6 "$@" ;; + *) ping "$@" ;; + esac +} -- 2.1.3 From 40912a71431e76ccdc39249c0e6d785353f087b8 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 1 Dec 2014 13:51:47 +1100 Subject: [PATCH 23/29] ctdb-tests: Match IPv6 connections in netstat output Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 3135a8c62551e09a3abada86335882a91d398747) --- ctdb/tests/complex/scripts/local.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash index fcd919b..ce84545 100644 --- a/ctdb/tests/complex/scripts/local.bash +++ b/ctdb/tests/complex/scripts/local.bash @@ -7,7 +7,7 @@ get_src_socket () local pid="$3" local prog="$4" - local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + local pat="^${proto}6?[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" out=$(netstat -tanp | egrep "$pat" | awk '{ print $4 }') -- 2.1.3 From 3804c28389aee7b92c6832eedefb7da02c05f606 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 1 Dec 2014 14:07:57 +1100 Subject: [PATCH 24/29] ctdb-tests: Generalise the gratarp and tickle sniffing code for IPv6 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs Pair-programmed-with: Amitay Isaacs (cherry picked from commit 0f3d9752c4677b2f3b5ee47a0b8f973b4260ef57) --- ctdb/tests/complex/scripts/local.bash | 80 +++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash index ce84545..4e0b1b4 100644 --- a/ctdb/tests/complex/scripts/local.bash +++ b/ctdb/tests/complex/scripts/local.bash @@ -1,5 +1,35 @@ # Hey Emacs, this is a -*- shell-script -*- !!! :-) +# Thanks/blame to Stephen Rothwell for suggesting that this can be +# done in the shell. ;-) +ipv6_to_hex () +{ + local addr="$1" + + # Replace "::" by something special. + local foo="${addr/::/:@:}" + + # Join the groups of digits together, 0-padding each group of + # digits out to 4 digits, and count the number of (non-@) groups + local out="" + local count=0 + local i + for i in $(IFS=":" ; echo $foo ) ; do + if [ "$i" = "@" ] ; then + out="${out}@" + else + out="${out}$(printf '%04x' 0x${i})" + count=$(($count + 4)) + fi + done + + # Replace '@' with correct number of zeroes + local zeroes=$(printf "%0$((32 - $count))x" 0) + echo "${out/@/${zeroes}}" +} + +####################################### + get_src_socket () { local proto="$1" @@ -119,14 +149,14 @@ tcpdump_show () tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null } -tcptickle_sniff_start () +tcp4tickle_sniff_start () { local src="$1" local dst="$2" local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" - local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 + local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14:2] == 1234)" # win == 1234 local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" @@ -134,6 +164,33 @@ tcptickle_sniff_start () tcpdump_start "$filter" } +# tcp[] does not work for IPv6 (in some versions of tcpdump) +tcp6tickle_sniff_start () +{ + local src="$1" + local dst="$2" + + local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" + local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" + local tickle_ack="${in} and (ip6[53] & tcp-ack != 0) and (ip6[54:2] == 1234)" # win == 1234 + local ack_ack="${out} and (ip6[53] & tcp-ack != 0)" + tcptickle_reset="${in} and ip6[53] & tcp-rst != 0" + local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" + + tcpdump_start "$filter" +} + +tcptickle_sniff_start () +{ + local src="$1" + local dst="$2" + + case "$dst" in + *:*) tcp6tickle_sniff_start "$src" "$dst" ;; + *) tcp4tickle_sniff_start "$src" "$dst" ;; + esac +} + tcptickle_sniff_wait_show () { tcpdump_wait 1 "$tcptickle_reset" @@ -142,11 +199,28 @@ tcptickle_sniff_wait_show () tcpdump_show } -gratarp_sniff_start () +gratarp4_sniff_start () { tcpdump_start "arp host ${test_ip}" } +gratarp6_sniff_start () +{ + local neighbor_advertisement="icmp6 and ip6[40] == 136" + local hex=$(ipv6_to_hex "$test_ip") + local match_target="ip6[48:4] == 0x${hex:0:8} and ip6[52:4] == 0x${hex:8:8} and ip6[56:4] == 0x${hex:16:8} and ip6[60:4] == 0x${hex:24:8}" + + tcpdump_start "${neighbor_advertisement} and ${match_target}" +} + +gratarp_sniff_start () +{ + case "$test_ip" in + *:*) gratarp6_sniff_start ;; + *) gratarp4_sniff_start ;; + esac +} + gratarp_sniff_wait_show () { tcpdump_wait 2 -- 2.1.3 From 5896d6dc6c1faf02e09d4525269340202fdf9a42 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Sat, 29 Nov 2014 20:01:20 +1100 Subject: [PATCH 25/29] ctdb-tests: Use ip neigh command instead of arp Extend select_test_node_and_ips() to set $test_prefix in addition to $test_ip. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit a7e7d95de9cf09652cef63d65484bbb0faa42e1c) --- ctdb/tests/complex/33_gratuitous_arp.sh | 10 +++++----- ctdb/tests/complex/41_failover_ping_discrete.sh | 8 ++++---- ctdb/tests/complex/42_failover_ssh_hostname.sh | 4 ++-- ctdb/tests/scripts/integration.bash | 5 +++++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ctdb/tests/complex/33_gratuitous_arp.sh b/ctdb/tests/complex/33_gratuitous_arp.sh index a51aa0c..2ee2e06 100755 --- a/ctdb/tests/complex/33_gratuitous_arp.sh +++ b/ctdb/tests/complex/33_gratuitous_arp.sh @@ -54,14 +54,14 @@ ctdb_restart_when_done select_test_node_and_ips echo "Removing ${test_ip} from the local ARP table..." -arp -d $test_ip >/dev/null 2>&1 || true +ip neigh flush "$test_prefix" >/dev/null 2>&1 || true echo "Pinging ${test_ip}..." ping_wrapper -q -n -c 1 $test_ip echo "Getting MAC address associated with ${test_ip}..." -original_mac=$(arp -n $test_ip | awk '$2 == "ether" {print $3}') -[ $? -eq 0 ] +original_mac=$(ip neigh show $test_prefix | awk '$4 == "lladdr" {print $5}') +[ -n "$original_mac" ] || die "Couldn't get MAC address for ${test_prefix}" echo "MAC address is: ${original_mac}" @@ -74,8 +74,8 @@ wait_until_node_has_status $test_node disabled gratarp_sniff_wait_show echo "Getting MAC address associated with ${test_ip} again..." -new_mac=$(arp -n $test_ip | awk '$2 == "ether" {print $3}') -[ $? -eq 0 ] +new_mac=$(ip neigh show $test_prefix | awk '$4 == "lladdr" {print $5}') +[ -n "$new_mac" ] || die "Couldn't get MAC address for ${test_prefix}" echo "MAC address is: ${new_mac}" diff --git a/ctdb/tests/complex/41_failover_ping_discrete.sh b/ctdb/tests/complex/41_failover_ping_discrete.sh index 1fe7c1f..22780c4 100755 --- a/ctdb/tests/complex/41_failover_ping_discrete.sh +++ b/ctdb/tests/complex/41_failover_ping_discrete.sh @@ -47,8 +47,8 @@ ctdb_restart_when_done select_test_node_and_ips -echo "Removing ${test_ip} from the local ARP table..." -arp -d $test_ip >/dev/null 2>&1 || true +echo "Removing ${test_ip} from the local neighbor table..." +ip neigh flush "$test_prefix" >/dev/null 2>&1 || true echo "Pinging ${test_ip}..." ping_wrapper -q -n -c 1 $test_ip @@ -61,8 +61,8 @@ wait_until_node_has_status $test_node disabled gratarp_sniff_wait_show -echo "Removing ${test_ip} from the local ARP table again..." -arp -d $test_ip >/dev/null 2>&1 || true +echo "Removing ${test_ip} from the local neighbor table again..." +ip neigh flush "$test_prefix" >/dev/null 2>&1 || true echo "Pinging ${test_ip} again..." ping_wrapper -q -n -c 1 $test_ip diff --git a/ctdb/tests/complex/42_failover_ssh_hostname.sh b/ctdb/tests/complex/42_failover_ssh_hostname.sh index defe15a..a74c632 100755 --- a/ctdb/tests/complex/42_failover_ssh_hostname.sh +++ b/ctdb/tests/complex/42_failover_ssh_hostname.sh @@ -47,8 +47,8 @@ ctdb_restart_when_done select_test_node_and_ips -echo "Removing ${test_ip} from the local ARP table..." -arp -d $test_ip >/dev/null 2>&1 || true +echo "Removing ${test_ip} from the local neighbor table..." +ip neigh flush "$test_prefix" >/dev/null 2>&1 || true echo "SSHing to ${test_ip} and running hostname..." original_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 2ae0342..6bec78e 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -186,6 +186,11 @@ _select_test_node_and_ips () echo "Selected node ${test_node} with IPs: ${test_node_ips}." test_ip="${test_node_ips%% *}" + case "$test_ip" in + *:*) test_prefix="${test_ip}/128" ;; + *) test_prefix="${test_ip}/32" ;; + esac + [ -n "$test_node" ] || return 1 } -- 2.1.3 From 5492dc8b0d8e9edb3df46439f73ca7a3b56ebc24 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 1 Dec 2014 13:30:29 +1100 Subject: [PATCH 26/29] ctdb-tests: Make tcpdump output more verbose This helps with debugging. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 49449f66fca52d5044d2e486570562df866adf57) --- ctdb/tests/complex/scripts/local.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash index 4e0b1b4..9c41af2 100644 --- a/ctdb/tests/complex/scripts/local.bash +++ b/ctdb/tests/complex/scripts/local.bash @@ -146,7 +146,7 @@ tcpdump_show () { local filter="${1:-${tcpdump_filter}}" - tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null + tcpdump -n -e -vv -XX -r $tcpdump_filename "$filter" 2>/dev/null } tcp4tickle_sniff_start () -- 2.1.3 From 4903c634399971bd2be1b50ac88753766d26b172 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2014 12:09:12 +1100 Subject: [PATCH 27/29] ctdb-tests: More debug on SSH failure Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 3a2c8bb906fac4e2611a28ead6b4290ddc93de54) --- ctdb/tests/complex/42_failover_ssh_hostname.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ctdb/tests/complex/42_failover_ssh_hostname.sh b/ctdb/tests/complex/42_failover_ssh_hostname.sh index a74c632..8c0ac10 100755 --- a/ctdb/tests/complex/42_failover_ssh_hostname.sh +++ b/ctdb/tests/complex/42_failover_ssh_hostname.sh @@ -51,8 +51,9 @@ echo "Removing ${test_ip} from the local neighbor table..." ip neigh flush "$test_prefix" >/dev/null 2>&1 || true echo "SSHing to ${test_ip} and running hostname..." -original_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) -[ $? -eq 0 ] +if ! original_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) ; then + die "Failed to get original hostname via SSH..." +fi echo "Hostname is: ${original_hostname}" @@ -65,8 +66,12 @@ wait_until_node_has_status $test_node disabled gratarp_sniff_wait_show echo "SSHing to ${test_ip} and running hostname (again)..." -new_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) -[ $? -eq 0 ] +if ! new_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) ; then + echo "Failed to get new hostname via SSH..." + echo "DEBUG:" + ip neigh show + exit 1 +fi echo "Hostname is: ${new_hostname}" -- 2.1.3 From 80319559a82d0ca68903ae18d9d3316d4418b9d4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 2 Dec 2014 10:57:12 +1100 Subject: [PATCH 28/29] ctdb-daemon: Gratuitous ARP equivalent for IPv6 is neighbor advertisement Not neighbour solicitation. See: https://tools.ietf.org/html/rfc4861#section-4.4 Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs (cherry picked from commit 7f3f3b15d52c4047cbcb1c3e81f65675c8708f5f) --- ctdb/common/system_linux.c | 49 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/ctdb/common/system_linux.c b/ctdb/common/system_linux.c index 9aaa1fd..97a57ac 100644 --- a/ctdb/common/system_linux.c +++ b/ctdb/common/system_linux.c @@ -75,9 +75,14 @@ int ctdb_sys_send_arp(const ctdb_sock_addr *addr, const char *iface) struct ether_header *eh; struct arphdr *ah; struct ip6_hdr *ip6; - struct nd_neighbor_solicit *nd_ns; + struct nd_neighbor_advert *nd_na; + struct nd_opt_hdr *nd_oh; struct ifreq if_hwaddr; - unsigned char buffer[78]; /* ipv6 neigh solicitation size */ + /* Size of IPv6 neighbor advertisement (with option) */ + unsigned char buffer[sizeof(struct ether_header) + + sizeof(struct ip6_hdr) + + sizeof(struct nd_neighbor_advert) + + sizeof(struct nd_opt_hdr) + ETH_ALEN]; char *ptr; char bdcast[] = {0xff,0xff,0xff,0xff,0xff,0xff}; struct ifreq ifr; @@ -219,31 +224,45 @@ int ctdb_sys_send_arp(const ctdb_sock_addr *addr, const char *iface) memset(buffer, 0 , sizeof(buffer)); eh = (struct ether_header *)buffer; - memset(eh->ether_dhost, 0xff, ETH_ALEN); + /* Ethernet multicast: 33:33:00:00:00:01 (see RFC2464, + * section 7) - note zeroes above! */ + eh->ether_dhost[0] = eh->ether_dhost[1] = 0x33; + eh->ether_dhost[5] = 0x01; memcpy(eh->ether_shost, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN); eh->ether_type = htons(ETHERTYPE_IP6); ip6 = (struct ip6_hdr *)(eh+1); ip6->ip6_vfc = 0x60; - ip6->ip6_plen = htons(sizeof(*nd_ns)); + ip6->ip6_plen = htons(sizeof(*nd_na) + + sizeof(struct nd_opt_hdr) + + ETH_ALEN); ip6->ip6_nxt = IPPROTO_ICMPV6; ip6->ip6_hlim = 255; - ip6->ip6_dst = addr->ip6.sin6_addr; - - nd_ns = (struct nd_neighbor_solicit *)(ip6+1); - nd_ns->nd_ns_type = ND_NEIGHBOR_SOLICIT; - nd_ns->nd_ns_code = 0; - nd_ns->nd_ns_reserved = 0; - nd_ns->nd_ns_target = addr->ip6.sin6_addr; - - nd_ns->nd_ns_cksum = tcp_checksum6((uint16_t *)nd_ns, ntohs(ip6->ip6_plen), ip6); + ip6->ip6_src = addr->ip6.sin6_addr; + /* all-nodes multicast */ + inet_pton(AF_INET6, "ff02::1", &ip6->ip6_dst); + + nd_na = (struct nd_neighbor_advert *)(ip6+1); + nd_na->nd_na_type = ND_NEIGHBOR_ADVERT; + nd_na->nd_na_code = 0; + nd_na->nd_na_flags_reserved = ND_NA_FLAG_OVERRIDE; + nd_na->nd_na_target = addr->ip6.sin6_addr; + /* Option: Target link-layer address */ + nd_oh = (struct nd_opt_hdr *)(nd_na+1); + nd_oh->nd_opt_type = ND_OPT_TARGET_LINKADDR; + nd_oh->nd_opt_len = 1; + memcpy(&(nd_oh+1)[0], if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN); + + nd_na->nd_na_cksum = tcp_checksum6((uint16_t *)nd_na, + ntohs(ip6->ip6_plen), ip6); sall.sll_family = AF_PACKET; sall.sll_halen = 6; - memcpy(&sall.sll_addr[0], bdcast, sall.sll_halen); + memcpy(&sall.sll_addr[0], &eh->ether_dhost[0], sall.sll_halen); sall.sll_protocol = htons(ETH_P_ALL); sall.sll_ifindex = ifr.ifr_ifindex; - ret = sendto(s, buffer, 78, 0, (struct sockaddr *)&sall, sizeof(sall)); + ret = sendto(s, buffer, sizeof(buffer), + 0, (struct sockaddr *)&sall, sizeof(sall)); if (ret < 0 ){ close(s); DEBUG(DEBUG_CRIT,(__location__ " failed sendto\n")); -- 2.1.3 From 86830d2bf07b3583b412a96484629699d180de0b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2014 12:10:07 +1100 Subject: [PATCH 29/29] ctdb-tests: Need to drop public IPs in kill-failover tests These tests simulate a dead node rather than a CTDB failure, so drop IP addresses when killing a "node" to avoid problems with duplicates. To cope with a CTDB failure a watchdog would be needed to ensure that the public IPs are dropped when CTDB dies. Let's not do that now. Signed-off-by: Martin Schwenke Reviewed-by: Amitay Isaacs Autobuild-User(master): Martin Schwenke Autobuild-Date(master): Fri Dec 5 23:29:39 CET 2014 on sn-devel-104 (cherry picked from commit a07af1bb56a76d1a7ab856547c801499fc17b21b) --- ctdb/tests/complex/31_nfs_tickle.sh | 7 ++++-- ctdb/tests/complex/45_failover_nfs_kill.sh | 7 +++++- ctdb/tests/scripts/integration.bash | 38 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/ctdb/tests/complex/31_nfs_tickle.sh b/ctdb/tests/complex/31_nfs_tickle.sh index 5aeb870..fbb30c1 100755 --- a/ctdb/tests/complex/31_nfs_tickle.sh +++ b/ctdb/tests/complex/31_nfs_tickle.sh @@ -89,9 +89,12 @@ wait_until $(($update_interval * 2)) \ tcptickle_sniff_start $src_socket "${test_ip}:${test_port}" # We need to be nasty to make that the node being failed out doesn't -# get a chance to send any tickles and confuse our sniff. +# get a chance to send any tickles and confuse our sniff. IPs also +# need to be dropped because we're simulating a dead node rather than +# a CTDB failure. To properly handle a CTDB failure we would need a +# watchdog to drop the IPs when CTDB disappears. echo "Killing ctdbd on ${test_node}..." -try_command_on_node $test_node killall -9 ctdbd +try_command_on_node -v $test_node "killall -9 ctdbd ; $CTDB_TEST_WRAPPER drop_ips ${test_node_ips}" wait_until_node_has_status $test_node disconnected diff --git a/ctdb/tests/complex/45_failover_nfs_kill.sh b/ctdb/tests/complex/45_failover_nfs_kill.sh index 52b423f..aada371 100755 --- a/ctdb/tests/complex/45_failover_nfs_kill.sh +++ b/ctdb/tests/complex/45_failover_nfs_kill.sh @@ -61,7 +61,12 @@ gratarp_sniff_start echo "Killing node $test_node" try_command_on_node $test_node $CTDB getpid pid=${out#*:} -try_command_on_node $test_node kill -9 $pid +# We need to be nasty to make that the node being failed out doesn't +# get a chance to send any tickles or doing anything else clever. IPs +# also need to be dropped because we're simulating a dead node rather +# than a CTDB failure. To properly handle a CTDB failure we would +# need a watchdog to drop the IPs when CTDB disappears. +try_command_on_node -v $test_node "kill -9 $pid ; $CTDB_TEST_WRAPPER drop_ips ${test_node_ips}" wait_until_node_has_status $test_node disconnected gratarp_sniff_wait_show diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 6bec78e..0d27c93 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -687,6 +687,44 @@ nfs_test_cleanup () ####################################### +# If the given IP is hosted then print 2 items: maskbits and iface +ip_maskbits_iface () +{ + _addr="$1" + + case "$_addr" in + *:*) _family="inet6" ; _bits=128 ;; + *) _family="inet" ; _bits=32 ;; + esac + + ip addr show to "${_addr}/${_bits}" 2>/dev/null | \ + awk -v family="${_family}" \ + 'NR == 1 { iface = gensub(":$", "", 1, $2) } \ + $1 ~ /inet/ { print gensub(".*/", "", 1, $2), iface, family }' +} + +drop_ip () +{ + _addr="${1%/*}" # Remove optional maskbits + + set -- $(ip_maskbits_iface $_addr) + if [ -n "$1" ] ; then + _maskbits="$1" + _iface="$2" + echo "Removing public address $_addr/$_maskbits from device $_iface" + ip addr del "$_ip/$_maskbits" dev "$_iface" >/dev/null 2>&1 || true + fi +} + +drop_ips () +{ + for _ip ; do + drop_ip "$_ip" + done +} + +####################################### + # $1: pnn, $2: DB name db_get_path () { -- 2.1.3