From 99397029071f355d7806f3f54d4893466929ac24 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Mon, 9 Jul 2018 14:53:00 +0200 Subject: [PATCH 1/6] ctdb/build: link ctdb_mutex_ceph_rados_helper against ceph-common ceph-common linkage is needed with new versions of Ceph. Also respect the --libcephfs_dir= parameter when provided. Signed-off-by: David Disseldorp Reviewed-by: Amitay Isaacs (cherry picked from commit bd64af6b8861f892e6ae2840a493f037d1e0a06c) --- ctdb/wscript | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ctdb/wscript b/ctdb/wscript index 05044cebdeb..addf8ece173 100644 --- a/ctdb/wscript +++ b/ctdb/wscript @@ -85,6 +85,10 @@ def set_options(opt): opt.add_option('--enable-etcd-reclock', help=("Enable etcd recovery lock helper (default=no)"), action="store_true", dest='ctdb_etcd_reclock', default=False) + + opt.add_option('--with-libcephfs', + help=("Directory under which libcephfs is installed"), + action="store", dest='libcephfs_dir', default=None) opt.add_option('--enable-ceph-reclock', help=("Enable Ceph CTDB recovery lock helper (default=no)"), action="store_true", dest='ctdb_ceph_reclock', default=False) @@ -237,8 +241,16 @@ def configure(conf): conf.env.etcd_reclock = have_etcd_reclock if Options.options.ctdb_ceph_reclock: + # Use custom libcephfs library path if provided. XXX The top level build + # explicitly sets LIBPATH_CEPH-COMMON when libcephfs_dir isn't provided. + if Options.options.libcephfs_dir: + conf.env['CPPPATH_RADOS'] = Options.options.libcephfs_dir + '/include' + conf.env['LIBPATH_RADOS'] = Options.options.libcephfs_dir + '/lib' + conf.env['LIBPATH_CEPH-COMMON'] = conf.env['LIBPATH_RADOS'] + '/ceph' + if (conf.CHECK_HEADERS('rados/librados.h', False, False, 'rados') and conf.CHECK_LIB('rados', shlib=True)): + conf.CHECK_LIB('ceph-common', shlib=True) Logs.info('Building with Ceph librados recovery lock support') conf.define('HAVE_LIBRADOS', 1) else: @@ -665,7 +677,7 @@ def build(bld): if bld.env.HAVE_LIBRADOS: bld.SAMBA_BINARY('ctdb_mutex_ceph_rados_helper', source='utils/ceph/ctdb_mutex_ceph_rados_helper.c', - deps='talloc tevent rados', + deps='talloc tevent rados ceph-common', includes='include', install_path='${CTDB_HELPER_BINDIR}') -- 2.13.7 From bc395d43fcb33894c0e0859050f61690fab844d4 Mon Sep 17 00:00:00 2001 From: Samuel Cabrero Date: Fri, 15 Jun 2018 18:15:53 +0200 Subject: [PATCH 2/6] ctdb_mutex_ceph_rados_helper: Set SIGINT signal handler Set a handler for SIGINT to release the lock. Signed-off-by: Samuel Cabrero Reviewed-by: David Disseldorp Reviewed-by: Amitay Isaacs (cherry picked from commit 85706bd27535eaa4ec653f99b1910fbd8f2aab88) --- ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c index 326a0b02519..b5cd1ebad12 100644 --- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c +++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c @@ -147,7 +147,8 @@ struct ctdb_mutex_rados_state { const char *object; int ppid; struct tevent_context *ev; - struct tevent_signal *sig_ev; + struct tevent_signal *sigterm_ev; + struct tevent_signal *sigint_ev; struct tevent_timer *timer_ev; rados_t ceph_cluster; rados_ioctx_t ioctx; @@ -269,11 +270,21 @@ int main(int argc, char *argv[]) } /* wait for sigterm */ - cmr_state->sig_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGTERM, 0, + cmr_state->sigterm_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGTERM, 0, ctdb_mutex_rados_sigterm_cb, cmr_state); - if (cmr_state->sig_ev == NULL) { - fprintf(stderr, "Failed to create signal event\n"); + if (cmr_state->sigterm_ev == NULL) { + fprintf(stderr, "Failed to create term signal event\n"); + fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); + ret = -ENOMEM; + goto err_state_free; + } + + cmr_state->sigint_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGINT, 0, + ctdb_mutex_rados_sigterm_cb, + cmr_state); + if (cmr_state->sigint_ev == NULL) { + fprintf(stderr, "Failed to create int signal event\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; goto err_state_free; -- 2.13.7 From 5fcbb90718a128a62e6d42a4e07aecc972157a5b Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Thu, 19 Jul 2018 18:46:27 +0200 Subject: [PATCH 3/6] ctdb_mutex_ceph_rados_helper: use talloc destructor for cleanup Signed-off-by: David Disseldorp Reviewed-by: Samuel Cabrero Reviewed-by: Amitay Isaacs (cherry picked from commit 8d30fd591600ac17c742cd78c7bc4056bba6b877) --- ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 51 ++++++++++++-------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c index b5cd1ebad12..9c31e310b91 100644 --- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c +++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c @@ -88,13 +88,6 @@ static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name, return 0; } -static void ctdb_mutex_rados_ctx_destroy(rados_t ceph_cluster, - rados_ioctx_t ioctx) -{ - rados_ioctx_destroy(ioctx); - rados_shutdown(ceph_cluster); -} - static int ctdb_mutex_rados_lock(rados_ioctx_t *ioctx, const char *oid) { @@ -162,18 +155,13 @@ static void ctdb_mutex_rados_sigterm_cb(struct tevent_context *ev, void *private_data) { struct ctdb_mutex_rados_state *cmr_state = private_data; - int ret; + int ret = 0; if (!cmr_state->holding_mutex) { fprintf(stderr, "Sigterm callback invoked without mutex!\n"); ret = -EINVAL; - goto err_ctx_cleanup; } - ret = ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); -err_ctx_cleanup: - ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, - cmr_state->ioctx); talloc_free(cmr_state); exit(ret ? 1 : 0); } @@ -184,7 +172,7 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, void *private_data) { struct ctdb_mutex_rados_state *cmr_state = private_data; - int ret; + int ret = 0; if (!cmr_state->holding_mutex) { fprintf(stderr, "Timer callback invoked without mutex!\n"); @@ -205,15 +193,26 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, return; } - /* parent ended, drop lock and exit */ - ret = ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); + /* parent ended, drop lock (via destructor) and exit */ err_ctx_cleanup: - ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, - cmr_state->ioctx); talloc_free(cmr_state); exit(ret ? 1 : 0); } +static int ctdb_mutex_rados_state_destroy(struct ctdb_mutex_rados_state *cmr_state) +{ + if (cmr_state->holding_mutex) { + ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); + } + if (cmr_state->ioctx != NULL) { + rados_ioctx_destroy(cmr_state->ioctx); + } + if (cmr_state->ceph_cluster != NULL) { + rados_shutdown(cmr_state->ceph_cluster); + } + return 0; +} + int main(int argc, char *argv[]) { int ret; @@ -241,6 +240,7 @@ int main(int argc, char *argv[]) goto err_out; } + talloc_set_destructor(cmr_state, ctdb_mutex_rados_state_destroy); cmr_state->ceph_cluster_name = argv[1]; cmr_state->ceph_auth_name = argv[2]; cmr_state->pool_name = argv[3]; @@ -258,7 +258,7 @@ int main(int argc, char *argv[]) */ fprintf(stderr, "%s: PPID == 1\n", progname); ret = -EPIPE; - goto err_state_free; + goto err_ctx_cleanup; } cmr_state->ev = tevent_context_init(cmr_state); @@ -266,7 +266,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "tevent_context_init failed\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; - goto err_state_free; + goto err_ctx_cleanup; } /* wait for sigterm */ @@ -277,7 +277,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "Failed to create term signal event\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; - goto err_state_free; + goto err_ctx_cleanup; } cmr_state->sigint_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGINT, 0, @@ -287,7 +287,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "Failed to create int signal event\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; - goto err_state_free; + goto err_ctx_cleanup; } /* periodically check parent */ @@ -299,7 +299,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "Failed to create timer event\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; - goto err_state_free; + goto err_ctx_cleanup; } ret = ctdb_mutex_rados_ctx_create(cmr_state->ceph_cluster_name, @@ -309,7 +309,7 @@ int main(int argc, char *argv[]) &cmr_state->ioctx); if (ret < 0) { fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); - goto err_state_free; + goto err_ctx_cleanup; } ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object); @@ -330,9 +330,6 @@ int main(int argc, char *argv[]) goto err_ctx_cleanup; } err_ctx_cleanup: - ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, - cmr_state->ioctx); -err_state_free: talloc_free(cmr_state); err_out: return ret ? 1 : 0; -- 2.13.7 From 8b07db89c15ef58270e616fa9dbb05ce995742f1 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Tue, 17 Jul 2018 23:36:36 +0200 Subject: [PATCH 4/6] ctdb_mutex_ceph_rados_helper: rename timer_ev to ppid_timer_ev In preparation for adding a lock refresh timer. Signed-off-by: David Disseldorp Reviewed-by: Samuel Cabrero Reviewed-by: Amitay Isaacs (cherry picked from commit 91a89c146453ca203a83dc2ba555bb93276c4d7f) --- ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c index 9c31e310b91..9f020bf57d5 100644 --- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c +++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c @@ -142,7 +142,7 @@ struct ctdb_mutex_rados_state { struct tevent_context *ev; struct tevent_signal *sigterm_ev; struct tevent_signal *sigint_ev; - struct tevent_timer *timer_ev; + struct tevent_timer *ppid_timer_ev; rados_t ceph_cluster; rados_ioctx_t ioctx; }; @@ -166,10 +166,10 @@ static void ctdb_mutex_rados_sigterm_cb(struct tevent_context *ev, exit(ret ? 1 : 0); } -static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, - struct tevent_timer *te, - struct timeval current_time, - void *private_data) +static void ctdb_mutex_rados_ppid_timer_cb(struct tevent_context *ev, + struct tevent_timer *te, + struct timeval current_time, + void *private_data) { struct ctdb_mutex_rados_state *cmr_state = private_data; int ret = 0; @@ -182,11 +182,12 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, if ((kill(cmr_state->ppid, 0) == 0) || (errno != ESRCH)) { /* parent still around, keep waiting */ - cmr_state->timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, + cmr_state->ppid_timer_ev = tevent_add_timer(cmr_state->ev, + cmr_state, tevent_timeval_current_ofs(5, 0), - ctdb_mutex_rados_timer_cb, - cmr_state); - if (cmr_state->timer_ev == NULL) { + ctdb_mutex_rados_ppid_timer_cb, + cmr_state); + if (cmr_state->ppid_timer_ev == NULL) { fprintf(stderr, "Failed to create timer event\n"); /* rely on signal cb */ } @@ -291,11 +292,11 @@ int main(int argc, char *argv[]) } /* periodically check parent */ - cmr_state->timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, + cmr_state->ppid_timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, tevent_timeval_current_ofs(5, 0), - ctdb_mutex_rados_timer_cb, + ctdb_mutex_rados_ppid_timer_cb, cmr_state); - if (cmr_state->timer_ev == NULL) { + if (cmr_state->ppid_timer_ev == NULL) { fprintf(stderr, "Failed to create timer event\n"); fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); ret = -ENOMEM; -- 2.13.7 From 4495ca87960d2d6efdb4f6caca9e1b9cb4bf75b7 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Thu, 19 Jul 2018 11:55:23 +0200 Subject: [PATCH 5/6] ctdb_mutex_ceph_rados_helper: fix deadlock via lock renewals RADOS locks without expiry persist indefinitely. This results in CTDB deadlock during failover if the recovery master dies unexpectedly, as subsequently elected recovery master nodes can't obtain the recovery lock. Avoid deadlock by using a lock expiration time (10s by default), and renewing it periodically. Bug: https://bugzilla.samba.org/show_bug.cgi?id=13540 Signed-off-by: David Disseldorp Reviewed-by: Samuel Cabrero Reviewed-by: Amitay Isaacs (cherry picked from commit ce289e89e5c469cf2c5626dc7f2666b945dba3bd) --- ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 107 ++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 9 deletions(-) diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c index 9f020bf57d5..7ef76c26e02 100644 --- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c +++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c @@ -1,7 +1,7 @@ /* CTDB mutex helper using Ceph librados locks - Copyright (C) David Disseldorp 2016 + Copyright (C) David Disseldorp 2016-2018 Based on ctdb_mutex_fcntl_helper.c, which is: Copyright (C) Martin Schwenke 2015 @@ -29,6 +29,11 @@ #define CTDB_MUTEX_CEPH_LOCK_NAME "ctdb_reclock_mutex" #define CTDB_MUTEX_CEPH_LOCK_COOKIE CTDB_MUTEX_CEPH_LOCK_NAME #define CTDB_MUTEX_CEPH_LOCK_DESC "CTDB recovery lock" +/* + * During failover it may take up to seconds before the + * newly elected recovery master can obtain the lock. + */ +#define CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT 10 #define CTDB_MUTEX_STATUS_HOLDING "0" #define CTDB_MUTEX_STATUS_CONTENDED "1" @@ -89,16 +94,19 @@ static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name, } static int ctdb_mutex_rados_lock(rados_ioctx_t *ioctx, - const char *oid) + const char *oid, + uint64_t lock_duration_s, + uint8_t flags) { int ret; + struct timeval tv = { lock_duration_s, 0 }; ret = rados_lock_exclusive(ioctx, oid, - CTDB_MUTEX_CEPH_LOCK_NAME, + CTDB_MUTEX_CEPH_LOCK_NAME, CTDB_MUTEX_CEPH_LOCK_COOKIE, CTDB_MUTEX_CEPH_LOCK_DESC, - NULL, /* infinite duration */ - 0); + lock_duration_s == 0 ? NULL : &tv, + flags); if ((ret == -EEXIST) || (ret == -EBUSY)) { /* lock contention */ return ret; @@ -138,11 +146,13 @@ struct ctdb_mutex_rados_state { const char *ceph_auth_name; const char *pool_name; const char *object; + uint64_t lock_duration_s; int ppid; struct tevent_context *ev; struct tevent_signal *sigterm_ev; struct tevent_signal *sigint_ev; struct tevent_timer *ppid_timer_ev; + struct tevent_timer *renew_timer_ev; rados_t ceph_cluster; rados_ioctx_t ioctx; }; @@ -200,6 +210,49 @@ err_ctx_cleanup: exit(ret ? 1 : 0); } +#define USECS_IN_SEC 1000000 + +static void ctdb_mutex_rados_lock_renew_timer_cb(struct tevent_context *ev, + struct tevent_timer *te, + struct timeval current_time, + void *private_data) +{ + struct ctdb_mutex_rados_state *cmr_state = private_data; + struct timeval tv; + int ret; + + ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object, + cmr_state->lock_duration_s, + LIBRADOS_LOCK_FLAG_RENEW); + if (ret == -EBUSY) { + /* should never get -EEXIST on renewal */ + fprintf(stderr, "Lock contention during renew: %d\n", ret); + goto err_ctx_cleanup; + } else if (ret < 0) { + fprintf(stderr, "Lock renew failed\n"); + goto err_ctx_cleanup; + } + + tv = tevent_timeval_current_ofs(0, + cmr_state->lock_duration_s * (USECS_IN_SEC / 2)); + cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev, + cmr_state, + tv, + ctdb_mutex_rados_lock_renew_timer_cb, + cmr_state); + if (cmr_state->renew_timer_ev == NULL) { + fprintf(stderr, "Failed to create timer event\n"); + goto err_ctx_cleanup; + } + + return; + +err_ctx_cleanup: + /* drop lock (via destructor) and exit */ + talloc_free(cmr_state); + exit(1); +} + static int ctdb_mutex_rados_state_destroy(struct ctdb_mutex_rados_state *cmr_state) { if (cmr_state->holding_mutex) { @@ -221,9 +274,10 @@ int main(int argc, char *argv[]) progname = argv[0]; - if (argc != 5) { + if ((argc != 5) && (argc != 6)) { fprintf(stderr, "Usage: %s " - " \n", + " " + "[lock duration secs]\n", progname); ret = -EINVAL; goto err_out; @@ -246,6 +300,19 @@ int main(int argc, char *argv[]) cmr_state->ceph_auth_name = argv[2]; cmr_state->pool_name = argv[3]; cmr_state->object = argv[4]; + if (argc == 6) { + /* optional lock duration provided */ + char *endptr = NULL; + cmr_state->lock_duration_s = strtoull(argv[5], &endptr, 0); + if ((endptr == argv[5]) || (*endptr != '\0')) { + fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); + ret = -EINVAL; + goto err_ctx_cleanup; + } + } else { + cmr_state->lock_duration_s + = CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT; + } cmr_state->ppid = getppid(); if (cmr_state->ppid == 1) { @@ -313,7 +380,9 @@ int main(int argc, char *argv[]) goto err_ctx_cleanup; } - ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object); + ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object, + cmr_state->lock_duration_s, + 0); if ((ret == -EEXIST) || (ret == -EBUSY)) { fprintf(stdout, CTDB_MUTEX_STATUS_CONTENDED); goto err_ctx_cleanup; @@ -321,8 +390,28 @@ int main(int argc, char *argv[]) fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); goto err_ctx_cleanup; } - cmr_state->holding_mutex = true; + + if (cmr_state->lock_duration_s != 0) { + /* + * renew (reobtain) the lock, using a period of half the lock + * duration. Convert to usecs to avoid rounding. + */ + struct timeval tv = tevent_timeval_current_ofs(0, + cmr_state->lock_duration_s * (USECS_IN_SEC / 2)); + cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev, + cmr_state, + tv, + ctdb_mutex_rados_lock_renew_timer_cb, + cmr_state); + if (cmr_state->renew_timer_ev == NULL) { + fprintf(stderr, "Failed to create timer event\n"); + fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); + ret = -ENOMEM; + goto err_ctx_cleanup; + } + } + fprintf(stdout, CTDB_MUTEX_STATUS_HOLDING); /* wait for the signal / timer events to do their work */ -- 2.13.7 From cb973e28cf483c747c20f06b4be69938212f0026 Mon Sep 17 00:00:00 2001 From: David Disseldorp Date: Fri, 20 Jul 2018 17:20:08 +0200 Subject: [PATCH 6/6] ctdb: add expiry test for ctdb_mutex_ceph_rados_helper Kill the ctdb_mutex_ceph_rados_helper with SIGKILL and then confirm that the lock is automatically released following expiry. Signed-off-by: David Disseldorp Reviewed-by: Samuel Cabrero Reviewed-by: Amitay Isaacs Autobuild-User(master): David Disseldorp Autobuild-Date(master): Thu Aug 9 16:26:36 CEST 2018 on sn-devel-144 (cherry picked from commit 4abf348ec4cbb78d3216d5e8c5f3020d4499f10a) --- ctdb/utils/ceph/test_ceph_rados_reclock.sh | 57 +++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/ctdb/utils/ceph/test_ceph_rados_reclock.sh b/ctdb/utils/ceph/test_ceph_rados_reclock.sh index 1adacf6a27c..68b44ff997f 100755 --- a/ctdb/utils/ceph/test_ceph_rados_reclock.sh +++ b/ctdb/utils/ceph/test_ceph_rados_reclock.sh @@ -46,7 +46,9 @@ which ctdb_mutex_ceph_rados_helper || exit 1 TMP_DIR="$(mktemp --directory)" || exit 1 rados -p "$POOL" rm "$OBJECT" -(ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \ +# explicitly disable lock expiry (duration=0), to ensure that we don't get +# intermittent failures (due to renewal) from the lock state diff further down +(ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" 0 \ > ${TMP_DIR}/first) & locker_pid=$! @@ -78,6 +80,9 @@ LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_first)" LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_first)" [ "$LOCKER_DESC" == "CTDB recovery lock" ] \ || _fail "unexpected locker description: $LOCKER_DESC" +LOCKER_EXP="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_first)" +[ "$LOCKER_EXP" == "0.000000" ] \ + || _fail "unexpected locker expiration: $LOCKER_EXP" # second attempt while first is still holding the lock - expect failure ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \ @@ -145,6 +150,56 @@ third_out=$(cat ${TMP_DIR}/third) [ "$third_out" == "0" ] \ || _fail "expected lock acquisition (0), but got $third_out" +# test renew / expire behaviour using a 1s expiry (update period = 500ms) +exec >${TMP_DIR}/forth -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" \ + "$POOL" "$OBJECT" 1 & +locker_pid=$! + +sleep 1 + +rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ + > ${TMP_DIR}/lock_state_fifth_a +#echo "with lock fifth: `cat ${TMP_DIR}/lock_state_fifth_a`" + +LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fifth_a)" +[ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \ + || _fail "unexpected lock name: $LOCK_NAME" +LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fifth_a)" +[ "$LOCK_TYPE" == "exclusive" ] \ + || _fail "unexpected lock type: $LOCK_TYPE" +LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fifth_a)" +[ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT" +LOCKER_EXP_A="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_a)" +[ "$LOCKER_EXP_A" != "0.000000" ] \ + || _fail "unexpected locker expiration: $LOCKER_EXP_A" +sleep 1 # sleep until renewal +rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ + > ${TMP_DIR}/lock_state_fifth_b +LOCKER_EXP_B="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_b)" +[ "$LOCKER_EXP_B" != "0.000000" ] \ + || _fail "unexpected locker expiration: $LOCKER_EXP_B" +#echo "lock expiration before renewal $LOCKER_EXP_A, after renewal $LOCKER_EXP_B" +[ "$LOCKER_EXP_B" != "$LOCKER_EXP_A" ] \ + || _fail "locker expiration matches: $LOCKER_EXP_B" + +# no chance to drop the lock, rely on expiry +kill -KILL $locker_pid || exit 1 +wait $locker_pid &> /dev/null +sleep 1 # sleep until lock expiry + +rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ + > ${TMP_DIR}/lock_state_sixth +#echo "lock expiry sixth: `cat ${TMP_DIR}/lock_state_sixth`" + +LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_sixth)" +[ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \ + || _fail "unexpected lock name: $LOCK_NAME" +LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_sixth)" +[ "$LOCK_TYPE" == "exclusive" ] \ + || _fail "unexpected lock type: $LOCK_TYPE" +LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_sixth)" +[ $LOCK_COUNT -eq 0 ] || _fail "expected 0 locks in rados state, got $LOCK_COUNT" + rm ${TMP_DIR}/* rmdir $TMP_DIR -- 2.13.7