The Samba-Bugzilla – Attachment 14415 Details for
Bug 13540
deadlock with ctdb_mutex_ceph_rados_helper
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
clean cherry-pick for 4.9.next
bso13540_fix_ctdb_mutex_ceph_rados_helper_deadlock_49t.patchset (text/plain), 23.57 KB, created by
David Disseldorp
on 2018-08-13 21:37:52 UTC
(
hide
)
Description:
clean cherry-pick for 4.9.next
Filename:
MIME Type:
Creator:
David Disseldorp
Created:
2018-08-13 21:37:52 UTC
Size:
23.57 KB
patch
obsolete
>From 99397029071f355d7806f3f54d4893466929ac24 Mon Sep 17 00:00:00 2001 >From: David Disseldorp <ddiss@samba.org> >Date: Mon, 9 Jul 2018 14:53:00 +0200 >Subject: [PATCH 1/6] ctdb/build: link ctdb_mutex_ceph_rados_helper against > ceph-common > >ceph-common linkage is needed with new versions of Ceph. >Also respect the --libcephfs_dir=<path> parameter when provided. > >Signed-off-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> >(cherry picked from commit bd64af6b8861f892e6ae2840a493f037d1e0a06c) >--- > ctdb/wscript | 14 +++++++++++++- > 1 file changed, 13 insertions(+), 1 deletion(-) > >diff --git a/ctdb/wscript b/ctdb/wscript >index 05044cebdeb..addf8ece173 100644 >--- a/ctdb/wscript >+++ b/ctdb/wscript >@@ -85,6 +85,10 @@ def set_options(opt): > opt.add_option('--enable-etcd-reclock', > help=("Enable etcd recovery lock helper (default=no)"), > action="store_true", dest='ctdb_etcd_reclock', default=False) >+ >+ opt.add_option('--with-libcephfs', >+ help=("Directory under which libcephfs is installed"), >+ action="store", dest='libcephfs_dir', default=None) > opt.add_option('--enable-ceph-reclock', > help=("Enable Ceph CTDB recovery lock helper (default=no)"), > action="store_true", dest='ctdb_ceph_reclock', default=False) >@@ -237,8 +241,16 @@ def configure(conf): > conf.env.etcd_reclock = have_etcd_reclock > > if Options.options.ctdb_ceph_reclock: >+ # Use custom libcephfs library path if provided. XXX The top level build >+ # explicitly sets LIBPATH_CEPH-COMMON when libcephfs_dir isn't provided. >+ if Options.options.libcephfs_dir: >+ conf.env['CPPPATH_RADOS'] = Options.options.libcephfs_dir + '/include' >+ conf.env['LIBPATH_RADOS'] = Options.options.libcephfs_dir + '/lib' >+ conf.env['LIBPATH_CEPH-COMMON'] = conf.env['LIBPATH_RADOS'] + '/ceph' >+ > if (conf.CHECK_HEADERS('rados/librados.h', False, False, 'rados') and > conf.CHECK_LIB('rados', shlib=True)): >+ conf.CHECK_LIB('ceph-common', shlib=True) > Logs.info('Building with Ceph librados recovery lock support') > conf.define('HAVE_LIBRADOS', 1) > else: >@@ -665,7 +677,7 @@ def build(bld): > if bld.env.HAVE_LIBRADOS: > bld.SAMBA_BINARY('ctdb_mutex_ceph_rados_helper', > source='utils/ceph/ctdb_mutex_ceph_rados_helper.c', >- deps='talloc tevent rados', >+ deps='talloc tevent rados ceph-common', > includes='include', > install_path='${CTDB_HELPER_BINDIR}') > >-- >2.13.7 > > >From bc395d43fcb33894c0e0859050f61690fab844d4 Mon Sep 17 00:00:00 2001 >From: Samuel Cabrero <scabrero@suse.de> >Date: Fri, 15 Jun 2018 18:15:53 +0200 >Subject: [PATCH 2/6] ctdb_mutex_ceph_rados_helper: Set SIGINT signal handler > >Set a handler for SIGINT to release the lock. > >Signed-off-by: Samuel Cabrero <scabrero@suse.de> >Reviewed-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> >(cherry picked from commit 85706bd27535eaa4ec653f99b1910fbd8f2aab88) >--- > ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 19 +++++++++++++++---- > 1 file changed, 15 insertions(+), 4 deletions(-) > >diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >index 326a0b02519..b5cd1ebad12 100644 >--- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >+++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >@@ -147,7 +147,8 @@ struct ctdb_mutex_rados_state { > const char *object; > int ppid; > struct tevent_context *ev; >- struct tevent_signal *sig_ev; >+ struct tevent_signal *sigterm_ev; >+ struct tevent_signal *sigint_ev; > struct tevent_timer *timer_ev; > rados_t ceph_cluster; > rados_ioctx_t ioctx; >@@ -269,11 +270,21 @@ int main(int argc, char *argv[]) > } > > /* wait for sigterm */ >- cmr_state->sig_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGTERM, 0, >+ cmr_state->sigterm_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGTERM, 0, > ctdb_mutex_rados_sigterm_cb, > cmr_state); >- if (cmr_state->sig_ev == NULL) { >- fprintf(stderr, "Failed to create signal event\n"); >+ if (cmr_state->sigterm_ev == NULL) { >+ fprintf(stderr, "Failed to create term signal event\n"); >+ fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); >+ ret = -ENOMEM; >+ goto err_state_free; >+ } >+ >+ cmr_state->sigint_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGINT, 0, >+ ctdb_mutex_rados_sigterm_cb, >+ cmr_state); >+ if (cmr_state->sigint_ev == NULL) { >+ fprintf(stderr, "Failed to create int signal event\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; > goto err_state_free; >-- >2.13.7 > > >From 5fcbb90718a128a62e6d42a4e07aecc972157a5b Mon Sep 17 00:00:00 2001 >From: David Disseldorp <ddiss@samba.org> >Date: Thu, 19 Jul 2018 18:46:27 +0200 >Subject: [PATCH 3/6] ctdb_mutex_ceph_rados_helper: use talloc destructor for > cleanup > >Signed-off-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Samuel Cabrero <scabrero@suse.de> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> >(cherry picked from commit 8d30fd591600ac17c742cd78c7bc4056bba6b877) >--- > ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 51 ++++++++++++-------------- > 1 file changed, 24 insertions(+), 27 deletions(-) > >diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >index b5cd1ebad12..9c31e310b91 100644 >--- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >+++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >@@ -88,13 +88,6 @@ static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name, > return 0; > } > >-static void ctdb_mutex_rados_ctx_destroy(rados_t ceph_cluster, >- rados_ioctx_t ioctx) >-{ >- rados_ioctx_destroy(ioctx); >- rados_shutdown(ceph_cluster); >-} >- > static int ctdb_mutex_rados_lock(rados_ioctx_t *ioctx, > const char *oid) > { >@@ -162,18 +155,13 @@ static void ctdb_mutex_rados_sigterm_cb(struct tevent_context *ev, > void *private_data) > { > struct ctdb_mutex_rados_state *cmr_state = private_data; >- int ret; >+ int ret = 0; > > if (!cmr_state->holding_mutex) { > fprintf(stderr, "Sigterm callback invoked without mutex!\n"); > ret = -EINVAL; >- goto err_ctx_cleanup; > } > >- ret = ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); >-err_ctx_cleanup: >- ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, >- cmr_state->ioctx); > talloc_free(cmr_state); > exit(ret ? 1 : 0); > } >@@ -184,7 +172,7 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, > void *private_data) > { > struct ctdb_mutex_rados_state *cmr_state = private_data; >- int ret; >+ int ret = 0; > > if (!cmr_state->holding_mutex) { > fprintf(stderr, "Timer callback invoked without mutex!\n"); >@@ -205,15 +193,26 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, > return; > } > >- /* parent ended, drop lock and exit */ >- ret = ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); >+ /* parent ended, drop lock (via destructor) and exit */ > err_ctx_cleanup: >- ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, >- cmr_state->ioctx); > talloc_free(cmr_state); > exit(ret ? 1 : 0); > } > >+static int ctdb_mutex_rados_state_destroy(struct ctdb_mutex_rados_state *cmr_state) >+{ >+ if (cmr_state->holding_mutex) { >+ ctdb_mutex_rados_unlock(cmr_state->ioctx, cmr_state->object); >+ } >+ if (cmr_state->ioctx != NULL) { >+ rados_ioctx_destroy(cmr_state->ioctx); >+ } >+ if (cmr_state->ceph_cluster != NULL) { >+ rados_shutdown(cmr_state->ceph_cluster); >+ } >+ return 0; >+} >+ > int main(int argc, char *argv[]) > { > int ret; >@@ -241,6 +240,7 @@ int main(int argc, char *argv[]) > goto err_out; > } > >+ talloc_set_destructor(cmr_state, ctdb_mutex_rados_state_destroy); > cmr_state->ceph_cluster_name = argv[1]; > cmr_state->ceph_auth_name = argv[2]; > cmr_state->pool_name = argv[3]; >@@ -258,7 +258,7 @@ int main(int argc, char *argv[]) > */ > fprintf(stderr, "%s: PPID == 1\n", progname); > ret = -EPIPE; >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > cmr_state->ev = tevent_context_init(cmr_state); >@@ -266,7 +266,7 @@ int main(int argc, char *argv[]) > fprintf(stderr, "tevent_context_init failed\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > /* wait for sigterm */ >@@ -277,7 +277,7 @@ int main(int argc, char *argv[]) > fprintf(stderr, "Failed to create term signal event\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > cmr_state->sigint_ev = tevent_add_signal(cmr_state->ev, cmr_state, SIGINT, 0, >@@ -287,7 +287,7 @@ int main(int argc, char *argv[]) > fprintf(stderr, "Failed to create int signal event\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > /* periodically check parent */ >@@ -299,7 +299,7 @@ int main(int argc, char *argv[]) > fprintf(stderr, "Failed to create timer event\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > ret = ctdb_mutex_rados_ctx_create(cmr_state->ceph_cluster_name, >@@ -309,7 +309,7 @@ int main(int argc, char *argv[]) > &cmr_state->ioctx); > if (ret < 0) { > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); >- goto err_state_free; >+ goto err_ctx_cleanup; > } > > ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object); >@@ -330,9 +330,6 @@ int main(int argc, char *argv[]) > goto err_ctx_cleanup; > } > err_ctx_cleanup: >- ctdb_mutex_rados_ctx_destroy(cmr_state->ceph_cluster, >- cmr_state->ioctx); >-err_state_free: > talloc_free(cmr_state); > err_out: > return ret ? 1 : 0; >-- >2.13.7 > > >From 8b07db89c15ef58270e616fa9dbb05ce995742f1 Mon Sep 17 00:00:00 2001 >From: David Disseldorp <ddiss@samba.org> >Date: Tue, 17 Jul 2018 23:36:36 +0200 >Subject: [PATCH 4/6] ctdb_mutex_ceph_rados_helper: rename timer_ev to > ppid_timer_ev > >In preparation for adding a lock refresh timer. > >Signed-off-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Samuel Cabrero <scabrero@suse.de> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> >(cherry picked from commit 91a89c146453ca203a83dc2ba555bb93276c4d7f) >--- > ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 25 +++++++++++++------------ > 1 file changed, 13 insertions(+), 12 deletions(-) > >diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >index 9c31e310b91..9f020bf57d5 100644 >--- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >+++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >@@ -142,7 +142,7 @@ struct ctdb_mutex_rados_state { > struct tevent_context *ev; > struct tevent_signal *sigterm_ev; > struct tevent_signal *sigint_ev; >- struct tevent_timer *timer_ev; >+ struct tevent_timer *ppid_timer_ev; > rados_t ceph_cluster; > rados_ioctx_t ioctx; > }; >@@ -166,10 +166,10 @@ static void ctdb_mutex_rados_sigterm_cb(struct tevent_context *ev, > exit(ret ? 1 : 0); > } > >-static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, >- struct tevent_timer *te, >- struct timeval current_time, >- void *private_data) >+static void ctdb_mutex_rados_ppid_timer_cb(struct tevent_context *ev, >+ struct tevent_timer *te, >+ struct timeval current_time, >+ void *private_data) > { > struct ctdb_mutex_rados_state *cmr_state = private_data; > int ret = 0; >@@ -182,11 +182,12 @@ static void ctdb_mutex_rados_timer_cb(struct tevent_context *ev, > > if ((kill(cmr_state->ppid, 0) == 0) || (errno != ESRCH)) { > /* parent still around, keep waiting */ >- cmr_state->timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, >+ cmr_state->ppid_timer_ev = tevent_add_timer(cmr_state->ev, >+ cmr_state, > tevent_timeval_current_ofs(5, 0), >- ctdb_mutex_rados_timer_cb, >- cmr_state); >- if (cmr_state->timer_ev == NULL) { >+ ctdb_mutex_rados_ppid_timer_cb, >+ cmr_state); >+ if (cmr_state->ppid_timer_ev == NULL) { > fprintf(stderr, "Failed to create timer event\n"); > /* rely on signal cb */ > } >@@ -291,11 +292,11 @@ int main(int argc, char *argv[]) > } > > /* periodically check parent */ >- cmr_state->timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, >+ cmr_state->ppid_timer_ev = tevent_add_timer(cmr_state->ev, cmr_state, > tevent_timeval_current_ofs(5, 0), >- ctdb_mutex_rados_timer_cb, >+ ctdb_mutex_rados_ppid_timer_cb, > cmr_state); >- if (cmr_state->timer_ev == NULL) { >+ if (cmr_state->ppid_timer_ev == NULL) { > fprintf(stderr, "Failed to create timer event\n"); > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > ret = -ENOMEM; >-- >2.13.7 > > >From 4495ca87960d2d6efdb4f6caca9e1b9cb4bf75b7 Mon Sep 17 00:00:00 2001 >From: David Disseldorp <ddiss@samba.org> >Date: Thu, 19 Jul 2018 11:55:23 +0200 >Subject: [PATCH 5/6] ctdb_mutex_ceph_rados_helper: fix deadlock via lock > renewals > >RADOS locks without expiry persist indefinitely. This results in CTDB >deadlock during failover if the recovery master dies unexpectedly, as >subsequently elected recovery master nodes can't obtain the recovery >lock. >Avoid deadlock by using a lock expiration time (10s by default), and >renewing it periodically. > >Bug: https://bugzilla.samba.org/show_bug.cgi?id=13540 > >Signed-off-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Samuel Cabrero <scabrero@suse.de> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> >(cherry picked from commit ce289e89e5c469cf2c5626dc7f2666b945dba3bd) >--- > ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c | 107 ++++++++++++++++++++++--- > 1 file changed, 98 insertions(+), 9 deletions(-) > >diff --git a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >index 9f020bf57d5..7ef76c26e02 100644 >--- a/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >+++ b/ctdb/utils/ceph/ctdb_mutex_ceph_rados_helper.c >@@ -1,7 +1,7 @@ > /* > CTDB mutex helper using Ceph librados locks > >- Copyright (C) David Disseldorp 2016 >+ Copyright (C) David Disseldorp 2016-2018 > > Based on ctdb_mutex_fcntl_helper.c, which is: > Copyright (C) Martin Schwenke 2015 >@@ -29,6 +29,11 @@ > #define CTDB_MUTEX_CEPH_LOCK_NAME "ctdb_reclock_mutex" > #define CTDB_MUTEX_CEPH_LOCK_COOKIE CTDB_MUTEX_CEPH_LOCK_NAME > #define CTDB_MUTEX_CEPH_LOCK_DESC "CTDB recovery lock" >+/* >+ * During failover it may take up to <lock duration> seconds before the >+ * newly elected recovery master can obtain the lock. >+ */ >+#define CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT 10 > > #define CTDB_MUTEX_STATUS_HOLDING "0" > #define CTDB_MUTEX_STATUS_CONTENDED "1" >@@ -89,16 +94,19 @@ static int ctdb_mutex_rados_ctx_create(const char *ceph_cluster_name, > } > > static int ctdb_mutex_rados_lock(rados_ioctx_t *ioctx, >- const char *oid) >+ const char *oid, >+ uint64_t lock_duration_s, >+ uint8_t flags) > { > int ret; >+ struct timeval tv = { lock_duration_s, 0 }; > > ret = rados_lock_exclusive(ioctx, oid, >- CTDB_MUTEX_CEPH_LOCK_NAME, >+ CTDB_MUTEX_CEPH_LOCK_NAME, > CTDB_MUTEX_CEPH_LOCK_COOKIE, > CTDB_MUTEX_CEPH_LOCK_DESC, >- NULL, /* infinite duration */ >- 0); >+ lock_duration_s == 0 ? NULL : &tv, >+ flags); > if ((ret == -EEXIST) || (ret == -EBUSY)) { > /* lock contention */ > return ret; >@@ -138,11 +146,13 @@ struct ctdb_mutex_rados_state { > const char *ceph_auth_name; > const char *pool_name; > const char *object; >+ uint64_t lock_duration_s; > int ppid; > struct tevent_context *ev; > struct tevent_signal *sigterm_ev; > struct tevent_signal *sigint_ev; > struct tevent_timer *ppid_timer_ev; >+ struct tevent_timer *renew_timer_ev; > rados_t ceph_cluster; > rados_ioctx_t ioctx; > }; >@@ -200,6 +210,49 @@ err_ctx_cleanup: > exit(ret ? 1 : 0); > } > >+#define USECS_IN_SEC 1000000 >+ >+static void ctdb_mutex_rados_lock_renew_timer_cb(struct tevent_context *ev, >+ struct tevent_timer *te, >+ struct timeval current_time, >+ void *private_data) >+{ >+ struct ctdb_mutex_rados_state *cmr_state = private_data; >+ struct timeval tv; >+ int ret; >+ >+ ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object, >+ cmr_state->lock_duration_s, >+ LIBRADOS_LOCK_FLAG_RENEW); >+ if (ret == -EBUSY) { >+ /* should never get -EEXIST on renewal */ >+ fprintf(stderr, "Lock contention during renew: %d\n", ret); >+ goto err_ctx_cleanup; >+ } else if (ret < 0) { >+ fprintf(stderr, "Lock renew failed\n"); >+ goto err_ctx_cleanup; >+ } >+ >+ tv = tevent_timeval_current_ofs(0, >+ cmr_state->lock_duration_s * (USECS_IN_SEC / 2)); >+ cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev, >+ cmr_state, >+ tv, >+ ctdb_mutex_rados_lock_renew_timer_cb, >+ cmr_state); >+ if (cmr_state->renew_timer_ev == NULL) { >+ fprintf(stderr, "Failed to create timer event\n"); >+ goto err_ctx_cleanup; >+ } >+ >+ return; >+ >+err_ctx_cleanup: >+ /* drop lock (via destructor) and exit */ >+ talloc_free(cmr_state); >+ exit(1); >+} >+ > static int ctdb_mutex_rados_state_destroy(struct ctdb_mutex_rados_state *cmr_state) > { > if (cmr_state->holding_mutex) { >@@ -221,9 +274,10 @@ int main(int argc, char *argv[]) > > progname = argv[0]; > >- if (argc != 5) { >+ if ((argc != 5) && (argc != 6)) { > fprintf(stderr, "Usage: %s <Ceph Cluster> <Ceph user> " >- "<RADOS pool> <RADOS object>\n", >+ "<RADOS pool> <RADOS object> " >+ "[lock duration secs]\n", > progname); > ret = -EINVAL; > goto err_out; >@@ -246,6 +300,19 @@ int main(int argc, char *argv[]) > cmr_state->ceph_auth_name = argv[2]; > cmr_state->pool_name = argv[3]; > cmr_state->object = argv[4]; >+ if (argc == 6) { >+ /* optional lock duration provided */ >+ char *endptr = NULL; >+ cmr_state->lock_duration_s = strtoull(argv[5], &endptr, 0); >+ if ((endptr == argv[5]) || (*endptr != '\0')) { >+ fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); >+ ret = -EINVAL; >+ goto err_ctx_cleanup; >+ } >+ } else { >+ cmr_state->lock_duration_s >+ = CTDB_MUTEX_CEPH_LOCK_DURATION_SECS_DEFAULT; >+ } > > cmr_state->ppid = getppid(); > if (cmr_state->ppid == 1) { >@@ -313,7 +380,9 @@ int main(int argc, char *argv[]) > goto err_ctx_cleanup; > } > >- ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object); >+ ret = ctdb_mutex_rados_lock(cmr_state->ioctx, cmr_state->object, >+ cmr_state->lock_duration_s, >+ 0); > if ((ret == -EEXIST) || (ret == -EBUSY)) { > fprintf(stdout, CTDB_MUTEX_STATUS_CONTENDED); > goto err_ctx_cleanup; >@@ -321,8 +390,28 @@ int main(int argc, char *argv[]) > fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); > goto err_ctx_cleanup; > } >- > cmr_state->holding_mutex = true; >+ >+ if (cmr_state->lock_duration_s != 0) { >+ /* >+ * renew (reobtain) the lock, using a period of half the lock >+ * duration. Convert to usecs to avoid rounding. >+ */ >+ struct timeval tv = tevent_timeval_current_ofs(0, >+ cmr_state->lock_duration_s * (USECS_IN_SEC / 2)); >+ cmr_state->renew_timer_ev = tevent_add_timer(cmr_state->ev, >+ cmr_state, >+ tv, >+ ctdb_mutex_rados_lock_renew_timer_cb, >+ cmr_state); >+ if (cmr_state->renew_timer_ev == NULL) { >+ fprintf(stderr, "Failed to create timer event\n"); >+ fprintf(stdout, CTDB_MUTEX_STATUS_ERROR); >+ ret = -ENOMEM; >+ goto err_ctx_cleanup; >+ } >+ } >+ > fprintf(stdout, CTDB_MUTEX_STATUS_HOLDING); > > /* wait for the signal / timer events to do their work */ >-- >2.13.7 > > >From cb973e28cf483c747c20f06b4be69938212f0026 Mon Sep 17 00:00:00 2001 >From: David Disseldorp <ddiss@samba.org> >Date: Fri, 20 Jul 2018 17:20:08 +0200 >Subject: [PATCH 6/6] ctdb: add expiry test for ctdb_mutex_ceph_rados_helper > >Kill the ctdb_mutex_ceph_rados_helper with SIGKILL and then confirm >that the lock is automatically released following expiry. > >Signed-off-by: David Disseldorp <ddiss@samba.org> >Reviewed-by: Samuel Cabrero <scabrero@suse.de> >Reviewed-by: Amitay Isaacs <amitay@gmail.com> > >Autobuild-User(master): David Disseldorp <ddiss@samba.org> >Autobuild-Date(master): Thu Aug 9 16:26:36 CEST 2018 on sn-devel-144 > >(cherry picked from commit 4abf348ec4cbb78d3216d5e8c5f3020d4499f10a) >--- > ctdb/utils/ceph/test_ceph_rados_reclock.sh | 57 +++++++++++++++++++++++++++++- > 1 file changed, 56 insertions(+), 1 deletion(-) > >diff --git a/ctdb/utils/ceph/test_ceph_rados_reclock.sh b/ctdb/utils/ceph/test_ceph_rados_reclock.sh >index 1adacf6a27c..68b44ff997f 100755 >--- a/ctdb/utils/ceph/test_ceph_rados_reclock.sh >+++ b/ctdb/utils/ceph/test_ceph_rados_reclock.sh >@@ -46,7 +46,9 @@ which ctdb_mutex_ceph_rados_helper || exit 1 > TMP_DIR="$(mktemp --directory)" || exit 1 > rados -p "$POOL" rm "$OBJECT" > >-(ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \ >+# explicitly disable lock expiry (duration=0), to ensure that we don't get >+# intermittent failures (due to renewal) from the lock state diff further down >+(ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" 0 \ > > ${TMP_DIR}/first) & > locker_pid=$! > >@@ -78,6 +80,9 @@ LOCKER_COOKIE="$(jq -r '.lockers[0].cookie' ${TMP_DIR}/lock_state_first)" > LOCKER_DESC="$(jq -r '.lockers[0].description' ${TMP_DIR}/lock_state_first)" > [ "$LOCKER_DESC" == "CTDB recovery lock" ] \ > || _fail "unexpected locker description: $LOCKER_DESC" >+LOCKER_EXP="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_first)" >+[ "$LOCKER_EXP" == "0.000000" ] \ >+ || _fail "unexpected locker expiration: $LOCKER_EXP" > > # second attempt while first is still holding the lock - expect failure > ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" "$POOL" "$OBJECT" \ >@@ -145,6 +150,56 @@ third_out=$(cat ${TMP_DIR}/third) > [ "$third_out" == "0" ] \ > || _fail "expected lock acquisition (0), but got $third_out" > >+# test renew / expire behaviour using a 1s expiry (update period = 500ms) >+exec >${TMP_DIR}/forth -- ctdb_mutex_ceph_rados_helper "$CLUSTER" "$USER" \ >+ "$POOL" "$OBJECT" 1 & >+locker_pid=$! >+ >+sleep 1 >+ >+rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ >+ > ${TMP_DIR}/lock_state_fifth_a >+#echo "with lock fifth: `cat ${TMP_DIR}/lock_state_fifth_a`" >+ >+LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_fifth_a)" >+[ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \ >+ || _fail "unexpected lock name: $LOCK_NAME" >+LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_fifth_a)" >+[ "$LOCK_TYPE" == "exclusive" ] \ >+ || _fail "unexpected lock type: $LOCK_TYPE" >+LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_fifth_a)" >+[ $LOCK_COUNT -eq 1 ] || _fail "expected 1 lock in rados state, got $LOCK_COUNT" >+LOCKER_EXP_A="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_a)" >+[ "$LOCKER_EXP_A" != "0.000000" ] \ >+ || _fail "unexpected locker expiration: $LOCKER_EXP_A" >+sleep 1 # sleep until renewal >+rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ >+ > ${TMP_DIR}/lock_state_fifth_b >+LOCKER_EXP_B="$(jq -r '.lockers[0].expiration' ${TMP_DIR}/lock_state_fifth_b)" >+[ "$LOCKER_EXP_B" != "0.000000" ] \ >+ || _fail "unexpected locker expiration: $LOCKER_EXP_B" >+#echo "lock expiration before renewal $LOCKER_EXP_A, after renewal $LOCKER_EXP_B" >+[ "$LOCKER_EXP_B" != "$LOCKER_EXP_A" ] \ >+ || _fail "locker expiration matches: $LOCKER_EXP_B" >+ >+# no chance to drop the lock, rely on expiry >+kill -KILL $locker_pid || exit 1 >+wait $locker_pid &> /dev/null >+sleep 1 # sleep until lock expiry >+ >+rados -p "$POOL" lock info "$OBJECT" ctdb_reclock_mutex \ >+ > ${TMP_DIR}/lock_state_sixth >+#echo "lock expiry sixth: `cat ${TMP_DIR}/lock_state_sixth`" >+ >+LOCK_NAME="$(jq -r '.name' ${TMP_DIR}/lock_state_sixth)" >+[ "$LOCK_NAME" == "ctdb_reclock_mutex" ] \ >+ || _fail "unexpected lock name: $LOCK_NAME" >+LOCK_TYPE="$(jq -r '.type' ${TMP_DIR}/lock_state_sixth)" >+[ "$LOCK_TYPE" == "exclusive" ] \ >+ || _fail "unexpected lock type: $LOCK_TYPE" >+LOCK_COUNT="$(jq -r '.lockers | length' ${TMP_DIR}/lock_state_sixth)" >+[ $LOCK_COUNT -eq 0 ] || _fail "expected 0 locks in rados state, got $LOCK_COUNT" >+ > rm ${TMP_DIR}/* > rmdir $TMP_DIR > >-- >2.13.7 >
<b>You cannot view the attachment while viewing its details because your browser does not support IFRAMEs. <a href="attachment.cgi?id=14415">View the attachment on a separate page</a>.</b>
View Attachment As Raw
Flags:
amitay
:
review+
Actions:
View
Attachments on
bug 13540
:
14413
|
14414
| 14415