From 852625d8c9d9bd79eacf4f2889675ddd809aee5a Mon Sep 17 00:00:00 2001 From: Poornima G Date: Thu, 30 Oct 2014 17:14:51 +0530 Subject: [PATCH 1/9] vfs_glusterfs: In vfs_gluster_sys_acl_get_file/fd, reduce the number of getxattr calls. Signed-off-by: Poornima G Reviewed-by: Ira Cooper Reviewed-by: Guenther Deschner (cherry picked from commit a4fa9ca5a7a4c0b770079ab126f8172ff6d6851c) --- source3/modules/vfs_glusterfs.c | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index ba2d8e8..c47013b 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -1006,6 +1006,8 @@ static int vfs_gluster_set_offline(struct vfs_handle_struct *handle, #define GLUSTER_ACL_HEADER_SIZE 4 #define GLUSTER_ACL_ENTRY_SIZE 8 +#define GLUSTER_ACL_SIZE(n) (GLUSTER_ACL_HEADER_SIZE + (n * GLUSTER_ACL_ENTRY_SIZE)) + static SMB_ACL_T gluster_to_smb_acl(const char *buf, size_t xattr_size, TALLOC_CTX *mem_ctx) { @@ -1275,7 +1277,7 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_file(struct vfs_handle_struct *handle, struct smb_acl_t *result; char *buf; const char *key; - ssize_t ret; + ssize_t ret, size = GLUSTER_ACL_SIZE(20); switch (type) { case SMB_ACL_TYPE_ACCESS: @@ -1289,13 +1291,22 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_file(struct vfs_handle_struct *handle, return NULL; } - ret = glfs_getxattr(handle->data, path_p, key, 0, 0); - if (ret <= 0) { + buf = alloca(size); + if (!buf) { return NULL; } - buf = alloca(ret); - ret = glfs_getxattr(handle->data, path_p, key, buf, ret); + ret = glfs_getxattr(handle->data, path_p, key, buf, size); + if (ret == -1 && errno == ERANGE) { + ret = glfs_getxattr(handle->data, path_p, key, 0, 0); + if (ret > 0) { + buf = alloca(ret); + if (!buf) { + return NULL; + } + ret = glfs_getxattr(handle->data, path_p, key, buf, ret); + } + } if (ret <= 0) { return NULL; } @@ -1310,18 +1321,29 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_fd(struct vfs_handle_struct *handle, TALLOC_CTX *mem_ctx) { struct smb_acl_t *result; - int ret; + ssize_t ret, size = GLUSTER_ACL_SIZE(20); char *buf; glfs_fd_t *glfd; glfd = *(glfs_fd_t **)VFS_FETCH_FSP_EXTENSION(handle, fsp); - ret = glfs_fgetxattr(glfd, "system.posix_acl_access", 0, 0); - if (ret <= 0) { + + buf = alloca(size); + if (!buf) { return NULL; } - buf = alloca(ret); - ret = glfs_fgetxattr(glfd, "system.posix_acl_access", buf, ret); + ret = glfs_fgetxattr(glfd, "system.posix_acl_access", buf, size); + if (ret == -1 && errno == ERANGE) { + ret = glfs_fgetxattr(glfd, "system.posix_acl_access", 0, 0); + if (ret > 0) { + buf = alloca(ret); + if (!buf) { + return NULL; + } + ret = glfs_fgetxattr(glfd, "system.posix_acl_access", + buf, ret); + } + } if (ret <= 0) { return NULL; } -- 1.9.3 From 151865c39295155dd0b25c86dc57dbfee50af8a4 Mon Sep 17 00:00:00 2001 From: raghavendra talur Date: Wed, 10 Dec 2014 12:26:34 +0530 Subject: [PATCH 2/9] vfs_glusterfs: Remember the connect path too for reopening. As Samba allows us to share subdirs, lets re-use preopened connections to glusterfs only if they are for same volume AND same connectpath. Signed-off-by: raghavendra talur Reviewed-by: Guenther Deschner Reviewed-by: Michael Adam (cherry picked from commit 2fb4317352a165d0dd4af78fcf0a576fa246eab2) --- source3/modules/vfs_glusterfs.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index c47013b..2dd9d71 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -78,13 +78,14 @@ static void smb_stat_ex_from_stat(struct stat_ex *dst, const struct stat *src) static struct glfs_preopened { char *volume; + char *connectpath; glfs_t *fs; int ref; struct glfs_preopened *next, *prev; } *glfs_preopened; -static int glfs_set_preopened(const char *volume, glfs_t *fs) +static int glfs_set_preopened(const char *volume, const char *connectpath, glfs_t *fs) { struct glfs_preopened *entry = NULL; @@ -101,6 +102,13 @@ static int glfs_set_preopened(const char *volume, glfs_t *fs) return -1; } + entry->connectpath = talloc_strdup(entry, connectpath); + if (entry->connectpath == NULL) { + talloc_free(entry); + errno = ENOMEM; + return -1; + } + entry->fs = fs; entry->ref = 1; @@ -109,12 +117,14 @@ static int glfs_set_preopened(const char *volume, glfs_t *fs) return 0; } -static glfs_t *glfs_find_preopened(const char *volume) +static glfs_t *glfs_find_preopened(const char *volume, const char *connectpath) { struct glfs_preopened *entry = NULL; for (entry = glfs_preopened; entry; entry = entry->next) { - if (strcmp(entry->volume, volume) == 0) { + if (strcmp(entry->volume, volume) == 0 && + strcmp(entry->connectpath, connectpath) == 0) + { entry->ref++; return entry->fs; } @@ -176,7 +186,7 @@ static int vfs_gluster_connect(struct vfs_handle_struct *handle, volume = service; } - fs = glfs_find_preopened(volume); + fs = glfs_find_preopened(volume, handle->conn->connectpath); if (fs) { goto done; } @@ -214,7 +224,7 @@ static int vfs_gluster_connect(struct vfs_handle_struct *handle, goto done; } - ret = glfs_set_preopened(volume, fs); + ret = glfs_set_preopened(volume, handle->conn->connectpath, fs); if (ret < 0) { DEBUG(0, ("%s: Failed to register volume (%s)\n", volume, strerror(errno))); -- 1.9.3 From 30d03b5a232d1c37aaaf928d3f18c210a328c622 Mon Sep 17 00:00:00 2001 From: raghavendra talur Date: Wed, 10 Dec 2014 12:30:10 +0530 Subject: [PATCH 3/9] vfs_glusterfs: Set connectpath as snapdir-entry-path. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapview client in gluster would require connectpath in order add snapdir entry to readdir result. Signed-off-by: raghavendra talur Reviewed-by: Guenther Deschner Reviewed-by: Michael Adam Autobuild-User(master): Günther Deschner Autobuild-Date(master): Wed Dec 10 16:38:27 CET 2014 on sn-devel-104 (cherry picked from commit a12ae815b01b372cbf8697d32d5d45d7b2d5f93d) --- source3/modules/vfs_glusterfs.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 2dd9d71..c222164 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -210,6 +210,17 @@ static int vfs_gluster_connect(struct vfs_handle_struct *handle, goto done; } + + ret = glfs_set_xlator_option(fs, "*-snapview-client", + "snapdir-entry-path", + handle->conn->connectpath); + if (ret < 0) { + DEBUG(0, ("%s: Failed to set xlator option:" + " snapdir-entry-path\n", volume)); + glfs_fini(fs); + return -1; + } + ret = glfs_set_logging(fs, logfile, loglevel); if (ret < 0) { DEBUG(0, ("%s: Failed to set logfile %s loglevel %d\n", -- 1.9.3 From aefa9110b6d199eccfb599d5c281e1d4eb2597bf Mon Sep 17 00:00:00 2001 From: Poornima G Date: Fri, 12 Dec 2014 14:11:49 +0100 Subject: [PATCH 4/9] vfs_glusterfs: Change sys_get_acl_file/fd to return ACLs corresponding to mode bits when there are no ACLs set. Signed-off-by: Poornima G Reviewed-by: Guenther Deschner Reviewed-by: Michael Adam (cherry picked from commit 26b3544251babdfcdf5ada338a4ed39ff18bc47a) --- source3/modules/vfs_glusterfs.c | 59 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index c222164..1b6304e 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -1029,6 +1029,39 @@ static int vfs_gluster_set_offline(struct vfs_handle_struct *handle, #define GLUSTER_ACL_SIZE(n) (GLUSTER_ACL_HEADER_SIZE + (n * GLUSTER_ACL_ENTRY_SIZE)) +static SMB_ACL_T mode_to_smb_acls(const struct stat *mode, TALLOC_CTX *mem_ctx) +{ + struct smb_acl_t *result; + int count; + + count = 3; + result = sys_acl_init(mem_ctx); + if (!result) { + errno = ENOMEM; + return NULL; + } + + result->acl = talloc_array(result, struct smb_acl_entry, count); + if (!result->acl) { + errno = ENOMEM; + talloc_free(result); + return NULL; + } + + result->count = count; + + result->acl[0].a_type = SMB_ACL_USER_OBJ; + result->acl[0].a_perm = (mode->st_mode & S_IRWXU) >> 6;; + + result->acl[1].a_type = SMB_ACL_GROUP_OBJ; + result->acl[1].a_perm = (mode->st_mode & S_IRWXG) >> 3;; + + result->acl[2].a_type = SMB_ACL_OTHER; + result->acl[2].a_perm = mode->st_mode & S_IRWXO;; + + return result; +} + static SMB_ACL_T gluster_to_smb_acl(const char *buf, size_t xattr_size, TALLOC_CTX *mem_ctx) { @@ -1296,6 +1329,7 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_file(struct vfs_handle_struct *handle, TALLOC_CTX *mem_ctx) { struct smb_acl_t *result; + struct stat st; char *buf; const char *key; ssize_t ret, size = GLUSTER_ACL_SIZE(20); @@ -1328,6 +1362,18 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_file(struct vfs_handle_struct *handle, ret = glfs_getxattr(handle->data, path_p, key, buf, ret); } } + + /* retrieving the ACL from the xattr has finally failed, do a + * mode-to-acl mapping */ + + if (ret == -1 && errno == ENODATA) { + ret = glfs_stat(handle->data, path_p, &st); + if (ret == 0) { + result = mode_to_smb_acls(&st, mem_ctx); + return result; + } + } + if (ret <= 0) { return NULL; } @@ -1342,6 +1388,7 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_fd(struct vfs_handle_struct *handle, TALLOC_CTX *mem_ctx) { struct smb_acl_t *result; + struct stat st; ssize_t ret, size = GLUSTER_ACL_SIZE(20); char *buf; glfs_fd_t *glfd; @@ -1365,6 +1412,18 @@ static SMB_ACL_T vfs_gluster_sys_acl_get_fd(struct vfs_handle_struct *handle, buf, ret); } } + + /* retrieving the ACL from the xattr has finally failed, do a + * mode-to-acl mapping */ + + if (ret == -1 && errno == ENODATA) { + ret = glfs_fstat(glfd, &st); + if (ret == 0) { + result = mode_to_smb_acls(&st, mem_ctx); + return result; + } + } + if (ret <= 0) { return NULL; } -- 1.9.3 From 490b11807d1ad94cbecbce69a178f02b241711f2 Mon Sep 17 00:00:00 2001 From: Poornima G Date: Thu, 11 Dec 2014 07:35:10 +0530 Subject: [PATCH 5/9] vfs_glusterfs: Implement AIO support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Poornima G Reviewed-by: Guenther Deschner Reviewed-by: Michael Adam Autobuild-User(master): Günther Deschner Autobuild-Date(master): Wed Dec 17 16:35:37 CET 2014 on sn-devel-104 (cherry picked from commit 051b81aac81c7ccb234da221e9f8272b70b265ce) --- source3/modules/vfs_glusterfs.c | 254 ++++++++++++++++++++++++++++++++++++++-- source3/wscript | 1 + 2 files changed, 243 insertions(+), 12 deletions(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 1b6304e..68aea40 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -26,8 +26,6 @@ * @brief Samba VFS module for glusterfs * * @todo - * - AIO support\n - * See, for example \c vfs_aio_linux.c in the \c sourc3/modules directory * - sendfile/recvfile support * * A Samba VFS module for GlusterFS, based on Gluster's libgfapi. @@ -42,9 +40,24 @@ #include #include "api/glfs.h" #include "lib/util/dlinklist.h" +#include "lib/util/tevent_unix.h" +#ifdef HAVE_SYS_EVENTFD_H +#include +#endif +#include +#include "smbd/globals.h" #define DEFAULT_VOLFILE_SERVER "localhost" +#ifdef HAVE_EVENTFD +static pthread_mutex_t lock_req_list = PTHREAD_MUTEX_INITIALIZER; +static int event_fd = -1; +static struct tevent_fd *aio_read_event = NULL; +static struct tevent_req **req_producer_list = NULL; +static struct tevent_req **req_consumer_list = NULL; +static uint64_t req_counter = 0; +#endif + /** * Helper to convert struct stat to struct stat_ex. */ @@ -482,20 +495,174 @@ static ssize_t vfs_gluster_pread(struct vfs_handle_struct *handle, return glfs_pread(*(glfs_fd_t **)VFS_FETCH_FSP_EXTENSION(handle, fsp), data, n, offset, 0); } +struct glusterfs_aio_state { + ssize_t ret; + int err; +}; + +/* + * This function is the callback that will be called on glusterfs + * threads once the async IO submitted is complete. To notify + * Samba of the completion we use eventfd mechanism. + */ +static void aio_glusterfs_done(glfs_fd_t *fd, ssize_t ret, void *data) +{ +#if HAVE_EVENTFD + struct tevent_req *req = NULL; + struct glusterfs_aio_state *state = NULL; + int i, sts = 0; + uint64_t u = 1; + + req = talloc_get_type_abort(data, struct tevent_req); + state = tevent_req_data(req, struct glusterfs_aio_state); + + if (ret < 0) { + state->ret = -1; + state->err = errno; + } else { + state->ret = ret; + state->err = 0; + } + + /* + * Store the reqs that needs to be completed by calling + * tevent_req_done(). tevent_req_done() cannot be called + * here, as it is not designed to be executed in the + * multithread environment, tevent_req_done() should be + * executed from the smbd main thread. + */ + pthread_mutex_lock (&lock_req_list); + { + for (i = 0 ; i < aio_pending_size ; i++) { + if(!req_producer_list[i]) { + req_producer_list[i] = req; + req_counter = req_counter + 1; + break; + } + } + } + pthread_mutex_unlock (&lock_req_list); + + /* + * For a bunch of fops notify only once + */ + if (req_counter == 1) { + sts = write (event_fd, &u, sizeof(uint64_t)); + if (sts < 0 && errno == EAGAIN) + DEBUG(0,("\nWRITE: reached max value")); + } + return; +#endif +} + +#ifdef HAVE_EVENTFD +static void aio_tevent_fd_done(struct tevent_context *event_ctx, + struct tevent_fd *fde, + uint16 flags, void *data) +{ + struct tevent_req *req = NULL; + struct tevent_req **temp = NULL; + int i = 0, sts = 0; + uint64_t u = 0; + + sts = read (event_fd, &u, sizeof(uint64_t)); + if (sts < 0 && errno == EAGAIN) + DEBUG(0,("\nREAD: eventfd read failed (%s)",strerror(errno))); + + pthread_mutex_lock (&lock_req_list); + { + temp = req_producer_list; + req_producer_list = req_consumer_list; + req_consumer_list = temp; + req_counter = 0; + } + pthread_mutex_unlock (&lock_req_list); + + for (i = 0 ; i < aio_pending_size ; i++) { + req = req_consumer_list[i]; + if (req) { + tevent_req_done(req); + req_consumer_list[i] = 0; + } + } + return; +} +#endif + +static bool init_gluster_aio(struct vfs_handle_struct *handle) +{ +#ifdef HAVE_EVENTFD + if (event_fd != -1) { + /* + * Already initialized. + */ + return true; + } + + event_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (event_fd == -1) { + goto fail; + } + + aio_read_event = tevent_add_fd(handle->conn->sconn->ev_ctx, + NULL, + event_fd, + TEVENT_FD_READ, + aio_tevent_fd_done, + NULL); + if (aio_read_event == NULL) { + goto fail; + } + + req_producer_list = talloc_zero_array(NULL, struct tevent_req *, + aio_pending_size); + req_consumer_list = talloc_zero_array(NULL, struct tevent_req *, + aio_pending_size); + + return true; +fail: + TALLOC_FREE(aio_read_event); + if (event_fd != -1) { + close(event_fd); + event_fd = -1; + } +#endif + return false; +} + static struct tevent_req *vfs_gluster_pread_send(struct vfs_handle_struct *handle, TALLOC_CTX *mem_ctx, struct tevent_context *ev, files_struct *fsp, void *data, size_t n, off_t offset) { + struct tevent_req *req = NULL; + struct glusterfs_aio_state *state = NULL; + int ret = 0; + +#ifndef HAVE_EVENTFD errno = ENOTSUP; return NULL; -} +#endif -static ssize_t vfs_gluster_pread_recv(struct tevent_req *req, int *err) -{ - errno = ENOTSUP; - return -1; + req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); + if (req == NULL) { + return NULL; + } + + if (!init_gluster_aio(handle)) { + tevent_req_error(req, EIO); + return tevent_req_post(req, ev); + } + ret = glfs_pread_async(*(glfs_fd_t **)VFS_FETCH_FSP_EXTENSION(handle, + fsp), data, n, offset, 0, aio_glusterfs_done, + req); + if (ret < 0) { + tevent_req_error(req, -ret); + return tevent_req_post(req, ev); + } + + return req; } static ssize_t vfs_gluster_write(struct vfs_handle_struct *handle, @@ -518,14 +685,53 @@ static struct tevent_req *vfs_gluster_pwrite_send(struct vfs_handle_struct const void *data, size_t n, off_t offset) { + struct tevent_req *req = NULL; + struct glusterfs_aio_state *state = NULL; + int ret = 0; + +#ifndef HAVE_EVENTFD errno = ENOTSUP; return NULL; +#endif + + req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); + if (req == NULL) { + return NULL; + } + if (!init_gluster_aio(handle)) { + tevent_req_error(req, EIO); + return tevent_req_post(req, ev); + } + ret = glfs_pwrite_async(*(glfs_fd_t **)VFS_FETCH_FSP_EXTENSION(handle, + fsp), data, n, offset, 0, aio_glusterfs_done, + req); + if (ret < 0) { + tevent_req_error(req, -ret); + return tevent_req_post(req, ev); + } + return req; } -static ssize_t vfs_gluster_pwrite_recv(struct tevent_req *req, int *err) +static ssize_t vfs_gluster_recv(struct tevent_req *req, int *err) { + struct glusterfs_aio_state *state = NULL; + +#ifndef HAVE_EVENTFD errno = ENOTSUP; return -1; +#endif + state = tevent_req_data(req, struct glusterfs_aio_state); + if (state == NULL) { + return -1; + } + + if (tevent_req_is_unix_error(req, err)) { + return -1; + } + if (state->ret == -1) { + *err = state->err; + } + return state->ret; } static off_t vfs_gluster_lseek(struct vfs_handle_struct *handle, @@ -570,14 +776,38 @@ static struct tevent_req *vfs_gluster_fsync_send(struct vfs_handle_struct struct tevent_context *ev, files_struct *fsp) { + struct tevent_req *req = NULL; + struct glusterfs_aio_state *state = NULL; + int ret = 0; + +#ifndef HAVE_EVENTFD errno = ENOTSUP; return NULL; +#endif + + req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); + if (req == NULL) { + return NULL; + } + if (!init_gluster_aio(handle)) { + tevent_req_error(req, EIO); + return tevent_req_post(req, ev); + } + ret = glfs_fsync_async(*(glfs_fd_t **)VFS_FETCH_FSP_EXTENSION(handle, + fsp), aio_glusterfs_done, req); + if (ret < 0) { + tevent_req_error(req, -ret); + return tevent_req_post(req, ev); + } + return req; } static int vfs_gluster_fsync_recv(struct tevent_req *req, int *err) { - errno = ENOTSUP; - return -1; + /* + * Use implicit conversion ssize_t->int + */ + return vfs_gluster_recv(req, err); } static int vfs_gluster_stat(struct vfs_handle_struct *handle, @@ -1530,11 +1760,11 @@ static struct vfs_fn_pointers glusterfs_fns = { .read_fn = vfs_gluster_read, .pread_fn = vfs_gluster_pread, .pread_send_fn = vfs_gluster_pread_send, - .pread_recv_fn = vfs_gluster_pread_recv, + .pread_recv_fn = vfs_gluster_recv, .write_fn = vfs_gluster_write, .pwrite_fn = vfs_gluster_pwrite, .pwrite_send_fn = vfs_gluster_pwrite_send, - .pwrite_recv_fn = vfs_gluster_pwrite_recv, + .pwrite_recv_fn = vfs_gluster_recv, .lseek_fn = vfs_gluster_lseek, .sendfile_fn = vfs_gluster_sendfile, .recvfile_fn = vfs_gluster_recvfile, diff --git a/source3/wscript b/source3/wscript index bf025fc..b317a32 100644 --- a/source3/wscript +++ b/source3/wscript @@ -534,6 +534,7 @@ return acl_get_perm_np(permset_d, perm); conf.DEFINE('HAVE_NO_AIO', '1') if host_os.rfind('linux') > -1: + conf.CHECK_FUNCS('eventfd') conf.CHECK_FUNCS_IN('io_submit', 'aio') conf.CHECK_CODE(''' struct io_event ioev; -- 1.9.3 From 566a61511938892ee0515fb66ff42ea05c9ad11c Mon Sep 17 00:00:00 2001 From: raghavendra talur Date: Wed, 10 Dec 2014 21:15:03 +0530 Subject: [PATCH 6/9] vfs/glusterfs: Change xattr key to match gluster key. Gluster changed the key used to do get_real_filename in 3.5. This changes Samba's glusterfs vfs-module to use the same key. This will render SMB_VFS_GET_REALFILENAME() working again under 3.5 and will remove support for that functionality for Gluster 3.4 and older. Signed-off-by: raghavendra talur Reviewed-by: Ira Cooper Reviewed-by: Michael Adam (cherry picked from commit af9ec9fea5a730023cdee6e236f9585e3a18b0e6) --- source3/modules/vfs_glusterfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 68aea40..10c3a22 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -1110,7 +1110,7 @@ static int vfs_gluster_get_real_filename(struct vfs_handle_struct *handle, } snprintf(key_buf, NAME_MAX + 64, - "user.glusterfs.get_real_filename:%s", name); + "glusterfs.get_real_filename:%s", name); ret = glfs_getxattr(handle->data, path, key_buf, val_buf, NAME_MAX + 1); if (ret == -1) { -- 1.9.3 From fa1921d85b5ba91970a02304e38e3e334af16331 Mon Sep 17 00:00:00 2001 From: Ira Cooper Date: Mon, 19 Jan 2015 23:08:17 -0500 Subject: [PATCH 7/9] vfs_glusterfs: Replace eventfd with pipes, for AIO use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipes clean up the AIO implementation substantially, due to the fact that they implement a natural ithread safe queue instead of us creating our own queue. Signed-off-by: Ira Cooper Signed-off-by: Poornima G Reviewed-by: Günther Deschner Reviewed-by: Michael Adam Autobuild-User(master): Ira Cooper Autobuild-Date(master): Wed Jan 21 20:40:11 CET 2015 on sn-devel-104 (cherry picked from commit 4c3a3d9e6adc95d0f0e1f6030b2406613d9f9f53) --- source3/modules/vfs_glusterfs.c | 136 ++++++++++++---------------------------- source3/wscript | 1 - 2 files changed, 40 insertions(+), 97 deletions(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 10c3a22..1d3abd8 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -41,22 +41,14 @@ #include "api/glfs.h" #include "lib/util/dlinklist.h" #include "lib/util/tevent_unix.h" -#ifdef HAVE_SYS_EVENTFD_H -#include -#endif -#include +#include "lib/tevent/tevent_internal.h" #include "smbd/globals.h" #define DEFAULT_VOLFILE_SERVER "localhost" -#ifdef HAVE_EVENTFD -static pthread_mutex_t lock_req_list = PTHREAD_MUTEX_INITIALIZER; -static int event_fd = -1; +static int read_fd = -1; +static int write_fd = -1; static struct tevent_fd *aio_read_event = NULL; -static struct tevent_req **req_producer_list = NULL; -static struct tevent_req **req_consumer_list = NULL; -static uint64_t req_counter = 0; -#endif /** * Helper to convert struct stat to struct stat_ex. @@ -503,15 +495,13 @@ struct glusterfs_aio_state { /* * This function is the callback that will be called on glusterfs * threads once the async IO submitted is complete. To notify - * Samba of the completion we use eventfd mechanism. + * Samba of the completion we use a pipe pased queue. */ static void aio_glusterfs_done(glfs_fd_t *fd, ssize_t ret, void *data) { -#if HAVE_EVENTFD struct tevent_req *req = NULL; struct glusterfs_aio_state *state = NULL; - int i, sts = 0; - uint64_t u = 1; + int sts = 0; req = talloc_get_type_abort(data, struct tevent_req); state = tevent_req_data(req, struct glusterfs_aio_state); @@ -525,88 +515,65 @@ static void aio_glusterfs_done(glfs_fd_t *fd, ssize_t ret, void *data) } /* - * Store the reqs that needs to be completed by calling - * tevent_req_done(). tevent_req_done() cannot be called - * here, as it is not designed to be executed in the - * multithread environment, tevent_req_done() should be + * Write the pointer to each req that needs to be completed + * by calling tevent_req_done(). tevent_req_done() cannot + * be called here, as it is not designed to be executed + * in the multithread environment, tevent_req_done() must be * executed from the smbd main thread. */ - pthread_mutex_lock (&lock_req_list); - { - for (i = 0 ; i < aio_pending_size ; i++) { - if(!req_producer_list[i]) { - req_producer_list[i] = req; - req_counter = req_counter + 1; - break; - } - } - } - pthread_mutex_unlock (&lock_req_list); - /* - * For a bunch of fops notify only once - */ - if (req_counter == 1) { - sts = write (event_fd, &u, sizeof(uint64_t)); - if (sts < 0 && errno == EAGAIN) - DEBUG(0,("\nWRITE: reached max value")); + sts = sys_write (write_fd, &req, sizeof(struct tevent_req *)); + if (sts < 0) { + DEBUG(0,("\nWrite to pipe failed (%s)", strerror(errno))); } + return; -#endif } -#ifdef HAVE_EVENTFD +/* + * Read each req off the pipe and process it. + */ static void aio_tevent_fd_done(struct tevent_context *event_ctx, struct tevent_fd *fde, uint16 flags, void *data) { struct tevent_req *req = NULL; - struct tevent_req **temp = NULL; - int i = 0, sts = 0; - uint64_t u = 0; - - sts = read (event_fd, &u, sizeof(uint64_t)); - if (sts < 0 && errno == EAGAIN) - DEBUG(0,("\nREAD: eventfd read failed (%s)",strerror(errno))); - - pthread_mutex_lock (&lock_req_list); - { - temp = req_producer_list; - req_producer_list = req_consumer_list; - req_consumer_list = temp; - req_counter = 0; - } - pthread_mutex_unlock (&lock_req_list); - - for (i = 0 ; i < aio_pending_size ; i++) { - req = req_consumer_list[i]; - if (req) { - tevent_req_done(req); - req_consumer_list[i] = 0; - } + int sts = 0; + + sts = sys_read(read_fd, &req, sizeof(struct tevent_req *)); + if (sts < 0) { + DEBUG(0,("\nRead from pipe failed (%s)", strerror(errno))); + } + + if (req) { + tevent_req_done(req); } return; } -#endif static bool init_gluster_aio(struct vfs_handle_struct *handle) { -#ifdef HAVE_EVENTFD - if (event_fd != -1) { + int fds[2]; + int ret = -1; + + if (read_fd != -1) { /* * Already initialized. */ return true; } - event_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); - if (event_fd == -1) { + ret = pipe(fds); + if (ret == -1) { goto fail; } + read_fd = fds[0]; + write_fd = fds[1]; + aio_read_event = tevent_add_fd(handle->conn->sconn->ev_ctx, NULL, - event_fd, + read_fd, TEVENT_FD_READ, aio_tevent_fd_done, NULL); @@ -614,19 +581,15 @@ static bool init_gluster_aio(struct vfs_handle_struct *handle) goto fail; } - req_producer_list = talloc_zero_array(NULL, struct tevent_req *, - aio_pending_size); - req_consumer_list = talloc_zero_array(NULL, struct tevent_req *, - aio_pending_size); - return true; fail: TALLOC_FREE(aio_read_event); - if (event_fd != -1) { - close(event_fd); - event_fd = -1; + if (read_fd != -1) { + close(read_fd); + close(write_fd); + read_fd = -1; + write_fd = -1; } -#endif return false; } @@ -640,11 +603,6 @@ static struct tevent_req *vfs_gluster_pread_send(struct vfs_handle_struct struct glusterfs_aio_state *state = NULL; int ret = 0; -#ifndef HAVE_EVENTFD - errno = ENOTSUP; - return NULL; -#endif - req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); if (req == NULL) { return NULL; @@ -689,11 +647,6 @@ static struct tevent_req *vfs_gluster_pwrite_send(struct vfs_handle_struct struct glusterfs_aio_state *state = NULL; int ret = 0; -#ifndef HAVE_EVENTFD - errno = ENOTSUP; - return NULL; -#endif - req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); if (req == NULL) { return NULL; @@ -716,10 +669,6 @@ static ssize_t vfs_gluster_recv(struct tevent_req *req, int *err) { struct glusterfs_aio_state *state = NULL; -#ifndef HAVE_EVENTFD - errno = ENOTSUP; - return -1; -#endif state = tevent_req_data(req, struct glusterfs_aio_state); if (state == NULL) { return -1; @@ -780,11 +729,6 @@ static struct tevent_req *vfs_gluster_fsync_send(struct vfs_handle_struct struct glusterfs_aio_state *state = NULL; int ret = 0; -#ifndef HAVE_EVENTFD - errno = ENOTSUP; - return NULL; -#endif - req = tevent_req_create(mem_ctx, &state, struct glusterfs_aio_state); if (req == NULL) { return NULL; diff --git a/source3/wscript b/source3/wscript index b317a32..bf025fc 100644 --- a/source3/wscript +++ b/source3/wscript @@ -534,7 +534,6 @@ return acl_get_perm_np(permset_d, perm); conf.DEFINE('HAVE_NO_AIO', '1') if host_os.rfind('linux') > -1: - conf.CHECK_FUNCS('eventfd') conf.CHECK_FUNCS_IN('io_submit', 'aio') conf.CHECK_CODE(''' struct io_event ioev; -- 1.9.3 From 187895823ce20191835640f6d159ecaf995fd143 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Wed, 21 Jan 2015 22:07:53 +0100 Subject: [PATCH 8/9] vfs:glusterfs: whitespace fix. Signed-off-by: Michael Adam Reviewed-by: Jeremy Allison Autobuild-User(master): Jeremy Allison Autobuild-Date(master): Thu Jan 22 03:20:17 CET 2015 on sn-devel-104 (cherry picked from commit cde5d380e192487ae77734afcb3eb3d327aafb98) --- source3/modules/vfs_glusterfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 1d3abd8..96f5559 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -522,9 +522,9 @@ static void aio_glusterfs_done(glfs_fd_t *fd, ssize_t ret, void *data) * executed from the smbd main thread. */ - sts = sys_write (write_fd, &req, sizeof(struct tevent_req *)); - if (sts < 0) { - DEBUG(0,("\nWrite to pipe failed (%s)", strerror(errno))); + sts = sys_write(write_fd, &req, sizeof(struct tevent_req *)); + if (sts < 0) { + DEBUG(0,("\nWrite to pipe failed (%s)", strerror(errno))); } return; -- 1.9.3 From df7e1863257f8d58558d26607981d768e2c4785a Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Wed, 21 Jan 2015 20:49:24 +0100 Subject: [PATCH 9/9] vfs: Fix a typo Signed-off-by: Volker Lendecke Reviewed-by: Andreas Schneider Autobuild-User(master): Andreas Schneider Autobuild-Date(master): Thu Jan 22 13:14:38 CET 2015 on sn-devel-104 (cherry picked from commit db63e4887c81c3e458194004353d7ef92c67b393) --- source3/modules/vfs_glusterfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/modules/vfs_glusterfs.c b/source3/modules/vfs_glusterfs.c index 96f5559..35a9c8d 100644 --- a/source3/modules/vfs_glusterfs.c +++ b/source3/modules/vfs_glusterfs.c @@ -495,7 +495,7 @@ struct glusterfs_aio_state { /* * This function is the callback that will be called on glusterfs * threads once the async IO submitted is complete. To notify - * Samba of the completion we use a pipe pased queue. + * Samba of the completion we use a pipe based queue. */ static void aio_glusterfs_done(glfs_fd_t *fd, ssize_t ret, void *data) { -- 1.9.3