From 3f8a9f638f70d4236d3c6581535f6d0554d80615 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 24 Jun 2021 14:13:50 +0200 Subject: [PATCH 1/5] replace: copy_file_range() BUG: https://bugzilla.samba.org/show_bug.cgi?id=12033 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 4dcc04228df233be15efe9c31bcbaba822b140c4) --- lib/replace/replace.c | 25 +++++++++++++++++++++++++ lib/replace/replace.h | 10 ++++++++++ lib/replace/wscript | 12 +++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/replace/replace.c b/lib/replace/replace.c index 99b18e82590..0652cb4e6d6 100644 --- a/lib/replace/replace.c +++ b/lib/replace/replace.c @@ -1056,3 +1056,28 @@ const char *rep_getprogname(void) #endif /* HAVE_PROGRAM_INVOCATION_SHORT_NAME */ } #endif /* HAVE_GETPROGNAME */ + +#ifndef HAVE_COPY_FILE_RANGE +# ifdef HAVE_SYSCALL_COPY_FILE_RANGE +# include +# endif +ssize_t rep_copy_file_range(int fd_in, + loff_t *off_in, + int fd_out, + loff_t *off_out, + size_t len, + unsigned int flags) +{ +# ifdef HAVE_SYSCALL_COPY_FILE_RANGE + return syscall(__NR_copy_file_range, + fd_in, + off_in, + fd_out, + off_out, + len, + flags); +# endif /* HAVE_SYSCALL_COPY_FILE_RANGE */ + errno = ENOSYS; + return -1; +} +#endif /* HAVE_COPY_FILE_RANGE */ diff --git a/lib/replace/replace.h b/lib/replace/replace.h index 6c78311f2d3..12ba6b43612 100644 --- a/lib/replace/replace.h +++ b/lib/replace/replace.h @@ -964,6 +964,16 @@ int rep_memset_s(void *dest, size_t destsz, int ch, size_t count); const char *rep_getprogname(void); #endif +#ifndef HAVE_COPY_FILE_RANGE +#define copy_file_range rep_copy_file_range +ssize_t rep_copy_file_range(int fd_in, + loff_t *off_in, + int fd_out, + loff_t *off_out, + size_t len, + unsigned int flags); +#endif /* HAVE_COPY_FILE_RANGE */ + #ifndef FALL_THROUGH # ifdef HAVE_FALLTHROUGH_ATTRIBUTE # define FALL_THROUGH __attribute__ ((fallthrough)) diff --git a/lib/replace/wscript b/lib/replace/wscript index 64f305d6df0..ebe79317750 100644 --- a/lib/replace/wscript +++ b/lib/replace/wscript @@ -465,6 +465,16 @@ samba_dist.DIST_DIRS('lib/replace buildtools:buildtools third_party/waf:third_pa conf.CHECK_FUNCS('getpwent_r getpwnam_r getpwuid_r epoll_create') conf.CHECK_FUNCS('port_create') conf.CHECK_FUNCS('getprogname') + if not conf.CHECK_FUNCS('copy_file_range'): + conf.CHECK_CODE(''' +#include +#include +syscall(SYS_copy_file_range,0,NULL,0,NULL,0,0); + ''', + 'HAVE_SYSCALL_COPY_FILE_RANGE', + msg='Checking whether we have copy_file_range system call') + if conf.CONFIG_SET('HAVE_COPY_FILE_RANGE') or conf.CONFIG_SET('HAVE_SYSCALL_COPY_FILE_RANGE'): + conf.DEFINE('USE_COPY_FILE_RANGE', 1) conf.SET_TARGET_TYPE('attr', 'EMPTY') @@ -848,7 +858,7 @@ REPLACEMENT_FUNCTIONS = { 'strsep', 'strtok_r', 'strtoll', 'strtoull', 'setenv', 'unsetenv', 'utime', 'utimes', 'dup2', 'chown', 'link', 'readlink', 'symlink', 'lchown', 'realpath', 'memmem', 'vdprintf', - 'dprintf', 'get_current_dir_name', + 'dprintf', 'get_current_dir_name', 'copy_file_range', 'strerror_r', 'clock_gettime', 'memset_s'], 'timegm.c': ['timegm'], # Note: C99_VSNPRINTF is not a function, but a special condition -- 2.31.1 From 70b45186db94821685f8625a7ba8c30dbf166152 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Fri, 25 Jun 2021 15:47:38 +0200 Subject: [PATCH 2/5] vfs_default: properly track written bytes for copy-chunk No change in behavour, this just makes the logic slightly more understandable. In theory it would also allow the logic to be adjusted for allowing short reads which is not quite clear from MS-SMB2 if we should allow it. The file could be truncated while we're reading it. BUG: https://bugzilla.samba.org/show_bug.cgi?id=12033 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit e2d524d4baee193b0d03df3e7edda48b3cb4bddf) --- source3/modules/vfs_default.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source3/modules/vfs_default.c b/source3/modules/vfs_default.c index cf5e1cbc296..8203f5554f9 100644 --- a/source3/modules/vfs_default.c +++ b/source3/modules/vfs_default.c @@ -1953,6 +1953,7 @@ struct vfswrap_offload_write_state { off_t dst_off; off_t to_copy; off_t remaining; + off_t copied; size_t next_io_size; }; @@ -2263,6 +2264,7 @@ static void vfswrap_offload_write_write_done(struct tevent_req *subreq) tevent_req_nterror(req, NT_STATUS_INTERNAL_ERROR); return; } + state->copied += nwritten; state->remaining -= nwritten; if (state->remaining == 0) { tevent_req_done(req); @@ -2299,7 +2301,7 @@ static NTSTATUS vfswrap_offload_write_recv(struct vfs_handle_struct *handle, return status; } - *copied = state->to_copy; + *copied = state->copied; DBG_DEBUG("copy chunk copied %lu\n", (unsigned long)*copied); tevent_req_received(req); -- 2.31.1 From 9522c653f987c051b94472f6ff36cfa8eb2512e4 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Sat, 26 Jun 2021 12:21:19 +0200 Subject: [PATCH 3/5] lib: add sys_io_ranges_overlap() BUG: https://bugzilla.samba.org/show_bug.cgi?id=12033 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit 4f1a02909b8694dcc30fd5c7c6772fcfa1092ed9) --- lib/util/sys_rw.c | 25 ++++++++ lib/util/sys_rw.h | 2 + lib/util/tests/test_sys_rw.c | 110 +++++++++++++++++++++++++++++++++++ lib/util/wscript_build | 6 ++ selftest/tests.py | 2 + 5 files changed, 145 insertions(+) create mode 100644 lib/util/tests/test_sys_rw.c diff --git a/lib/util/sys_rw.c b/lib/util/sys_rw.c index d74395fc409..02aaa871a39 100644 --- a/lib/util/sys_rw.c +++ b/lib/util/sys_rw.c @@ -48,6 +48,31 @@ bool sys_valid_io_range(off_t offset, size_t length) return true; } +bool sys_io_ranges_overlap(size_t c1, off_t o1, + size_t c2, off_t o2) +{ + if (c1 == 0 || c2 == 0) { + return false; + } + if (o2 < o1) { + /* + * o1 + * |····c1····| + * o2 + * |····c2···| ? + */ + return (o2 + c2 > o1); + } else { + /* + * o1 + * |····c1···| + * o2 + * |····c2····| ? + */ + return (o1 + c1 > o2); + } +} + /******************************************************************* A read wrapper that will deal with EINTR/EWOULDBLOCK ********************************************************************/ diff --git a/lib/util/sys_rw.h b/lib/util/sys_rw.h index b224ecb30ac..6693d34de57 100644 --- a/lib/util/sys_rw.h +++ b/lib/util/sys_rw.h @@ -28,6 +28,8 @@ struct iovec; bool sys_valid_io_range(off_t offset, size_t length); +bool sys_io_ranges_overlap(size_t c1, off_t o1, + size_t c2, off_t o2); ssize_t sys_read(int fd, void *buf, size_t count); void sys_read_v(int fd, void *buf, size_t count); ssize_t sys_write(int fd, const void *buf, size_t count); diff --git a/lib/util/tests/test_sys_rw.c b/lib/util/tests/test_sys_rw.c new file mode 100644 index 00000000000..551a6a09bda --- /dev/null +++ b/lib/util/tests/test_sys_rw.c @@ -0,0 +1,110 @@ +/* + * Unix SMB/CIFS implementation. + * + * Unit test for sys_rw.c + * + * Copyright (C) Ralph Böhme 2021 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + */ + +#include +#include +#include +#include +#include + +#include "lib/replace/replace.h" +#include "system/dir.h" + +#include "lib/util/sys_rw.c" + +static void test_sys_io_ranges_overlap(void **state) +{ + bool overlap; + + /* + * sys_io_ranges_overlap() args are: + * + * src size, src offset, dst size, dst offset + */ + + /* src and dst size 0 => no overlap */ + overlap = sys_io_ranges_overlap(0, 0, 0, 0); + assert_false(overlap); + + /* dst size 0 => no overlap */ + overlap = sys_io_ranges_overlap(1, 0, 0, 0); + assert_false(overlap); + + /* src size 0 => no overlap */ + overlap = sys_io_ranges_overlap(0, 0, 1, 0); + assert_false(overlap); + + /* same range => overlap */ + overlap = sys_io_ranges_overlap(1, 0, 1, 0); + assert_true(overlap); + + /* + * |.| + * |.| + * src before dst => no overlap + */ + overlap = sys_io_ranges_overlap(1, 0, 1, 1); + assert_false(overlap); + + /* + * |..| + * |..| + * src into dst => overlap + */ + overlap = sys_io_ranges_overlap(2, 0, 2, 1); + assert_true(overlap); + + /* + * |....| + * |..| + * src encompasses dst => overlap + */ + overlap = sys_io_ranges_overlap(4, 0, 1, 2); + assert_true(overlap); + + + /* + * |..| + * |..| + * dst into src => overlap + */ + overlap = sys_io_ranges_overlap(2, 1, 2, 0); + assert_true(overlap); + + /* + * |..| + * |....| + * dst encompasses src => overlap + */ + overlap = sys_io_ranges_overlap(2, 1, 4, 0); + assert_true(overlap); +} + +int main(int argc, char **argv) +{ + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_sys_io_ranges_overlap), + }; + + cmocka_set_message_output(CM_OUTPUT_SUBUNIT); + + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/lib/util/wscript_build b/lib/util/wscript_build index 5a8a04965ec..8f35bc38392 100644 --- a/lib/util/wscript_build +++ b/lib/util/wscript_build @@ -317,3 +317,9 @@ bld.SAMBA_LIBRARY('genrand', deps='cmocka replace talloc samba-util', local_include=False, for_selftest=True) + + bld.SAMBA_BINARY('test_sys_rw', + source='tests/test_sys_rw.c', + deps='cmocka replace samba-util', + local_include=False, + for_selftest=True) diff --git a/selftest/tests.py b/selftest/tests.py index 2b65943b2ed..301962c36c6 100644 --- a/selftest/tests.py +++ b/selftest/tests.py @@ -400,6 +400,8 @@ plantestsuite("samba.unittests.util", "none", [os.path.join(bindir(), "default/lib/util/test_util")]) plantestsuite("samba.unittests.memcache", "none", [os.path.join(bindir(), "default/lib/util/test_memcache")]) +plantestsuite("samba.unittests.sys_rw", "none", + [os.path.join(bindir(), "default/lib/util/test_sys_rw")]) plantestsuite("samba.unittests.ntlm_check", "none", [os.path.join(bindir(), "default/libcli/auth/test_ntlm_check")]) plantestsuite("samba.unittests.gnutls", "none", -- 2.31.1 From 2f9805a08c29f084ea71aaf916a37c1664f90722 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Mon, 28 Jun 2021 15:50:32 +0200 Subject: [PATCH 4/5] smbd: use sys_io_ranges_overlap() in fsctl_dup_extents_check_overlap() BUG: https://bugzilla.samba.org/show_bug.cgi?id=12033 Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison (cherry picked from commit e72be5213335ab1ea0f9f396ab071669231c151b) --- source3/smbd/smb2_ioctl_filesys.c | 38 +++++-------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/source3/smbd/smb2_ioctl_filesys.c b/source3/smbd/smb2_ioctl_filesys.c index a398920bfed..a54b0fbc095 100644 --- a/source3/smbd/smb2_ioctl_filesys.c +++ b/source3/smbd/smb2_ioctl_filesys.c @@ -30,6 +30,7 @@ #include "../librpc/ndr/libndr.h" #include "librpc/gen_ndr/ndr_ioctl.h" #include "smb2_ioctl_private.h" +#include "lib/util/sys_rw.h" #undef DBGC_CLASS #define DBGC_CLASS DBGC_SMB2 @@ -96,43 +97,16 @@ static NTSTATUS fsctl_dup_extents_check_overlap(struct files_struct *src_fsp, struct files_struct *dst_fsp, struct fsctl_dup_extents_to_file *dup_extents) { - uint64_t src_off_last; - uint64_t tgt_off_last; - if (!file_id_equal(&src_fsp->file_id, &dst_fsp->file_id)) { /* src and dest refer to different files */ return NT_STATUS_OK; } - if (dup_extents->byte_count == 0) { - /* no range to overlap */ - return NT_STATUS_OK; - } - - /* - * [MS-FSCC] 2.3.8 FSCTL_DUPLICATE_EXTENTS_TO_FILE Reply - * STATUS_NOT_SUPPORTED: - * The source and target destination ranges overlap on the same file. - */ - - src_off_last = dup_extents->source_off + dup_extents->byte_count - 1; - if ((dup_extents->target_off >= dup_extents->source_off) - && (dup_extents->target_off <= src_off_last)) { - /* - * src: |-----------| - * tgt: |-----------| - */ - return NT_STATUS_NOT_SUPPORTED; - } - - - tgt_off_last = dup_extents->target_off + dup_extents->byte_count - 1; - if ((tgt_off_last >= dup_extents->source_off) - && (tgt_off_last <= src_off_last)) { - /* - * src: |-----------| - * tgt: |-----------| - */ + if (sys_io_ranges_overlap(dup_extents->byte_count, + dup_extents->source_off, + dup_extents->byte_count, + dup_extents->target_off)) + { return NT_STATUS_NOT_SUPPORTED; } -- 2.31.1 From 53abbfc4c1669214531724ca33c1ae4b8bf62376 Mon Sep 17 00:00:00 2001 From: Ralph Boehme Date: Thu, 24 Jun 2021 16:21:42 +0200 Subject: [PATCH 5/5] vfs_default: use copy_file_range() Original file on an XFS filesystem: $ ls -l /mnt/test/1048578-file -rw-rw-r--. 1 slow slow 1048578 Jun 25 11:40 /mnt/test/1048578-file $ xfs_bmap /mnt/test/1048578-file /mnt/test/1048578-file: 0: [0..2055]: 192..2247 Copy created with cp --reflink=never: $ xfs_bmap /mnt/test/1048578-file-reflink-never /mnt/test/1048578-file-reflink-never: 0: [0..2055]: 2248..4303 Copy created with cp --reflink=always $ xfs_bmap /mnt/test/1048578-file-reflink-always /mnt/test/1048578-file-reflink-always: 0: [0..2055]: 192..2247 Copy done from a Windows client: $ xfs_bmap /mnt/test/1048578-file\ -\ Copy /mnt/test/1048578-file - Copy: 0: [0..2055]: 192..2247 BUG: https://bugzilla.samba.org/show_bug.cgi?id=12033 RN: smbd should support copy_file_range() for FSCTL_SRV_COPYCHUNK Signed-off-by: Ralph Boehme Reviewed-by: Jeremy Allison Autobuild-User(master): Jeremy Allison Autobuild-Date(master): Wed Jun 30 17:40:23 UTC 2021 on sn-devel-184 (backported from commit accaa2f1f67a7f064a4ce03a120d7b2f8e847ccf) [slow@samba.org: fsp_get_io_fd() is new in 4.14] --- source3/modules/vfs_default.c | 134 ++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/source3/modules/vfs_default.c b/source3/modules/vfs_default.c index 8203f5554f9..682cbbb625e 100644 --- a/source3/modules/vfs_default.c +++ b/source3/modules/vfs_default.c @@ -1973,6 +1973,7 @@ static void vfswrap_offload_write_cleanup(struct tevent_req *req, state->dst_fsp = NULL; } +static NTSTATUS vfswrap_offload_copy_file_range(struct tevent_req *req); static NTSTATUS vfswrap_offload_write_loop(struct tevent_req *req); static struct tevent_req *vfswrap_offload_write_send( @@ -2112,6 +2113,16 @@ static struct tevent_req *vfswrap_offload_write_send( return tevent_req_post(req, ev); } + status = vfswrap_offload_copy_file_range(req); + if (NT_STATUS_IS_OK(status)) { + tevent_req_done(req); + return tevent_req_post(req, ev); + } + if (!NT_STATUS_EQUAL(status, NT_STATUS_MORE_PROCESSING_REQUIRED)) { + tevent_req_nterror(req, status); + return tevent_req_post(req, ev); + } + state->buf = talloc_array(state, uint8_t, num); if (tevent_req_nomem(state->buf, req)) { return tevent_req_post(req, ev); @@ -2126,6 +2137,129 @@ static struct tevent_req *vfswrap_offload_write_send( return req; } +static NTSTATUS vfswrap_offload_copy_file_range(struct tevent_req *req) +{ + struct vfswrap_offload_write_state *state = tevent_req_data( + req, struct vfswrap_offload_write_state); + struct lock_struct lck; + ssize_t nwritten; + NTSTATUS status; + bool same_file; + bool ok; + +#ifndef USE_COPY_FILE_RANGE + return NT_STATUS_MORE_PROCESSING_REQUIRED; +#endif + + same_file = file_id_equal(&state->src_fsp->file_id, + &state->dst_fsp->file_id); + if (same_file && + sys_io_ranges_overlap(state->remaining, + state->src_off, + state->remaining, + state->dst_off)) + { + return NT_STATUS_MORE_PROCESSING_REQUIRED; + } + + if (is_named_stream(state->src_fsp->fsp_name) || + is_named_stream(state->dst_fsp->fsp_name)) + { + return NT_STATUS_MORE_PROCESSING_REQUIRED; + } + + init_strict_lock_struct(state->src_fsp, + state->src_fsp->op->global->open_persistent_id, + state->src_off, + state->remaining, + READ_LOCK, + &lck); + + ok = SMB_VFS_STRICT_LOCK_CHECK(state->src_fsp->conn, + state->src_fsp, + &lck); + if (!ok) { + return NT_STATUS_FILE_LOCK_CONFLICT; + } + + ok = change_to_user_and_service_by_fsp(state->dst_fsp); + if (!ok) { + return NT_STATUS_INTERNAL_ERROR; + } + + init_strict_lock_struct(state->dst_fsp, + state->dst_fsp->op->global->open_persistent_id, + state->dst_off, + state->remaining, + WRITE_LOCK, + &lck); + + ok = SMB_VFS_STRICT_LOCK_CHECK(state->dst_fsp->conn, + state->dst_fsp, + &lck); + if (!ok) { + return NT_STATUS_FILE_LOCK_CONFLICT; + } + + while (state->remaining > 0) { + nwritten = copy_file_range(state->src_fsp->fh->fd, + &state->src_off, + state->dst_fsp->fh->fd, + &state->dst_off, + state->remaining, + 0); + if (nwritten == -1) { + DBG_DEBUG("copy_file_range src [%s]:[%jd] dst [%s]:[%jd] " + "n [%jd] failed: %s\n", + fsp_str_dbg(state->src_fsp), + (intmax_t)state->src_off, + fsp_str_dbg(state->dst_fsp), + (intmax_t)state->dst_off, + (intmax_t)state->remaining, + strerror(errno)); + switch (errno) { + case EXDEV: + status = NT_STATUS_MORE_PROCESSING_REQUIRED; + break; + default: + status = map_nt_error_from_unix(errno); + if (NT_STATUS_EQUAL( + status, + NT_STATUS_MORE_PROCESSING_REQUIRED)) + { + /* Avoid triggering the fallback */ + status = NT_STATUS_INTERNAL_ERROR; + } + break; + } + return status; + } + + if (state->remaining < nwritten) { + DBG_DEBUG("copy_file_range src [%s] dst [%s] " + "n [%jd] remaining [%jd]\n", + fsp_str_dbg(state->src_fsp), + fsp_str_dbg(state->dst_fsp), + (intmax_t)nwritten, + (intmax_t)state->remaining); + return NT_STATUS_INTERNAL_ERROR; + } + + if (nwritten == 0) { + break; + } + state->copied += nwritten; + state->remaining -= nwritten; + } + + /* + * Tell the req cleanup function there's no need to call + * change_to_user_and_service_by_fsp() on the dst handle. + */ + state->dst_fsp = NULL; + return NT_STATUS_OK; +} + static void vfswrap_offload_write_read_done(struct tevent_req *subreq); static NTSTATUS vfswrap_offload_write_loop(struct tevent_req *req) -- 2.31.1