The Samba-Bugzilla – Attachment 13169 Details for
Bug 12755
[patch] Improve execution speed on Windows; with Win32 API calls
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Improve wall-time execution speed on Windows
0001-Improve-execution-speed-on-Windows.patch (text/plain), 13.59 KB, created by
Joseph Benden
on 2017-04-22 19:04:44 UTC
(
hide
)
Description:
Improve wall-time execution speed on Windows
Filename:
MIME Type:
Creator:
Joseph Benden
Created:
2017-04-22 19:04:44 UTC
Size:
13.59 KB
patch
obsolete
>From 13a9e0012d0935850b114f132b5be3764c189880 Mon Sep 17 00:00:00 2001 >From: Joseph Benden <joe@benden.us> >Date: Sat, 22 Apr 2017 10:54:09 -0700 >Subject: [PATCH] Improve execution speed on Windows > >Improves speed of execution under Windows by utilizing the Win32 APIs >for directory traversals and path information queries (stat/lstat >replacement.) >--- > configure.ac | 21 ++++++++++ > flist.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- > mkproto.pl | 2 +- > rsync.h | 15 +++++++ > syscall.c | 79 +++++++++++++++++++++++++++++++++++ > util.c | 88 +++++++++++++++++++++++++++++++++++++++ > 6 files changed, 334 insertions(+), 4 deletions(-) > >diff --git a/configure.ac b/configure.ac >index e01e124..20aba15 100644 >--- a/configure.ac >+++ b/configure.ac >@@ -336,6 +336,27 @@ case $host_os in > * ) AC_MSG_RESULT(no);; > esac > >+AC_ARG_WITH(w32api-include-path, >+ AS_HELP_STRING([--with-w32api-include-path=PATH],[set location of w32api headers (default: /usr/include/w32api)]), >+ [ W32API_INCLUDE_PATH="$with_w32api_include_path" ], >+ [ W32API_INCLUDE_PATH="/usr/include/w32api" ]) >+ >+AC_CHECK_SIZEOF([void*]) >+ >+AC_MSG_CHECKING([whether platform should use Win32 API]) >+case $host_os in >+ *cygwin*|*msys* ) AC_MSG_RESULT(yes) >+ AS_IF([test "$ac_cv_sizeof_voidp" -eq 4], >+ [AC_DEFINE(_WIN32, 1, >+ [Define to 1 if Win32 API should be used])], >+ [AC_DEFINE(_WIN64, 1, >+ [Define to 1 if Win64 API should be used])]) >+ dnl Include W32API Headers for Cygwin and Msys >+ CFLAGS="-I${W32API_INCLUDE_PATH} $CFLAGS" >+ ;; >+ * ) AC_MSG_RESULT(no);; >+esac >+ > AC_C_BIGENDIAN > AC_HEADER_DIRENT > AC_HEADER_TIME >diff --git a/flist.c b/flist.c >index 28553fc..c4e07e6 100644 >--- a/flist.c >+++ b/flist.c >@@ -1687,16 +1687,90 @@ static void interpret_stat_error(const char *fname, int is_dir) > static void send_directory(int f, struct file_list *flist, char *fbuf, int len, > int flags) > { >- struct dirent *di; >+#ifdef _IS_WINDOWS >+ WIN32_FIND_DATAA ffd; >+ LARGE_INTEGER filesize; >+ char szDir[MAX_PATH]; >+ unsigned szDir_len; >+ HANDLE hFind = INVALID_HANDLE_VALUE; >+ DWORD dwError = 0; >+ char *szFname = (char*) fbuf; >+#ifdef __CYGWIN__ >+ char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fbuf); >+ if (!winpath) { >+ errno = ENOMEM; >+ return; >+ } >+ szFname = winpath; >+#endif >+#else >+ struct dirent *di = NULL; >+ DIR *d = NULL; >+#endif > unsigned remainder; >- char *p; >- DIR *d; >+ char *p = NULL; > int divert_dirs = (flags & FLAG_DIVERT_DIRS) != 0; > int start = flist->used; > int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; > > assert(flist != NULL); > >+#ifdef _IS_WINDOWS >+ // subtract an additional two character for the later strlcat! >+ if ((szDir_len = strlcpy(szDir, szFname, MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2)) >= MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2) { >+ io_error |= IOERR_GENERAL; >+ rprintf(FERROR_XFER, >+ "filename overflows max-path len by %u: %s\n", >+ szDir_len - MAX_PATH - _WIN_PATH_SIZE_INTERNAL - 2 + 1, szFname); >+ errno = EOVERFLOW; >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ return; >+ } >+ >+ if ((szDir_len = strlcat(szDir, "\\*", MAX_PATH - _WIN_PATH_SIZE_INTERNAL)) >= MAX_PATH - _WIN_PATH_SIZE_INTERNAL) { >+ io_error |= IOERR_GENERAL; >+ rprintf(FERROR_XFER, >+ "filename overflows max-path len by %u: %s\n", >+ szDir_len - MAX_PATH - _WIN_PATH_SIZE_INTERNAL + 1, szDir); >+ errno = EOVERFLOW; >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ return; >+ } >+ >+ if (DEBUG_GTE(TIME, 3)) { >+ rprintf(FINFO, >+ "[debug] FindFirstFileA on '%s' from '%s'; current error code %d\n", >+ szDir, >+ fbuf, >+ GetLastError()); >+ } >+ >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ >+ SetLastError(ERROR_SUCCESS); >+ >+ if ((hFind = FindFirstFileA(szDir, &ffd)) == INVALID_HANDLE_VALUE) { >+ if (GetLastError() == ERROR_FILE_NOT_FOUND) { >+ errno = ENOENT; >+ if (am_sender) /* Can abuse this for vanished error w/ENOENT: */ >+ interpret_stat_error(fbuf, True); >+ return; >+ } >+ errno = EINVAL; >+ io_error |= IOERR_GENERAL; >+ rprintf(FERROR_XFER, >+ "unable to open '%s'; FindFirstFileA failed with error code %d\n", >+ szDir, >+ GetLastError()); >+ return; >+ } >+#else > if (!(d = opendir(fbuf))) { > if (errno == ENOENT) { > if (am_sender) /* Can abuse this for vanished error w/ENOENT: */ >@@ -1707,6 +1781,7 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, > rsyserr(FERROR_XFER, errno, "opendir %s failed", full_fname(fbuf)); > return; > } >+#endif > > p = fbuf + len; > if (len == 1 && *fbuf == '/') >@@ -1718,9 +1793,39 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, > } else > remainder = 0; > >+#ifdef _IS_WINDOWS >+ for (errno = 0, dwError = GetLastError(); dwError == ERROR_SUCCESS; dwError = (FindNextFileA(hFind, &ffd) != 0 ? ERROR_SUCCESS : GetLastError())) { >+#else > for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) { >+#endif > unsigned name_len; >+#ifdef _IS_WINDOWS >+ STRUCT_STAT sst; >+ sst.st_uid = 0; >+ sst.st_gid = 0; >+ filesize.LowPart = ffd.nFileSizeLow; >+ filesize.HighPart = ffd.nFileSizeHigh; >+ sst.st_size = filesize.QuadPart; >+ sst.st_blocks = (sst.st_size / 512ULL) + 1ULL; >+ sst.st_blksize = 4096; >+ sst.st_rdev = 0; >+ sst.st_nlink = 0; >+ sst.st_dev = 0; >+ sst.st_ino = 0; >+ sst.st_mode = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) ? S_IFDIR : S_IFREG; >+ sst.st_mode = ((ffd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) ? S_IFLNK : sst.st_mode; >+ sst.st_mode += (S_IRWXU | S_IRWXG | S_IRWXO); >+ sst.st_atime = win32_filetime_to_epoch(&ffd.ftLastAccessTime); >+ sst.st_ctime = win32_filetime_to_epoch(&ffd.ftCreationTime); >+ sst.st_mtime = win32_filetime_to_epoch(&ffd.ftLastWriteTime); >+ char *dname = ffd.cFileName; >+ >+ if (DEBUG_GTE(TIME, 1)) { >+ rprintf(FINFO, "path %s with mode 0x%x and mtime %ld (dir: %d)\n", dname, sst.st_mode, sst.st_mtime, ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) > 0) ? 1 : 0); >+ } >+#else > char *dname = d_name(di); >+#endif > if (dname[0] == '.' && (dname[1] == '\0' > || (dname[1] == '.' && dname[2] == '\0'))) > continue; >@@ -1743,17 +1848,39 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, > continue; > } > >+#ifdef _IS_WINDOWS >+ send_file_name(f, flist, fbuf, &sst, flags, filter_level); >+#else > send_file_name(f, flist, fbuf, NULL, flags, filter_level); >+#endif > } > > fbuf[len] = '\0'; > >+#ifdef _IS_WINDOWS >+ if (DEBUG_GTE(TIME, 3)) { >+ rprintf(FINFO, >+ "[debug] FindFirstFileA loop finished with error code %d inside '%s'\n", >+ GetLastError(), >+ full_fname(fbuf)); >+ } >+ >+ if (dwError != 0 && dwError != ERROR_NO_MORE_FILES) { >+ io_error |= IOERR_GENERAL; >+ rsyserr(FERROR_XFER, dwError, "FindNextFileA(%s)", full_fname(fbuf)); >+ } >+#else > if (errno) { > io_error |= IOERR_GENERAL; > rsyserr(FERROR_XFER, errno, "readdir(%s)", full_fname(fbuf)); > } >+#endif > >+#ifdef _IS_WINDOWS >+ FindClose(hFind); >+#else > closedir(d); >+#endif > > if (f >= 0 && recurse && !divert_dirs) { > int i, end = flist->used - 1; >diff --git a/mkproto.pl b/mkproto.pl >index cdeb2ea..e3d65be 100644 >--- a/mkproto.pl >+++ b/mkproto.pl >@@ -28,7 +28,7 @@ while (<>) { > $func = $3; > $arg = $1 eq 'LOCAL' ? 'int module_id' : 'void'; > $protos .= "$ret$func($arg);\n"; >- } elsif (/^static|^extern/ || /[;]/ || !/^[A-Za-z][A-Za-z0-9_]* /) { >+ } elsif (/^static|^extern/ || /[;]/ || /win32/ || !/^[A-Za-z][A-Za-z0-9_]* /) { > ; > } elsif (/[(].*[)][ \t]*$/) { > s/$/;/; >diff --git a/rsync.h b/rsync.h >index 1720293..c527944 100644 >--- a/rsync.h >+++ b/rsync.h >@@ -1007,6 +1007,21 @@ typedef struct { > #define ACL_READY(sx) ((sx).acc_acl != NULL) > #define XATTR_READY(sx) ((sx).xattr != NULL) > >+#if defined(_WIN32) || defined(_WIN64) >+#define _IS_WINDOWS >+#define _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR 10000000ULL >+#define _WIN_SECONDS_TO_UNIX_EPOCH 11644473600ULL >+#define _WIN_PATH_SIZE_INTERNAL 12 >+ >+#include <windows.h> >+#ifdef __CYGWIN__ >+#include <sys/cygwin.h> >+#endif >+ >+/* Forward Declarations */ >+extern time_t win32_filetime_to_epoch(const FILETIME *ft); >+#endif >+ > #include "proto.h" > > #ifndef SUPPORT_XATTRS >diff --git a/syscall.c b/syscall.c >index d89eacd..4394768 100644 >--- a/syscall.c >+++ b/syscall.c >@@ -317,15 +317,93 @@ int do_mkstemp(char *template, mode_t perms) > > int do_stat(const char *fname, STRUCT_STAT *st) > { >+#ifdef _IS_WINDOWS >+ ULARGE_INTEGER filesize; >+ WIN32_FILE_ATTRIBUTE_DATA fad; >+ char *szFname = (char*) fname; >+#ifdef __CYGWIN__ >+ char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fname); >+ if (!winpath) { >+ errno = ENOMEM; >+ return -1; >+ } >+ szFname = winpath; >+#endif >+ >+ if (DEBUG_GTE(TIME, 1)) { >+ rprintf(FINFO, "do_stat on '%s'\n", fname); >+ } >+ >+ if (GetFileAttributesExA(szFname, GetFileExInfoStandard, &fad) == 0) { >+ if (DEBUG_GTE(TIME, 1)) { >+ rprintf(FINFO, "do_stat on '%s' errored with error code %d\n", fname, GetLastError()); >+ } >+ switch (GetLastError()) { >+ case ERROR_FILE_NOT_FOUND: >+ case ERROR_PATH_NOT_FOUND: >+ errno = ENOENT; >+ break; >+ case ERROR_ACCESS_DENIED: >+ case ERROR_NETWORK_ACCESS_DENIED: >+ errno = EACCES; >+ break; >+ case ERROR_INVALID_HANDLE: >+ errno = EBADF; >+ break; >+ case ERROR_TOO_MANY_OPEN_FILES: >+ case ERROR_OUTOFMEMORY: >+ errno = ENOMEM; >+ break; >+ case ERROR_BAD_LENGTH: >+ errno = ENAMETOOLONG; >+ break; >+ case ERROR_INVALID_PARAMETER: >+ errno = EFAULT; >+ break; >+ default: >+ errno = EINVAL; >+ } >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ return -1; >+ } >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ >+ st->st_uid = 0; >+ st->st_gid = 0; >+ filesize.LowPart = fad.nFileSizeLow; >+ filesize.HighPart = fad.nFileSizeHigh; >+ st->st_size = filesize.QuadPart; >+ st->st_blocks = (st->st_size / 512ULL) + 1ULL; >+ st->st_blksize = 4096; >+ st->st_rdev = 0; >+ st->st_nlink = 0; >+ st->st_dev = 0; >+ st->st_ino = 0; >+ st->st_mode = ((fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) > 0) ? S_IFDIR : S_IFREG; >+ st->st_mode += (S_IRWXU | S_IRWXG | S_IRWXO); >+ st->st_atime = win32_filetime_to_epoch(&fad.ftLastAccessTime); >+ st->st_ctime = win32_filetime_to_epoch(&fad.ftCreationTime); >+ st->st_mtime = win32_filetime_to_epoch(&fad.ftLastWriteTime); >+ >+ return 0; >+#else > #ifdef USE_STAT64_FUNCS > return stat64(fname, st); > #else > return stat(fname, st); > #endif >+#endif > } > > int do_lstat(const char *fname, STRUCT_STAT *st) > { >+#ifdef _IS_WINDOWS1 >+return do_stat(fname, st); >+#else > #ifdef SUPPORT_LINKS > # ifdef USE_STAT64_FUNCS > return lstat64(fname, st); >@@ -335,6 +413,7 @@ int do_lstat(const char *fname, STRUCT_STAT *st) > #else > return do_stat(fname, st); > #endif >+#endif > } > > int do_fstat(int fd, STRUCT_STAT *st) >diff --git a/util.c b/util.c >index 49c5b71..df3eed7 100644 >--- a/util.c >+++ b/util.c >@@ -115,11 +115,36 @@ void print_child_argv(const char *prefix, char **cmd) > rprintf(FCLIENT, " (%d args)\n", cnt); > } > >+#ifdef _IS_WINDOWS >+time_t win32_filetime_to_epoch(const FILETIME *ft) >+{ >+ ULARGE_INTEGER liFileTime; >+ ULONGLONG llSeconds; >+ time_t retval; >+ >+ if (!ft) return (time_t) -1; >+ >+ liFileTime.LowPart = ft->dwLowDateTime; >+ liFileTime.HighPart = ft->dwHighDateTime; >+ llSeconds = ((ULONGLONG) liFileTime.QuadPart / (ULONGLONG) _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR - _WIN_SECONDS_TO_UNIX_EPOCH); >+ retval = (time_t) llSeconds; >+ >+ if (llSeconds != (ULONGLONG) retval) { >+ // value exceed POSIX epoch time, fail >+ return (time_t) -1; >+ } >+ >+ return retval; >+} >+#endif >+ > /* This returns 0 for success, 1 for a symlink if symlink time-setting > * is not possible, or -1 for any other error. */ > int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) > { >+#ifndef _IS_WINDOWS > static int switch_step = 0; >+#endif > > if (DEBUG_GTE(TIME, 1)) { > rprintf(FINFO, "set modtime of %s to (%ld) %s", >@@ -127,6 +152,68 @@ int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) > asctime(localtime(&modtime))); > } > >+#ifdef _IS_WINDOWS >+ HANDLE hFile; >+ ULARGE_INTEGER uliModtime; >+ FILETIME ftModtime; >+ DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; >+ char *szFname = (char*) fname; >+#ifdef __CYGWIN__ >+ char *winpath = cygwin_create_path(CCP_POSIX_TO_WIN_A, fname); >+ if (!winpath) { >+ errno = ENOMEM; >+ return -1; >+ } >+ szFname = winpath; >+#endif >+ >+ if (S_ISDIR(mode)) { >+ dwFlagsAndAttributes = FILE_FLAG_BACKUP_SEMANTICS; >+ } >+ >+ // convert POSIX epoch time to win32 FILETIME >+ uliModtime.QuadPart = (((ULONGLONG) modtime + (ULONGLONG) _WIN_SECONDS_TO_UNIX_EPOCH) * (ULONGLONG) _WIN_FILETIME_TO_UTC_EPOCH_DIVISOR); >+ ftModtime.dwLowDateTime = uliModtime.LowPart; >+ ftModtime.dwHighDateTime = uliModtime.HighPart; >+ >+ hFile = CreateFileA(szFname, // file to open >+ FILE_WRITE_ATTRIBUTES, // file opts >+ FILE_SHARE_WRITE, // share opts >+ NULL, //default security >+ OPEN_EXISTING, // existing file only >+ dwFlagsAndAttributes, // normal file >+ NULL); // no attr. template >+ >+ if (hFile == INVALID_HANDLE_VALUE) { >+ // error opening.... >+ rprintf(FERROR, "failed to open inside of set modtime of %s to (%ld) %s: error code %d\n", >+ szFname, (long)modtime, >+ asctime(localtime(&modtime)), >+ GetLastError()); >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ return -1; >+ } >+ >+ if (SetFileTime(hFile, NULL, NULL, &ftModtime) == 0) { >+ // error setting >+ rprintf(FERROR, "failed to set modtime of %s to (%ld) %s: error code %d\n", >+ szFname, (long)modtime, >+ asctime(localtime(&modtime)), >+ GetLastError()); >+ CloseHandle(hFile); >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+ return -1; >+ } >+ >+ CloseHandle(hFile); >+#ifdef __CYGWIN__ >+ free(winpath); >+#endif >+#else > switch (switch_step) { > #ifdef HAVE_UTIMENSAT > #include "case_N.h" >@@ -168,6 +255,7 @@ int set_modtime(const char *fname, time_t modtime, uint32 mod_nsec, mode_t mode) > > return -1; > } >+#endif > > return 0; > } >-- >2.10.2.windows.1 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 12755
: 13169