Attachment 12914 Details for Bug 12571 – patch for 4.6, 4.5 and 4.4

[patch] patch for 4.6, 4.5 and 4.4

0001-s3-vfs-Only-walk-the-directory-once-in-open_and_sort.patch (text/plain), 3.24 KB, created by Andreas Schneider on 2017-02-10 12:37:24 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Andreas Schneider

Created: 2017-02-10 12:37:24 UTC

Size: 3.24 KB

patch

obsolete

>From ae288f738e15ccf226ad27928ad2a21d97ad4860 Mon Sep 17 00:00:00 2001
>From: Andreas Schneider <asn@samba.org>
>Date: Thu, 9 Feb 2017 15:05:01 +0100
>Subject: [PATCH] s3-vfs: Only walk the directory once in open_and_sort_dir()
>
>On a slow filesystem or network filesystem this can make a huge
>difference.
>
>BUG: https://bugzilla.samba.org/show_bug.cgi?id=12571
>
>Signed-off-by: Andreas Schneider <asn@samba.org>
>Reviewed-by: Jeremy Allison <jra@samba.org>
>(cherry picked from commit 6c3aaccd0eb648e31fd2717aaca0187966e125d5)
>---
> source3/modules/vfs_dirsort.c | 63 ++++++++++++++++++++++++++-----------------
> 1 file changed, 39 insertions(+), 24 deletions(-)
>
>diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c
>index 4a3e152039f..cd5597a3b40 100644
>--- a/source3/modules/vfs_dirsort.c
>+++ b/source3/modules/vfs_dirsort.c
>@@ -65,8 +65,10 @@ static bool get_sorted_dir_mtime(vfs_handle_struct *handle,
> static bool open_and_sort_dir(vfs_handle_struct *handle,
> 				struct dirsort_privates *data)
> {
>-	unsigned int i = 0;
>-	unsigned int total_count = 0;
>+	uint32_t total_count = 0;
>+	/* This should be enough for most use cases */
>+	uint32_t dirent_allocated = 64;
>+	struct dirent *dp;
> 
> 	data->number_of_entries = 0;
> 
>@@ -74,38 +76,51 @@ static bool open_and_sort_dir(vfs_handle_struct *handle,
> 		return false;
> 	}
> 
>-	while (SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL)
>-	       != NULL) {
>-		total_count++;
>-	}
>-
>-	if (total_count == 0) {
>+	dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
>+	if (dp == NULL) {
> 		return false;
> 	}
> 
>-	/* Open the underlying directory and count the number of entries
>-	   Skip back to the beginning as we'll read it again */
>-	SMB_VFS_NEXT_REWINDDIR(handle, data->source_directory);
>-
> 	/* Set up an array and read the directory entries into it */
> 	TALLOC_FREE(data->directory_list); /* destroy previous cache if needed */
> 	data->directory_list = talloc_zero_array(data,
>-					struct dirent,
>-					total_count);
>-	if (!data->directory_list) {
>+						 struct dirent,
>+						 dirent_allocated);
>+	if (data->directory_list == NULL) {
> 		return false;
> 	}
>-	for (i = 0; i < total_count; i++) {
>-		struct dirent *dp = SMB_VFS_NEXT_READDIR(handle,
>-						data->source_directory,
>-						NULL);
>-		if (dp == NULL) {
>-			break;
>+
>+	do {
>+		if (total_count >= dirent_allocated) {
>+			struct dirent *dlist;
>+
>+			/*
>+			 * Be memory friendly.
>+			 *
>+			 * We should not double the amount of memory. With a lot
>+			 * of files we reach easily 50MB, and doubling will
>+			 * get much bigger just for a few files more.
>+			 *
>+			 * For 200k files this means 50 memory reallocations.
>+			 */
>+			dirent_allocated += 4096;
>+
>+			dlist = talloc_realloc(data,
>+					       data->directory_list,
>+					       struct dirent,
>+					       dirent_allocated);
>+			if (dlist == NULL) {
>+				break;
>+			}
>+			data->directory_list = dlist;
> 		}
>-		data->directory_list[i] = *dp;
>-	}
>+		data->directory_list[total_count] = *dp;
>+
>+		total_count++;
>+		dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
>+	} while (dp != NULL);
> 
>-	data->number_of_entries = i;
>+	data->number_of_entries = total_count;
> 
> 	/* Sort the directory entries by name */
> 	TYPESAFE_QSORT(data->directory_list, data->number_of_entries, compare_dirent);
>-- 
>2.11.0
>

Flags:

jra: review+

Actions: View

Attachments on bug 12571: 12914 | 12922 | 12923 | 12924