Attachment 2096 Details for Bug 4030 – implementation of --filelimit

[patch] implementation of --filelimit

filelimit.diff (text/plain), 4.98 KB, created by Volker Lendecke on 2006-08-17 14:59:18 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Volker Lendecke

Created: 2006-08-17 14:59:18 UTC

Size: 4.98 KB

patch

obsolete

>diff -urN rsync-2.6.8-orig/flist.c rsync-2.6.8/flist.c
>--- rsync-2.6.8-orig/flist.c	2006-04-21 18:36:30.000000000 +0200
>+++ rsync-2.6.8/flist.c	2006-08-17 21:41:31.302338500 +0200
>@@ -57,6 +57,7 @@
> extern int copy_unsafe_links;
> extern int protocol_version;
> extern int sanitize_paths;
>+extern int filelimit;
> extern struct stats stats;
> extern struct file_list *the_file_list;
> 
>@@ -306,6 +307,50 @@
> 		out_of_memory("flist_expand");
> }
> 
>+/**
>+ * Sleep to not overload the disk subsytem while creating the file tree
>+ **/
>+
>+static void sleep_for_filelimit(void)
>+{
>+	static struct timeval prior_tv;
>+	static long total_written = 0; 
>+	struct timeval tv, start_tv;
>+	long elapsed_usec, sleep_usec;
>+
>+#define ONE_SEC	1000000L /* # of microseconds in a second */
>+
>+	if (filelimit == 0) {
>+		return;
>+	}
>+
>+	total_written += 1;
>+
>+	gettimeofday(&start_tv, NULL);
>+	if (prior_tv.tv_sec) {
>+		elapsed_usec = (start_tv.tv_sec - prior_tv.tv_sec) * ONE_SEC
>+			     + (start_tv.tv_usec - prior_tv.tv_usec);
>+		total_written -= elapsed_usec * filelimit / (ONE_SEC);
>+		if (total_written < 0)
>+			total_written = 0;
>+	}
>+
>+	sleep_usec = total_written * (ONE_SEC) / filelimit;
>+	if (sleep_usec < ONE_SEC / 10) {
>+		prior_tv = start_tv;
>+		return;
>+	}
>+
>+	tv.tv_sec  = sleep_usec / ONE_SEC;
>+	tv.tv_usec = sleep_usec % ONE_SEC;
>+	select(0, NULL, NULL, NULL, &tv);
>+
>+	gettimeofday(&prior_tv, NULL);
>+	elapsed_usec = (prior_tv.tv_sec - start_tv.tv_sec) * ONE_SEC
>+		     + (prior_tv.tv_usec - start_tv.tv_usec);
>+	total_written = (sleep_usec - elapsed_usec) * filelimit / (ONE_SEC);
>+}
>+
> static void send_file_entry(struct file_struct *file, int f)
> {
> 	unsigned short flags;
>@@ -320,6 +365,8 @@
> 	char fname[MAXPATHLEN];
> 	int l1, l2;
> 
>+	sleep_for_filelimit();
>+
> 	if (f < 0)
> 		return;
> 
>diff -urN rsync-2.6.8-orig/lib/dummy rsync-2.6.8/lib/dummy
>--- rsync-2.6.8-orig/lib/dummy	1970-01-01 01:00:00.000000000 +0100
>+++ rsync-2.6.8/lib/dummy	2006-08-17 21:04:03.685871250 +0200
>@@ -0,0 +1,2 @@
>+This is a dummy file to ensure that the lib directory gets created
>+by configure when a VPATH is used.
>diff -urN rsync-2.6.8-orig/options.c rsync-2.6.8/options.c
>--- rsync-2.6.8-orig/options.c	2006-03-29 01:09:36.000000000 +0200
>+++ rsync-2.6.8/options.c	2006-08-17 21:37:03.305589750 +0200
>@@ -99,6 +99,7 @@
> int size_only = 0;
> int daemon_bwlimit = 0;
> int bwlimit = 0;
>+int filelimit = 0;
> int fuzzy_basis = 0;
> size_t bwlimit_writemax = 0;
> int ignore_existing = 0;
>@@ -363,6 +364,7 @@
>   rprintf(F,"     --password-file=FILE    read password from FILE\n");
>   rprintf(F,"     --list-only             list the files instead of copying them\n");
>   rprintf(F,"     --bwlimit=KBPS          limit I/O bandwidth; KBytes per second\n");
>+  rprintf(F,"     --filelimit=n           limit list creation to n files per second\n");
>   rprintf(F,"     --write-batch=FILE      write a batched update to FILE\n");
>   rprintf(F,"     --only-write-batch=FILE like --write-batch but w/o updating destination\n");
>   rprintf(F,"     --read-batch=FILE       read a batched update from FILE\n");
>@@ -495,6 +497,7 @@
>   {"log-format",       0,  POPT_ARG_STRING, &log_format, 0, 0, 0 },
>   {"itemize-changes", 'i', POPT_ARG_NONE,   0, 'i', 0, 0 },
>   {"bwlimit",          0,  POPT_ARG_INT,    &bwlimit, 0, 0, 0 },
>+  {"filelimit",        0,  POPT_ARG_INT,    &filelimit, 0, 0, 0 },
>   {"backup",          'b', POPT_ARG_NONE,   &make_backups, 0, 0, 0 },
>   {"backup-dir",       0,  POPT_ARG_STRING, &backup_dir, 0, 0, 0 },
>   {"suffix",           0,  POPT_ARG_STRING, &backup_suffix, 0, 0, 0 },
>@@ -1619,6 +1622,12 @@
> 		args[ac++] = arg;
> 	}
> 
>+	if (filelimit) {
>+		if (asprintf(&arg, "--filelimit=%d", filelimit) < 0)
>+			goto oom;
>+		args[ac++] = arg;
>+	}
>+
> 	if (backup_dir) {
> 		args[ac++] = "--backup-dir";
> 		args[ac++] = backup_dir;
>diff -urN rsync-2.6.8-orig/rsync.1 rsync-2.6.8/rsync.1
>--- rsync-2.6.8-orig/rsync.1	2006-04-22 17:38:37.000000000 +0200
>+++ rsync-2.6.8/rsync.1	2006-08-17 21:54:38.607542000 +0200
>@@ -447,6 +447,7 @@
>      --password-file=FILE    read password from FILE
>      --list-only             list the files instead of copying them
>      --bwlimit=KBPS          limit I/O bandwidth; KBytes per second
>+     --filelimit=n           limit list creation to n files per second
>      --write-batch=FILE      write a batched update to FILE
>      --only-write-batch=FILE like --write-batch but w/o updating dest
>      --read-batch=FILE       read a batched update from FILE
>@@ -1930,6 +1931,12 @@
> result is an average transfer rate equaling the specified limit\&. A value
> of zero specifies no limit\&.
> .IP 
>+.IP "\fB--filelimit=n\fP" 
>+This option allows you to limit the creation of the file list to n per
>+second\&. This might be useful for busy servers that have limited disk I/O 
>+bandwidth where a full-speed directory traversal would saturate the number
>+of disk transactions per second\&. A value of zero specifies no limit\&.
>+.IP 
> .IP "\fB--write-batch=FILE\fP" 
> Record a file that can later be applied to
> another identical destination with \fB--read-batch\fP\&. See the "BATCH MODE"

Actions: View

Attachments on bug 4030: 2096