Attachment 13681 Details for Bug 13083 – Use both partial file + basis file

[patch] Use both partial file + basis file

rsync_dual_basis.patch (text/plain), 9.60 KB, created by Ben RUBSON on 2017-10-12 14:28:32 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ben RUBSON

Created: 2017-10-12 14:28:32 UTC

Size: 9.60 KB

patch

obsolete

>--- util.c.orig	2015-12-21 19:54:02.000000000 +0100
>+++ util.c	2017-10-10 23:34:49.000000000 +0200
>@@ -1186,7 +1186,7 @@
> 		}
> 	} else
> 		fn = fname;
>-	if ((int)pathjoin(t, sz, partial_dir, fn) >= sz)
>+	if ((int)pathjoin(t, sz, partial_dir, fn) > sz)
> 		return NULL;
> 	if (daemon_filter_list.head) {
> 		t = strrchr(partial_fname, '/');
>--- generator.c.orig	2017-10-01 17:10:28.000000000 +0200
>+++ generator.c	2017-10-12 15:13:27.000000000 +0200
>@@ -682,16 +682,32 @@
>  *
>  * Generate approximately one checksum every block_len bytes.
>  */
>-static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
>+static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy, OFF_T total_size, int fd_cmplnk, OFF_T len_cmplnk)
> {
> 	int32 i;
>+	int32 pad_sums = 0;
> 	struct map_struct *mapbuf;
> 	struct sum_struct sum;
> 	OFF_T offset = 0;
> 
>-	sum_sizes_sqroot(&sum, len);
>+	sum_sizes_sqroot(&sum, len + len_cmplnk);
>+	if (len_cmplnk > 0 && (sum.count < 0 || append_mode > 0)) {
>+		len_cmplnk = 0;
>+		sum_sizes_sqroot(&sum, len);
>+	}
> 	if (sum.count < 0)
> 		return -1;
>+
>+	/* Let's send some dummy 0 blocks after the first checksumed (partial) file
>+	 * so that sender will see the second file's blocks as if they were after
>+	 * the end of the final file, thus it will be able to fully use them.
>+	 * Useful when inplace is used. */
>+	if (len_cmplnk > 0 && total_size > len) {
>+		pad_sums = total_size / sum.blength + (total_size % sum.blength != 0)
>+		           - (len / sum.blength + (len % sum.blength != 0));
>+		sum.count += pad_sums;
>+	}
>+
> 	write_sum_head(f_out, &sum);
> 
> 	if (append_mode > 0 && f_copy < 0)
>@@ -717,6 +733,39 @@
> 				continue;
> 		}
> 
>+		/* Time to checksum second file */
>+		if (n1 < sum.blength && len_cmplnk > 0) {
>+			/* Stop copy */
>+			f_copy = -1;
>+			/* Load the second mapfuf */
>+			if (mapbuf)
>+				unmap_file(mapbuf);
>+			mapbuf = map_file(fd_cmplnk, len_cmplnk, MAX_MAP_SIZE, sum.blength);
>+			len = len_cmplnk;
>+			len_cmplnk = 0; /* don't enter this if{} anymore */
>+			/* Don't checksum the last partial block if we have a second partial file
>+			 * as it will not have the correct size.
>+			 * Send a dummy O block instead. */
>+			i--;
>+			char sum2[SUM_LENGTH] = {0};
>+			if (n1 > 0) {
>+				write_int(f_out, 0);
>+				write_buf(f_out, sum2, sum.s2length);
>+				i++;
>+				offset = sum.blength - n1;
>+			} else {
>+				offset = 0;
>+			}
>+			/* Sens the dummy 0 blocks between the 2 files */
>+			while (pad_sums > 0) {
>+				write_int(f_out, 0);
>+				write_buf(f_out, sum2, sum.s2length);
>+				i++;
>+				pad_sums--;
>+			}
>+			continue;
>+		}
>+
> 		sum1 = get_checksum1(map, n1);
> 		get_checksum2(map, n1, sum2);
> 
>@@ -1186,13 +1235,15 @@
> 	static struct file_list *fuzzy_dirlist[MAX_BASIS_DIRS+1];
> 	static int need_fuzzy_dirlist = 0;
> 	struct file_struct *fuzzy_file = NULL;
>-	int fd = -1, f_copy = -1;
>+	int fd = -1, f_copy = -1, fd_cmplnk = -1;
> 	stat_x sx, real_sx;
> 	STRUCT_STAT partial_st;
>+	STRUCT_STAT cmplnk_st;
> 	struct file_struct *back_file = NULL;
> 	int statret, real_ret, stat_errno;
> 	char *fnamecmp, *partialptr, *backupptr = NULL;
> 	char fnamecmpbuf[MAXPATHLEN];
>+	char fnamecmplnk[MAXPATHLEN];
> 	uchar fnamecmp_type;
> 	int del_opts = delete_mode || force_delete ? DEL_RECURSE : 0;
> 	int is_dir = !S_ISDIR(file->mode) ? 0
>@@ -1713,14 +1764,6 @@
> 	real_sx.st = sx.st; /* Don't copy xattr/acl pointers, as they would free wrong. */
> 	real_ret = statret;
> 
>-	if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
>-	    && link_stat(partialptr, &partial_st, 0) == 0
>-	    && S_ISREG(partial_st.st_mode)) {
>-		if (statret != 0)
>-			goto prepare_to_open;
>-	} else
>-		partialptr = NULL;
>-
> 	if (statret != 0 && fuzzy_basis) {
> 		/* Sets fnamecmp_type to FNAMECMP_FUZZY or above. */
> 		fuzzy_file = find_fuzzy(file, fuzzy_dirlist, &fnamecmp_type);
>@@ -1736,6 +1779,14 @@
> 		}
> 	}
> 
>+	if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
>+	    && link_stat(partialptr, &partial_st, 0) == 0
>+	    && S_ISREG(partial_st.st_mode)) {
>+		if (statret != 0)
>+			goto prepare_to_open;
>+	} else
>+		partialptr = NULL;
>+
> 	if (statret != 0) {
> #ifdef SUPPORT_HARD_LINKS
> 		if (preserve_hard_links && F_HLINK_NOT_LAST(file)) {
>@@ -1783,7 +1834,18 @@
> 	}
> 
>   prepare_to_open:
>+	fnamecmplnk[0] = '\0';
>+	cmplnk_st.st_size = 0;
> 	if (partialptr) {
>+		/* We have a partial file, and also a basis file.
>+		   Let's then link the basis file so that both generator and
>+		   receiver will be able to use it right after the partial one. */
>+		if (statret == 0) {
>+			strcpy(fnamecmplnk, partialptr);
>+			strcat(fnamecmplnk, ".");
>+			unlink(fnamecmplnk);
>+			symlink(fnamecmp, fnamecmplnk);
>+		}
> 		sx.st = partial_st;
> 		fnamecmp = partialptr;
> 		fnamecmp_type = FNAMECMP_PARTIAL_DIR;
>@@ -1836,6 +1898,18 @@
> 		statret = real_ret = -1;
> 		goto notify_others;
> 	}
>+	if (strlen(fnamecmplnk) > 0) {
>+		if ((fd_cmplnk = do_open(fnamecmplnk, O_RDONLY, 0)) < 0) {
>+			rsyserr(FERROR, errno, "generator failed to open second basis %s, continuing",
>+				full_fname(fnamecmplnk));
>+		} else {
>+			if (do_stat(fnamecmplnk, &cmplnk_st) < 0) {
>+				cmplnk_st.st_size = 0;
>+				rsyserr(FERROR, errno, "generator failed to stat second basis %s, continuing",
>+					full_fname(fnamecmplnk));
>+			}
>+		}
>+	}
> 
> 	if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) {
> 		if (!(backupptr = get_backup_name(fname))) {
>@@ -1907,11 +1981,11 @@
> 
> 	if (statret != 0 || whole_file)
> 		write_sum_head(f_out, NULL);
>-	else if (sx.st.st_size <= 0) {
>+	else if (sx.st.st_size < 0) {
> 		write_sum_head(f_out, NULL);
> 		close(fd);
> 	} else {
>-		if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy) < 0) {
>+		if (generate_and_send_sums(fd, sx.st.st_size, f_out, f_copy, F_LENGTH(file), fd_cmplnk, cmplnk_st.st_size) < 0) {
> 			rprintf(FWARNING,
> 			    "WARNING: file is too large for checksum sending: %s\n",
> 			    fnamecmp);
>@@ -1921,6 +1995,8 @@
> 	}
> 
>   cleanup:
>+	if (fd_cmplnk != -1)
>+		close(fd_cmplnk);
> 	if (back_file) {
> 		int save_preserve_xattrs = preserve_xattrs;
> 		if (f_copy >= 0)
>--- receiver.c.orig	2017-10-02 17:05:17.000000000 +0200
>+++ receiver.c	2017-10-12 15:02:43.000000000 +0200
>@@ -229,16 +229,18 @@
> }
> 
> static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
>-			const char *fname, int fd, OFF_T total_size)
>+			const char *fname, int fd, OFF_T total_size, char *cmplnk, int fd_cmplnk, OFF_T len_cmplnk)
> {
> 	static char file_sum1[MAX_DIGEST_LEN];
> 	struct map_struct *mapbuf;
>+	struct map_struct *mapbuf_cmplnk;
> 	struct sum_struct sum;
> 	int32 len;
> 	OFF_T offset = 0;
> 	OFF_T offset2;
> 	char *data;
> 	int32 i;
>+	int32 pad_size = 0;
> 	char *map = NULL;
> #ifdef SUPPORT_PREALLOCATION
> #ifdef PREALLOCATE_NEEDS_TRUNCATE
>@@ -269,6 +271,22 @@
> 	} else
> 		mapbuf = NULL;
> 
>+	if (fd_cmplnk >= 0 && len_cmplnk > 0) {
>+		int32 read_size = MAX(sum.blength * 2, 16*1024);
>+		mapbuf_cmplnk = map_file(fd_cmplnk, len_cmplnk, read_size, sum.blength);
>+		if (DEBUG_GTE(DELTASUM, 2)) {
>+			rprintf(FINFO, "recv mapped %s of size %s\n",
>+				cmplnk, big_num(len_cmplnk));
>+		}
>+	} else
>+		mapbuf_cmplnk = NULL;
>+
>+	if (len_cmplnk > 0 && total_size > size_r) {
>+		pad_size = total_size / sum.blength + (total_size % sum.blength != 0)
>+		           - (size_r / sum.blength + (size_r % sum.blength != 0));
>+		pad_size *= sum.blength;
>+	}
>+
> 	sum_init(checksum_seed);
> 
> 	if (append_mode > 0) {
>@@ -338,15 +356,20 @@
> 				updating_basis_or_equiv && offset == offset2 ? " (seek)" : "");
> 		}
> 
>-		if (mapbuf) {
>+		if (mapbuf && offset2 <= size_r) {
> 			map = map_ptr(mapbuf,offset2,len);
>-
>+			see_token(map, len);
>+			sum_update(map, len);
>+		}
>+		else if (mapbuf_cmplnk && offset2 > size_r) {
>+			offset2 -= (size_r + pad_size);
>+			map = map_ptr(mapbuf_cmplnk,offset2,len);
> 			see_token(map, len);
> 			sum_update(map, len);
> 		}
> 
> 		if (updating_basis_or_equiv) {
>-			if (offset == offset2 && fd != -1) {
>+			if (offset == offset2 && offset2 < size_r && fd != -1) {
> 				OFF_T pos;
> 				if (flush_write_file(fd) < 0)
> 					goto report_write_error;
>@@ -398,6 +421,8 @@
> 
> 	if (mapbuf)
> 		unmap_file(mapbuf);
>+	if (mapbuf_cmplnk)
>+		unmap_file(mapbuf_cmplnk);
> 
> 	read_buf(f_in, sender_file_sum, checksum_len);
> 	if (DEBUG_GTE(DELTASUM, 2))
>@@ -410,7 +435,7 @@
> 
> static void discard_receive_data(int f_in, OFF_T length)
> {
>-	receive_data(f_in, NULL, -1, 0, NULL, -1, length);
>+	receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL, -1, 0);
> }
> 
> static void handle_delayed_updates(char *local_name)
>@@ -525,6 +550,9 @@
> 	char fnametmp[MAXPATHLEN];
> 	char *fnamecmp, *partialptr;
> 	char fnamecmpbuf[MAXPATHLEN];
>+	char fnamecmplnk[MAXPATHLEN];
>+	int fd_cmplnk = -1;
>+	STRUCT_STAT cmplnk_st;
> 	uchar fnamecmp_type;
> 	struct file_struct *file;
> 	int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i;
>@@ -851,6 +879,22 @@
> 			continue;
> 		}
> 
>+		/* open second basis file if it exists */
>+		fnamecmplnk[0] = '\0';
>+		cmplnk_st.st_size = 0;
>+		if (fd1 != -1 && partialptr) {
>+			strcpy(fnamecmplnk, partialptr);
>+			strcat(fnamecmplnk, ".");
>+			if ((fd_cmplnk = do_open(fnamecmplnk, O_RDONLY, 0)) >= 0) {
>+				if (do_stat(fnamecmplnk, &cmplnk_st) < 0) {
>+					cmplnk_st.st_size = 0;
>+					rsyserr(FERROR, errno, "receiver failed to stat second basis %s, continuing",
>+						full_fname(fnamecmplnk));
>+				}
>+				unlink(fnamecmplnk);
>+			}
>+		}
>+
> 		/* log the transfer */
> 		if (log_before_transfer)
> 			log_item(FCLIENT, file, iflags, NULL);
>@@ -859,10 +903,12 @@
> 
> 		/* recv file data */
> 		recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
>-				       fname, fd2, F_LENGTH(file));
>+				       fname, fd2, F_LENGTH(file), fnamecmplnk, fd_cmplnk, cmplnk_st.st_size);
> 
> 		log_item(log_code, file, iflags, NULL);
> 
>+		if (fd_cmplnk != -1)
>+			close(fd_cmplnk);
> 		if (fd1 != -1)
> 			close(fd1);
> 		if (close(fd2) < 0) {

Actions: View

Attachments on bug 13083: 13681 | 13958