add is_partial_backup flag

alubennikova · alubennikova · commit 45220bad01ce · 2017-01-24T17:25:30.000+03:00
diff --git a/contrib/pg_probackup/backup.c b/contrib/pg_probackup/backup.c
@@ -1286,7 +1286,7 @@ backup_files(void *arg)
 					continue;
 				}
 			}
-			else if (is_compressed_data_file(file, arguments->files))
+			else if (is_compressed_data_file(file))
 			{
 				size_t skip_size = 0;
 				if (backup_compressed_file_partially(file, arguments, &skip_size))
@@ -1398,13 +1398,6 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
 
 				pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
 
-// 				if (is_compressed_data_file(&tmp_file, list_file))
-// 				{
-// 					elog(NOTICE, "file %s is compressed, don't remove it from list", tmp_file.path);
-// 					pg_free(tmp_file.path);
-// 					break;
-// 				}
-
 				if (pre_search_file != NULL)
 				{
 					search_file = *pre_search_file;
diff --git a/contrib/pg_probackup/data.c b/contrib/pg_probackup/data.c
@@ -513,43 +513,12 @@ restore_compressed_file(const char *from_root,
 						const char *to_root,
 						pgFile *file)
 {
-	char	to_path[MAXPGPATH];
-	pgFile tmp_file;
-
-	join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
-	tmp_file.path = psprintf("%s.cfm", to_path);
-
-	FileMap* map;
-	int md = open(tmp_file.path, O_RDWR|PG_BINARY, 0);
-	if (md < 0)
-	{
-		elog(LOG, "restore_compressed_file(). cannot open cfm file '%s'", tmp_file.path);
-		copy_file(from_root, to_root, file);
-		pfree(tmp_file.path);
-		return;
-	}
-
-	elog(NOTICE, "restore_compressed_file(). map %s", tmp_file.path);
-	map = cfs_mmap(md);
-	if (map == MAP_FAILED)
-	{
-		elog(LOG, "restore_compressed_file(). cfs_compression_ration failed to map file %s: %m", tmp_file.path);
-		if (close(md) < 0)
-			elog(LOG, "restore_compressed_file(). CFS failed to close file %s: %m", tmp_file.path);
-		pfree(tmp_file.path);
-		return;
-	}
-
-	if (map->generation != file->generation)
+	if (file->is_partial_copy == 0)
 		copy_file(from_root, to_root, file);
-	else
+	else if (file->is_partial_copy == 1)
 		restore_file_partly(from_root, to_root, file);
-
-	if (cfs_munmap(map) < 0)
-		elog(LOG, "restore_compressed_file(). CFS failed to unmap file %s: %m", tmp_file.path);
-	if (close(md) < 0)
-		elog(LOG, "restore_compressed_file(). CFS failed to close file %s: %m", tmp_file.path);
-	pfree(tmp_file.path);
+	else
+		elog(ERROR, "restore_compressed_file()");
 }
 
 /*
@@ -700,22 +669,14 @@ restore_data_file(const char *from_root,
 	fclose(out);
 }
 
-/*  */
+/* If someone's want to use this function before correct
+ * generation values is set, he can look up for corresponding
+ * .cfm file in the file_list
+ */
 bool
-is_compressed_data_file(pgFile *file, parray *file_list)
+is_compressed_data_file(pgFile *file)
 {
-//	return (file->generation != -1);
-	pgFile map_file;
-	pgFile **pre_search_file;
-
-	map_file.path = psprintf("%s.cfm", file->path);
-	pre_search_file = (pgFile **) parray_bsearch(file_list, &map_file, pgFileComparePath);
-	pg_free(map_file.path);
-
-	if (pre_search_file != NULL)
-		return true;
-
-	return false;
+	return (file->generation != -1);
 }
 
 bool
@@ -958,10 +919,8 @@ copy_file_partly(const char *from_root, const char *to_root,
 			 strerror(errno_tmp));
 	}
 
+	file->is_partial_copy = 1;
 	elog(NOTICE, "copy_file_partly(). %s file->write_size %lu", to_path, file->write_size);
-// 	pgFile newfile;
-// 	newfile.path = pg_strdup(to_path);
-// 	file->crc = pgFileGetCRC(&newfile);
 
 	fclose(in);
 	fclose(out);
diff --git a/contrib/pg_probackup/dir.c b/contrib/pg_probackup/dir.c
@@ -93,6 +93,7 @@ pgFileNew(const char *path, bool omit_symlink)
 	file->segno = 0;
 	file->path = pgut_malloc(strlen(path) + 1);
 	file->generation = -1;
+	file->is_partial_copy = 0;
 	strcpy(file->path, path);		/* enough buffer size guaranteed */
 
 	return file;
@@ -523,7 +524,7 @@ dir_print_file_list(FILE *out, const parray *files, const char *root, const char
 			fprintf(out, " %s", timestamp);
 		}
 
-		fprintf(out, " %d\n", file->generation);
+		fprintf(out, " %d %d\n", file->generation, file->is_partial_copy);
 	}
 }
 
@@ -550,18 +551,19 @@ dir_read_file_list(const char *root, const char *file_txt)
 		char			path[MAXPGPATH];
 		char			type;
 		int				generation;
+		int				is_partial_copy;
 		unsigned long	write_size;
 		pg_crc32		crc;
 		unsigned int	mode;	/* bit length of mode_t depends on platforms */
 		struct tm		tm;
 		pgFile			*file;
 
 		memset(&tm, 0, sizeof(tm));
-		if (sscanf(buf, "%s %c %lu %u %o %d-%d-%d %d:%d:%d %d",
+		if (sscanf(buf, "%s %c %lu %u %o %d-%d-%d %d:%d:%d %d %d",
 			path, &type, &write_size, &crc, &mode,
 			&tm.tm_year, &tm.tm_mon, &tm.tm_mday,
 			&tm.tm_hour, &tm.tm_min, &tm.tm_sec,
-			&generation) != 12)
+			&generation, &is_partial_copy) != 13)
 		{
 			elog(ERROR, "invalid format found in \"%s\"",
 				file_txt);
@@ -587,6 +589,7 @@ dir_read_file_list(const char *root, const char *file_txt)
 			((type == 'f' || type == 'F') ? S_IFREG :
 			 type == 'd' ? S_IFDIR : type == 'l' ? S_IFLNK : 0);
 		file->generation = generation;
+		file->is_partial_copy = is_partial_copy;
 		file->size = 0;
 		file->read_size = 0;
 		file->write_size = write_size;
diff --git a/contrib/pg_probackup/pg_probackup.h b/contrib/pg_probackup/pg_probackup.h
@@ -73,6 +73,8 @@ typedef struct pgFile
 	int		segno;			/* Segment number for ptrack */
 	int		generation;		/* Generation of compressed file.
 							 * -1 for non-compressed files */
+	int	is_partial_copy; /* for compressed files.
+						  * 1 if backed up via copy_file_partly()  */
 	volatile uint32 lock;
 	datapagemap_t pagemap;
 } pgFile;
@@ -317,7 +319,7 @@ extern bool backup_data_file(const char *from_root, const char *to_root,
 							 pgFile *file, const XLogRecPtr *lsn);
 extern void restore_data_file(const char *from_root, const char *to_root,
 							  pgFile *file, pgBackup *backup);
-extern bool is_compressed_data_file(pgFile *file, parray *file_list);
+extern bool is_compressed_data_file(pgFile *file);
 extern bool backup_compressed_file_partially(pgFile *file,
 											 void *arg,
 											 size_t *skip_size);

Original file line number	Diff line number	Diff line change
`@@ -1286,7 +1286,7 @@ backup_files(void *arg)`
`1286`	`1286`	`continue;`
`1287`	`1287`	`}`
`1288`	`1288`	`}`
`1289`		`- else if (is_compressed_data_file(file, arguments->files))`
	`1289`	`+ else if (is_compressed_data_file(file))`
`1290`	`1290`	`{`
`1291`	`1291`	`size_t skip_size = 0;`
`1292`	`1292`	`if (backup_compressed_file_partially(file, arguments, &skip_size))`
`@@ -1398,13 +1398,6 @@ add_files(parray files, const char root, bool add_root, bool is_pgdata)`
`1398`	`1398`
`1399`	`1399`	`pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);`
`1400`	`1400`
`1401`		`-// if (is_compressed_data_file(&tmp_file, list_file))`
`1402`		`-// {`
`1403`		`-// elog(NOTICE, "file %s is compressed, don't remove it from list", tmp_file.path);`
`1404`		`-// pg_free(tmp_file.path);`
`1405`		`-// break;`
`1406`		`-// }`
`1407`		`-`
`1408`	`1401`	`if (pre_search_file != NULL)`
`1409`	`1402`	`{`
`1410`	`1403`	`search_file = *pre_search_file;`