[PBCKP-146] - fio_get_crc32 - add "missing_ok" parameter

funny-falcon · funny-falcon · commit 4730857b7946 · 2022-10-13T14:27:40.000+03:00
diff --git a/src/archive.c b/src/archive.c
@@ -512,8 +512,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d
 		pg_crc32 crc32_src;
 		pg_crc32 crc32_dst;
 
-		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
-		crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false);
+		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
+		crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false);
 
 		if (crc32_src == crc32_dst)
 		{
@@ -760,9 +760,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir,
 		pg_crc32 crc32_src;
 		pg_crc32 crc32_dst;
 
-		/* TODO: what if one of them goes missing? */
-		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
-		crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true);
+		crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
+		crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false);
 
 		if (crc32_src == crc32_dst)
 		{
diff --git a/src/data.c b/src/data.c
@@ -801,8 +801,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
 		(prev_file && file->exists_in_prev &&
 		 file->mtime <= parent_backup_time))
 	{
-
-		file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
+		/*
+		 * file could be deleted under our feets.
+		 * But then backup_non_data_file_internal will handle it safely
+		 */
+		file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
 
 		/* ...and checksum is the same... */
 		if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1327,7 +1330,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 	if (already_exists)
 	{
 		/* compare checksums of already existing file and backup file */
-		pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false);
+		pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
 
 		if (file_crc == tmp_file->crc)
 		{
diff --git a/src/utils/file.c b/src/utils/file.c
@@ -1355,9 +1355,15 @@ fio_sync(char const* path, fio_location location)
 	}
 }
 
+enum {
+	GET_CRC32_DECOMPRESS = 1,
+	GET_CRC32_MISSING_OK = 2
+};
+
 /* Get crc32 of file */
 pg_crc32
-fio_get_crc32(const char *file_path, fio_location location, bool decompress)
+fio_get_crc32(const char *file_path, fio_location location,
+			  bool decompress, bool missing_ok)
 {
 	if (fio_is_remote(location))
 	{
@@ -1370,7 +1376,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
 		hdr.arg = 0;
 
 		if (decompress)
-			hdr.arg = 1;
+			hdr.arg = GET_CRC32_DECOMPRESS;
+		if (missing_ok)
+			hdr.arg |= GET_CRC32_MISSING_OK;
 
 		IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
 		IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len);
@@ -1381,9 +1389,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
 	else
 	{
 		if (decompress)
-			return pgFileGetCRCgz(file_path, true, true);
+			return pgFileGetCRCgz(file_path, true, missing_ok);
 		else
-			return pgFileGetCRC(file_path, true, true);
+			return pgFileGetCRC(file_path, true, missing_ok);
 	}
 }
 
@@ -3380,10 +3388,10 @@ fio_communicate(int in, int out)
 			break;
 		  case FIO_GET_CRC32:
 			/* calculate crc32 for a file */
-			if (hdr.arg == 1)
-				crc = pgFileGetCRCgz(buf, true, true);
+			if ((hdr.arg & GET_CRC32_DECOMPRESS))
+				crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			else
-				crc = pgFileGetCRC(buf, true, true);
+				crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
 			IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
 			break;
 		  case FIO_GET_CHECKSUM_MAP:
diff --git a/src/utils/file.h b/src/utils/file.h
@@ -120,7 +120,8 @@ extern int     fio_truncate(int fd, off_t size);
 extern int     fio_close(int fd);
 extern void    fio_disconnect(void);
 extern int     fio_sync(char const* path, fio_location location);
-extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress);
+extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
+							  bool decompress, bool missing_ok);
 
 extern int     fio_rename(char const* old_path, char const* new_path, fio_location location);
 extern int     fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);
