From 1b9311e98109094e7bd71457176381fa9493a95f Mon Sep 17 00:00:00 2001 From: wuyf41619 Date: Thu, 14 Jul 2022 11:09:01 +0800 Subject: [PATCH 1/3] Repair backup failed (the value of segment_size is greater than 2G) --- src/catalog.c | 4 ++-- src/data.c | 28 ++++++++++++++-------------- src/pg_probackup.h | 10 +++++----- src/utils/file.c | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 9d817913e..4ba379389 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1117,7 +1117,7 @@ get_backup_filelist(pgBackup *backup, bool strict) file->segno = (int) segno; if (get_control_value_int64(buf, "n_blocks", &n_blocks, false)) - file->n_blocks = (int) n_blocks; + file->n_blocks = (int64) n_blocks; if (get_control_value_int64(buf, "n_headers", &n_headers, false)) file->n_headers = (int) n_headers; @@ -2568,7 +2568,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks > 0) - len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); + len += sprintf(line+len, ",\"n_blocks\":\"%ld\"", file->n_blocks); if (file->n_headers > 0) { diff --git a/src/data.c b/src/data.c index e5a551127..129124a45 100644 --- a/src/data.c +++ b/src/data.c @@ -300,7 +300,7 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, while (!page_is_valid && try_again--) { /* read the block */ - int read_len = fio_pread(in, page, blknum * BLCKSZ); + int read_len = fio_pread(in, page, ((int64)blknum) * BLCKSZ); /* The block could have been truncated. It is fine. */ if (read_len == 0) @@ -609,7 +609,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } - file->read_size = rc * BLCKSZ; + file->read_size = ((int64)rc) * BLCKSZ; /* refresh n_blocks for FULL and DELTA */ if (backup_mode == BACKUP_MODE_FULL || @@ -758,7 +758,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } - file->read_size = rc * BLCKSZ; + file->read_size = ((int64)rc) * BLCKSZ; /* Determine that file didn`t changed in case of incremental catchup */ if (backup_mode != BACKUP_MODE_FULL && @@ -1072,7 +1072,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers if (fio_fseek(out, 0) < 0) elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); - if (fio_ftruncate(out, blknum * BLCKSZ) != 0) + if (fio_ftruncate(out, ((int64)blknum) * BLCKSZ) != 0) elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); break; @@ -1123,7 +1123,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", + elog(ERROR, "Cannot seek to offset %ld of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; @@ -1158,7 +1158,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * When restoring file from FULL backup, pages are written sequentially, * so there is no need to issue fseek for every page. */ - write_pos = blknum * BLCKSZ; + write_pos = ((int64)blknum) * BLCKSZ; if (cur_pos_out != write_pos) { @@ -1742,7 +1742,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); else - elog(INFO, "Seek to %u", headers[n_hdr].pos); + elog(INFO, "Seek to %ld", headers[n_hdr].pos); cur_pos_in = headers[n_hdr].pos; } @@ -1879,7 +1879,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* read local data file and construct map with block checksums */ PageState* get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) + int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { PageState *checksum_map = NULL; FILE *in = NULL; @@ -1894,7 +1894,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); @@ -1948,7 +1948,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, /* return bitmap of valid blocks, bitmap is empty, then NULL is returned */ datapagemap_t * get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) + int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) { FILE *in = NULL; BlockNumber blknum = 0; @@ -1965,7 +1965,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", + elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); @@ -2296,9 +2296,9 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, else if (rc == PageIsOk) { - if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", - blknum * BLCKSZ, to_fullpath, strerror(errno)); + if (fseek(out, ((int64)blknum) * BLCKSZ, SEEK_SET) != 0) + elog(ERROR, "Cannot seek to position %ld in destination file \"%s\": %s", + ((int64)blknum) * BLCKSZ, to_fullpath, strerror(errno)); if (write_page(file, out, curr_page) != BLCKSZ) elog(ERROR, "File: \"%s\", cannot write at block %u: %s", diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7eb62466f..c5e17d975 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -267,7 +267,7 @@ typedef struct pgFile Oid relOid; /* relOid extracted from path, if applicable */ ForkName forkName; /* forkName extracted from path, if applicable */ int segno; /* Segment number for ptrack */ - int n_blocks; /* number of blocks in the data file in data directory */ + int64 n_blocks; /* number of blocks in the data file in data directory */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ @@ -677,8 +677,8 @@ typedef struct BackupPageHeader typedef struct BackupPageHeader2 { XLogRecPtr lsn; - int32 block; /* block number */ - int32 pos; /* position in backup file */ + int64 block; /* block number */ + int64 pos; /* position in backup file */ uint16 checksum; } BackupPageHeader2; @@ -1115,9 +1115,9 @@ extern bool create_empty_file(fio_location from_location, const char *to_root, fio_location to_location, pgFile *file); extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); + int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); + int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); diff --git a/src/utils/file.c b/src/utils/file.c index 7103c8f1d..180b8456e 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2138,7 +2138,7 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - if (fio_fseek(out, blknum * BLCKSZ) < 0) + if (fio_fseek(out, ((int64)blknum) * BLCKSZ) < 0) { elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); @@ -2196,7 +2196,7 @@ fio_send_pages_impl(int out, char* buf) datapagemap_iterator_t *iter = NULL; /* page headers */ int32 hdr_num = -1; - int32 cur_pos_out = 0; + int64 cur_pos_out = 0; BackupPageHeader2 *headers = NULL; /* open source file */ From 45c729725397bb0d95aa382471e9c50d6c3dc6b3 Mon Sep 17 00:00:00 2001 From: wuyf41619 Date: Thu, 14 Jul 2022 11:09:01 +0800 Subject: [PATCH 2/3] Repair backup failed (the value of segment_size is greater than 2G) --- src/catalog.c | 7 +- src/data.c | 158 ++++++++++++++++++++++++++++++++++++++++----- src/merge.c | 2 +- src/pg_probackup.h | 22 +++++-- src/utils/file.c | 10 +-- src/validate.c | 4 +- 6 files changed, 173 insertions(+), 30 deletions(-) diff --git a/src/catalog.c b/src/catalog.c index 4ba379389..06061efb4 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -2340,6 +2340,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) fio_fprintf(out, "program-version = %s\n", backup->program_version); if (backup->server_version[0] != '\0') fio_fprintf(out, "server-version = %s\n", backup->server_version); + fio_fprintf(out,"large-file = %u\n",backup->large_file); fio_fprintf(out, "\n#Result backup info\n"); fio_fprintf(out, "timelineid = %d\n", backup->tli); @@ -2568,7 +2569,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); if (file->n_blocks > 0) - len += sprintf(line+len, ",\"n_blocks\":\"%ld\"", file->n_blocks); + len += sprintf(line+len, ",\"n_blocks\":\"" INT64_FORMAT "\"", file->n_blocks); if (file->n_headers > 0) { @@ -2632,6 +2633,7 @@ readBackupControlFile(const char *path) char *merge_dest_backup = NULL; char *program_version = NULL; char *server_version = NULL; + bool large_file = false; char *compress_alg = NULL; int parsed_options; @@ -2667,6 +2669,7 @@ readBackupControlFile(const char *path) {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, {'s', 0, "note", &backup->note, SOURCE_FILE_STRICT}, {'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT}, + {'b',0, "large-file", &large_file,SOURCE_FILE_STRICT}, {0} }; @@ -2780,6 +2783,7 @@ readBackupControlFile(const char *path) if (compress_alg) backup->compress_alg = parse_compress_alg(compress_alg); + backup->large_file = large_file; return backup; } @@ -2936,6 +2940,7 @@ pgBackupInit(pgBackup *backup) backup->files = NULL; backup->note = NULL; backup->content_crc = 0; + backup->large_file = true; } /* free pgBackup object */ diff --git a/src/data.c b/src/data.c index 129124a45..b20459b4b 100644 --- a/src/data.c +++ b/src/data.c @@ -34,6 +34,9 @@ typedef struct DataPage static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, pg_crc32 *crc, bool use_crc32c); +static BackupPageHeader2_v1* +get_data_file_headers_v1(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); + #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ static int32 @@ -489,7 +492,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat CompressAlg calg, int clevel, uint32 checksum_version, HeaderMap *hdr_map, bool is_merge) { - int rc; + int64 rc; bool use_pagemap; char *errmsg = NULL; BlockNumber err_blknum = 0; @@ -609,7 +612,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat elog(ERROR, "Cannot read file \"%s\"", from_fullpath); } - file->read_size = ((int64)rc) * BLCKSZ; + file->read_size = rc * BLCKSZ; /* refresh n_blocks for FULL and DELTA */ if (backup_mode == BACKUP_MODE_FULL || @@ -905,7 +908,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, if (use_headers && tmp_file->n_headers > 0) headers = get_data_file_headers(&(backup->hdr_map), tmp_file, parse_program_version(backup->program_version), - true); + true, + backup->large_file); if (use_headers && !headers && tmp_file->n_headers > 0) elog(ERROR, "Failed to get page headers for file \"%s\"", from_fullpath); @@ -949,7 +953,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, + const char *from_fullpath, const char *to_fullpath, int64 nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, datapagemap_t *lsn_map, BackupPageHeader2 *headers) { @@ -1123,7 +1127,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_in != headers[n_hdr].pos) { if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %ld of \"%s\": %s", + elog(ERROR, "Cannot seek to offset " INT64_FORMAT " of \"%s\": %s", headers[n_hdr].pos, from_fullpath, strerror(errno)); cur_pos_in = headers[n_hdr].pos; @@ -1672,7 +1676,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, /* Valiate pages of datafile in backup one by one */ bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map) + uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map, bool large_file) { size_t read_len = 0; bool is_valid = true; @@ -1693,7 +1697,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot open file \"%s\": %s", fullpath, strerror(errno)); - headers = get_data_file_headers(hdr_map, file, backup_version, false); + headers = get_data_file_headers(hdr_map, file, backup_version, false, large_file); if (!headers && file->n_headers > 0) { @@ -1742,7 +1746,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); else - elog(INFO, "Seek to %ld", headers[n_hdr].pos); + elog(INFO, "Seek to " INT64_FORMAT, headers[n_hdr].pos); cur_pos_in = headers[n_hdr].pos; } @@ -1894,7 +1898,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s", + elog(ERROR, "Cannot truncate file to blknum " INT64_FORMAT " \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); @@ -1965,7 +1969,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* truncate up to blocks */ if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s", + elog(ERROR, "Cannot truncate file to blknum " INT64_FORMAT " \"%s\": %s", n_blocks, fullpath, strerror(errno)); setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); @@ -2032,10 +2036,10 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, return false; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "Odd size page found at offset %ld of \"%s\"", + "Odd size page found at offset " INT64_FORMAT " of \"%s\"", ftello(in), fullpath); else - elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", + elog(ERROR, "Cannot read header at offset " INT64_FORMAT " of \"%s\": %s", ftello(in), fullpath, strerror(errno)); } @@ -2358,20 +2362,142 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, return n_blocks_read; } +BackupPageHeader2* +get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict, bool large_file) +{ + bool success = false; + FILE *in = NULL; + size_t read_len = 0; + pg_crc32 hdr_crc; + BackupPageHeader2 *headers = NULL; + /* header decompression */ + int z_len = 0; + char *zheaders = NULL; + const char *errormsg = NULL; + + if (backup_version < 20400) + return NULL; + + if (file->n_headers <= 0) + return NULL; + + if(!large_file) + { + BackupPageHeader2_v1 *tmp_headers; + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + tmp_headers = get_data_file_headers_v1(hdr_map, file, backup_version, strict); + headers = pgut_malloc(read_len); + memset(headers, 0, read_len); + if(!tmp_headers) + { + return NULL; + } + for(int i=0; in_headers+1; i++) + { + headers[i].block = tmp_headers[i].block; + headers[i].lsn = tmp_headers[i].lsn; + headers[i].pos = tmp_headers[i].pos; + headers[i].checksum = tmp_headers[i].checksum; + } + return headers; + } + /* TODO: consider to make this descriptor thread-specific */ + in = fopen(hdr_map->path, PG_BINARY_R); + + if (!in) + { + elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + return NULL; + } + /* disable buffering for header file */ + setvbuf(in, NULL, _IONBF, 0); + + if (fseeko(in, file->hdr_off, SEEK_SET)) + { + elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", + file->hdr_off, hdr_map->path, strerror(errno)); + goto cleanup; + } + + /* + * The actual number of headers in header file is n+1, last one is a dummy header, + * used for calculation of read_len for actual last header. + */ + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + + /* allocate memory for compressed headers */ + zheaders = pgut_malloc(file->hdr_size); + memset(zheaders, 0, file->hdr_size); + + if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) + { + elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", + file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); + goto cleanup; + } + + /* allocate memory for uncompressed headers */ + headers = pgut_malloc(read_len); + memset(headers, 0, read_len); + + z_len = do_decompress(headers, read_len, zheaders, file->hdr_size, + ZLIB_COMPRESS, &errormsg); + if (z_len <= 0) + { + if (errormsg) + elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %s", + file->rel_path, errormsg); + else + elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %i", + file->rel_path, z_len); + + goto cleanup; + } + + /* validate checksum */ + INIT_FILE_CRC32(true, hdr_crc); + COMP_FILE_CRC32(true, hdr_crc, headers, read_len); + FIN_FILE_CRC32(true, hdr_crc); + + if (hdr_crc != file->hdr_crc) + { + elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " + "offset: %llu, len: %lu, current: %u, expected: %u", + file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); + goto cleanup; + } + + success = true; + +cleanup: + + pg_free(zheaders); + if (in && fclose(in)) + elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); + + if (!success) + { + pg_free(headers); + headers = NULL; + } + + return headers; +} + /* * Attempt to open header file, read content and return as * array of headers. * TODO: some access optimizations would be great here: * less fseeks, buffering, descriptor sharing, etc. */ -BackupPageHeader2* -get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) +BackupPageHeader2_v1* +get_data_file_headers_v1(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) { bool success = false; FILE *in = NULL; size_t read_len = 0; pg_crc32 hdr_crc; - BackupPageHeader2 *headers = NULL; + BackupPageHeader2_v1 *headers = NULL; /* header decompression */ int z_len = 0; char *zheaders = NULL; @@ -2405,7 +2531,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b * The actual number of headers in header file is n+1, last one is a dummy header, * used for calculation of read_len for actual last header. */ - read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + read_len = (file->n_headers+1) * sizeof(BackupPageHeader2_v1); /* allocate memory for compressed headers */ zheaders = pgut_malloc(file->hdr_size); diff --git a/src/merge.c b/src/merge.c index ff39c2510..2d5357345 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1085,7 +1085,7 @@ merge_files(void *arg) tmp_file->n_headers = file->n_headers; headers = get_data_file_headers(&(arguments->full_backup->hdr_map), file, parse_program_version(arguments->full_backup->program_version), - true); + true, arguments->full_backup->large_file); /* sanity */ if (!headers && file->n_headers > 0) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index c5e17d975..e1e9c1711 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -522,6 +522,7 @@ struct pgBackup /* map used for access to page headers */ HeaderMap hdr_map; + bool large_file; /* file's size is greate 2G*/ }; /* Recovery target for restore and validate subcommands */ @@ -673,8 +674,17 @@ typedef struct BackupPageHeader int32 compressed_size; } BackupPageHeader; -/* 4MB for 1GB file */ -typedef struct BackupPageHeader2 +/* 4MB for 1GB file ,for version 1*/ +typedef struct BackupPageHeader2_v1 +{ + XLogRecPtr lsn; + int32 block; /* block number */ + int32 pos; /* position in backup file */ + uint16 checksum; +} BackupPageHeader2_v1; + +/* 4MB for 2GB file ,for last version*/ +typedef struct BackupPageHeader2_v2 { XLogRecPtr lsn; int64 block; /* block number */ @@ -1103,7 +1113,7 @@ extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *o const char *to_fullpath, bool use_bitmap, PageState *checksum_map, XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers); extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, + const char *from_fullpath, const char *to_fullpath, int64 nblocks, datapagemap_t *map, PageState *checksum_map, int checksum_version, datapagemap_t *lsn_map, BackupPageHeader2 *headers); extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, @@ -1119,9 +1129,9 @@ extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); + uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map, bool large_file); -extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); +extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict,bool large_file); extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); extern void init_header_map(pgBackup *backup); extern void cleanup_header_map(HeaderMap *hdr_map); @@ -1236,7 +1246,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); extern void pgut_setenv(const char *key, const char *val); extern void pgut_unsetenv(const char *key); -extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, +extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_version, diff --git a/src/utils/file.c b/src/utils/file.c index 180b8456e..978e79556 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -2180,7 +2180,7 @@ fio_send_pages_impl(int out, char* buf) { FILE *in = NULL; BlockNumber blknum = 0; - int current_pos = 0; + int64 current_pos = 0; BlockNumber n_blocks_read = 0; PageState page_st; char read_buffer[BLCKSZ+1]; @@ -2267,11 +2267,11 @@ fio_send_pages_impl(int out, char* buf) * Optimize stdio buffer usage, fseek only when current position * does not match the position of requested block. */ - if (current_pos != blknum*BLCKSZ) + if (current_pos != ((int64)blknum)*BLCKSZ) { - current_pos = blknum*BLCKSZ; + current_pos = ((int64)blknum)*BLCKSZ; if (fseek(in, current_pos, SEEK_SET) != 0) - elog(ERROR, "fseek to position %u is failed on remote file '%s': %s", + elog(ERROR, "fseek to position " INT64_FORMAT " is failed on remote file '%s': %s", current_pos, from_fullpath, strerror(errno)); } @@ -2954,7 +2954,7 @@ fio_list_dir(parray *files, const char *root, bool exclude, } PageState * -fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, +fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) { if (fio_is_remote(location)) diff --git a/src/validate.c b/src/validate.c index 4044ac158..9aa12018f 100644 --- a/src/validate.c +++ b/src/validate.c @@ -39,6 +39,7 @@ typedef struct * 0 means there is no error, 1 - there is an error. */ int ret; + bool large_file; } validate_files_arg; /* @@ -152,6 +153,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) arg->backup_version = parse_program_version(backup->program_version); arg->external_prefix = external_prefix; arg->hdr_map = &(backup->hdr_map); + arg->large_file = backup->large_file; // arg->dbOid_exclude_list = dbOid_exclude_list; /* By default there are some error */ threads_args[i].ret = 1; @@ -362,7 +364,7 @@ pgBackupValidateFiles(void *arg) if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn, arguments->checksum_version, arguments->backup_version, - arguments->hdr_map)) + arguments->hdr_map,arguments->large_file)) arguments->corrupted = true; } } From b41e02cbdea399b8c884e39c6fc9413dd3fcdaa0 Mon Sep 17 00:00:00 2001 From: wuyf41619 Date: Mon, 17 Oct 2022 20:38:39 +0800 Subject: [PATCH 3/3] fix Memory leak --- src/data.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data.c b/src/data.c index b20459b4b..3fa90fb9c 100644 --- a/src/data.c +++ b/src/data.c @@ -2399,6 +2399,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b headers[i].pos = tmp_headers[i].pos; headers[i].checksum = tmp_headers[i].checksum; } + pg_free(tmp_headers); return headers; } /* TODO: consider to make this descriptor thread-specific */ pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy