Skip to content

Commit 693bffe

Browse files
funny-falconfunny-falcon
committed
[PBCKP-146] fix cfm truncated crc calculation in delta/page backup
- On backup we should compare truncated crc with previous version. - copying remote file didn't honor "don't truncate first 64 bytes" rule. - crc calculation didn't honoer "don't truncate first 64 bytes" rule.
1 parent 8570825 commit 693bffe

File tree

4 files changed

+48
-13
lines changed

4 files changed

+48
-13
lines changed

‎src/data.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,10 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
806806
* file could be deleted under our feets.
807807
* But then backup_non_data_file_internal will handle it safely
808808
*/
809-
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
809+
if (file->forkName != cfm)
810+
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
811+
else
812+
file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true);
810813

811814
/* ...and checksum is the same... */
812815
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1334,7 +1337,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13341337
pg_crc32 file_crc;
13351338
if (tmp_file->forkName == cfm &&
13361339
tmp_file->uncompressed_size > tmp_file->write_size)
1337-
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST);
1340+
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false);
13381341
else
13391342
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13401343

‎src/pg_probackup.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1082,7 +1082,7 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path);
10821082
extern void pgFileFree(void *file);
10831083

10841084
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok);
1085-
extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c);
1085+
extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok);
10861086
extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok);
10871087

10881088
extern int pgFileMapComparePath(const void *f1, const void *f2);

‎src/utils/file.c

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,8 +1377,6 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
13771377
{
13781378
if (decompress && truncated)
13791379
elog(ERROR, "Could not calculate CRC for compressed truncated file");
1380-
if (missing_ok && truncated)
1381-
elog(ERROR, "CRC calculation for missing truncated file is forbidden");
13821380

13831381
if (fio_is_remote(location))
13841382
{
@@ -1408,7 +1406,7 @@ fio_get_crc32_ex(const char *file_path, fio_location location,
14081406
if (decompress)
14091407
return pgFileGetCRCgz(file_path, true, missing_ok);
14101408
else if (truncated)
1411-
return pgFileGetCRCTruncated(file_path, true);
1409+
return pgFileGetCRCTruncated(file_path, true, missing_ok);
14121410
else
14131411
return pgFileGetCRC(file_path, true, missing_ok);
14141412
}
@@ -1422,9 +1420,10 @@ fio_get_crc32(const char *file_path, fio_location location,
14221420
}
14231421

14241422
pg_crc32
1425-
fio_get_crc32_truncated(const char *file_path, fio_location location)
1423+
fio_get_crc32_truncated(const char *file_path, fio_location location,
1424+
bool missing_ok)
14261425
{
1427-
return fio_get_crc32_ex(file_path, location, false, false, true);
1426+
return fio_get_crc32_ex(file_path, location, false, missing_ok, true);
14281427
}
14291428

14301429
/* Remove file */
@@ -3003,6 +3002,7 @@ fio_send_file_impl(int out, char const* path)
30033002
fio_header hdr;
30043003
char *buf = pgut_malloc(CHUNK_SIZE);
30053004
size_t read_len = 0;
3005+
int64_t read_size = 0;
30063006
char *errormsg = NULL;
30073007

30083008
/* open source file for read */
@@ -3066,7 +3066,19 @@ fio_send_file_impl(int out, char const* path)
30663066
if (read_len > 0)
30673067
{
30683068
/* send chunk */
3069-
size_t non_zero_len = find_zero_tail(buf, read_len);
3069+
int64_t non_zero_len = find_zero_tail(buf, read_len);
3070+
/*
3071+
* It is dirty trick to silence warnings in CFS GC process:
3072+
* backup at least cfs header size bytes.
3073+
*/
3074+
if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
3075+
read_size + read_len > 0)
3076+
{
3077+
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3078+
read_size + read_len);
3079+
non_zero_len -= read_size;
3080+
}
3081+
30703082
if (non_zero_len > 0)
30713083
{
30723084
hdr.cop = FIO_PAGE;
@@ -3082,6 +3094,8 @@ fio_send_file_impl(int out, char const* path)
30823094
hdr.arg = read_len - non_zero_len;
30833095
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
30843096
}
3097+
3098+
read_size += read_len;
30853099
}
30863100

30873101
if (feof(fp))
@@ -3166,7 +3180,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
31663180
* Read the local file to compute CRC for it extened to real_size.
31673181
*/
31683182
pg_crc32
3169-
pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
3183+
pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok)
31703184
{
31713185
FILE *fp;
31723186
char *buf;
@@ -3180,6 +3194,15 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
31803194
fp = fopen(file_path, PG_BINARY_R);
31813195
if (fp == NULL)
31823196
{
3197+
if (errno == ENOENT)
3198+
{
3199+
if (missing_ok)
3200+
{
3201+
FIN_FILE_CRC32(use_crc32c, st.crc);
3202+
return st.crc;
3203+
}
3204+
}
3205+
31833206
elog(ERROR, "Cannot open file \"%s\": %s",
31843207
file_path, strerror(errno));
31853208
}
@@ -3200,6 +3223,14 @@ pgFileGetCRCTruncated(const char *file_path, bool use_crc32c)
32003223
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
32013224

32023225
non_zero_len = find_zero_tail(buf, len);
3226+
/* same trick as in fio_send_file */
3227+
if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY &&
3228+
st.read_size + len > 0)
3229+
{
3230+
non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY,
3231+
st.read_size + len);
3232+
non_zero_len -= st.read_size;
3233+
}
32033234
if (non_zero_len)
32043235
{
32053236
fio_send_file_crc(&st, buf, non_zero_len);
@@ -3894,12 +3925,12 @@ fio_communicate(int in, int out)
38943925
break;
38953926
case FIO_GET_CRC32:
38963927
Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 ||
3897-
(hdr.arg & GET_CRC32_TRUNCATED) == GET_CRC32_TRUNCATED);
3928+
(hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED);
38983929
/* calculate crc32 for a file */
38993930
if ((hdr.arg & GET_CRC32_DECOMPRESS))
39003931
crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39013932
else if ((hdr.arg & GET_CRC32_TRUNCATED))
3902-
crc = pgFileGetCRCTruncated(buf, true);
3933+
crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39033934
else
39043935
crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
39053936
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));

‎src/utils/file.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ extern void fio_disconnect(void);
123123
extern int fio_sync(char const* path, fio_location location);
124124
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
125125
bool decompress, bool missing_ok);
126-
extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location);
126+
extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location,
127+
bool missing_ok);
127128

128129
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
129130
extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy