Skip to content

Commit 0c7c78b

Browse files
funny-falconfunny-falcon
committed
Merge branch 'REL_2_5-PBCKP-146-2' into REL_2_5
2 parents 34af025 + e36924a commit 0c7c78b

File tree

12 files changed

+727
-256
lines changed

12 files changed

+727
-256
lines changed

‎src/archive.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
13751375
#ifdef HAVE_LIBZ
13761376
/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
13771377
if (IsXLogFileName(filename))
1378-
rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg);
1378+
rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg);
13791379
if (rc == FILE_MISSING)
13801380
#endif
13811381
/* ... failing that, use uncompressed */
1382-
rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
1382+
rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg);
13831383

13841384
/* When not in prefetch mode, try to use partial file */
13851385
if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
13891389
#ifdef HAVE_LIBZ
13901390
/* '.gz.partial' goes first ... */
13911391
snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
1392-
rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg);
1392+
rc = fio_send_file_gz(from_partial, out, &errmsg);
13931393
if (rc == FILE_MISSING)
13941394
#endif
13951395
{
13961396
/* ... failing that, use '.partial' */
13971397
snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
1398-
rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
1398+
rc = fio_send_file(from_partial, out, false, NULL, &errmsg);
13991399
}
14001400

14011401
if (rc == SEND_OK)

‎src/catalog.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
10691069
char linked[MAXPGPATH];
10701070
char compress_alg_string[MAXPGPATH];
10711071
int64 write_size,
1072+
uncompressed_size,
10721073
mode, /* bit length of mode_t depends on platforms */
10731074
is_datafile,
10741075
is_cfs,
@@ -1132,6 +1133,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
11321133
if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
11331134
file->hdr_size = (int) hdr_size;
11341135

1136+
if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
1137+
file->uncompressed_size = uncompressed_size;
1138+
else
1139+
file->uncompressed_size = write_size;
1140+
11351141
if (file->external_dir_num == 0)
11361142
set_forkname(file);
11371143

@@ -2561,6 +2567,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
25612567
file->external_dir_num,
25622568
file->dbOid);
25632569

2570+
if (file->uncompressed_size != 0 &&
2571+
file->uncompressed_size != file->write_size)
2572+
len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
2573+
file->uncompressed_size);
2574+
25642575
if (file->is_datafile)
25652576
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
25662577

‎src/data.c

Lines changed: 36 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
799799
* and its mtime is less than parent backup start time ... */
800800
if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
801801
(prev_file && file->exists_in_prev &&
802+
file->size == prev_file->size &&
802803
file->mtime <= parent_backup_time))
803804
{
804805
/*
@@ -1330,7 +1331,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13301331
if (already_exists)
13311332
{
13321333
/* compare checksums of already existing file and backup file */
1333-
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
1334+
pg_crc32 file_crc;
1335+
if (tmp_file->forkName == cfm &&
1336+
tmp_file->uncompressed_size > tmp_file->write_size)
1337+
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST);
1338+
else
1339+
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13341340

13351341
if (file_crc == tmp_file->crc)
13361342
{
@@ -1387,10 +1393,12 @@ backup_non_data_file_internal(const char *from_fullpath,
13871393
const char *to_fullpath, pgFile *file,
13881394
bool missing_ok)
13891395
{
1390-
FILE *in = NULL;
13911396
FILE *out = NULL;
1392-
ssize_t read_len = 0;
1393-
char *buf = NULL;
1397+
char *errmsg = NULL;
1398+
int rc;
1399+
bool cut_zero_tail;
1400+
1401+
cut_zero_tail = file->forkName == cfm;
13941402

13951403
INIT_FILE_CRC32(true, file->crc);
13961404

@@ -1412,107 +1420,44 @@ backup_non_data_file_internal(const char *from_fullpath,
14121420

14131421
/* backup remote file */
14141422
if (fio_is_remote(FIO_DB_HOST))
1415-
{
1416-
char *errmsg = NULL;
1417-
int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
1423+
rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
1424+
else
1425+
rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
14181426

1419-
/* handle errors */
1420-
if (rc == FILE_MISSING)
1421-
{
1422-
/* maybe deleted, it's not error in case of backup */
1423-
if (missing_ok)
1424-
{
1425-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1426-
file->write_size = FILE_NOT_FOUND;
1427-
goto cleanup;
1428-
}
1429-
else
1430-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1431-
}
1432-
else if (rc == WRITE_FAILED)
1433-
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1434-
else if (rc != SEND_OK)
1427+
/* handle errors */
1428+
if (rc == FILE_MISSING)
1429+
{
1430+
/* maybe deleted, it's not error in case of backup */
1431+
if (missing_ok)
14351432
{
1436-
if (errmsg)
1437-
elog(ERROR, "%s", errmsg);
1438-
else
1439-
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
1433+
elog(LOG, "File \"%s\" is not found", from_fullpath);
1434+
file->write_size = FILE_NOT_FOUND;
1435+
goto cleanup;
14401436
}
1441-
1442-
pg_free(errmsg);
1437+
else
1438+
elog(ERROR, "File \"%s\" is not found", from_fullpath);
14431439
}
1444-
/* backup local file */
1445-
else
1440+
else if (rc == WRITE_FAILED)
1441+
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1442+
else if (rc != SEND_OK)
14461443
{
1447-
/* open source file for read */
1448-
in = fopen(from_fullpath, PG_BINARY_R);
1449-
if (in == NULL)
1450-
{
1451-
/* maybe deleted, it's not error in case of backup */
1452-
if (errno == ENOENT)
1453-
{
1454-
if (missing_ok)
1455-
{
1456-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1457-
file->write_size = FILE_NOT_FOUND;
1458-
goto cleanup;
1459-
}
1460-
else
1461-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1462-
}
1463-
1464-
elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
1465-
strerror(errno));
1466-
}
1467-
1468-
/* disable stdio buffering for local input/output files to avoid triple buffering */
1469-
setvbuf(in, NULL, _IONBF, BUFSIZ);
1470-
setvbuf(out, NULL, _IONBF, BUFSIZ);
1471-
1472-
/* allocate 64kB buffer */
1473-
buf = pgut_malloc(CHUNK_SIZE);
1474-
1475-
/* copy content and calc CRC */
1476-
for (;;)
1477-
{
1478-
read_len = fread(buf, 1, CHUNK_SIZE, in);
1479-
1480-
if (ferror(in))
1481-
elog(ERROR, "Cannot read from file \"%s\": %s",
1482-
from_fullpath, strerror(errno));
1483-
1484-
if (read_len > 0)
1485-
{
1486-
if (fwrite(buf, 1, read_len, out) != read_len)
1487-
elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
1488-
strerror(errno));
1489-
1490-
/* update CRC */
1491-
COMP_FILE_CRC32(true, file->crc, buf, read_len);
1492-
file->read_size += read_len;
1493-
}
1494-
1495-
if (feof(in))
1496-
break;
1497-
}
1444+
if (errmsg)
1445+
elog(ERROR, "%s", errmsg);
1446+
else
1447+
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
14981448
}
14991449

1500-
file->write_size = (int64) file->read_size;
1501-
1502-
if (file->write_size > 0)
1503-
file->uncompressed_size = file->write_size;
1450+
file->uncompressed_size = file->read_size;
15041451

15051452
cleanup:
1453+
if (errmsg != NULL)
1454+
pg_free(errmsg);
1455+
15061456
/* finish CRC calculation and store into pgFile */
15071457
FIN_FILE_CRC32(true, file->crc);
15081458

1509-
if (in && fclose(in))
1510-
elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
1511-
15121459
if (out && fclose(out))
15131460
elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
1514-
1515-
pg_free(buf);
15161461
}
15171462

15181463
/*

‎src/dir.c

Lines changed: 1 addition & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -262,137 +262,6 @@ pgFileDelete(mode_t mode, const char *full_path)
262262
}
263263
}
264264

265-
/*
266-
* Read the local file to compute its CRC.
267-
* We cannot make decision about file decompression because
268-
* user may ask to backup already compressed files and we should be
269-
* obvious about it.
270-
*/
271-
pg_crc32
272-
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
273-
{
274-
FILE *fp;
275-
pg_crc32 crc = 0;
276-
char *buf;
277-
size_t len = 0;
278-
279-
INIT_FILE_CRC32(use_crc32c, crc);
280-
281-
/* open file in binary read mode */
282-
fp = fopen(file_path, PG_BINARY_R);
283-
if (fp == NULL)
284-
{
285-
if (errno == ENOENT)
286-
{
287-
if (missing_ok)
288-
{
289-
FIN_FILE_CRC32(use_crc32c, crc);
290-
return crc;
291-
}
292-
}
293-
294-
elog(ERROR, "Cannot open file \"%s\": %s",
295-
file_path, strerror(errno));
296-
}
297-
298-
/* disable stdio buffering */
299-
setvbuf(fp, NULL, _IONBF, BUFSIZ);
300-
buf = pgut_malloc(STDIO_BUFSIZE);
301-
302-
/* calc CRC of file */
303-
for (;;)
304-
{
305-
if (interrupted)
306-
elog(ERROR, "interrupted during CRC calculation");
307-
308-
len = fread(buf, 1, STDIO_BUFSIZE, fp);
309-
310-
if (ferror(fp))
311-
elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno));
312-
313-
/* update CRC */
314-
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
315-
316-
if (feof(fp))
317-
break;
318-
}
319-
320-
FIN_FILE_CRC32(use_crc32c, crc);
321-
fclose(fp);
322-
pg_free(buf);
323-
324-
return crc;
325-
}
326-
327-
/*
328-
* Read the local file to compute its CRC.
329-
* We cannot make decision about file decompression because
330-
* user may ask to backup already compressed files and we should be
331-
* obvious about it.
332-
*/
333-
pg_crc32
334-
pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok)
335-
{
336-
gzFile fp;
337-
pg_crc32 crc = 0;
338-
int len = 0;
339-
int err;
340-
char *buf;
341-
342-
INIT_FILE_CRC32(use_crc32c, crc);
343-
344-
/* open file in binary read mode */
345-
fp = gzopen(file_path, PG_BINARY_R);
346-
if (fp == NULL)
347-
{
348-
if (errno == ENOENT)
349-
{
350-
if (missing_ok)
351-
{
352-
FIN_FILE_CRC32(use_crc32c, crc);
353-
return crc;
354-
}
355-
}
356-
357-
elog(ERROR, "Cannot open file \"%s\": %s",
358-
file_path, strerror(errno));
359-
}
360-
361-
buf = pgut_malloc(STDIO_BUFSIZE);
362-
363-
/* calc CRC of file */
364-
for (;;)
365-
{
366-
if (interrupted)
367-
elog(ERROR, "interrupted during CRC calculation");
368-
369-
len = gzread(fp, buf, STDIO_BUFSIZE);
370-
371-
if (len <= 0)
372-
{
373-
/* we either run into eof or error */
374-
if (gzeof(fp))
375-
break;
376-
else
377-
{
378-
const char *err_str = NULL;
379-
380-
err_str = gzerror(fp, &err);
381-
elog(ERROR, "Cannot read from compressed file %s", err_str);
382-
}
383-
}
384-
385-
/* update CRC */
386-
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
387-
}
388-
389-
FIN_FILE_CRC32(use_crc32c, crc);
390-
gzclose(fp);
391-
pg_free(buf);
392-
393-
return crc;
394-
}
395-
396265
void
397266
pgFileFree(void *file)
398267
{
@@ -1812,7 +1681,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
18121681
FIO_BACKUP_HOST);
18131682
file->crc = pgFileGetCRC(database_map_path, true, false);
18141683
file->write_size = file->size;
1815-
file->uncompressed_size = file->read_size;
1684+
file->uncompressed_size = file->size;
18161685

18171686
parray_append(backup_files_list, file);
18181687
}

‎src/merge.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1078,7 +1078,7 @@ merge_files(void *arg)
10781078
tmp_file->hdr_crc = file->hdr_crc;
10791079
}
10801080
else
1081-
tmp_file->uncompressed_size = tmp_file->write_size;
1081+
tmp_file->uncompressed_size = tmp_file->uncompressed_size;
10821082

10831083
/* Copy header metadata from old map into a new one */
10841084
tmp_file->n_headers = file->n_headers;

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy