Skip to content

Commit 0bdffbc

Browse files
author
Artur Zakirov
committed
Merge branch 'simplify_backup_data_file' into ptrack
2 parents 4cb028c + 547470a commit 0bdffbc

File tree

5 files changed

+649
-300
lines changed

5 files changed

+649
-300
lines changed

‎backup.c

Lines changed: 173 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,6 +1105,75 @@ backup_cleanup(bool fatal, void *userdata)
11051105
}
11061106
}
11071107

1108+
/* Count bytes in file */
1109+
static long
1110+
file_size(const char *file)
1111+
{
1112+
long r;
1113+
FILE *f = fopen(file, "r");
1114+
1115+
if (!f)
1116+
{
1117+
elog(ERROR, "pg_probackup: could not open file \"%s\" for reading: %s\n",
1118+
file, strerror(errno));
1119+
return -1;
1120+
}
1121+
fseek(f, 0, SEEK_END);
1122+
r = ftell(f);
1123+
fclose(f);
1124+
return r;
1125+
}
1126+
1127+
/*
1128+
* Find corresponding file in previous backup.
1129+
* Compare generations and return true if we don't need full copy
1130+
* of the file, but just part of it.
1131+
*
1132+
* skip_size - size of the file in previous backup. We can skip it
1133+
* and copy just remaining part of the file.
1134+
*/
1135+
bool
1136+
backup_compressed_file_partially(pgFile *file, void *arg, size_t *skip_size)
1137+
{
1138+
bool result = false;
1139+
pgFile *prev_file = NULL;
1140+
size_t current_file_size;
1141+
backup_files_args *arguments = (backup_files_args *) arg;
1142+
1143+
if (arguments->prev_files)
1144+
{
1145+
pgFile **p = (pgFile **) parray_bsearch(arguments->prev_files,
1146+
file, pgFileComparePath);
1147+
if (p)
1148+
prev_file = *p;
1149+
1150+
/* If file's gc generation has changed since last backup, just copy it*/
1151+
if (prev_file && prev_file->generation == file->generation)
1152+
{
1153+
current_file_size = file_size(file->path);
1154+
1155+
if (prev_file->write_size == BYTES_INVALID)
1156+
return false;
1157+
1158+
*skip_size = prev_file->write_size;
1159+
1160+
if (current_file_size >= prev_file->write_size)
1161+
{
1162+
elog(LOG, "Backup file %s partially: prev_size %lu, current_size %lu",
1163+
file->path, prev_file->write_size, current_file_size);
1164+
result = true;
1165+
}
1166+
else
1167+
elog(ERROR, "Something is wrong with %s. current_file_size %lu, prev %lu",
1168+
file->path, current_file_size, prev_file->write_size);
1169+
}
1170+
else
1171+
elog(LOG, "Copy full %s.", file->path);
1172+
}
1173+
1174+
return result;
1175+
}
1176+
11081177
/*
11091178
* Take differential backup at page level.
11101179
*/
@@ -1203,9 +1272,47 @@ backup_files(void *arg)
12031272
}
12041273

12051274
/* copy the file into backup */
1206-
if (!(file->is_datafile
1207-
? backup_data_file(arguments->from_root, arguments->to_root, file, arguments->lsn)
1208-
: copy_file(arguments->from_root, arguments->to_root, file)))
1275+
if (file->is_datafile)
1276+
{
1277+
if (!backup_data_file(arguments->from_root,
1278+
arguments->to_root, file,
1279+
arguments->lsn))
1280+
{
1281+
/* record as skipped file in file_xxx.txt */
1282+
file->write_size = BYTES_INVALID;
1283+
elog(LOG, "skip");
1284+
continue;
1285+
}
1286+
}
1287+
else if (is_compressed_data_file(file))
1288+
{
1289+
size_t skip_size = 0;
1290+
if (backup_compressed_file_partially(file, arguments, &skip_size))
1291+
{
1292+
/* backup cfs segment partly */
1293+
if (!copy_file_partly(arguments->from_root,
1294+
arguments->to_root,
1295+
file, skip_size))
1296+
{
1297+
/* record as skipped file in file_xxx.txt */
1298+
file->write_size = BYTES_INVALID;
1299+
elog(LOG, "skip");
1300+
continue;
1301+
}
1302+
}
1303+
else if (!copy_file(arguments->from_root,
1304+
arguments->to_root,
1305+
file))
1306+
{
1307+
/* record as skipped file in file_xxx.txt */
1308+
file->write_size = BYTES_INVALID;
1309+
elog(LOG, "skip");
1310+
continue;
1311+
}
1312+
}
1313+
else if (!copy_file(arguments->from_root,
1314+
arguments->to_root,
1315+
file))
12091316
{
12101317
/* record as skipped file in file_xxx.txt */
12111318
file->write_size = BYTES_INVALID;
@@ -1254,14 +1361,14 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12541361
relative = file->path + strlen(root) + 1;
12551362
if (is_pgdata &&
12561363
!path_is_prefix_of_path("base", relative) &&
1257-
/*!path_is_prefix_of_path("global", relative) &&*/
1364+
/*!path_is_prefix_of_path("global", relative) &&*/ //TODO What's wrong with this line?
12581365
!path_is_prefix_of_path("pg_tblspc", relative))
12591366
continue;
12601367

12611368
/* Get file name from path */
12621369
fname = last_dir_separator(relative);
12631370

1264-
/* Remove temp tables */
1371+
/* Remove temp tables from the list */
12651372
if (fname[0] == 't' && isdigit(fname[1]))
12661373
{
12671374
pgFileFree(file);
@@ -1271,7 +1378,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12711378
}
12721379

12731380
path_len = strlen(file->path);
1274-
/* Get link ptrack file to realations files */
1381+
/* Get link ptrack file to relations files */
12751382
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
12761383
{
12771384
pgFile *search_file;
@@ -1280,12 +1387,15 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12801387
while(true) {
12811388
pgFile tmp_file;
12821389
tmp_file.path = pg_strdup(file->path);
1283-
/* I hope segno not more than 999999 */
1390+
1391+
/* Segno fits into 6 digits since it is not more than 4000 */
12841392
if (segno > 0)
12851393
sprintf(tmp_file.path+path_len-7, ".%d", segno);
12861394
else
12871395
tmp_file.path[path_len-7] = '\0';
1396+
12881397
pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
1398+
12891399
if (pre_search_file != NULL)
12901400
{
12911401
search_file = *pre_search_file;
@@ -1299,6 +1409,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
12991409
segno++;
13001410
}
13011411

1412+
/* Remove ptrack file itself from backup list */
13021413
pgFileFree(file);
13031414
parray_remove(list_file, i);
13041415
i--;
@@ -1307,12 +1418,7 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
13071418

13081419
/* compress map file it is not data file */
13091420
if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0)
1310-
{
1311-
if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK ||
1312-
current.backup_mode == BACKUP_MODE_DIFF_PAGE)
1313-
elog(ERROR, "You can't use incremental backup with compress tablespace");
13141421
continue;
1315-
}
13161422

13171423
/* name of data file start with digit */
13181424
if (fname == NULL)
@@ -1358,11 +1464,34 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
13581464
pgFile tmp_file;
13591465
tmp_file.path = pg_strdup(file->path);
13601466
tmp_file.path[path_len-4] = '\0';
1361-
pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
1467+
pre_search_file = (pgFile **) parray_bsearch(list_file,
1468+
&tmp_file, pgFileComparePath);
13621469
if (pre_search_file != NULL)
13631470
{
1471+
FileMap* map;
1472+
int md = open(file->path, O_RDWR|PG_BINARY, 0);
1473+
if (md < 0)
1474+
elog(ERROR, "add_files(). cannot open cfm file '%s'", file->path);
1475+
1476+
map = cfs_mmap(md);
1477+
if (map == MAP_FAILED)
1478+
{
1479+
elog(LOG, "add_files(). cfs_compression_ration failed to map file %s: %m", file->path);
1480+
close(md);
1481+
break;
1482+
}
1483+
1484+
(*pre_search_file)->generation = map->generation;
13641485
(*pre_search_file)->is_datafile = false;
1486+
1487+
if (cfs_munmap(map) < 0)
1488+
elog(LOG, "add_files(). CFS failed to unmap file %s: %m", file->path);
1489+
if (close(md) < 0)
1490+
elog(LOG, "add_files(). CFS failed to close file %s: %m", file->path);
13651491
}
1492+
else
1493+
elog(ERROR, "corresponding segment '%s' is not found", tmp_file.path);
1494+
13661495
pg_free(tmp_file.path);
13671496
}
13681497
}
@@ -1636,3 +1765,34 @@ StreamLog(void *arg)
16361765
PQfinish(conn);
16371766
conn = NULL;
16381767
}
1768+
1769+
1770+
FileMap* cfs_mmap(int md)
1771+
{
1772+
FileMap* map;
1773+
#ifdef WIN32
1774+
HANDLE mh = CreateFileMapping(_get_osfhandle(md), NULL, PAGE_READWRITE,
1775+
0, (DWORD)sizeof(FileMap), NULL);
1776+
if (mh == NULL)
1777+
return (FileMap*)MAP_FAILED;
1778+
1779+
map = (FileMap*)MapViewOfFile(mh, FILE_MAP_ALL_ACCESS, 0, 0, 0);
1780+
CloseHandle(mh);
1781+
if (map == NULL)
1782+
return (FileMap*)MAP_FAILED;
1783+
1784+
#else
1785+
map = (FileMap*)mmap(NULL, sizeof(FileMap),
1786+
PROT_WRITE | PROT_READ, MAP_SHARED, md, 0);
1787+
#endif
1788+
return map;
1789+
}
1790+
1791+
int cfs_munmap(FileMap* map)
1792+
{
1793+
#ifdef WIN32
1794+
return UnmapViewOfFile(map) ? 0 : -1;
1795+
#else
1796+
return munmap(map, sizeof(FileMap));
1797+
#endif
1798+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy