Skip to content

Commit 884e8b0

Browse files
dlepikhovadlepikhova
authored
[pbckp-128] dry-run option for catchup (#477)
* Added dry-run option for catchup. Run catchup without affect on the files and WAL
1 parent 7be2e73 commit 884e8b0

File tree

5 files changed

+220
-32
lines changed

5 files changed

+220
-32
lines changed

‎src/catchup.c

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
*
33
* catchup.c: sync DB cluster
44
*
5-
* Copyright (c) 2021, Postgres Professional
5+
* Copyright (c) 2022, Postgres Professional
66
*
77
*-------------------------------------------------------------------------
88
*/
@@ -507,16 +507,20 @@ catchup_multithreaded_copy(int num_threads,
507507
/* Run threads */
508508
thread_interrupted = false;
509509
threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
510-
for (i = 0; i < num_threads; i++)
510+
if (!dry_run)
511511
{
512-
elog(VERBOSE, "Start thread num: %i", i);
513-
pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i]));
512+
for (i = 0; i < num_threads; i++)
513+
{
514+
elog(VERBOSE, "Start thread num: %i", i);
515+
pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i]));
516+
}
514517
}
515518

516519
/* Wait threads */
517520
for (i = 0; i < num_threads; i++)
518521
{
519-
pthread_join(threads[i], NULL);
522+
if (!dry_run)
523+
pthread_join(threads[i], NULL);
520524
all_threads_successful &= threads_args[i].completed;
521525
transfered_bytes_result += threads_args[i].transfered_bytes;
522526
}
@@ -706,9 +710,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
706710

707711
/* Start stream replication */
708712
join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR);
709-
fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST);
710-
start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt,
711-
current.start_lsn, current.tli, false);
713+
if (!dry_run)
714+
{
715+
fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST);
716+
start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt,
717+
current.start_lsn, current.tli, false);
718+
}
719+
else
720+
elog(INFO, "WAL streaming skipping with --dry-run option");
712721

713722
source_filelist = parray_new();
714723

@@ -779,9 +788,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
779788

780789
/* Build the page map from ptrack information */
781790
make_pagemap_from_ptrack_2(source_filelist, source_conn,
782-
source_node_info.ptrack_schema,
783-
source_node_info.ptrack_version_num,
784-
dest_redo.lsn);
791+
source_node_info.ptrack_schema,
792+
source_node_info.ptrack_version_num,
793+
dest_redo.lsn);
785794
time(&end_time);
786795
elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec",
787796
difftime(end_time, start_time));
@@ -820,9 +829,9 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
820829
char dirpath[MAXPGPATH];
821830

822831
join_path_components(dirpath, dest_pgdata, file->rel_path);
823-
824832
elog(VERBOSE, "Create directory '%s'", dirpath);
825-
fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST);
833+
if (!dry_run)
834+
fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST);
826835
}
827836
else
828837
{
@@ -853,15 +862,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
853862
elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"",
854863
linked_path, to_path);
855864

856-
/* create tablespace directory */
857-
if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0)
858-
elog(ERROR, "Could not create tablespace directory \"%s\": %s",
859-
linked_path, strerror(errno));
860-
861-
/* create link to linked_path */
862-
if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0)
863-
elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s",
864-
linked_path, to_path, strerror(errno));
865+
if (!dry_run)
866+
{
867+
/* create tablespace directory */
868+
if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0)
869+
elog(ERROR, "Could not create tablespace directory \"%s\": %s",
870+
linked_path, strerror(errno));
871+
872+
/* create link to linked_path */
873+
if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0)
874+
elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s",
875+
linked_path, to_path, strerror(errno));
876+
}
865877
}
866878
}
867879

@@ -930,7 +942,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
930942
char fullpath[MAXPGPATH];
931943

932944
join_path_components(fullpath, dest_pgdata, file->rel_path);
933-
fio_delete(file->mode, fullpath, FIO_LOCAL_HOST);
945+
if (!dry_run)
946+
{
947+
fio_delete(file->mode, fullpath, FIO_LOCAL_HOST);
948+
}
934949
elog(VERBOSE, "Deleted file \"%s\"", fullpath);
935950

936951
/* shrink dest pgdata list */
@@ -961,7 +976,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
961976
catchup_isok = transfered_datafiles_bytes != -1;
962977

963978
/* at last copy control file */
964-
if (catchup_isok)
979+
if (catchup_isok && !dry_run)
965980
{
966981
char from_fullpath[MAXPGPATH];
967982
char to_fullpath[MAXPGPATH];
@@ -972,7 +987,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
972987
transfered_datafiles_bytes += source_pg_control_file->size;
973988
}
974989

975-
if (!catchup_isok)
990+
if (!catchup_isok && !dry_run)
976991
{
977992
char pretty_time[20];
978993
char pretty_transfered_data_bytes[20];
@@ -1010,14 +1025,18 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10101025
pg_free(stop_backup_query_text);
10111026
}
10121027

1013-
wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, &current);
1028+
if (!dry_run)
1029+
wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, &current);
10141030

10151031
#if PG_VERSION_NUM >= 90600
10161032
/* Write backup_label */
10171033
Assert(stop_backup_result.backup_label_content != NULL);
1018-
pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label",
1019-
stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len,
1020-
NULL);
1034+
if (!dry_run)
1035+
{
1036+
pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label",
1037+
stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len,
1038+
NULL);
1039+
}
10211040
free(stop_backup_result.backup_label_content);
10221041
stop_backup_result.backup_label_content = NULL;
10231042
stop_backup_result.backup_label_content_len = 0;
@@ -1040,6 +1059,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10401059
#endif
10411060

10421061
/* wait for end of wal streaming and calculate wal size transfered */
1062+
if (!dry_run)
10431063
{
10441064
parray *wal_files_list = NULL;
10451065
wal_files_list = parray_new();
@@ -1091,17 +1111,17 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
10911111
}
10921112

10931113
/* Sync all copied files unless '--no-sync' flag is used */
1094-
if (sync_dest_files)
1114+
if (sync_dest_files && !dry_run)
10951115
catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file);
10961116
else
10971117
elog(WARNING, "Files are not synced to disk");
10981118

10991119
/* Cleanup */
1100-
if (dest_filelist)
1120+
if (dest_filelist && !dry_run)
11011121
{
11021122
parray_walk(dest_filelist, pgFileFree);
1103-
parray_free(dest_filelist);
11041123
}
1124+
parray_free(dest_filelist);
11051125
parray_walk(source_filelist, pgFileFree);
11061126
parray_free(source_filelist);
11071127
pgFileFree(source_pg_control_file);

‎src/help.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ help_pg_probackup(void)
261261
printf(_(" [--remote-proto] [--remote-host]\n"));
262262
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
263263
printf(_(" [--ssh-options]\n"));
264+
printf(_(" [--dry-run]\n"));
264265
printf(_(" [--help]\n"));
265266

266267
if ((PROGRAM_URL || PROGRAM_EMAIL))
@@ -1047,6 +1048,7 @@ help_catchup(void)
10471048
printf(_(" [--remote-proto] [--remote-host]\n"));
10481049
printf(_(" [--remote-port] [--remote-path] [--remote-user]\n"));
10491050
printf(_(" [--ssh-options]\n"));
1051+
printf(_(" [--dry-run]\n"));
10501052
printf(_(" [--help]\n\n"));
10511053

10521054
printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n"));
@@ -1081,4 +1083,6 @@ help_catchup(void)
10811083
printf(_(" --remote-user=username user name for ssh connection (default: current user)\n"));
10821084
printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n"));
10831085
printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n"));
1086+
1087+
printf(_(" --dry-run perform a trial run without any changes\n\n"));
10841088
}

‎tests/catchup.py

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1455,3 +1455,157 @@ def test_config_exclusion(self):
14551455
dst_pg.stop()
14561456
#self.assertEqual(1, 0, 'Stop test')
14571457
self.del_test_dir(module_name, self.fname)
1458+
1459+
#########################################
1460+
# --dry-run
1461+
#########################################
1462+
def test_dry_run_catchup_full(self):
1463+
"""
1464+
Test dry-run option for full catchup
1465+
"""
1466+
# preparation 1: source
1467+
src_pg = self.make_simple_node(
1468+
base_dir = os.path.join(module_name, self.fname, 'src'),
1469+
set_replication = True
1470+
)
1471+
src_pg.slow_start()
1472+
1473+
# preparation 2: make clean shutdowned lagging behind replica
1474+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1475+
1476+
src_pg.pgbench_init(scale = 10)
1477+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1478+
pgbench.wait()
1479+
1480+
# save the condition before dry-run
1481+
content_before = self.pgdata_content(dst_pg.data_dir)
1482+
1483+
# do full catchup
1484+
self.catchup_node(
1485+
backup_mode = 'FULL',
1486+
source_pgdata = src_pg.data_dir,
1487+
destination_node = dst_pg,
1488+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run']
1489+
)
1490+
1491+
# compare data dirs before and after catchup
1492+
self.compare_pgdata(
1493+
content_before,
1494+
self.pgdata_content(dst_pg.data_dir)
1495+
)
1496+
1497+
# Cleanup
1498+
src_pg.stop()
1499+
self.del_test_dir(module_name, self.fname)
1500+
1501+
def test_dry_run_catchup_ptrack(self):
1502+
"""
1503+
Test dry-run option for catchup in incremental ptrack mode
1504+
"""
1505+
if not self.ptrack:
1506+
return unittest.skip('Skipped because ptrack support is disabled')
1507+
1508+
# preparation 1: source
1509+
src_pg = self.make_simple_node(
1510+
base_dir = os.path.join(module_name, self.fname, 'src'),
1511+
set_replication = True,
1512+
ptrack_enable = True,
1513+
initdb_params = ['--data-checksums']
1514+
)
1515+
src_pg.slow_start()
1516+
src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack")
1517+
1518+
src_pg.pgbench_init(scale = 10)
1519+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1520+
pgbench.wait()
1521+
1522+
# preparation 2: make clean shutdowned lagging behind replica
1523+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1524+
self.catchup_node(
1525+
backup_mode = 'FULL',
1526+
source_pgdata = src_pg.data_dir,
1527+
destination_node = dst_pg,
1528+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream']
1529+
)
1530+
self.set_replica(src_pg, dst_pg)
1531+
dst_options = {}
1532+
dst_options['port'] = str(dst_pg.port)
1533+
self.set_auto_conf(dst_pg, dst_options)
1534+
dst_pg.slow_start(replica = True)
1535+
dst_pg.stop()
1536+
1537+
# save the condition before dry-run
1538+
content_before = self.pgdata_content(dst_pg.data_dir)
1539+
1540+
# do incremental catchup
1541+
self.catchup_node(
1542+
backup_mode = 'PTRACK',
1543+
source_pgdata = src_pg.data_dir,
1544+
destination_node = dst_pg,
1545+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run']
1546+
)
1547+
1548+
# compare data dirs before and after cathup
1549+
self.compare_pgdata(
1550+
content_before,
1551+
self.pgdata_content(dst_pg.data_dir)
1552+
)
1553+
1554+
# Cleanup
1555+
src_pg.stop()
1556+
self.del_test_dir(module_name, self.fname)
1557+
1558+
def test_dry_run_catchup_delta(self):
1559+
"""
1560+
Test dry-run option for catchup in incremental delta mode
1561+
"""
1562+
1563+
# preparation 1: source
1564+
src_pg = self.make_simple_node(
1565+
base_dir = os.path.join(module_name, self.fname, 'src'),
1566+
set_replication = True,
1567+
initdb_params = ['--data-checksums'],
1568+
pg_options = { 'wal_log_hints': 'on' }
1569+
)
1570+
src_pg.slow_start()
1571+
1572+
src_pg.pgbench_init(scale = 10)
1573+
pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum'])
1574+
pgbench.wait()
1575+
1576+
# preparation 2: make clean shutdowned lagging behind replica
1577+
dst_pg = self.make_empty_node(os.path.join(module_name, self.fname, 'dst'))
1578+
self.catchup_node(
1579+
backup_mode = 'FULL',
1580+
source_pgdata = src_pg.data_dir,
1581+
destination_node = dst_pg,
1582+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream']
1583+
)
1584+
self.set_replica(src_pg, dst_pg)
1585+
dst_options = {}
1586+
dst_options['port'] = str(dst_pg.port)
1587+
self.set_auto_conf(dst_pg, dst_options)
1588+
dst_pg.slow_start(replica = True)
1589+
dst_pg.stop()
1590+
1591+
# save the condition before dry-run
1592+
content_before = self.pgdata_content(dst_pg.data_dir)
1593+
1594+
# do delta catchup
1595+
self.catchup_node(
1596+
backup_mode = 'DELTA',
1597+
source_pgdata = src_pg.data_dir,
1598+
destination_node = dst_pg,
1599+
options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', "--dry-run"]
1600+
)
1601+
1602+
# compare data dirs before and after cathup
1603+
self.compare_pgdata(
1604+
content_before,
1605+
self.pgdata_content(dst_pg.data_dir)
1606+
)
1607+
1608+
# Cleanup
1609+
src_pg.stop()
1610+
self.del_test_dir(module_name, self.fname)
1611+

‎tests/expected/option_help.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
178178
[--remote-proto] [--remote-host]
179179
[--remote-port] [--remote-path] [--remote-user]
180180
[--ssh-options]
181+
[--dry-run]
181182
[--help]
182183

183184
Read the website for details <https://github.com/postgrespro/pg_probackup>.

‎travis/run_tests.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,20 @@ source pyenv/bin/activate
100100
pip3 install testgres
101101

102102
echo "############### Testing:"
103+
echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA}
104+
echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION}
105+
echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD}
106+
echo PGPROBACKUPBIN=${PGPROBACKUPBIN}
107+
echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE}
108+
echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB}
109+
echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK}
103110
if [ "$MODE" = "basic" ]; then
104111
export PG_PROBACKUP_TEST_BASIC=ON
112+
echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC}
105113
python3 -m unittest -v tests
106114
python3 -m unittest -v tests.init
107115
else
116+
echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC}
108117
python3 -m unittest -v tests.$MODE
109118
fi
110119

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy