Skip to content

Commit b87ca18

Browse files
dlepikhovakulaginmindrups
authored
[Issue #265][PGPRO-5421] archive-push backward compatibility (#437)
Restore the --wal-file-path option of the archive-push command (it was ignored since a196073) Co-authored-by: Mikhail A. Kulagin <m.kulagin@postgrespro.ru> Co-authored-by: Elena Indrupskaya <e.indrupskaya@postgrespro.ru>
1 parent 5b6ca62 commit b87ca18

File tree

17 files changed

+338
-86
lines changed

17 files changed

+338
-86
lines changed

‎doc/pgprobackup.xml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
131131
<arg choice="plain"><option>archive-push</option></arg>
132132
<arg choice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
133133
<arg choice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
134+
<arg choice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
134135
<arg choice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
135136
<arg rep="repeat"><replaceable>option</replaceable></arg>
136137
</cmdsynopsis>
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53675368
Provides the path to the WAL file in
53685369
<parameter>archive_command</parameter> and
53695370
<parameter>restore_command</parameter>. Use the <literal>%p</literal>
5370-
variable as the value for this option for correct processing.
5371+
variable as the value for this option or explicitly specify the path to a file
5372+
outside of the data directory. If you skip this option, the path
5373+
specified in <filename>pg_probackup.conf</filename> will be used.
53715374
</para>
53725375
</listitem>
53735376
</varlistentry>
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53805383
<parameter>archive_command</parameter> and
53815384
<parameter>restore_command</parameter>. Use the <literal>%f</literal>
53825385
variable as the value for this option for correct processing.
5386+
If the value of <option>--wal-file-path</option> is a path
5387+
outside of the data directory, explicitly specify the filename.
53835388
</para>
53845389
</listitem>
53855390
</varlistentry>

‎src/archive.c

Lines changed: 14 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* archive.c: - pg_probackup specific archive commands for archive backups.
44
*
55
*
6-
* Portions Copyright (c) 2018-2019, Postgres Professional
6+
* Portions Copyright (c) 2018-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -113,15 +113,13 @@ static parray *setup_push_filelist(const char *archive_status_dir,
113113
* Where archlog_path is $BACKUP_PATH/wal/instance_name
114114
*/
115115
void
116-
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
116+
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
117117
char *wal_file_name, int batch_size, bool overwrite,
118118
bool no_sync, bool no_ready_rename)
119119
{
120120
uint64 i;
121-
char current_dir[MAXPGPATH];
122-
char pg_xlog_dir[MAXPGPATH];
123-
char archive_status_dir[MAXPGPATH];
124-
uint64 system_id;
121+
/* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
122+
char archive_status_dir[MAXPGPATH] = "";
125123
bool is_compress = false;
126124

127125
/* arrays with meta info for multi threaded backup */
@@ -141,31 +139,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
141139
parray *batch_files = NULL;
142140
int n_threads;
143141

144-
if (wal_file_name == NULL)
145-
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
146-
147-
if (!getcwd(current_dir, sizeof(current_dir)))
148-
elog(ERROR, "getcwd() error");
149-
150-
/* verify that archive-push --instance parameter is valid */
151-
system_id = get_system_identifier(current_dir, FIO_DB_HOST);
152-
153-
if (instance->pgdata == NULL)
154-
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
155-
156-
if (system_id != instance->system_identifier)
157-
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
158-
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
159-
wal_file_name, instanceState->instance_name, instance->system_identifier, system_id);
160-
161-
if (instance->compress_alg == PGLZ_COMPRESS)
162-
elog(ERROR, "Cannot use pglz for WAL compression");
163-
164-
join_path_components(pg_xlog_dir, current_dir, XLOGDIR);
165-
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
166-
167-
/* Create 'archlog_path' directory. Do nothing if it already exists. */
168-
//fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
142+
if (!no_ready_rename || batch_size > 1)
143+
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
169144

170145
#ifdef HAVE_LIBZ
171146
if (instance->compress_alg == ZLIB_COMPRESS)
@@ -204,12 +179,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
204179
{
205180
int rc;
206181
WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i);
182+
bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0;
207183

208-
rc = push_file(xlogfile, archive_status_dir,
184+
rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir,
209185
pg_xlog_dir, instanceState->instance_wal_subdir_path,
210186
overwrite, no_sync,
211187
instance->archive_timeout,
212-
no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false,
188+
no_ready_rename || first_wal,
213189
is_compress && IsXLogFileName(xlogfile->name) ? true : false,
214190
instance->compress_level);
215191
if (rc == 0)
@@ -233,7 +209,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
233209
arg->first_filename = wal_file_name;
234210
arg->archive_dir = instanceState->instance_wal_subdir_path;
235211
arg->pg_xlog_dir = pg_xlog_dir;
236-
arg->archive_status_dir = archive_status_dir;
212+
arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL;
237213
arg->overwrite = overwrite;
238214
arg->compress = is_compress;
239215
arg->no_sync = no_sync;
@@ -276,7 +252,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
276252

277253
/* Note, that we are leaking memory here,
278254
* because pushing into archive is a very
279-
* time-sensetive operation, so we skip freeing stuff.
255+
* time-sensitive operation, so we skip freeing stuff.
280256
*/
281257

282258
push_done:
@@ -356,9 +332,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
356332
int compress_level)
357333
{
358334
int rc;
359-
char wal_file_dummy[MAXPGPATH];
360-
361-
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
362335

363336
elog(LOG, "pushing file \"%s\"", xlogfile->name);
364337

@@ -375,11 +348,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
375348
#endif
376349

377350
/* take '--no-ready-rename' flag into account */
378-
if (!no_ready_rename)
351+
if (!no_ready_rename && archive_status_dir != NULL)
379352
{
353+
char wal_file_dummy[MAXPGPATH];
380354
char wal_file_ready[MAXPGPATH];
381355
char wal_file_done[MAXPGPATH];
382356

357+
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
383358
snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready");
384359
snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done");
385360

‎src/backup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
943943
uint64 system_id_conn;
944944
uint64 system_id_pgdata;
945945

946-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
946+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
947947
system_id_conn = get_remote_system_identifier(conn);
948948

949949
/* for checkdb check only system_id_pgdata and system_id_conn */

‎src/catchup.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons
4848

4949
/* Get WAL segments size and system ID of source PG instance */
5050
instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata);
51-
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST);
51+
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false);
5252
current.start_time = time(NULL);
5353

5454
strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version));
@@ -163,15 +163,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
163163
uint64 source_conn_id, source_id, dest_id;
164164

165165
source_conn_id = get_remote_system_identifier(source_conn);
166-
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */
166+
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */
167167

168168
if (source_conn_id != source_id)
169169
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
170170
source_conn_id, source_pgdata, source_id);
171171

172172
if (current.backup_mode != BACKUP_MODE_FULL)
173173
{
174-
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST);
174+
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false);
175175
if (source_conn_id != dest_id)
176176
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
177177
source_conn_id, dest_pgdata, dest_id);

‎src/help.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ help_pg_probackup(void)
227227

228228
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
229229
printf(_(" --wal-file-name=wal-file-name\n"));
230+
printf(_(" [--wal-file-path=wal-file-path]\n"));
230231
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
231232
printf(_(" [--archive-timeout=timeout]\n"));
232233
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
937938
{
938939
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
939940
printf(_(" --wal-file-name=wal-file-name\n"));
941+
printf(_(" [--wal-file-path=wal-file-path]\n"));
940942
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
941943
printf(_(" [--archive-timeout=timeout]\n"));
942944
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
951953
printf(_(" --instance=instance_name name of the instance to delete\n"));
952954
printf(_(" --wal-file-name=wal-file-name\n"));
953955
printf(_(" name of the file to copy into WAL archive\n"));
956+
printf(_(" --wal-file-path=wal-file-path\n"));
957+
printf(_(" relative destination path of the WAL archive\n"));
954958
printf(_(" -j, --threads=NUM number of parallel threads\n"));
955959
printf(_(" --batch-size=NUM number of files to be copied\n"));
956960
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
981985
help_archive_get(void)
982986
{
983987
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
984-
printf(_(" --wal-file-path=wal-file-path\n"));
985988
printf(_(" --wal-file-name=wal-file-name\n"));
989+
printf(_(" [--wal-file-path=wal-file-path]\n"));
986990
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
987991
printf(_(" [--no-validate-wal]\n"));
988992
printf(_(" [--remote-proto] [--remote-host]\n"));

‎src/init.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
5757
"(-D, --pgdata)");
5858

5959
/* Read system_identifier from PGDATA */
60-
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST);
60+
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false);
6161
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
6262
instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata);
6363

‎src/pg_probackup.c

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
* which includes info about pgdata directory and connection.
3636
*
3737
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
38-
* Portions Copyright (c) 2015-2019, Postgres Professional
38+
* Portions Copyright (c) 2015-2021, Postgres Professional
3939
*
4040
*-------------------------------------------------------------------------
4141
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
151151
static char *wal_file_name;
152152
static bool file_overwrite = false;
153153
static bool no_ready_rename = false;
154+
static char archive_push_xlog_dir[MAXPGPATH] = "";
154155

155156
/* archive get options */
156157
static char *prefetch_dir;
@@ -788,14 +789,105 @@ main(int argc, char *argv[])
788789
current.stream = stream_wal = true;
789790
if (instance_config.external_dir_str)
790791
elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd));
791-
// TODO проверить instance_config.conn_opt
792+
// TODO check instance_config.conn_opt
792793
}
793794

794795
/* sanity */
795796
if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate)
796797
elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command",
797798
get_subcmd_name(backup_subcmd));
798799

800+
if (backup_subcmd == ARCHIVE_PUSH_CMD)
801+
{
802+
/* Check archive-push parameters and construct archive_push_xlog_dir
803+
*
804+
* There are 4 cases:
805+
* 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
806+
* (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
807+
* in this case we can use batches and threads
808+
* 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
809+
* in this case we can use this path, as well as batches and thread
810+
* 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
811+
* in this case we can use this path, as well as batches and thread
812+
* (replica for example, see test_archive_push_sanity)
813+
* 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
814+
* disable optimizations and work with user specified path
815+
*/
816+
bool check_system_id = true;
817+
uint64 system_id;
818+
char current_dir[MAXPGPATH];
819+
820+
if (wal_file_name == NULL)
821+
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
822+
823+
if (instance_config.pgdata == NULL)
824+
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
825+
826+
/* TODO may be remove in preference of checking inside compress_init()? */
827+
if (instance_config.compress_alg == PGLZ_COMPRESS)
828+
elog(ERROR, "Cannot use pglz for WAL compression");
829+
830+
if (!getcwd(current_dir, sizeof(current_dir)))
831+
elog(ERROR, "getcwd() error");
832+
833+
if (wal_file_path == NULL)
834+
{
835+
/* 1st case */
836+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, false);
837+
join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR);
838+
}
839+
else
840+
{
841+
/*
842+
* Usually we get something like
843+
* wal_file_path = "pg_wal/0000000100000000000000A1"
844+
* wal_file_name = "0000000100000000000000A1"
845+
* instance_config.pgdata = "/pgdata/.../node/data"
846+
* We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
847+
* and compare this directories.
848+
* Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
849+
*/
850+
char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
851+
join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
852+
if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST))
853+
{
854+
/* 2nd case */
855+
system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false);
856+
/* archive_push_xlog_dir already have right value */
857+
}
858+
else
859+
{
860+
if (strlen(stripped_wal_file_path) < MAXPGPATH)
861+
strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH);
862+
else
863+
elog(ERROR, "Value specified to --wal_file_path is too long");
864+
865+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, true);
866+
/* 3rd case if control file present -- i.e. system_id != 0 */
867+
868+
if (system_id == 0)
869+
{
870+
/* 4th case */
871+
check_system_id = false;
872+
873+
if (batch_size > 1 || num_threads > 1 || !no_ready_rename)
874+
{
875+
elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
876+
batch_size = 1;
877+
num_threads = 1;
878+
no_ready_rename = true;
879+
}
880+
}
881+
}
882+
pfree(stripped_wal_file_path);
883+
}
884+
885+
if (check_system_id && system_id != instance_config.system_identifier)
886+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
887+
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
888+
wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id);
889+
}
890+
799891
#if PG_VERSION_NUM >= 100000
800892
if (temp_slot && perm_slot)
801893
elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -819,7 +911,7 @@ main(int argc, char *argv[])
819911
switch (backup_subcmd)
820912
{
821913
case ARCHIVE_PUSH_CMD:
822-
do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name,
914+
do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name,
823915
batch_size, file_overwrite, no_sync, no_ready_rename);
824916
break;
825917
case ARCHIVE_GET_CMD:

‎src/pg_probackup.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,7 @@ extern int do_init(CatalogState *catalogState);
889889
extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance);
890890

891891
/* in archive.c */
892-
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
892+
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
893893
char *wal_file_name, int batch_size, bool overwrite,
894894
bool no_sync, bool no_ready_rename);
895895
extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path,
@@ -1153,7 +1153,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T
11531153
extern TimeLineID get_current_timeline(PGconn *conn);
11541154
extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe);
11551155
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
1156-
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location);
1156+
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe);
11571157
extern uint64 get_remote_system_identifier(PGconn *conn);
11581158
extern uint32 get_data_checksum_version(bool safe);
11591159
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);

‎src/restore.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
21862186
*/
21872187
elog(INFO, "Trying to read pg_control file in destination directory");
21882188

2189-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
2189+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
21902190

21912191
if (system_id_pgdata == instance_config.system_identifier)
21922192
system_id_match = true;

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy