Skip to content

Commit cf131fa

Browse files
nathan-bossartnathan-bossart
committed
initdb: Add --no-sync-data-files.
This new option instructs initdb to skip synchronizing any files in database directories, the database directories themselves, and the tablespace directories, i.e., everything in the base/ subdirectory and any other tablespace directories. Other files, such as those in pg_wal/ and pg_xact/, will still be synchronized unless --no-sync is also specified. --no-sync-data-files is primarily intended for internal use by tools that separately ensure the skipped files are synchronized to disk. A follow-up commit will use this to help optimize pg_upgrade's file transfer step. The --sync-method=fsync implementation of this option makes use of a new exclude_dir parameter for walkdir(). When not NULL, exclude_dir specifies a directory to skip processing. The --sync-method=syncfs implementation of this option just skips synchronizing the non-default tablespace directories. This means that initdb will still synchronize some or all of the database files, but there's not much we can do about that. Discussion: https://postgr.es/m/Zyvop-LxLXBLrZil%40nathan
1 parent 650ab8a commit cf131fa

File tree

9 files changed

+96
-37
lines changed

9 files changed

+96
-37
lines changed

‎doc/src/sgml/ref/initdb.sgml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,33 @@ PostgreSQL documentation
527527
</listitem>
528528
</varlistentry>
529529

530+
<varlistentry id="app-initdb-option-no-sync-data-files">
531+
<term><option>--no-sync-data-files</option></term>
532+
<listitem>
533+
<para>
534+
By default, <command>initdb</command> safely writes all database files
535+
to disk. This option instructs <command>initdb</command> to skip
536+
synchronizing all files in the individual database directories, the
537+
database directories themselves, and the tablespace directories, i.e.,
538+
everything in the <filename>base</filename> subdirectory and any other
539+
tablespace directories. Other files, such as those in
540+
<literal>pg_wal</literal> and <literal>pg_xact</literal>, will still be
541+
synchronized unless the <option>--no-sync</option> option is also
542+
specified.
543+
</para>
544+
<para>
545+
Note that if <option>--no-sync-data-files</option> is used in
546+
conjuction with <option>--sync-method=syncfs</option>, some or all of
547+
the aforementioned files and directories will be synchronized because
548+
<literal>syncfs</literal> processes entire file systems.
549+
</para>
550+
<para>
551+
This option is primarily intended for internal use by tools that
552+
separately ensure the skipped files are synchronized to disk.
553+
</para>
554+
</listitem>
555+
</varlistentry>
556+
530557
<varlistentry id="app-initdb-option-no-instructions">
531558
<term><option>--no-instructions</option></term>
532559
<listitem>

‎src/bin/initdb/initdb.c

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ static bool data_checksums = true;
168168
static char *xlog_dir = NULL;
169169
static int wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
170170
static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
171+
static bool sync_data_files = true;
171172

172173

173174
/* internal vars */
@@ -2566,6 +2567,7 @@ usage(const char *progname)
25662567
printf(_(" -L DIRECTORY where to find the input files\n"));
25672568
printf(_(" -n, --no-clean do not clean up after errors\n"));
25682569
printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
2570+
printf(_(" --no-sync-data-files do not sync files within database directories\n"));
25692571
printf(_(" --no-instructions do not print instructions for next steps\n"));
25702572
printf(_(" -s, --show show internal settings, then exit\n"));
25712573
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
@@ -3208,6 +3210,7 @@ main(int argc, char *argv[])
32083210
{"icu-rules", required_argument, NULL, 18},
32093211
{"sync-method", required_argument, NULL, 19},
32103212
{"no-data-checksums", no_argument, NULL, 20},
3213+
{"no-sync-data-files", no_argument, NULL, 21},
32113214
{NULL, 0, NULL, 0}
32123215
};
32133216

@@ -3402,6 +3405,9 @@ main(int argc, char *argv[])
34023405
case 20:
34033406
data_checksums = false;
34043407
break;
3408+
case 21:
3409+
sync_data_files = false;
3410+
break;
34053411
default:
34063412
/* getopt_long already emitted a complaint */
34073413
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -3453,7 +3459,7 @@ main(int argc, char *argv[])
34533459

34543460
fputs(_("syncing data to disk ... "), stdout);
34553461
fflush(stdout);
3456-
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method);
3462+
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files);
34573463
check_ok();
34583464
return 0;
34593465
}
@@ -3516,7 +3522,7 @@ main(int argc, char *argv[])
35163522
{
35173523
fputs(_("syncing data to disk ... "), stdout);
35183524
fflush(stdout);
3519-
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method);
3525+
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files);
35203526
check_ok();
35213527
}
35223528
else

‎src/bin/initdb/t/001_initdb.pl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
'checksums are enabled in control file');
7777

7878
command_ok([ 'initdb', '--sync-only', $datadir ], 'sync only');
79+
command_ok([ 'initdb', '--sync-only', '--no-sync-data-files', $datadir ], '--no-sync-data-files');
7980
command_fails([ 'initdb', $datadir ], 'existing data directory');
8081

8182
if ($supports_syncfs)

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2310,7 +2310,7 @@ BaseBackup(char *compression_algorithm, char *compression_detail,
23102310
}
23112311
else
23122312
{
2313-
(void) sync_pgdata(basedir, serverVersion, sync_method);
2313+
(void) sync_pgdata(basedir, serverVersion, sync_method, true);
23142314
}
23152315
}
23162316

‎src/bin/pg_checksums/pg_checksums.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,7 @@ main(int argc, char *argv[])
633633
if (do_sync)
634634
{
635635
pg_log_info("syncing data directory");
636-
sync_pgdata(DataDir, PG_VERSION_NUM, sync_method);
636+
sync_pgdata(DataDir, PG_VERSION_NUM, sync_method, true);
637637
}
638638

639639
pg_log_info("updating control file");

‎src/bin/pg_combinebackup/pg_combinebackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ main(int argc, char *argv[])
424424
else
425425
{
426426
pg_log_debug("recursively fsyncing \"%s\"", opt.output);
427-
sync_pgdata(opt.output, version * 10000, opt.sync_method);
427+
sync_pgdata(opt.output, version * 10000, opt.sync_method, true);
428428
}
429429
}
430430

‎src/bin/pg_rewind/file_ops.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ sync_target_dir(void)
296296
if (!do_sync || dry_run)
297297
return;
298298

299-
sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method);
299+
sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method, true);
300300
}
301301

302302

‎src/common/file_utils.c

Lines changed: 55 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ static int pre_sync_fname(const char *fname, bool isdir);
5050
#endif
5151
static void walkdir(const char *path,
5252
int (*action) (const char *fname, bool isdir),
53-
bool process_symlinks);
53+
bool process_symlinks,
54+
const char *exclude_dir);
5455

5556
#ifdef HAVE_SYNCFS
5657

@@ -93,11 +94,15 @@ do_syncfs(const char *path)
9394
* syncing, and might not have privileges to write at all.
9495
*
9596
* serverVersion indicates the version of the server to be sync'd.
97+
*
98+
* If sync_data_files is false, this function skips syncing "base/" and any
99+
* other tablespace directories.
96100
*/
97101
void
98102
sync_pgdata(const char *pg_data,
99103
int serverVersion,
100-
DataDirSyncMethod sync_method)
104+
DataDirSyncMethod sync_method,
105+
bool sync_data_files)
101106
{
102107
bool xlog_is_symlink;
103108
char pg_wal[MAXPGPATH];
@@ -147,30 +152,33 @@ sync_pgdata(const char *pg_data,
147152
do_syncfs(pg_data);
148153

149154
/* If any tablespaces are configured, sync each of those. */
150-
dir = opendir(pg_tblspc);
151-
if (dir == NULL)
152-
pg_log_error("could not open directory \"%s\": %m",
153-
pg_tblspc);
154-
else
155+
if (sync_data_files)
155156
{
156-
while (errno = 0, (de = readdir(dir)) != NULL)
157+
dir = opendir(pg_tblspc);
158+
if (dir == NULL)
159+
pg_log_error("could not open directory \"%s\": %m",
160+
pg_tblspc);
161+
else
157162
{
158-
char subpath[MAXPGPATH * 2];
163+
while (errno = 0, (de = readdir(dir)) != NULL)
164+
{
165+
char subpath[MAXPGPATH * 2];
159166

160-
if (strcmp(de->d_name, ".") == 0 ||
161-
strcmp(de->d_name, "..") == 0)
162-
continue;
167+
if (strcmp(de->d_name, ".") == 0 ||
168+
strcmp(de->d_name, "..") == 0)
169+
continue;
163170

164-
snprintf(subpath, sizeof(subpath), "%s/%s",
165-
pg_tblspc, de->d_name);
166-
do_syncfs(subpath);
167-
}
171+
snprintf(subpath, sizeof(subpath), "%s/%s",
172+
pg_tblspc, de->d_name);
173+
do_syncfs(subpath);
174+
}
168175

169-
if (errno)
170-
pg_log_error("could not read directory \"%s\": %m",
171-
pg_tblspc);
176+
if (errno)
177+
pg_log_error("could not read directory \"%s\": %m",
178+
pg_tblspc);
172179

173-
(void) closedir(dir);
180+
(void) closedir(dir);
181+
}
174182
}
175183

176184
/* If pg_wal is a symlink, process that too. */
@@ -182,15 +190,21 @@ sync_pgdata(const char *pg_data,
182190

183191
case DATA_DIR_SYNC_METHOD_FSYNC:
184192
{
193+
char *exclude_dir = NULL;
194+
195+
if (!sync_data_files)
196+
exclude_dir = psprintf("%s/base", pg_data);
197+
185198
/*
186199
* If possible, hint to the kernel that we're soon going to
187200
* fsync the data directory and its contents.
188201
*/
189202
#ifdef PG_FLUSH_DATA_WORKS
190-
walkdir(pg_data, pre_sync_fname, false);
203+
walkdir(pg_data, pre_sync_fname, false, exclude_dir);
191204
if (xlog_is_symlink)
192-
walkdir(pg_wal, pre_sync_fname, false);
193-
walkdir(pg_tblspc, pre_sync_fname, true);
205+
walkdir(pg_wal, pre_sync_fname, false, NULL);
206+
if (sync_data_files)
207+
walkdir(pg_tblspc, pre_sync_fname, true, NULL);
194208
#endif
195209

196210
/*
@@ -203,10 +217,14 @@ sync_pgdata(const char *pg_data,
203217
* get fsync'd twice. That's not an expected case so we don't
204218
* worry about optimizing it.
205219
*/
206-
walkdir(pg_data, fsync_fname, false);
220+
walkdir(pg_data, fsync_fname, false, exclude_dir);
207221
if (xlog_is_symlink)
208-
walkdir(pg_wal, fsync_fname, false);
209-
walkdir(pg_tblspc, fsync_fname, true);
222+
walkdir(pg_wal, fsync_fname, false, NULL);
223+
if (sync_data_files)
224+
walkdir(pg_tblspc, fsync_fname, true, NULL);
225+
226+
if (exclude_dir)
227+
pfree(exclude_dir);
210228
}
211229
break;
212230
}
@@ -245,10 +263,10 @@ sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method)
245263
* fsync the data directory and its contents.
246264
*/
247265
#ifdef PG_FLUSH_DATA_WORKS
248-
walkdir(dir, pre_sync_fname, false);
266+
walkdir(dir, pre_sync_fname, false, NULL);
249267
#endif
250268

251-
walkdir(dir, fsync_fname, false);
269+
walkdir(dir, fsync_fname, false, NULL);
252270
}
253271
break;
254272
}
@@ -264,18 +282,25 @@ sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method)
264282
* ignored in subdirectories, ie we intentionally don't pass down the
265283
* process_symlinks flag to recursive calls.
266284
*
285+
* If exclude_dir is not NULL, it specifies a directory path to skip
286+
* processing.
287+
*
267288
* Errors are reported but not considered fatal.
268289
*
269290
* See also walkdir in fd.c, which is a backend version of this logic.
270291
*/
271292
static void
272293
walkdir(const char *path,
273294
int (*action) (const char *fname, bool isdir),
274-
bool process_symlinks)
295+
bool process_symlinks,
296+
const char *exclude_dir)
275297
{
276298
DIR *dir;
277299
struct dirent *de;
278300

301+
if (exclude_dir && strcmp(exclude_dir, path) == 0)
302+
return;
303+
279304
dir = opendir(path);
280305
if (dir == NULL)
281306
{
@@ -299,7 +324,7 @@ walkdir(const char *path,
299324
(*action) (subpath, false);
300325
break;
301326
case PGFILETYPE_DIR:
302-
walkdir(subpath, action, false);
327+
walkdir(subpath, action, false, exclude_dir);
303328
break;
304329
default:
305330

‎src/include/common/file_utils.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct iovec; /* avoid including port/pg_iovec.h here */
3535
#ifdef FRONTEND
3636
extern int fsync_fname(const char *fname, bool isdir);
3737
extern void sync_pgdata(const char *pg_data, int serverVersion,
38-
DataDirSyncMethod sync_method);
38+
DataDirSyncMethod sync_method, bool sync_data_files);
3939
extern void sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method);
4040
extern int durable_rename(const char *oldfile, const char *newfile);
4141
extern int fsync_parent_path(const char *fname);

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy