Skip to content

Commit d93627b

Browse files
macdicemacdice
committed
Add --copy-file-range option to pg_upgrade.
The copy_file_range() system call is available on at least Linux and FreeBSD, and asks the kernel to use efficient ways to copy ranges of a file. Options available to the kernel include sharing block ranges (similar to --clone mode), and pushing down block copies to the storage layer. For automated testing, see PG_TEST_PG_UPGRADE_MODE. (Perhaps in a later commit we could consider setting this mode for one of the CI targets.) Reviewed-by: Peter Eisentraut <peter@eisentraut.org> Discussion: https://postgr.es/m/CA%2BhUKGKe7Hb0-UNih8VD5UNZy5-ojxFb3Pr3xSBBL8qj2M2%3DdQ%40mail.gmail.com
1 parent 2bce0ad commit d93627b

File tree

11 files changed

+120
-4
lines changed

11 files changed

+120
-4
lines changed

‎configure

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15259,7 +15259,7 @@ fi
1525915259
LIBS_including_readline="$LIBS"
1526015260
LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'`
1526115261

15262-
for ac_func in backtrace_symbols copyfile getifaddrs getpeerucred inet_pton kqueue mbstowcs_l memset_s posix_fallocate ppoll pthread_is_threaded_np setproctitle setproctitle_fast strchrnul strsignal syncfs sync_file_range uselocale wcstombs_l
15262+
for ac_func in backtrace_symbols copyfile copy_file_range getifaddrs getpeerucred inet_pton kqueue mbstowcs_l memset_s posix_fallocate ppoll pthread_is_threaded_np setproctitle setproctitle_fast strchrnul strsignal syncfs sync_file_range uselocale wcstombs_l
1526315263
do :
1526415264
as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
1526515265
ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"

‎configure.ac

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1749,6 +1749,7 @@ LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'`
17491749
AC_CHECK_FUNCS(m4_normalize([
17501750
backtrace_symbols
17511751
copyfile
1752+
copy_file_range
17521753
getifaddrs
17531754
getpeerucred
17541755
inet_pton

‎doc/src/sgml/ref/pgupgrade.sgml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,19 @@ PostgreSQL documentation
263263
</listitem>
264264
</varlistentry>
265265

266+
<varlistentry>
267+
<term><option>--copy-file-range</option></term>
268+
<listitem>
269+
<para>
270+
Use the <function>copy_file_range</function> system call for efficient
271+
copying. On some file systems this gives results similar to
272+
<option>--clone</option>, sharing physical disk blocks, while on others
273+
it may still copy blocks, but do so via an optimized path. At present,
274+
it is supported on Linux and FreeBSD.
275+
</para>
276+
</listitem>
277+
</varlistentry>
278+
266279
<varlistentry>
267280
<term><option>-?</option></term>
268281
<term><option>--help</option></term>

‎meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2420,6 +2420,7 @@ func_checks = [
24202420
['backtrace_symbols', {'dependencies': [execinfo_dep]}],
24212421
['clock_gettime', {'dependencies': [rt_dep], 'define': false}],
24222422
['copyfile'],
2423+
['copy_file_range'],
24232424
# gcc/clang's sanitizer helper library provides dlopen but not dlsym, thus
24242425
# when enabling asan the dlopen check doesn't notice that -ldl is actually
24252426
# required. Just checking for dlsym() ought to suffice.

‎src/bin/pg_upgrade/TESTING

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ export oldinstall=...otherversion/ (old version's install base path)
2020
See DETAILS below for more information about creation of the dump.
2121

2222
You can also test the different transfer modes (--copy, --link,
23-
--clone) by setting the environment variable PG_TEST_PG_UPGRADE_MODE
24-
to the respective command-line option, like
23+
--clone, --copy-file-range) by setting the environment variable
24+
PG_TEST_PG_UPGRADE_MODE to the respective command-line option, like
2525

2626
make check PG_TEST_PG_UPGRADE_MODE=--link
2727

‎src/bin/pg_upgrade/check.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,9 @@ check_new_cluster(void)
235235
break;
236236
case TRANSFER_MODE_COPY:
237237
break;
238+
case TRANSFER_MODE_COPY_FILE_RANGE:
239+
check_copy_file_range();
240+
break;
238241
case TRANSFER_MODE_LINK:
239242
check_hard_link();
240243
break;

‎src/bin/pg_upgrade/file.c

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "postgres_fe.h"
1111

1212
#include <sys/stat.h>
13+
#include <limits.h>
1314
#include <fcntl.h>
1415
#ifdef HAVE_COPYFILE_H
1516
#include <copyfile.h>
@@ -140,6 +141,45 @@ copyFile(const char *src, const char *dst,
140141
}
141142

142143

144+
/*
145+
* copyFileByRange()
146+
*
147+
* Copies a relation file from src to dst.
148+
* schemaName/relName are relation's SQL name (used for error messages only).
149+
*/
150+
void
151+
copyFileByRange(const char *src, const char *dst,
152+
const char *schemaName, const char *relName)
153+
{
154+
#ifdef HAVE_COPY_FILE_RANGE
155+
int src_fd;
156+
int dest_fd;
157+
ssize_t nbytes;
158+
159+
if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
160+
pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s",
161+
schemaName, relName, src, strerror(errno));
162+
163+
if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
164+
pg_file_create_mode)) < 0)
165+
pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s",
166+
schemaName, relName, dst, strerror(errno));
167+
168+
do
169+
{
170+
nbytes = copy_file_range(src_fd, NULL, dest_fd, NULL, SSIZE_MAX, 0);
171+
if (nbytes < 0)
172+
pg_fatal("error while copying relation \"%s.%s\": could not copy file range from \"%s\" to \"%s\": %s",
173+
schemaName, relName, src, dst, strerror(errno));
174+
}
175+
while (nbytes > 0);
176+
177+
close(src_fd);
178+
close(dest_fd);
179+
#endif
180+
}
181+
182+
143183
/*
144184
* linkFile()
145185
*
@@ -358,6 +398,44 @@ check_file_clone(void)
358398
unlink(new_link_file);
359399
}
360400

401+
void
402+
check_copy_file_range(void)
403+
{
404+
char existing_file[MAXPGPATH];
405+
char new_link_file[MAXPGPATH];
406+
407+
snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
408+
snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.copy_file_range_test", new_cluster.pgdata);
409+
unlink(new_link_file); /* might fail */
410+
411+
#if defined(HAVE_COPY_FILE_RANGE)
412+
{
413+
int src_fd;
414+
int dest_fd;
415+
416+
if ((src_fd = open(existing_file, O_RDONLY | PG_BINARY, 0)) < 0)
417+
pg_fatal("could not open file \"%s\": %s",
418+
existing_file, strerror(errno));
419+
420+
if ((dest_fd = open(new_link_file, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
421+
pg_file_create_mode)) < 0)
422+
pg_fatal("could not create file \"%s\": %s",
423+
new_link_file, strerror(errno));
424+
425+
if (copy_file_range(src_fd, NULL, dest_fd, NULL, SSIZE_MAX, 0) < 0)
426+
pg_fatal("could not copy file range between old and new data directories: %s",
427+
strerror(errno));
428+
429+
close(src_fd);
430+
close(dest_fd);
431+
}
432+
#else
433+
pg_fatal("copy_file_range not supported on this platform");
434+
#endif
435+
436+
unlink(new_link_file);
437+
}
438+
361439
void
362440
check_hard_link(void)
363441
{

‎src/bin/pg_upgrade/option.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ parseCommandLine(int argc, char *argv[])
5858
{"verbose", no_argument, NULL, 'v'},
5959
{"clone", no_argument, NULL, 1},
6060
{"copy", no_argument, NULL, 2},
61-
{"sync-method", required_argument, NULL, 3},
61+
{"copy-file-range", no_argument, NULL, 3},
62+
{"sync-method", required_argument, NULL, 4},
6263

6364
{NULL, 0, NULL, 0}
6465
};
@@ -203,6 +204,9 @@ parseCommandLine(int argc, char *argv[])
203204
break;
204205

205206
case 3:
207+
user_opts.transfer_mode = TRANSFER_MODE_COPY_FILE_RANGE;
208+
break;
209+
case 4:
206210
if (!parse_sync_method(optarg, &unused))
207211
exit(1);
208212
user_opts.sync_method = pg_strdup(optarg);
@@ -301,6 +305,7 @@ usage(void)
301305
printf(_(" -V, --version display version information, then exit\n"));
302306
printf(_(" --clone clone instead of copying files to new cluster\n"));
303307
printf(_(" --copy copy files to new cluster (default)\n"));
308+
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
304309
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
305310
printf(_(" -?, --help show this help, then exit\n"));
306311
printf(_("\n"

‎src/bin/pg_upgrade/pg_upgrade.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@ typedef enum
256256
{
257257
TRANSFER_MODE_CLONE,
258258
TRANSFER_MODE_COPY,
259+
TRANSFER_MODE_COPY_FILE_RANGE,
259260
TRANSFER_MODE_LINK,
260261
} transferMode;
261262

@@ -402,11 +403,14 @@ void cloneFile(const char *src, const char *dst,
402403
const char *schemaName, const char *relName);
403404
void copyFile(const char *src, const char *dst,
404405
const char *schemaName, const char *relName);
406+
void copyFileByRange(const char *src, const char *dst,
407+
const char *schemaName, const char *relName);
405408
void linkFile(const char *src, const char *dst,
406409
const char *schemaName, const char *relName);
407410
void rewriteVisibilityMap(const char *fromfile, const char *tofile,
408411
const char *schemaName, const char *relName);
409412
void check_file_clone(void);
413+
void check_copy_file_range(void);
410414
void check_hard_link(void);
411415

412416
/* fopen_priv() is no longer different from fopen() */

‎src/bin/pg_upgrade/relfilenumber.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ transfer_all_new_tablespaces(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr,
3737
case TRANSFER_MODE_COPY:
3838
prep_status_progress("Copying user relation files");
3939
break;
40+
case TRANSFER_MODE_COPY_FILE_RANGE:
41+
prep_status_progress("Copying user relation files with copy_file_range");
42+
break;
4043
case TRANSFER_MODE_LINK:
4144
prep_status_progress("Linking user relation files");
4245
break;
@@ -250,6 +253,11 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro
250253
old_file, new_file);
251254
copyFile(old_file, new_file, map->nspname, map->relname);
252255
break;
256+
case TRANSFER_MODE_COPY_FILE_RANGE:
257+
pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\" with copy_file_range",
258+
old_file, new_file);
259+
copyFileByRange(old_file, new_file, map->nspname, map->relname);
260+
break;
253261
case TRANSFER_MODE_LINK:
254262
pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"",
255263
old_file, new_file);

‎src/include/pg_config.h.in

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@
7878
/* Define to 1 if you have the <copyfile.h> header file. */
7979
#undef HAVE_COPYFILE_H
8080

81+
/* Define to 1 if you have the `copy_file_range' function. */
82+
#undef HAVE_COPY_FILE_RANGE
83+
8184
/* Define to 1 if you have the <crtdefs.h> header file. */
8285
#undef HAVE_CRTDEFS_H
8386

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy