Skip to content

Commit 172259a

Browse files
alvherrealvherre
committed
Verify roundtrip dump/restore of regression database
Add a test to pg_upgrade's test suite that verifies that dump-restore-dump of regression database produces equivalent output to dumping it directly. This was already being tested by running pg_upgrade itself, but non-binary-upgrade mode was not being covered. The regression database has accrued, over time, a sufficient collection of interesting objects to ensure good coverage, but there hasn't been a concerted effort to be completely exhaustive, so it is likely still possible to have more. This'd belong more naturally in the pg_dump test suite, but we chose to put it in src/bin/pg_upgrade/t/002_pg_upgrade.pl because we need a run of the regression tests which is already done here, so this has less total test runtime impact. Also, experiments have shown that using parallel dump/restore is slightly faster, so we use --format=directory -j2. This test has already reported pg_dump bugs, as fixed in fd41ba9, 74563f6, d611f8b, 4694aed. Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com> Reviewed-by: Michael Paquier <michael@paquier.xyz> Reviewed-by: Daniel Gustafsson <daniel@yesql.se> Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us> Reviewed-by: Álvaro Herrera <alvherre@alvh.no-ip.org> Discussion: https://www.postgresql.org/message-id/CAExHW5uF5V=Cjecx3_Z=7xfh4rg2Wf61PT+hfquzjBqouRzQJQ@mail.gmail.com
1 parent 764d501 commit 172259a

File tree

4 files changed

+254
-3
lines changed

4 files changed

+254
-3
lines changed

‎src/bin/pg_upgrade/t/002_pg_upgrade.pl

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@
1111

1212
use PostgreSQL::Test::Cluster;
1313
use PostgreSQL::Test::Utils;
14+
use PostgreSQL::Test::AdjustDump;
1415
use PostgreSQL::Test::AdjustUpgrade;
1516
use Test::More;
1617

1718
# Can be changed to test the other modes.
1819
my $mode = $ENV{PG_TEST_PG_UPGRADE_MODE} || '--copy';
1920

21+
my $tempdir = PostgreSQL::Test::Utils::tempdir;
22+
2023
# Generate a database with a name made of a range of ASCII characters.
2124
sub generate_db
2225
{
@@ -35,8 +38,8 @@ sub generate_db
3538
"created database with ASCII characters from $from_char to $to_char");
3639
}
3740

38-
# Filter the contents of a dump before its use in a content comparison.
39-
# This returns the path to the filtered dump.
41+
# Filter the contents of a dump before its use in a content comparison for
42+
# upgrade testing. This returns the path to the filtered dump.
4043
sub filter_dump
4144
{
4245
my ($is_old, $old_version, $dump_file) = @_;
@@ -60,6 +63,41 @@ sub filter_dump
6063
return $dump_file_filtered;
6164
}
6265

66+
# Dump database db from the given node in plain format and adjust it for
67+
# comparing dumps from the original and the restored database.
68+
#
69+
# file_prefix is used to create unique names for all dump files so that they
70+
# remain available for debugging in case the test fails.
71+
#
72+
# adjust_child_columns is passed to adjust_regress_dumpfile() which actually
73+
# adjusts the dump output.
74+
#
75+
# The name of the file containting adjusted dump is returned.
76+
sub get_dump_for_comparison
77+
{
78+
my ($node, $db, $file_prefix, $adjust_child_columns) = @_;
79+
80+
my $dumpfile = $tempdir . '/' . $file_prefix . '.sql';
81+
my $dump_adjusted = "${dumpfile}_adjusted";
82+
83+
open(my $dh, '>', $dump_adjusted)
84+
|| die "could not open $dump_adjusted for writing $!";
85+
86+
# Don't dump statistics, because there are still some bugs.
87+
$node->run_log(
88+
[
89+
'pg_dump', '--no-sync', '--no-statistics',
90+
'-d' => $node->connstr($db),
91+
'-f' => $dumpfile
92+
]);
93+
94+
print $dh adjust_regress_dumpfile(slurp_file($dumpfile),
95+
$adjust_child_columns);
96+
close($dh);
97+
98+
return $dump_adjusted;
99+
}
100+
63101
# The test of pg_upgrade requires two clusters, an old one and a new one
64102
# that gets upgraded. Before running the upgrade, a logical dump of the
65103
# old cluster is taken, and a second logical dump of the new one is taken
@@ -80,7 +118,6 @@ sub filter_dump
80118
}
81119

82120
# Paths to the dumps taken during the tests.
83-
my $tempdir = PostgreSQL::Test::Utils::tempdir;
84121
my $dump1_file = "$tempdir/dump1.sql";
85122
my $dump2_file = "$tempdir/dump2.sql";
86123

@@ -264,6 +301,60 @@ sub filter_dump
264301
is($rc, 0, 'regression tests pass');
265302
}
266303

304+
# Test that dump/restore of the regression database roundtrips cleanly. This
305+
# doesn't work well when the nodes are different versions, so skip it in that
306+
# case. Note that this isn't a pg_restore test, but it's convenient to do it
307+
# here because we've gone to the trouble of creating the regression database.
308+
#
309+
# Do this while the old cluster is running before it is shut down by the
310+
# upgrade test.
311+
SKIP:
312+
{
313+
my $dstnode = PostgreSQL::Test::Cluster->new('dst_node');
314+
315+
skip "different Postgres versions"
316+
if ($oldnode->pg_version != $dstnode->pg_version);
317+
skip "source node not using default install"
318+
if (defined $oldnode->install_path);
319+
320+
# Dump the original database for comparison later.
321+
my $src_dump =
322+
get_dump_for_comparison($oldnode, 'regression', 'src_dump', 1);
323+
324+
# Setup destination database cluster
325+
$dstnode->init(%node_params);
326+
# Stabilize stats for comparison.
327+
$dstnode->append_conf('postgresql.conf', 'autovacuum = off');
328+
$dstnode->start;
329+
330+
my $dump_file = "$tempdir/regression.dump";
331+
332+
# Use --create in dump and restore commands so that the restored
333+
# database has the same configurable variable settings as the original
334+
# database so that the dumps taken from both databases taken do not
335+
# differ because of locale changes. Additionally this provides test
336+
# coverage for --create option.
337+
#
338+
# Use directory format so that we can use parallel dump/restore.
339+
$oldnode->command_ok(
340+
[
341+
'pg_dump', '-Fd', '-j2', '--no-sync',
342+
'-d' => $oldnode->connstr('regression'),
343+
'--create', '-f' => $dump_file
344+
],
345+
'pg_dump on source instance');
346+
347+
$dstnode->command_ok(
348+
[ 'pg_restore', '--create', '-j2', '-d' => 'postgres', $dump_file ],
349+
'pg_restore to destination instance');
350+
351+
my $dst_dump =
352+
get_dump_for_comparison($dstnode, 'regression', 'dest_dump', 0);
353+
354+
compare_files($src_dump, $dst_dump,
355+
'dump outputs from original and restored regression databases match');
356+
}
357+
267358
# Initialize a new node for the upgrade.
268359
my $newnode = PostgreSQL::Test::Cluster->new('new_node');
269360

‎src/test/perl/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ install: all installdirs
2525
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/Kerberos.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
2626
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/Cluster.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
2727
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/BackgroundPsql.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
28+
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/AdjustDump.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
2829
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Test/AdjustUpgrade.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
2930
$(INSTALL_DATA) $(srcdir)/PostgreSQL/Version.pm '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
3031

@@ -35,6 +36,7 @@ uninstall:
3536
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Kerberos.pm'
3637
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/Cluster.pm'
3738
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/BackgroundPsql.pm'
39+
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustDump.pm'
3840
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Test/AdjustUpgrade.pm'
3941
rm -f '$(DESTDIR)$(pgxsdir)/$(subdir)/PostgreSQL/Version.pm'
4042

‎src/test/perl/PostgreSQL/Test/AdjustDump.pm

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Copyright (c) 2024-2025, PostgreSQL Global Development Group
2+
3+
=pod
4+
5+
=head1 NAME
6+
7+
PostgreSQL::Test::AdjustDump - helper module for dump/restore tests
8+
9+
=head1 SYNOPSIS
10+
11+
use PostgreSQL::Test::AdjustDump;
12+
13+
# Adjust contents of dump output file so that dump output from original
14+
# regression database and that from the restored regression database match
15+
$dump = adjust_regress_dumpfile($dump, $adjust_child_columns);
16+
17+
=head1 DESCRIPTION
18+
19+
C<PostgreSQL::Test::AdjustDump> encapsulates various hacks needed to
20+
compare the results of dump/restore tests.
21+
22+
=cut
23+
24+
package PostgreSQL::Test::AdjustDump;
25+
26+
use strict;
27+
use warnings FATAL => 'all';
28+
29+
use Exporter 'import';
30+
use Test::More;
31+
32+
our @EXPORT = qw(
33+
adjust_regress_dumpfile
34+
);
35+
36+
=pod
37+
38+
=head1 ROUTINES
39+
40+
=over
41+
42+
=item $dump = adjust_regress_dumpfile($dump, $adjust_child_columns)
43+
44+
Edit a dump output file, taken from the source regression database,
45+
to remove the known differences to a dump taken after restoring the
46+
same database.
47+
48+
Arguments:
49+
50+
=over
51+
52+
=item C<dump>: Contents of dump file
53+
54+
=item C<adjust_child_columns>: 1 indicates that the given dump file requires
55+
adjusting columns in the child tables; usually when the dump is from original
56+
database. 0 indicates no such adjustment is needed; usually when the dump is
57+
from restored database.
58+
59+
=back
60+
61+
Returns the adjusted dump text.
62+
63+
Adjustments Applied:
64+
65+
=over
66+
67+
=item Column reordering on child table creation
68+
69+
This rearranges the column declarations in the C<CREATE TABLE... INHERITS>
70+
statements in the dump file from original database so that they match those
71+
from the restored database.
72+
73+
Only executed if C<adjust_child_columns> is true.
74+
75+
Context: some regression tests purposefully create child tables in such a way
76+
that the order of their inherited columns differ from column orders of their
77+
respective parents. In the restored database, however, the order of their
78+
inherited columns are same as that of their respective parents. Thus the column
79+
orders of these child tables in the original database and those in the restored
80+
database differ, causing difference in the dump outputs. See
81+
C<MergeAttributes()> and C<dumpTableSchema()> for details.
82+
83+
=item Removal of problematic C<COPY> statements
84+
85+
Remove COPY statements to abnormal children tables.
86+
87+
Context: This difference is caused because of columns that are added to parent
88+
tables that already have children; because recreating the children tables puts
89+
the columns from the parent ahead of columns declared locally in children,
90+
these extra columns are in earlier position compared to the original database.
91+
Reordering columns on the entire C<COPY> data is impractical, so we just remove
92+
them.
93+
94+
=item Newline adjustment
95+
96+
Windows-style newlines are changed to Unix-style. Empty lines are trimmed.
97+
98+
=back
99+
100+
=cut
101+
102+
sub adjust_regress_dumpfile
103+
{
104+
my ($dump, $adjust_child_columns) = @_;
105+
106+
# use Unix newlines
107+
$dump =~ s/\r\n/\n/g;
108+
109+
# Adjust the CREATE TABLE ... INHERITS statements.
110+
if ($adjust_child_columns)
111+
{
112+
$dump =~ s/(^CREATE\sTABLE\sgenerated_stored_tests\.gtestxx_4\s\()
113+
(\n\s+b\sinteger),
114+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
115+
116+
$dump =~ s/(^CREATE\sTABLE\sgenerated_virtual_tests\.gtestxx_4\s\()
117+
(\n\s+b\sinteger),
118+
(\n\s+a\sinteger\sNOT\sNULL)/$1$3,$2/mgx;
119+
120+
$dump =~ s/(^CREATE\sTABLE\spublic\.test_type_diff2_c1\s\()
121+
(\n\s+int_four\sbigint),
122+
(\n\s+int_eight\sbigint),
123+
(\n\s+int_two\ssmallint)/$1$4,$2,$3/mgx;
124+
125+
$dump =~ s/(^CREATE\sTABLE\spublic\.test_type_diff2_c2\s\()
126+
(\n\s+int_eight\sbigint),
127+
(\n\s+int_two\ssmallint),
128+
(\n\s+int_four\sbigint)/$1$3,$4,$2/mgx;
129+
}
130+
131+
# Remove COPY statements with differing column order
132+
for my $table (
133+
'public\.b_star', 'public\.c_star',
134+
'public\.cc2', 'public\.d_star',
135+
'public\.e_star', 'public\.f_star',
136+
'public\.renamecolumnanother', 'public\.renamecolumnchild',
137+
'public\.test_type_diff2_c1', 'public\.test_type_diff2_c2',
138+
'public\.test_type_diff_c')
139+
{
140+
# This multiline pattern matches the whole COPY, up to the
141+
# terminating "\."
142+
$dump =~ s/^COPY $table \(.+?^\\\.$//sm;
143+
}
144+
145+
# Suppress blank lines, as some places in pg_dump emit more or fewer.
146+
$dump =~ s/\n\n+/\n/g;
147+
148+
return $dump;
149+
}
150+
151+
=pod
152+
153+
=back
154+
155+
=cut
156+
157+
1;

‎src/test/perl/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ install_data(
1313
'PostgreSQL/Test/Kerberos.pm',
1414
'PostgreSQL/Test/Cluster.pm',
1515
'PostgreSQL/Test/BackgroundPsql.pm',
16+
'PostgreSQL/Test/AdjustDump.pm',
1617
'PostgreSQL/Test/AdjustUpgrade.pm',
1718
install_dir: dir_pgxs / 'src/test/perl/PostgreSQL/Test')

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy