Advance input pointer when LZ4 compressing data

tvondra · tvondra · commit 1a05c1d25299 · 2023-05-17T16:49:34.000+02:00
LZ4File_write() did not advance the input pointer on subsequent invocations of LZ4F_compressUpdate(). As a result the generated compressed output would be a compressed version of the same input chunk. Tests failed to catch this error because the data would comfortably fit within the default buffer size, as a single chunk. Tests have been added to provide adequate coverage of multi-chunk compression. WriteDataToArchiveLZ4() which is also using LZ4F_compressUpdate() did not suffer from this omission. Author: Georgios Kokolatos <gkokolatos@pm.me> Reported-by: Michael Paquier <michael@paquier.xyz> Discussion: https://postgr.es/m/ZFhCyn4Gm2eu60rB%40paquier.xyz
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
@@ -17,7 +17,13 @@
 
 #include "pg_backup_archiver.h"
 
-/* Default size used for IO buffers */
+/*
+ * Default size used for IO buffers
+ *
+ * When changing this value, it's necessary to check the relevant test cases
+ * still exercise all the branches. This applies especially if the value is
+ * increased, in which case the overflow buffer may not be needed.
+ */
 #define DEFAULT_IO_BUFFER_SIZE	4096
 
 extern char *supports_compression(const pg_compress_specification compression_spec);
diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
@@ -588,6 +588,8 @@ LZ4Stream_write(const void *ptr, size_t size, CompressFileHandle *CFH)
 			errno = (errno) ? errno : ENOSPC;
 			return false;
 		}
+
+		ptr = ((const char *) ptr) + chunk;
 	}
 
 	return true;
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -3108,6 +3108,52 @@
 		},
 	},
 
+	'CREATE TABLE test_compression_method' => {
+		create_order => 110,
+		create_sql   => 'CREATE TABLE dump_test.test_compression_method (
+						   col1 text
+					   );',
+		regexp => qr/^
+			\QCREATE TABLE dump_test.test_compression_method (\E\n
+			\s+\Qcol1 text\E\n
+			\Q);\E
+			/xm,
+		like => {
+			%full_runs,
+			%dump_test_schema_runs,
+			section_pre_data     => 1,
+		},
+		unlike => {
+			exclude_dump_test_schema => 1,
+			only_dump_measurement    => 1,
+		},
+	},
+
+	# Insert enough data to surpass DEFAULT_IO_BUFFER_SIZE during
+	# (de)compression operations
+	'COPY test_compression_method' => {
+		create_order => 111,
+		create_sql   => 'INSERT INTO dump_test.test_compression_method (col1) '
+		  . 'SELECT string_agg(a::text, \'\') FROM generate_series(1,4096) a;',
+		regexp => qr/^
+			\QCOPY dump_test.test_compression_method (col1) FROM stdin;\E
+			\n(?:\d{15277}\n){1}\\\.\n
+			/xm,
+		like => {
+			%full_runs,
+			data_only                => 1,
+			section_data             => 1,
+			only_dump_test_schema    => 1,
+			test_schema_plus_large_objects    => 1,
+		},
+		unlike => {
+			binary_upgrade           => 1,
+			exclude_dump_test_schema => 1,
+			schema_only              => 1,
+			only_dump_measurement    => 1,
+		},
+	},
+
 	'CREATE TABLE fk_reference_test_table' => {
 		create_order => 21,
 		create_sql   => 'CREATE TABLE dump_test.fk_reference_test_table (

Original file line number	Diff line number	Diff line change
`@@ -588,6 +588,8 @@ LZ4Stream_write(const void ptr, size_t size, CompressFileHandle CFH)`
`588`	`588`	`errno = (errno) ? errno : ENOSPC;`
`589`	`589`	`return false;`
`590`	`590`	`}`
	`591`	`+`
	`592`	`+ ptr = ((const char *) ptr) + chunk;`
`591`	`593`	`}`
`592`	`594`
`593`	`595`	`return true;`