From db9dfe74b424a75911aef50e480add88c4950de5 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 31 Oct 2021 03:09:11 +0300 Subject: [PATCH 01/10] [PGPRO-5771] Replace macro COMP_CRC32C with handler function. Tags: ptrack --- engine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine.c b/engine.c index 42fa65a..649f7fa 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#include "port/pg_crc32c.h" +#include "common/checksum_helper.h" #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -73,7 +73,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32C(*crc, (char *) chunk, size); + pg_comp_crc32c_handler(crc, (char *) chunk, size); if (write(fd, chunk, size) != size) { @@ -193,7 +193,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); + pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); From c5d59068160ab0c7c8d7fd5b3bcf75852cab7db5 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 31 Oct 2021 16:11:26 +0300 Subject: [PATCH 02/10] [PGPRO-5771] Add pg_comp_crc32c_handler and restore pg_comp_crc32c pointer definition in REL_14_STABLE core patch. Tags: ptrack --- patches/REL_14_STABLE-ptrack-core.diff | 99 ++++++++++++++++---------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index 3491700..f08081a 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,14 +1,17 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 31aad7adad5d9f977e3a3623108ff81b82181d68 +Author: Anton A. Melnikov +Date: Fri Oct 29 08:45:13 2021 +0300 - add ptrack 2.0 + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c + pointer definition. + + Tags: ptrack diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 50ae1f16d0..721b926ad2 100644 +index e09108d0ec..6858c7b670 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -22,7 +25,7 @@ index 50ae1f16d0..721b926ad2 100644 /* end of list */ {NULL, false} }; -@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -240,6 +247,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,7 +38,7 @@ index 50ae1f16d0..721b926ad2 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 0cf598dd0c..c9c44a4ae7 100644 +index da8b7cbeca..6a817f45a5 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ @@ -58,7 +61,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index b4bca7eed6..90d479526c 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec @@ -70,7 +73,7 @@ index 0eacd461cd..c2ef404a1a 100644 /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); @@ -80,7 +83,7 @@ index 0eacd461cd..c2ef404a1a 100644 } /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); @@ -91,10 +94,10 @@ index 0eacd461cd..c2ef404a1a 100644 /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index bc3ceb2712..6c72d24c95 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -78,6 +78,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; @@ -103,7 +106,7 @@ index 3ded2cdd71..3a596a59f7 100644 /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -441,6 +443,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; @@ -114,10 +117,10 @@ index 3ded2cdd71..3a596a59f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 +index 6652a60ec3..051cd6359d 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c -@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; +@@ -659,7 +659,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; @@ -126,7 +129,7 @@ index 1683629ee3..d2fc154576 100644 char *role_string; diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 831cf42d3a..891302ba2f 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,10 +145,10 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index f8d5ecb6ad..4027bdf3df 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); @@ -153,7 +156,7 @@ index 233441837f..cf7bd073bf 100644 static void WriteEmptyXLOG(void); static void usage(void); -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); @@ -161,7 +164,7 @@ index 233441837f..cf7bd073bf 100644 WriteEmptyXLOG(); printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) +@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } @@ -216,10 +219,10 @@ index 233441837f..cf7bd073bf 100644 /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index fbb97b5cf1..6cd7f2ae3e 100644 +index 2618b4c957..bbba813fe7 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -230,11 +233,37 @@ index fbb97b5cf1..6cd7f2ae3e 100644 /* end of list */ {NULL, false} }; +diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c +index 431e247d59..3d042ebb77 100644 +--- a/src/common/checksum_helper.c ++++ b/src/common/checksum_helper.c +@@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) + Assert(retval <= PG_CHECKSUM_MAX_LENGTH); + return retval; + } ++ ++pg_crc32c pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} +diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h +index cac7570ea1..3f7287371b 100644 +--- a/src/include/common/checksum_helper.h ++++ b/src/include/common/checksum_helper.h +@@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); + extern int pg_checksum_update(pg_checksum_context *, const uint8 *input, + size_t len); + extern int pg_checksum_final(pg_checksum_context *, uint8 *output); ++extern pg_crc32c ++pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len); + + #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 +index 3f155ce4f8..ce8dc0bee5 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode +@@ -401,7 +401,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; @@ -244,25 +273,19 @@ index 72e3352398..5c2e016501 100644 #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 +index f3c4107ff9..05b2e63775 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) +@@ -70,7 +70,6 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 5d28f59c1d..0d3f04d8af 100644 +index 2c3936b0da..c6691c0be0 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ @@ -276,7 +299,7 @@ index 5d28f59c1d..0d3f04d8af 100644 extern void copy_file(char *fromfile, char *tofile); diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 07fd1bb7d0..5294811bc8 100644 +index 752b440864..61c18a169f 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ @@ -294,10 +317,10 @@ index 07fd1bb7d0..5294811bc8 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index e16ab8e711..88da9686eb 100644 +index fbdf34f762..455de202b6 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -50,6 +50,9 @@ typedef struct FileTag +@@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag; From dc3530b69eec224771c03b3c485cdc3deeacc411 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Tue, 2 Nov 2021 14:04:33 +0300 Subject: [PATCH 03/10] [PGPRO-5771] Add macros to use pg_comp_crc32c_handler() in windows only. Tags: ptrack --- engine.c | 13 +++++++++++++ pg_probackup | 1 + 2 files changed, 14 insertions(+) create mode 160000 pg_probackup diff --git a/engine.c b/engine.c index 649f7fa..e9b6bf7 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,11 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" +#ifdef WIN32 #include "common/checksum_helper.h" +#else +#include "port/pg_crc32c.h" +#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -73,7 +77,12 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { + +#ifdef WIN32 pg_comp_crc32c_handler(crc, (char *) chunk, size); +#else + COMP_CRC32C(*crc, (char *) chunk, size); +#endif if (write(fd, chunk, size) != size) { @@ -193,7 +202,11 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); +#ifdef WIN32 pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); +#else + COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); +#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/pg_probackup b/pg_probackup new file mode 160000 index 0000000..5b6ca62 --- /dev/null +++ b/pg_probackup @@ -0,0 +1 @@ +Subproject commit 5b6ca624170e1b7955c293ce0173a812b6402d80 From f9e9dcda0fa3e889aa609fa5900f33c580691288 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 7 Nov 2021 23:27:46 +0300 Subject: [PATCH 04/10] [PGPRO-5771] Rename pg_comp_crc32c_handler() to comp_crc32c() Tags: ptrack --- engine.c | 4 ++-- patches/REL_14_STABLE-ptrack-core.diff | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/engine.c b/engine.c index e9b6bf7..d6733c4 100644 --- a/engine.c +++ b/engine.c @@ -79,7 +79,7 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { #ifdef WIN32 - pg_comp_crc32c_handler(crc, (char *) chunk, size); + comp_crc32c(crc, (char *) chunk, size); #else COMP_CRC32C(*crc, (char *) chunk, size); #endif @@ -203,7 +203,7 @@ ptrackMapReadFromFile(const char *ptrack_path) INIT_CRC32C(crc); #ifdef WIN32 - pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); + comp_crc32c(&crc, (char *) ptrack_map, PtrackCrcOffset); #else COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); #endif diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index f08081a..bb2d95b 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,9 +1,9 @@ -commit 31aad7adad5d9f977e3a3623108ff81b82181d68 +commit 84a43938a670336fbb203b85c7f5ea15d642aec2 Author: Anton A. Melnikov Date: Fri Oct 29 08:45:13 2021 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c - pointer definition. + [PGPRO-5771] Add wrapper function for COMP_CRC32C and restore + pg_comp_crc32c pointer definition. Tags: ptrack @@ -234,7 +234,7 @@ index 2618b4c957..bbba813fe7 100644 {NULL, false} }; diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c -index 431e247d59..3d042ebb77 100644 +index 431e247d59..405247a597 100644 --- a/src/common/checksum_helper.c +++ b/src/common/checksum_helper.c @@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) @@ -242,13 +242,13 @@ index 431e247d59..3d042ebb77 100644 return retval; } + -+pg_crc32c pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len) ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) +{ + COMP_CRC32C(*crc, data, len); + return *crc; +} diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h -index cac7570ea1..3f7287371b 100644 +index cac7570ea1..8c002860cf 100644 --- a/src/include/common/checksum_helper.h +++ b/src/include/common/checksum_helper.h @@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); @@ -256,7 +256,7 @@ index cac7570ea1..3f7287371b 100644 size_t len); extern int pg_checksum_final(pg_checksum_context *, uint8 *output); +extern pg_crc32c -+pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len); ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h From 06b27cf1cada1d9e022ab25e0015c5e59f6e7595 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Mon, 8 Nov 2021 07:29:38 +0300 Subject: [PATCH 05/10] [PGPRO-5771] Move comp_crc32c() to checksum_impl.h There are no checksum_helper interface in 11 and 12 versions. Tags: ptrack --- engine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine.c b/engine.c index d6733c4..9fbec3e 100644 --- a/engine.c +++ b/engine.c @@ -32,7 +32,7 @@ #include "catalog/pg_tablespace.h" #include "miscadmin.h" #ifdef WIN32 -#include "common/checksum_helper.h" +#include "storage/checksum.h" #else #include "port/pg_crc32c.h" #endif From 099b0b0e928fd3220cb2ca93a5f94fdd198fa3a1 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Mon, 8 Nov 2021 08:32:26 +0300 Subject: [PATCH 06/10] [PGPRO-5771] Update core patches for REL_11_STABLE, REL_12_STABLE, REL_13_STABLE and REL_14_STABLE. Tags: ptrack --- patches/REL_11_STABLE-ptrack-core.diff | 138 +++++++++++++--------- patches/REL_12_STABLE-ptrack-core.diff | 152 +++++++++++++++---------- patches/REL_13_STABLE-ptrack-core.diff | 142 +++++++++++++---------- patches/REL_14_STABLE-ptrack-core.diff | 141 +++++++++++------------ 4 files changed, 321 insertions(+), 252 deletions(-) diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff index a8207f5..bf4a834 100644 --- a/patches/REL_11_STABLE-ptrack-core.diff +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -1,11 +1,17 @@ +commit 5255ff7a6051f56689e0db2a0fa7e9a6e4086d66 +Author: Anton A. Melnikov +Date: Mon Nov 8 11:11:45 2021 +0300 + + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3e53b3df6fb..f76bfc2a646 100644 +index 3e53b3df6f..f76bfc2a64 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -28,13 +34,13 @@ index 3e53b3df6fb..f76bfc2a646 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 4a0d23b11e3..d59009a4c8c 100644 +index 4a0d23b11e..d59009a4c8 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -43,7 +49,7 @@ index 4a0d23b11e3..d59009a4c8c 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -51,50 +57,50 @@ index 4a0d23b11e3..d59009a4c8c 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 200cc7f657a..d0dcb5c0287 100644 +index fa29e7041f..8bda639eda 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -39,6 +39,7 @@ #include "utils/memutils.h" #include "pg_trace.h" - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; - + /* intervals for calling AbsorbFsyncRequests in mdsync and mdpostckpt */ #define FSYNCS_PER_ABSORB 10 @@ -114,6 +115,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* * In some contexts (currently, standalone backends and the checkpointer) -@@ -558,6 +561,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -600,6 +603,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -851,6 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -893,6 +899,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -1329,6 +1338,9 @@ mdsync(void) +@@ -1371,6 +1380,9 @@ mdsync(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -102,29 +108,29 @@ index 200cc7f657a..d0dcb5c0287 100644 mdsync_in_progress = false; } diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 6fb403a5a8a..6e31ccb3e0f 100644 +index 611edf6ade..ec1c1212bd 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -516,6 +517,7 @@ main(int argc, char *argv[]) + +@@ -525,6 +526,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1201,6 +1203,57 @@ KillExistingArchiveStatus(void) +@@ -1210,6 +1212,57 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -176,17 +182,17 @@ index 6fb403a5a8a..6e31ccb3e0f 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 197163d5544..fc846e78175 100644 +index 8ea7fafa27..997168fcac 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -118,6 +118,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -195,57 +201,77 @@ index 197163d5544..fc846e78175 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 80241455357..50dca7bf6f4 100644 +index 7c9f319b67..1e29827030 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -367,7 +367,7 @@ typedef enum ProcessingMode +@@ -379,7 +379,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 9a26295c8e8..dc72b27a10d 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 433755e279..de06d3b0cf 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index a49d27febb..459c938018 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (checksum % 65535) + 1; + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 4fef3e21072..e55430879c3 100644 +index 4fef3e2107..e55430879c 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h -index 0298ed1a2bc..24c684771d0 100644 +index 0298ed1a2b..24c684771d 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -116,6 +116,17 @@ extern void AtEOXact_SMgr(void); /* internals: move me elsewhere -- ay 7/94 */ - + /* in md.c */ + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, diff --git a/patches/REL_12_STABLE-ptrack-core.diff b/patches/REL_12_STABLE-ptrack-core.diff index d8c00e0..ba246ce 100644 --- a/patches/REL_12_STABLE-ptrack-core.diff +++ b/patches/REL_12_STABLE-ptrack-core.diff @@ -1,11 +1,17 @@ +commit 0504f29dfe157db0bb24c3a1196476e58b1ee263 +Author: Anton A. Melnikov +Date: Mon Nov 8 10:53:11 2021 +0300 + + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3bc26568eb7..aa282bfe0ab 100644 +index efc458d80b..3479214b00 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -210,6 +210,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -16,7 +22,7 @@ index 3bc26568eb7..aa282bfe0ab 100644 /* end of list */ {NULL, false} }; -@@ -225,6 +232,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -224,6 +231,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -29,13 +35,13 @@ index 3bc26568eb7..aa282bfe0ab 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 30f6200a86f..53e3b22c3e4 100644 +index 30f6200a86..53e3b22c3e 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -44,7 +50,7 @@ index 30f6200a86f..53e3b22c3e4 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -52,46 +58,46 @@ index 30f6200a86f..53e3b22c3e4 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 050cee5f9a9..75cf67d464f 100644 +index 0c0e26b522..7d70db2778 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -86,6 +86,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -422,6 +424,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -461,6 +463,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -692,6 +697,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -731,6 +736,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index aff3e885f36..4fffa5df17c 100644 +index aff3e885f3..4fffa5df17 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ @@ -100,7 +106,7 @@ index aff3e885f36..4fffa5df17c 100644 @@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -108,19 +114,19 @@ index aff3e885f36..4fffa5df17c 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index a70e79c4891..712f985f3e8 100644 +index 1dd8c5674f..741e0c4e92 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -581,7 +581,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 03c3da3d730..fdfe5c1318e 100644 +index 2a749266ad..d8184e6ddc 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -113,6 +113,11 @@ static const struct exclude_list_item skip[] = { @@ -136,29 +142,29 @@ index 03c3da3d730..fdfe5c1318e 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 349347593cf..99d1e0a4fc0 100644 +index fc79896126..129c587f7e 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1121,6 +1123,53 @@ KillExistingArchiveStatus(void) +@@ -1130,6 +1132,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -206,17 +212,17 @@ index 349347593cf..99d1e0a4fc0 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 56f83d2fb2f..60bb7bf7a3b 100644 +index 72299f00ea..2548143da5 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -117,6 +117,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -225,58 +231,78 @@ index 56f83d2fb2f..60bb7bf7a3b 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 61a24c2e3c6..cbd46d0cb02 100644 +index 11e9f5fcc9..045a20f3f4 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -369,7 +369,7 @@ typedef enum ProcessingMode +@@ -381,7 +381,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index fbd079d2439..01682035e0b 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 7ef32a3baa..3778c8f773 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 4b87610796..e202b25ccc 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (checksum % 65535) + 1; + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 525cc6203e1..9481e1c5a88 100644 +index 525cc6203e..9481e1c5a8 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index df24b931613..b32c1e9500f 100644 +index df24b93161..b32c1e9500 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -288,13 +314,13 @@ index df24b931613..b32c1e9500f 100644 extern void mdinit(void); extern void mdclose(SMgrRelation reln, ForkNumber forknum); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index 16428c5f5fb..6b0cd8f8eea 100644 +index 16428c5f5f..6b0cd8f8ee 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_13_STABLE-ptrack-core.diff b/patches/REL_13_STABLE-ptrack-core.diff index 3491700..6cde3fa 100644 --- a/patches/REL_13_STABLE-ptrack-core.diff +++ b/patches/REL_13_STABLE-ptrack-core.diff @@ -1,8 +1,8 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 2ffe8b4a2c2cb65a327ac85a02f52d8b74f244f9 +Author: Anton A. Melnikov +Date: Mon Nov 8 10:36:20 2021 +0300 - add ptrack 2.0 + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 50ae1f16d0..721b926ad2 100644 @@ -11,7 +11,7 @@ index 50ae1f16d0..721b926ad2 100644 @@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -41,7 +41,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index 51617e17c1..ce9938e533 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -474,6 +476,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -760,6 +765,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index 4ce2e687d3..e9ba3fe903 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -76,6 +76,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -444,6 +446,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index 3ded2cdd71..3a596a59f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 +index f7a3297161..ff9157697f 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 52e873159d..5cc334b44c 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index c8cbd11fc2..f318d4c1a3 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) +@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,7 +212,7 @@ index 233441837f..cf7bd073bf 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -222,7 +222,7 @@ index fbb97b5cf1..6cd7f2ae3e 100644 @@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,36 +231,56 @@ index fbb97b5cf1..6cd7f2ae3e 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 +index 6b2b4343a0..66b6d24e0a 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode +@@ -400,7 +400,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 6e77744cbc..81a11cffcc 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 364acfafd2..f4813e22aa 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h @@ -268,13 +288,13 @@ index 5d28f59c1d..0d3f04d8af 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h @@ -282,7 +302,7 @@ index 07fd1bb7d0..5294811bc8 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -300,7 +320,7 @@ index e16ab8e711..88da9686eb 100644 @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index bb2d95b..8dfbac4 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,11 +1,8 @@ -commit 84a43938a670336fbb203b85c7f5ea15d642aec2 +commit 6b7ba7c634fb34bfaad63b0b9184fb57584880db Author: Anton A. Melnikov -Date: Fri Oct 29 08:45:13 2021 +0300 +Date: Mon Nov 8 09:19:07 2021 +0300 - [PGPRO-5771] Add wrapper function for COMP_CRC32C and restore - pg_comp_crc32c pointer definition. - - Tags: ptrack + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index e09108d0ec..6858c7b670 100644 @@ -14,7 +11,7 @@ index e09108d0ec..6858c7b670 100644 @@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -44,7 +41,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -53,7 +50,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -65,51 +62,51 @@ index b4bca7eed6..90d479526c 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ @@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* @@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index bc3ceb2712..6c72d24c95 100644 +index a12b357275..e52c2910f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -78,6 +78,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -79,6 +79,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -441,6 +443,9 @@ ProcessSyncRequests(void) +@@ -465,6 +467,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -123,11 +120,11 @@ index 6652a60ec3..051cd6359d 100644 @@ -659,7 +659,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 831cf42d3a..891302ba2f 100644 --- a/src/bin/pg_checksums/pg_checksums.c @@ -155,19 +152,19 @@ index f8d5ecb6ad..4027bdf3df 100644 +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - + @@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); @@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -215,7 +212,7 @@ index f8d5ecb6ad..4027bdf3df 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -225,7 +222,7 @@ index 2618b4c957..bbba813fe7 100644 @@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -233,32 +230,6 @@ index 2618b4c957..bbba813fe7 100644 /* end of list */ {NULL, false} }; -diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c -index 431e247d59..405247a597 100644 ---- a/src/common/checksum_helper.c -+++ b/src/common/checksum_helper.c -@@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) - Assert(retval <= PG_CHECKSUM_MAX_LENGTH); - return retval; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} -diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h -index cac7570ea1..8c002860cf 100644 ---- a/src/include/common/checksum_helper.h -+++ b/src/include/common/checksum_helper.h -@@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); - extern int pg_checksum_update(pg_checksum_context *, const uint8 *input, - size_t len); - extern int pg_checksum_final(pg_checksum_context *, uint8 *output); -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); - - #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 3f155ce4f8..ce8dc0bee5 100644 --- a/src/include/miscadmin.h @@ -266,24 +237,50 @@ index 3f155ce4f8..ce8dc0bee5 100644 @@ -401,7 +401,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index f3c4107ff9..05b2e63775 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -70,7 +70,6 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); - extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 80d2359192..ac55286088 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index ef30268330..19a0248538 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 2c3936b0da..c6691c0be0 100644 --- a/src/include/storage/copydir.h @@ -291,13 +288,13 @@ index 2c3936b0da..c6691c0be0 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 752b440864..61c18a169f 100644 --- a/src/include/storage/md.h @@ -305,7 +302,7 @@ index 752b440864..61c18a169f 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -323,7 +320,7 @@ index fbdf34f762..455de202b6 100644 @@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + From 0e000fa5a796cda3df6f2040adf5c6996400df7d Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 23:36:03 +0300 Subject: [PATCH 07/10] [PGPRO-5771] Revert changes for stable branches (last 6 commits). At first we decided to solve this task for master branch only. Tags: ptrack --- engine.c | 13 -- patches/REL_11_STABLE-ptrack-core.diff | 138 +++++++++------------ patches/REL_12_STABLE-ptrack-core.diff | 152 ++++++++++------------- patches/REL_13_STABLE-ptrack-core.diff | 142 +++++++++------------ patches/REL_14_STABLE-ptrack-core.diff | 164 +++++++++++-------------- pg_probackup | 1 - 6 files changed, 252 insertions(+), 358 deletions(-) delete mode 160000 pg_probackup diff --git a/engine.c b/engine.c index 9fbec3e..42fa65a 100644 --- a/engine.c +++ b/engine.c @@ -31,11 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#ifdef WIN32 -#include "storage/checksum.h" -#else #include "port/pg_crc32c.h" -#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,12 +73,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - -#ifdef WIN32 - comp_crc32c(crc, (char *) chunk, size); -#else COMP_CRC32C(*crc, (char *) chunk, size); -#endif if (write(fd, chunk, size) != size) { @@ -202,11 +193,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); -#ifdef WIN32 - comp_crc32c(&crc, (char *) ptrack_map, PtrackCrcOffset); -#else COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); -#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff index bf4a834..a8207f5 100644 --- a/patches/REL_11_STABLE-ptrack-core.diff +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -1,17 +1,11 @@ -commit 5255ff7a6051f56689e0db2a0fa7e9a6e4086d66 -Author: Anton A. Melnikov -Date: Mon Nov 8 11:11:45 2021 +0300 - - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. - diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3e53b3df6f..f76bfc2a64 100644 +index 3e53b3df6fb..f76bfc2a646 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -34,13 +28,13 @@ index 3e53b3df6f..f76bfc2a64 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 4a0d23b11e..d59009a4c8 100644 +index 4a0d23b11e3..d59009a4c8c 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -49,7 +43,7 @@ index 4a0d23b11e..d59009a4c8 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -57,50 +51,50 @@ index 4a0d23b11e..d59009a4c8 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index fa29e7041f..8bda639eda 100644 +index 200cc7f657a..d0dcb5c0287 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -39,6 +39,7 @@ #include "utils/memutils.h" #include "pg_trace.h" - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; - + /* intervals for calling AbsorbFsyncRequests in mdsync and mdpostckpt */ #define FSYNCS_PER_ABSORB 10 @@ -114,6 +115,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* * In some contexts (currently, standalone backends and the checkpointer) -@@ -600,6 +603,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -558,6 +561,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -893,6 +899,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -851,6 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -1371,6 +1380,9 @@ mdsync(void) +@@ -1329,6 +1338,9 @@ mdsync(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -108,29 +102,29 @@ index fa29e7041f..8bda639eda 100644 mdsync_in_progress = false; } diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 611edf6ade..ec1c1212bd 100644 +index 6fb403a5a8a..6e31ccb3e0f 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -525,6 +526,7 @@ main(int argc, char *argv[]) + +@@ -516,6 +517,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1210,6 +1212,57 @@ KillExistingArchiveStatus(void) +@@ -1201,6 +1203,57 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -182,17 +176,17 @@ index 611edf6ade..ec1c1212bd 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 8ea7fafa27..997168fcac 100644 +index 197163d5544..fc846e78175 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -118,6 +118,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -201,77 +195,57 @@ index 8ea7fafa27..997168fcac 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 7c9f319b67..1e29827030 100644 +index 80241455357..50dca7bf6f4 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -379,7 +379,7 @@ typedef enum ProcessingMode +@@ -367,7 +367,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 433755e279..de06d3b0cf 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index a49d27febb..459c938018 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (checksum % 65535) + 1; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 9a26295c8e8..dc72b27a10d 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 4fef3e2107..e55430879c 100644 +index 4fef3e21072..e55430879c3 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h -index 0298ed1a2b..24c684771d 100644 +index 0298ed1a2bc..24c684771d0 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -116,6 +116,17 @@ extern void AtEOXact_SMgr(void); /* internals: move me elsewhere -- ay 7/94 */ - + /* in md.c */ + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, diff --git a/patches/REL_12_STABLE-ptrack-core.diff b/patches/REL_12_STABLE-ptrack-core.diff index ba246ce..d8c00e0 100644 --- a/patches/REL_12_STABLE-ptrack-core.diff +++ b/patches/REL_12_STABLE-ptrack-core.diff @@ -1,17 +1,11 @@ -commit 0504f29dfe157db0bb24c3a1196476e58b1ee263 -Author: Anton A. Melnikov -Date: Mon Nov 8 10:53:11 2021 +0300 - - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. - diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index efc458d80b..3479214b00 100644 +index 3bc26568eb7..aa282bfe0ab 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -210,6 +210,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -22,7 +16,7 @@ index efc458d80b..3479214b00 100644 /* end of list */ {NULL, false} }; -@@ -224,6 +231,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -225,6 +232,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,13 +29,13 @@ index efc458d80b..3479214b00 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 30f6200a86..53e3b22c3e 100644 +index 30f6200a86f..53e3b22c3e4 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +44,7 @@ index 30f6200a86..53e3b22c3e 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,46 +52,46 @@ index 30f6200a86..53e3b22c3e 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0c0e26b522..7d70db2778 100644 +index 050cee5f9a9..75cf67d464f 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -86,6 +86,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -461,6 +463,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -422,6 +424,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -731,6 +736,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -692,6 +697,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index aff3e885f3..4fffa5df17 100644 +index aff3e885f36..4fffa5df17c 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ @@ -106,7 +100,7 @@ index aff3e885f3..4fffa5df17 100644 @@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +108,19 @@ index aff3e885f3..4fffa5df17 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1dd8c5674f..741e0c4e92 100644 +index a70e79c4891..712f985f3e8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -581,7 +581,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 2a749266ad..d8184e6ddc 100644 +index 03c3da3d730..fdfe5c1318e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -113,6 +113,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +136,29 @@ index 2a749266ad..d8184e6ddc 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index fc79896126..129c587f7e 100644 +index 349347593cf..99d1e0a4fc0 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1130,6 +1132,53 @@ KillExistingArchiveStatus(void) +@@ -1121,6 +1123,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,17 +206,17 @@ index fc79896126..129c587f7e 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 72299f00ea..2548143da5 100644 +index 56f83d2fb2f..60bb7bf7a3b 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -117,6 +117,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,78 +225,58 @@ index 72299f00ea..2548143da5 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 11e9f5fcc9..045a20f3f4 100644 +index 61a24c2e3c6..cbd46d0cb02 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -381,7 +381,7 @@ typedef enum ProcessingMode +@@ -369,7 +369,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 7ef32a3baa..3778c8f773 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index 4b87610796..e202b25ccc 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (checksum % 65535) + 1; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index fbd079d2439..01682035e0b 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 525cc6203e..9481e1c5a8 100644 +index 525cc6203e1..9481e1c5a88 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index df24b93161..b32c1e9500 100644 +index df24b931613..b32c1e9500f 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -314,13 +288,13 @@ index df24b93161..b32c1e9500 100644 extern void mdinit(void); extern void mdclose(SMgrRelation reln, ForkNumber forknum); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index 16428c5f5f..6b0cd8f8ee 100644 +index 16428c5f5fb..6b0cd8f8eea 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_13_STABLE-ptrack-core.diff b/patches/REL_13_STABLE-ptrack-core.diff index 6cde3fa..3491700 100644 --- a/patches/REL_13_STABLE-ptrack-core.diff +++ b/patches/REL_13_STABLE-ptrack-core.diff @@ -1,8 +1,8 @@ -commit 2ffe8b4a2c2cb65a327ac85a02f52d8b74f244f9 -Author: Anton A. Melnikov -Date: Mon Nov 8 10:36:20 2021 +0300 +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + add ptrack 2.0 diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 50ae1f16d0..721b926ad2 100644 @@ -11,7 +11,7 @@ index 50ae1f16d0..721b926ad2 100644 @@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -41,7 +41,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 51617e17c1..ce9938e533 100644 +index 0eacd461cd..c2ef404a1a 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -474,6 +476,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -760,6 +765,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 4ce2e687d3..e9ba3fe903 100644 +index 3ded2cdd71..3a596a59f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -76,6 +76,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -444,6 +446,9 @@ ProcessSyncRequests(void) +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index 4ce2e687d3..e9ba3fe903 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index f7a3297161..ff9157697f 100644 +index 1683629ee3..d2fc154576 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 52e873159d..5cc334b44c 100644 +index ffdc23945c..7ae95866ce 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index 52e873159d..5cc334b44c 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index c8cbd11fc2..f318d4c1a3 100644 +index 233441837f..cf7bd073bf 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,7 +212,7 @@ index c8cbd11fc2..f318d4c1a3 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -222,7 +222,7 @@ index fbb97b5cf1..6cd7f2ae3e 100644 @@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,56 +231,36 @@ index fbb97b5cf1..6cd7f2ae3e 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 6b2b4343a0..66b6d24e0a 100644 +index 72e3352398..5c2e016501 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -400,7 +400,7 @@ typedef enum ProcessingMode +@@ -388,7 +388,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 6e77744cbc..81a11cffcc 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index 364acfafd2..f4813e22aa 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (uint16) ((checksum % 65535) + 1); - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h @@ -288,13 +268,13 @@ index 5d28f59c1d..0d3f04d8af 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h @@ -302,7 +282,7 @@ index 07fd1bb7d0..5294811bc8 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -320,7 +300,7 @@ index e16ab8e711..88da9686eb 100644 @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index 8dfbac4..3491700 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,17 +1,17 @@ -commit 6b7ba7c634fb34bfaad63b0b9184fb57584880db -Author: Anton A. Melnikov -Date: Mon Nov 8 09:19:07 2021 +0300 +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + add ptrack 2.0 diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index e09108d0ec..6858c7b670 100644 +index 50ae1f16d0..721b926ad2 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -22,7 +22,7 @@ index e09108d0ec..6858c7b670 100644 /* end of list */ {NULL, false} }; -@@ -240,6 +247,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,13 +35,13 @@ index e09108d0ec..6858c7b670 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index da8b7cbeca..6a817f45a5 100644 +index 0cf598dd0c..c9c44a4ae7 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index da8b7cbeca..6a817f45a5 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index b4bca7eed6..90d479526c 100644 +index 0eacd461cd..c2ef404a1a 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index a12b357275..e52c2910f7 100644 +index 3ded2cdd71..3a596a59f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -79,6 +79,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -465,6 +467,9 @@ ProcessSyncRequests(void) +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index a12b357275..e52c2910f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 6652a60ec3..051cd6359d 100644 +index 1683629ee3..d2fc154576 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c -@@ -659,7 +659,6 @@ static char *recovery_target_xid_string; +@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 831cf42d3a..891302ba2f 100644 +index ffdc23945c..7ae95866ce 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index 831cf42d3a..891302ba2f 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index f8d5ecb6ad..4027bdf3df 100644 +index 233441837f..cf7bd073bf 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,17 +212,17 @@ index f8d5ecb6ad..4027bdf3df 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 2618b4c957..bbba813fe7 100644 +index fbb97b5cf1..6cd7f2ae3e 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,78 +231,58 @@ index 2618b4c957..bbba813fe7 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 3f155ce4f8..ce8dc0bee5 100644 +index 72e3352398..5c2e016501 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -401,7 +401,7 @@ typedef enum ProcessingMode +@@ -388,7 +388,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 80d2359192..ac55286088 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index ef30268330..19a0248538 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (uint16) ((checksum % 65535) + 1); - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 2c3936b0da..c6691c0be0 100644 +index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 752b440864..61c18a169f 100644 +index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -314,13 +294,13 @@ index 752b440864..61c18a169f 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index fbdf34f762..455de202b6 100644 +index e16ab8e711..88da9686eb 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -55,6 +55,9 @@ typedef struct FileTag +@@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/pg_probackup b/pg_probackup deleted file mode 160000 index 5b6ca62..0000000 --- a/pg_probackup +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5b6ca624170e1b7955c293ce0173a812b6402d80 From 6ad12a5db489c9baf619bd399f491fc10f025c8d Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 21:49:23 +0300 Subject: [PATCH 08/10] [PGPRO-5771] Fix make errors with current PG master. Tags: ptrack --- engine.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/engine.c b/engine.c index 42fa65a..b0860b1 100644 --- a/engine.c +++ b/engine.c @@ -39,6 +39,10 @@ #endif #include "storage/reinit.h" #include "storage/smgr.h" +#if PG_VERSION_NUM >= 150000 +#include "storage/fd.h" +#include "access/xlogrecovery.h" +#endif #include "utils/array.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" From 511e341a510d29b88c1dfe97ba093c5e42c75e91 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 22:16:45 +0300 Subject: [PATCH 09/10] [PGPRO-5771] Use common macro for Windows and Linux. Tags: ptrack --- engine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine.c b/engine.c index b0860b1..0d7933f 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#include "port/pg_crc32c.h" +#include "storage/checksum.h" #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,7 +77,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32C(*crc, (char *) chunk, size); + COMP_CRC32_COMMON(*crc, (char *) chunk, size); if (write(fd, chunk, size) != size) { @@ -197,7 +197,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); + COMP_CRC32_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); From 1ad0308c0a6932ee1f3b3282193b4569b1191e34 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 22:37:12 +0300 Subject: [PATCH 10/10] [PGPRO-5771] Use common crc32 macro with PG master only. Tags: ptrack --- engine.c | 16 +++- patches/master-ptrack-core.diff | 136 ++++++++++++++++---------------- 2 files changed, 84 insertions(+), 68 deletions(-) diff --git a/engine.c b/engine.c index 0d7933f..8f65557 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,11 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" +#if PG_VERSION_NUM >= 150000 #include "storage/checksum.h" +#else +#include "port/pg_crc32c.h" +#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,7 +81,11 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32_COMMON(*crc, (char *) chunk, size); +#if PG_VERSION_NUM >= 150000 + COMP_CRC32C_COMMON(*crc, (char *) chunk, size); +#else + COMP_CRC32C(*crc, (char *) chunk, size); +#endif if (write(fd, chunk, size) != size) { @@ -197,7 +205,11 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); +#if PG_VERSION_NUM >= 150000 + COMP_CRC32C_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); +#else + COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); +#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/patches/master-ptrack-core.diff b/patches/master-ptrack-core.diff index 3491700..a756ca0 100644 --- a/patches/master-ptrack-core.diff +++ b/patches/master-ptrack-core.diff @@ -1,14 +1,14 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 98d98792c0dbe589e8baee4c4100e0dc174ccacb +Author: Anton A. Melnikov +Date: Wed Apr 13 09:49:33 2022 +0300 - add ptrack 2.0 + Apply core patch on PG master diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 50ae1f16d0..721b926ad2 100644 +index 67489192a2..5e22017bd1 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -197,6 +197,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -22,7 +22,7 @@ index 50ae1f16d0..721b926ad2 100644 /* end of list */ {NULL, false} }; -@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -212,6 +219,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,7 +35,7 @@ index 50ae1f16d0..721b926ad2 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 0cf598dd0c..c9c44a4ae7 100644 +index 658fd95ba9..eee38eba17 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ @@ -58,7 +58,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index 286dd3f755..2c496b3a6d 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec @@ -70,7 +70,7 @@ index 0eacd461cd..c2ef404a1a 100644 /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -465,6 +467,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); @@ -80,7 +80,7 @@ index 0eacd461cd..c2ef404a1a 100644 } /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -757,6 +762,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); @@ -91,10 +91,10 @@ index 0eacd461cd..c2ef404a1a 100644 /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index c695d816fc..77e2ab1996 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -81,6 +81,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; @@ -103,7 +103,7 @@ index 3ded2cdd71..3a596a59f7 100644 /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -478,6 +480,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; @@ -113,23 +113,11 @@ index 3ded2cdd71..3a596a59f7 100644 /* Flag successful completion of ProcessSyncRequests */ sync_in_progress = false; } -diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 ---- a/src/backend/utils/misc/guc.c -+++ b/src/backend/utils/misc/guc.c -@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; - static char *recovery_target_name_string; - static char *recovery_target_lsn_string; - -- - /* should be static, but commands/variable.c needs to get at this */ - char *role_string; - diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 21dfe1b6ee..266ac1ef40 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c -@@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { +@@ -118,6 +118,11 @@ static const struct exclude_list_item skip[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -142,10 +130,10 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index d4772a2965..66456f7e44 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); @@ -153,7 +141,7 @@ index 233441837f..cf7bd073bf 100644 static void WriteEmptyXLOG(void); static void usage(void); -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) +@@ -488,6 +489,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); @@ -161,8 +149,8 @@ index 233441837f..cf7bd073bf 100644 WriteEmptyXLOG(); printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) - } +@@ -1036,6 +1038,53 @@ KillExistingArchiveStatus(void) + pg_fatal("could not close directory \"%s\": %m", ARCHSTATDIR); } +/* @@ -216,10 +204,10 @@ index 233441837f..cf7bd073bf 100644 /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index fbb97b5cf1..6cd7f2ae3e 100644 +index d61067f6b2..27ec23137d 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -157,6 +157,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -230,39 +218,55 @@ index fbb97b5cf1..6cd7f2ae3e 100644 /* end of list */ {NULL, false} }; -diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 ---- a/src/include/miscadmin.h -+++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode - NormalProcessing /* normal processing */ - } ProcessingMode; +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 1904fabd5a..f9306a6d01 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H --extern ProcessingMode Mode; -+extern PGDLLIMPORT ProcessingMode Mode; + #include "storage/block.h" ++#include "port/pg_crc32c.h" - #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) - #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,18 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT ++#ifdef WIN32 ++/* ++ * Wrapper function for COMP_CRC32C macro. Was added to avoid ++ * FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++ */ ++extern void ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ ++#define COMP_CRC32C_COMMON(crc, data, len) \ ++ comp_crc32c(&(crc), data, len) ++#else ++#define COMP_CRC32C_COMMON COMP_CRC32C ++#endif /* WIN32 */ ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 015f0f1f83..1c5a0ba351 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,11 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++#ifdef WIN32 ++void ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++} +#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 5d28f59c1d..0d3f04d8af 100644 +index 50a26edeb0..af1602f515 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ @@ -276,7 +280,7 @@ index 5d28f59c1d..0d3f04d8af 100644 extern void copy_file(char *fromfile, char *tofile); diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 07fd1bb7d0..5294811bc8 100644 +index 6e46d8d96a..f0967ef484 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ @@ -294,10 +298,10 @@ index 07fd1bb7d0..5294811bc8 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index e16ab8e711..88da9686eb 100644 +index 9737e1eb67..914ad86328 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -50,6 +50,9 @@ typedef struct FileTag +@@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy