From 2ae320a30d617128d96db32a861c6d5a46397f33 Mon Sep 17 00:00:00 2001 From: Daniel Shelepanov Date: Tue, 4 Oct 2022 12:18:52 +0300 Subject: [PATCH] [PBCKP-278] ptrack adapted to hadling cfs relations tags: cfs, ptrack --- engine.c | 76 +++++++++++++++++++ engine.h | 6 ++ ptrack.c | 42 ++++++++--- ptrack.h | 3 + t/002_cfs_compatibility.pl | 148 +++++++++++++++++++++++++++++++++++++ 5 files changed, 264 insertions(+), 11 deletions(-) create mode 100644 t/002_cfs_compatibility.pl diff --git a/engine.c b/engine.c index 2cd1908..23c3ab8 100644 --- a/engine.c +++ b/engine.c @@ -36,6 +36,10 @@ #include "catalog/pg_tablespace.h" #include "miscadmin.h" #include "port/pg_crc32c.h" +#ifdef PGPRO_EE +/* For file_is_in_cfs_tablespace() only. */ +#include "common/cfs_common.h" +#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -91,6 +95,44 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) } } +/* + * Determines whether given file path is a path to a cfm file. + */ +bool +is_cfm_file_path(const char *filepath) { + ssize_t len = strlen(filepath); + + // For this length checks we assume that the filename is at least + // 1 character longer than the corresponding extension ".cfm": + // strlen(".cfm") == 4 therefore we assume that the filename can't be + // shorter than 5 bytes, for example: "5.cfm". + return strlen(filepath) >= 5 && strcmp(&filepath[len-4], ".cfm") == 0; +} + +#ifdef PGPRO_EE +/* + * Determines the relation file size specified by fullpath as if it + * was not compressed. + */ +off_t +get_cfs_relation_file_decompressed_size(RelFileNodeBackend rnode, const char *fullpath, ForkNumber forknum) { + File fd; + int compressor; + off_t size; + + compressor = md_get_compressor_internal(rnode.node, rnode.backend, forknum); + fd = PathNameOpenFile(fullpath, O_RDWR | PG_BINARY, compressor); + + if(fd < 0) + return (off_t)-1; + + size = FileSize(fd); + FileClose(fd); + + return size; +} +#endif + /* * Delete ptrack files when ptrack is disabled. * @@ -498,8 +540,13 @@ assign_ptrack_map_size(int newval, void *extra) * For use in functions that copy directories bypassing buffer manager. */ static void +#ifdef PGPRO_EE +ptrack_mark_file(Oid dbOid, Oid tablespaceOid, + const char *filepath, const char *filename, bool is_cfs) +#else ptrack_mark_file(Oid dbOid, Oid tablespaceOid, const char *filepath, const char *filename) +#endif { RelFileNodeBackend rnode; ForkNumber forknum; @@ -508,6 +555,9 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid, struct stat stat_buf; int oidchars; char oidbuf[OIDCHARS + 1]; +#ifdef PGPRO_EE + off_t rel_size; +#endif /* Do not track temporary relations */ if (looks_like_temp_rel_name(filename)) @@ -526,6 +576,21 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid, oidbuf[oidchars] = '\0'; nodeRel(nodeOf(rnode)) = atooid(oidbuf); +#ifdef PGPRO_EE + // if current tablespace is cfs-compressed and md_get_compressor_internal + // returns the type of the compressing algorithm for filepath, then it + // needs to be de-compressed to obtain its size + if(is_cfs && md_get_compressor_internal(rnode.node, rnode.backend, forknum) != 0) { + rel_size = get_cfs_relation_file_decompressed_size(rnode, filepath, forknum); + + if(rel_size == (off_t)-1) { + elog(WARNING, "ptrack: could not open cfs-compressed relation file: %s", filepath); + return; + } + + nblocks = rel_size / BLCKSZ; + } else +#endif /* Compute number of blocks based on file size */ if (stat(filepath, &stat_buf) == 0) nblocks = stat_buf.st_size / BLCKSZ; @@ -546,6 +611,9 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid) { DIR *dir; struct dirent *de; +#ifdef PGPRO_EE + bool is_cfs; +#endif /* Do not walk during bootstrap and if ptrack is disabled */ if (ptrack_map_size == 0 @@ -554,6 +622,10 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid) || InitializingParallelWorker) return; +#ifdef PGPRO_EE + is_cfs = file_is_in_cfs_tablespace(path); +#endif + dir = AllocateDir(path); while ((de = ReadDirExtended(dir, path, LOG)) != NULL) @@ -581,7 +653,11 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid) } if (S_ISREG(fst.st_mode)) +#ifdef PGPRO_EE + ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name, is_cfs); +#else ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name); +#endif } FreeDir(dir); /* we ignore any error here */ diff --git a/engine.h b/engine.h index 5daf69a..4797a54 100644 --- a/engine.h +++ b/engine.h @@ -111,4 +111,10 @@ extern void ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid); extern void ptrack_mark_block(RelFileNodeBackend smgr_rnode, ForkNumber forkno, BlockNumber blkno); +extern bool is_cfm_file_path(const char *path); +#ifdef PGPRO_EE +extern off_t get_cfs_relation_file_decompressed_size(RelFileNodeBackend rnode, + const char *fullpath, ForkNumber forknum); +#endif + #endif /* PTRACK_ENGINE_H */ diff --git a/ptrack.c b/ptrack.c index d0cbb4a..933227a 100644 --- a/ptrack.c +++ b/ptrack.c @@ -251,14 +251,6 @@ ptrack_copydir_hook(const char *path) elog(DEBUG1, "ptrack_copydir_hook: spcOid %u, dbOid %u", spcOid, dbOid); -#ifdef PGPRO_EE - /* - * Currently, we do not track files from compressed tablespaces in ptrack. - */ - if (file_is_in_cfs_tablespace(path)) - elog(DEBUG1, "ptrack_copydir_hook: skipping changes tracking in the CFS tablespace %u", spcOid); - else -#endif ptrack_walkdir(path, spcOid, dbOid); if (prev_copydir_hook) @@ -302,6 +294,11 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) { DIR *dir; struct dirent *de; +#ifdef PGPRO_EE + bool is_cfs; + + is_cfs = file_is_in_cfs_tablespace(path); +#endif dir = AllocateDir(path); @@ -315,7 +312,8 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0 || - looks_like_temp_rel_name(de->d_name)) + looks_like_temp_rel_name(de->d_name) || + is_cfm_file_path(de->d_name)) continue; snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name); @@ -362,6 +360,10 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid) nodeSpc(pfl->relnode) = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid; pfl->path = GetRelationPath(dbOid, nodeSpc(pfl->relnode), nodeRel(pfl->relnode), InvalidBackendId, pfl->forknum); +#ifdef PGPRO_EE + pfl->is_cfs_compressed = is_cfs + && md_get_compressor_internal(pfl->relnode, InvalidBackendId, pfl->forknum) != 0; +#endif *filelist = lappend(*filelist, pfl); @@ -403,6 +405,10 @@ ptrack_filelist_getnext(PtScanCtx * ctx) ListCell *cell; char *fullpath; struct stat fst; + off_t rel_st_size = 0; +#ifdef PGPRO_EE + RelFileNodeBackend rnodebackend; +#endif /* No more file in the list */ if (list_length(ctx->filelist) == 0) @@ -449,14 +455,28 @@ ptrack_filelist_getnext(PtScanCtx * ctx) return ptrack_filelist_getnext(ctx); } +#ifdef PGPRO_EE + rnodebackend.node = ctx->bid.relnode; + rnodebackend.backend = InvalidBackendId; + + if(pfl->is_cfs_compressed) { + rel_st_size = get_cfs_relation_file_decompressed_size(rnodebackend, fullpath, pfl->forknum); + + // Could not open fullpath for some reason, trying the next file. + if(rel_st_size == -1) + return ptrack_filelist_getnext(ctx); + } else +#endif + rel_st_size = fst.st_size; + if (pfl->segno > 0) { - ctx->relsize = pfl->segno * RELSEG_SIZE + fst.st_size / BLCKSZ; + ctx->relsize = pfl->segno * RELSEG_SIZE + rel_st_size / BLCKSZ; ctx->bid.blocknum = pfl->segno * RELSEG_SIZE; } else /* Estimate relsize as size of first segment in blocks */ - ctx->relsize = fst.st_size / BLCKSZ; + ctx->relsize = rel_st_size / BLCKSZ; elog(DEBUG3, "ptrack: got file %s with size %u from the file list", pfl->path, ctx->relsize); diff --git a/ptrack.h b/ptrack.h index e42bfb8..eec03bb 100644 --- a/ptrack.h +++ b/ptrack.h @@ -78,6 +78,9 @@ typedef struct PtrackFileList_i ForkNumber forknum; int segno; char *path; +#ifdef PGPRO_EE + bool is_cfs_compressed; +#endif } PtrackFileList_i; #endif /* PTRACK_H */ diff --git a/t/002_cfs_compatibility.pl b/t/002_cfs_compatibility.pl new file mode 100644 index 0000000..31e31be --- /dev/null +++ b/t/002_cfs_compatibility.pl @@ -0,0 +1,148 @@ +use strict; +use warnings; +use Test::More; + +my $pg_15_modules; + +BEGIN +{ + $pg_15_modules = eval + { + require PostgreSQL::Test::Cluster; + require PostgreSQL::Test::Utils; + return 1; + }; + + unless (defined $pg_15_modules) + { + $pg_15_modules = 0; + + require PostgresNode; + require TestLib; + } +} + +note('PostgreSQL 15 modules are used: ' . ($pg_15_modules ? 'yes' : 'no')); + +my $node; +my $res_stdout; +my $res_stderr; + +# Create node. +# Older versions of PostgreSQL modules use get_new_node function. +# Newer use standard perl object constructor syntax. +eval +{ + if ($pg_15_modules) + { + $node = PostgreSQL::Test::Cluster->new("node"); + } + else + { + $node = PostgresNode::get_new_node("node"); + } +}; + +note "Test for handling a ptrack map in compressed relations"; + +my $psql_stdout; + +# Starting the node +$node->init; + +# Could not load ptrack module after postmaster start + +my $cfs_tblspc1 = $node->basedir."/cfs_tblspc1"; +my $cfs_tblspc2 = $node->basedir."/cfs_tblspc2"; +mkdir $cfs_tblspc1 or die; +mkdir $cfs_tblspc2 or die; +my $no_cfs_tblspc1 = $node->basedir."/no_cfs_tblspc1"; +my $no_cfs_tblspc2 = $node->basedir."/no_cfs_tblspc2"; +mkdir $no_cfs_tblspc1 or die; +mkdir $no_cfs_tblspc2 or die; + +$node->append_conf('postgresql.conf', qq{ + shared_preload_libraries = 'ptrack' + ptrack.map_size = 16 + wal_level = 'replica' +}); + +$node->start; + +# check cfs availability first +my $cfs_available = $node->safe_psql('postgres', + "select count(oid) from pg_proc where proname = 'cfs_version'"); + +if($cfs_available eq "0") { + $node->stop; + plan skip_all => "CFS is not supported by this PostgreSQL build"; +} else { + plan tests => 2; +} + +# Creating content +$node->safe_psql('postgres', qq| + create tablespace cfs_tblspc1 location '$cfs_tblspc1' with (compression=true); + create tablespace cfs_tblspc2 location '$cfs_tblspc2' with (compression=true); + create tablespace no_cfs_tblspc1 location '$no_cfs_tblspc1'; + create tablespace no_cfs_tblspc2 location '$no_cfs_tblspc2'; + + create database testing_cfs tablespace cfs_tblspc1; + create database testing_no_cfs tablespace no_cfs_tblspc1; +|); + +$node->safe_psql('testing_cfs', qq{ + create table testing(i int, text varchar); + insert into testing select 1, '1111111111111111111111111' from generate_series(1,10000000); +}); + +$node->safe_psql('testing_no_cfs', qq{ + create table testing_no(i int, text varchar); + insert into testing_no select 1, '1111111111111111111111111' from generate_series(1,10000000); +}); + +# creating ptrack +$node->safe_psql('postgres', "create extension ptrack"); + +# obtaining init lsn for further usage in ptrack_get_pagemapset +my $init_lsn = $node->safe_psql('postgres', 'select ptrack_init_lsn()'); + +# forcing copydir() hook by altering dbs tablespaces +$node->safe_psql('postgres', "alter database testing_cfs set tablespace cfs_tblspc2;"); +$node->safe_psql('postgres', "alter database testing_no_cfs set tablespace no_cfs_tblspc2;"); + +# obtaining relpath for cfs table +my $cfs_relpath = $node->safe_psql('testing_cfs', "select pg_relation_filepath('testing');"); + +# obtaining relpath for no-cfs table +my $no_cfs_relpath = $node->safe_psql('testing_no_cfs', "select pg_relation_filepath('testing_no');"); + +# select the pagecount sums and compare them (should be equal) +my $pagecount_sum_cfs = $node->safe_psql('postgres', + "select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';"); +my $pagecount_sum_no_cfs = $node->safe_psql('postgres', + "select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';"); + +is($pagecount_sum_cfs, $pagecount_sum_no_cfs, "pagecount sums don't match"); + +# forcing copydir() hook by altering dbs tablespaces back +$node->safe_psql('postgres', "alter database testing_cfs set tablespace cfs_tblspc1;"); +$node->safe_psql('postgres', "alter database testing_no_cfs set tablespace no_cfs_tblspc1;"); + +# obtaining new relpath for cfs table +$cfs_relpath = $node->safe_psql('testing_cfs', "select pg_relation_filepath('testing');"); + +# obtaining new relpath for no-cfs table +$no_cfs_relpath = $node->safe_psql('testing_no_cfs', "select pg_relation_filepath('testing_no');"); + +# select the pagecount sums and compare them (again, they should be equal) +$pagecount_sum_cfs = $node->safe_psql('postgres', + "select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';"); +$pagecount_sum_no_cfs = $node->safe_psql('postgres', + "select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';"); + +is($pagecount_sum_cfs, $pagecount_sum_no_cfs, "pagecount sums don't match"); + + +$node->stop; + pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy