Skip to content

Commit c7b5c52

Browse files
author
Daniel Shelepanov
committed
[PBCKP-278] ptrack adapted to hadling cfs relations
tags: cfs, ptrack
1 parent 936db26 commit c7b5c52

File tree

5 files changed

+259
-12
lines changed

5 files changed

+259
-12
lines changed

‎engine.c

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
*/
1919

2020
#include "postgres.h"
21-
2221
#include <unistd.h>
2322
#include <sys/stat.h>
2423

@@ -36,6 +35,10 @@
3635
#include "catalog/pg_tablespace.h"
3736
#include "miscadmin.h"
3837
#include "port/pg_crc32c.h"
38+
#ifdef PGPRO_EE
39+
/* For file_is_in_cfs_tablespace() only. */
40+
#include "common/cfs_common.h"
41+
#endif
3942
#include "storage/copydir.h"
4043
#if PG_VERSION_NUM >= 120000
4144
#include "storage/md.h"
@@ -91,6 +94,44 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size)
9194
}
9295
}
9396

97+
/*
98+
* Determines whether given file path is a path to a cfm file.
99+
*/
100+
bool
101+
is_cfm_file_path(const char *filepath) {
102+
ssize_t len = strlen(filepath);
103+
104+
// For this length checks we assume that the filename is at least
105+
// 1 character longer than the corresponding extension ".cfm":
106+
// strlen(".cfm") == 4 therefore we assume that the filename can't be
107+
// shorter than 5 bytes, for example: "5.cfm".
108+
return strlen(filepath) >= 5 && strcmp(&filepath[len-4], ".cfm") == 0;
109+
}
110+
111+
#ifdef PGPRO_EE
112+
/*
113+
* Determines the relation file size specified by fullpath as if it
114+
* was not compressed.
115+
*/
116+
off_t
117+
get_cfs_relation_file_decompressed_size(RelFileNodeBackend rnode, const char *fullpath, ForkNumber forknum) {
118+
File fd;
119+
int compressor;
120+
off_t size;
121+
122+
compressor = md_get_compressor_internal(rnode.node, rnode.backend, forknum);
123+
fd = PathNameOpenFile(fullpath, O_RDWR | PG_BINARY, compressor);
124+
125+
if(fd < 0)
126+
return (off_t)-1;
127+
128+
size = FileSize(fd);
129+
FileClose(fd);
130+
131+
return size;
132+
}
133+
#endif
134+
94135
/*
95136
* Delete ptrack files when ptrack is disabled.
96137
*
@@ -498,8 +539,13 @@ assign_ptrack_map_size(int newval, void *extra)
498539
* For use in functions that copy directories bypassing buffer manager.
499540
*/
500541
static void
542+
#ifdef PGPRO_EE
543+
ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
544+
const char *filepath, const char *filename, bool is_cfs)
545+
#else
501546
ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
502547
const char *filepath, const char *filename)
548+
#endif
503549
{
504550
RelFileNodeBackend rnode;
505551
ForkNumber forknum;
@@ -508,6 +554,9 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
508554
struct stat stat_buf;
509555
int oidchars;
510556
char oidbuf[OIDCHARS + 1];
557+
#ifdef PGPRO_EE
558+
off_t rel_size;
559+
#endif
511560

512561
/* Do not track temporary relations */
513562
if (looks_like_temp_rel_name(filename))
@@ -526,6 +575,21 @@ ptrack_mark_file(Oid dbOid, Oid tablespaceOid,
526575
oidbuf[oidchars] = '\0';
527576
nodeRel(nodeOf(rnode)) = atooid(oidbuf);
528577

578+
#ifdef PGPRO_EE
579+
// if current tablespace is cfs-compressed and md_get_compressor_internal
580+
// returns the type of the compressing algorithm for filepath, then it
581+
// needs to be de-compressed to obtain its size
582+
if(is_cfs && md_get_compressor_internal(rnode.node, rnode.backend, forknum) != 0) {
583+
rel_size = get_cfs_relation_file_decompressed_size(rnode, filepath, forknum);
584+
585+
if(rel_size == (off_t)-1) {
586+
elog(WARNING, "ptrack: could not open cfs-compressed relation file: %s", filepath);
587+
return;
588+
}
589+
590+
nblocks = rel_size / BLCKSZ;
591+
} else
592+
#endif
529593
/* Compute number of blocks based on file size */
530594
if (stat(filepath, &stat_buf) == 0)
531595
nblocks = stat_buf.st_size / BLCKSZ;
@@ -546,6 +610,9 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid)
546610
{
547611
DIR *dir;
548612
struct dirent *de;
613+
#ifdef PGPRO_EE
614+
bool is_cfs;
615+
#endif
549616

550617
/* Do not walk during bootstrap and if ptrack is disabled */
551618
if (ptrack_map_size == 0
@@ -554,6 +621,10 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid)
554621
|| InitializingParallelWorker)
555622
return;
556623

624+
#ifdef PGPRO_EE
625+
is_cfs = file_is_in_cfs_tablespace(path);
626+
#endif
627+
557628
dir = AllocateDir(path);
558629

559630
while ((de = ReadDirExtended(dir, path, LOG)) != NULL)
@@ -581,7 +652,11 @@ ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid)
581652
}
582653

583654
if (S_ISREG(fst.st_mode))
655+
#ifdef PGPRO_EE
656+
ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name, is_cfs);
657+
#else
584658
ptrack_mark_file(dbOid, tablespaceOid, subpath, de->d_name);
659+
#endif
585660
}
586661

587662
FreeDir(dir); /* we ignore any error here */

‎engine.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,4 +111,10 @@ extern void ptrack_walkdir(const char *path, Oid tablespaceOid, Oid dbOid);
111111
extern void ptrack_mark_block(RelFileNodeBackend smgr_rnode,
112112
ForkNumber forkno, BlockNumber blkno);
113113

114+
extern bool is_cfm_file_path(const char *path);
115+
#ifdef PGPRO_EE
116+
extern off_t get_cfs_relation_file_decompressed_size(RelFileNodeBackend rnode,
117+
const char *fullpath, ForkNumber forknum);
118+
#endif
119+
114120
#endif /* PTRACK_ENGINE_H */

‎ptrack.c

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -251,14 +251,6 @@ ptrack_copydir_hook(const char *path)
251251

252252
elog(DEBUG1, "ptrack_copydir_hook: spcOid %u, dbOid %u", spcOid, dbOid);
253253

254-
#ifdef PGPRO_EE
255-
/*
256-
* Currently, we do not track files from compressed tablespaces in ptrack.
257-
*/
258-
if (file_is_in_cfs_tablespace(path))
259-
elog(DEBUG1, "ptrack_copydir_hook: skipping changes tracking in the CFS tablespace %u", spcOid);
260-
else
261-
#endif
262254
ptrack_walkdir(path, spcOid, dbOid);
263255

264256
if (prev_copydir_hook)
@@ -302,6 +294,10 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
302294
{
303295
DIR *dir;
304296
struct dirent *de;
297+
#ifdef PGPRO_EE
298+
bool is_cfs;
299+
is_cfs = file_is_in_cfs_tablespace(path);
300+
#endif
305301

306302
dir = AllocateDir(path);
307303

@@ -315,7 +311,8 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
315311

316312
if (strcmp(de->d_name, ".") == 0 ||
317313
strcmp(de->d_name, "..") == 0 ||
318-
looks_like_temp_rel_name(de->d_name))
314+
looks_like_temp_rel_name(de->d_name) ||
315+
is_cfm_file_path(de->d_name))
319316
continue;
320317

321318
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
@@ -362,6 +359,10 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
362359
nodeSpc(pfl->relnode) = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid;
363360
pfl->path = GetRelationPath(dbOid, nodeSpc(pfl->relnode),
364361
nodeRel(pfl->relnode), InvalidBackendId, pfl->forknum);
362+
#ifdef PGPRO_EE
363+
pfl->is_cfs_compressed = is_cfs
364+
&& md_get_compressor_internal(pfl->relnode, InvalidBackendId, pfl->forknum) != 0;
365+
#endif
365366

366367
*filelist = lappend(*filelist, pfl);
367368

@@ -403,6 +404,10 @@ ptrack_filelist_getnext(PtScanCtx * ctx)
403404
ListCell *cell;
404405
char *fullpath;
405406
struct stat fst;
407+
off_t rel_st_size = 0;
408+
#ifdef PGPRO_EE
409+
RelFileNodeBackend rnodebackend;
410+
#endif
406411

407412
/* No more file in the list */
408413
if (list_length(ctx->filelist) == 0)
@@ -449,14 +454,24 @@ ptrack_filelist_getnext(PtScanCtx * ctx)
449454
return ptrack_filelist_getnext(ctx);
450455
}
451456

457+
#ifdef PGPRO_EE
458+
rnodebackend.node = ctx->bid.relnode;
459+
rnodebackend.backend = InvalidBackendId;
460+
461+
if(pfl->is_cfs_compressed)
462+
rel_st_size = get_cfs_relation_file_decompressed_size(rnodebackend, fullpath, pfl->forknum);
463+
else
464+
#endif
465+
rel_st_size = fst.st_size;
466+
452467
if (pfl->segno > 0)
453468
{
454-
ctx->relsize = pfl->segno * RELSEG_SIZE + fst.st_size / BLCKSZ;
469+
ctx->relsize = pfl->segno * RELSEG_SIZE + rel_st_size / BLCKSZ;
455470
ctx->bid.blocknum = pfl->segno * RELSEG_SIZE;
456471
}
457472
else
458473
/* Estimate relsize as size of first segment in blocks */
459-
ctx->relsize = fst.st_size / BLCKSZ;
474+
ctx->relsize = rel_st_size / BLCKSZ;
460475

461476
elog(DEBUG3, "ptrack: got file %s with size %u from the file list", pfl->path, ctx->relsize);
462477

‎ptrack.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ typedef struct PtrackFileList_i
7878
ForkNumber forknum;
7979
int segno;
8080
char *path;
81+
#ifdef PGPRO_EE
82+
bool is_cfs_compressed;
83+
#endif
8184
} PtrackFileList_i;
8285

8386
#endif /* PTRACK_H */

‎t/002_cfs_compatibility.pl

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
use strict;
2+
use warnings;
3+
use Test::More;
4+
5+
my $pg_15_modules;
6+
7+
BEGIN
8+
{
9+
$pg_15_modules = eval
10+
{
11+
require PostgreSQL::Test::Cluster;
12+
require PostgreSQL::Test::Utils;
13+
return 1;
14+
};
15+
16+
unless (defined $pg_15_modules)
17+
{
18+
$pg_15_modules = 0;
19+
20+
require PostgresNode;
21+
require TestLib;
22+
}
23+
}
24+
25+
note('PostgreSQL 15 modules are used: ' . ($pg_15_modules ? 'yes' : 'no'));
26+
27+
my $node;
28+
my $res_stdout;
29+
my $res_stderr;
30+
31+
# Create node.
32+
# Older versions of PostgreSQL modules use get_new_node function.
33+
# Newer use standard perl object constructor syntax.
34+
eval
35+
{
36+
if ($pg_15_modules)
37+
{
38+
$node = PostgreSQL::Test::Cluster->new("node");
39+
}
40+
else
41+
{
42+
$node = PostgresNode::get_new_node("node");
43+
}
44+
};
45+
46+
note "Test for handling a ptrack map in compressed relations";
47+
48+
my $psql_stdout;
49+
50+
# Starting the node
51+
$node->init;
52+
53+
# Could not load ptrack module after postmaster start
54+
55+
my $cfs_tblspc1 = $node->basedir."/cfs_tblspc1";
56+
my $cfs_tblspc2 = $node->basedir."/cfs_tblspc2";
57+
mkdir $cfs_tblspc1 or die;
58+
mkdir $cfs_tblspc2 or die;
59+
my $no_cfs_tblspc1 = $node->basedir."/no_cfs_tblspc1";
60+
my $no_cfs_tblspc2 = $node->basedir."/no_cfs_tblspc2";
61+
mkdir $no_cfs_tblspc1 or die;
62+
mkdir $no_cfs_tblspc2 or die;
63+
64+
$node->append_conf('postgresql.conf', qq{
65+
shared_preload_libraries = 'ptrack'
66+
ptrack.map_size = 16
67+
wal_level = 'replica'
68+
});
69+
70+
$node->start;
71+
72+
# check cfs availability first
73+
my $cfs_available = $node->safe_psql('postgres',
74+
"select count(oid) from pg_proc where proname = 'cfs_version'");
75+
76+
if($cfs_available eq "0") {
77+
$node->stop;
78+
plan skip_all => "CFS is not supported by this PostgreSQL build";
79+
} else {
80+
plan tests => 2;
81+
}
82+
83+
# Creating content
84+
$node->safe_psql('postgres', qq|
85+
create tablespace cfs_tblspc1 location '$cfs_tblspc1' with (compression=true);
86+
create tablespace cfs_tblspc2 location '$cfs_tblspc2' with (compression=true);
87+
create tablespace no_cfs_tblspc1 location '$no_cfs_tblspc1';
88+
create tablespace no_cfs_tblspc2 location '$no_cfs_tblspc2';
89+
90+
create database testing_cfs tablespace cfs_tblspc1;
91+
create database testing_no_cfs tablespace no_cfs_tblspc1;
92+
|);
93+
94+
$node->safe_psql('testing_cfs', qq{
95+
create table testing(i int, text varchar);
96+
insert into testing select 1, '1111111111111111111111111' from generate_series(1,10000000);
97+
});
98+
99+
$node->safe_psql('testing_no_cfs', qq{
100+
create table testing_no(i int, text varchar);
101+
insert into testing_no select 1, '1111111111111111111111111' from generate_series(1,10000000);
102+
});
103+
104+
# creating ptrack
105+
$node->safe_psql('postgres', "create extension ptrack");
106+
107+
# obtaining init lsn for further usage in ptrack_get_pagemapset
108+
my $init_lsn = $node->safe_psql('postgres', 'select ptrack_init_lsn()');
109+
110+
# forcing copydir() hook by altering dbs tablespaces
111+
$node->safe_psql('postgres', "alter database testing_cfs set tablespace cfs_tblspc2;");
112+
$node->safe_psql('postgres', "alter database testing_no_cfs set tablespace no_cfs_tblspc2;");
113+
114+
# obtaining relpath for cfs table
115+
my $cfs_relpath = $node->safe_psql('testing_cfs', "select pg_relation_filepath('testing');");
116+
117+
# obtaining relpath for no-cfs table
118+
my $no_cfs_relpath = $node->safe_psql('testing_no_cfs', "select pg_relation_filepath('testing_no');");
119+
120+
# select the pagecount sums and compare them (should be equal)
121+
my $pagecount_sum_cfs = $node->safe_psql('postgres',
122+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';");
123+
my $pagecount_sum_no_cfs = $node->safe_psql('postgres',
124+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';");
125+
126+
is($pagecount_sum_cfs, $pagecount_sum_no_cfs, "pagecount sums don't match");
127+
128+
# forcing copydir() hook by altering dbs tablespaces back
129+
$node->safe_psql('postgres', "alter database testing_cfs set tablespace cfs_tblspc1;");
130+
$node->safe_psql('postgres', "alter database testing_no_cfs set tablespace no_cfs_tblspc1;");
131+
132+
# obtaining new relpath for cfs table
133+
$cfs_relpath = $node->safe_psql('testing_cfs', "select pg_relation_filepath('testing');");
134+
135+
# obtaining new relpath for no-cfs table
136+
$no_cfs_relpath = $node->safe_psql('testing_no_cfs', "select pg_relation_filepath('testing_no');");
137+
138+
# select the pagecount sums and compare them (again, they should be equal)
139+
$pagecount_sum_cfs = $node->safe_psql('postgres',
140+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$cfs_relpath%';");
141+
$pagecount_sum_no_cfs = $node->safe_psql('postgres',
142+
"select sum(pagecount) from ptrack_get_pagemapset('$init_lsn'::pg_lsn) where path like '%$no_cfs_relpath%';");
143+
144+
is($pagecount_sum_cfs, $pagecount_sum_no_cfs, "pagecount sums don't match");
145+
146+
147+
$node->stop;
148+

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy