Skip to content

Commit 708c8e2

Browse files
ololobusololobus
authored
Merge pull request #6 from postgrespro/double_slot
Resolve issues #5 and #1: reduce number of collisions in the ptrack map
2 parents de8d655 + 9c132a3 commit 708c8e2

File tree

11 files changed

+162
-83
lines changed

11 files changed

+162
-83
lines changed

‎.gitignore

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
.deps
22
*.so
33
*.o
4-
ptrack--2.0.sql
54
Dockerfile
6-

‎Makefile

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,11 @@
22

33
MODULE_big = ptrack
44
OBJS = ptrack.o datapagemap.o engine.o $(WIN32RES)
5-
EXTENSION = ptrack
6-
EXTVERSION = 2.1
7-
DATA = ptrack.sql ptrack--2.0--2.1.sql
8-
DATA_built = $(EXTENSION)--$(EXTVERSION).sql
95
PGFILEDESC = "ptrack - block-level incremental backup engine"
106

11-
EXTRA_CLEAN = $(EXTENSION)--$(EXTVERSION).sql
7+
EXTENSION = ptrack
8+
EXTVERSION = 2.2
9+
DATA = ptrack--2.1.sql ptrack--2.0--2.1.sql ptrack--2.1--2.2.sql
1210

1311
TAP_TESTS = 1
1412

@@ -22,13 +20,3 @@ top_builddir = ../..
2220
include $(top_builddir)/src/Makefile.global
2321
include $(top_srcdir)/contrib/contrib-global.mk
2422
endif
25-
26-
$(EXTENSION)--$(EXTVERSION).sql: ptrack.sql
27-
cat $^ > $@
28-
29-
# temp-install: EXTRA_INSTALL=contrib/ptrack
30-
31-
# check-tap: temp-install
32-
# $(prove_check)
33-
34-
# check: check-tap

‎README.md

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ To disable `ptrack` and clean up all remaining service files set `ptrack.map_siz
6565

6666
* ptrack_version() — returns ptrack version string.
6767
* ptrack_init_lsn() — returns LSN of the last ptrack map initialization.
68-
* ptrack_get_pagemapset('LSN') — returns a set of changed data files with bitmaps of changed blocks since specified LSN.
68+
* ptrack_get_pagemapset(start_lsn pg_lsn) — returns a set of changed data files with bitmaps of changed blocks since specified `start_lsn`.
69+
* ptrack_get_change_stat(start_lsn pg_lsn) — returns statistic of changes (number of files, pages and size in MB) since specified `start_lsn`.
6970

7071
Usage example:
7172

@@ -102,6 +103,10 @@ Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSIO
102103
* Do `ALTER EXTENSION 'ptrack' UPDATE;`.
103104
* Restart your server.
104105

106+
#### Upgrading from 2.1.* to 2.2.*:
107+
108+
Since version 2.2 we use a different algorithm for tracking changed pages. Thus, data recorded in the `ptrack.map` using pre 2.2 versions of `ptrack` is incompatible with newer versions. After extension upgrade and server restart old `ptrack.map` will be discarded with `WARNING` and initialized from the scratch.
109+
105110
## Limitations
106111

107112
1. You can only use `ptrack` safely with `wal_level >= 'replica'`. Otherwise, you can lose tracking of some changes if crash-recovery occurs, since [certain commands are designed not to write WAL at all if wal_level is minimal](https://www.postgresql.org/docs/12/populate.html#POPULATE-PITR), but we only durably flush `ptrack` map at checkpoint time.

‎engine.c

Lines changed: 61 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ ptrackMapInit(void)
156156
sprintf(ptrack_path, "%s/%s", DataDir, PTRACK_PATH);
157157
sprintf(ptrack_mmap_path, "%s/%s", DataDir, PTRACK_MMAP_PATH);
158158

159+
ptrack_map_reinit:
160+
159161
/* Remove old PTRACK_MMAP_PATH file, if exists */
160162
if (ptrack_file_exists(ptrack_mmap_path))
161163
durable_unlink(ptrack_mmap_path, LOG);
@@ -175,18 +177,15 @@ ptrackMapInit(void)
175177
if (stat(ptrack_path, &stat_buf) == 0)
176178
{
177179
copy_file(ptrack_path, ptrack_mmap_path);
178-
is_new_map = false; /* flag to check checksum */
180+
is_new_map = false; /* flag to check map file format and checksum */
179181
ptrack_fd = BasicOpenFile(ptrack_mmap_path, O_RDWR | PG_BINARY);
180-
if (ptrack_fd < 0)
181-
elog(ERROR, "ptrack init: failed to open map file \"%s\": %m", ptrack_mmap_path);
182182
}
183183
else
184-
{
185184
/* Create new file for PTRACK_MMAP_PATH */
186185
ptrack_fd = BasicOpenFile(ptrack_mmap_path, O_RDWR | O_CREAT | PG_BINARY);
187-
if (ptrack_fd < 0)
188-
elog(ERROR, "ptrack init: failed to open map file \"%s\": %m", ptrack_mmap_path);
189-
}
186+
187+
if (ptrack_fd < 0)
188+
elog(ERROR, "ptrack init: failed to open map file \"%s\": %m", ptrack_mmap_path);
190189

191190
#ifdef WIN32
192191
{
@@ -227,7 +226,20 @@ ptrackMapInit(void)
227226
elog(ERROR, "ptrack init: wrong map format of file \"%s\"", ptrack_path);
228227

229228
/* Check ptrack version inside old ptrack map */
230-
/* No-op for now, but may be used for future compatibility checks */
229+
if (ptrack_map->version_num != PTRACK_VERSION_NUM)
230+
{
231+
ereport(WARNING,
232+
(errcode(ERRCODE_DATA_CORRUPTED),
233+
errmsg("ptrack init: map format version %d in the file \"%s\" is incompatible with loaded version %d",
234+
ptrack_map->version_num, ptrack_path, PTRACK_VERSION_NUM),
235+
errdetail("Deleting file \"%s\" and reinitializing ptrack map.", ptrack_path)));
236+
237+
/* Clean up everything and try again */
238+
ptrackCleanFilesAndMap();
239+
240+
is_new_map = true;
241+
goto ptrack_map_reinit;
242+
}
231243

232244
/* Check CRC */
233245
INIT_CRC32C(crc);
@@ -378,7 +390,7 @@ ptrackCheckpoint(void)
378390
/*
379391
* We are writing ptrack map values to file, but we want to simply map it
380392
* into the memory with mmap after a crash/restart. That way, we have to
381-
* write values taking into account all paddings/allignments.
393+
* write values taking into account all paddings/alignments.
382394
*
383395
* Write both magic and varsion_num at once.
384396
*/
@@ -435,7 +447,7 @@ ptrackCheckpoint(void)
435447
* going to overflow. */
436448

437449
/*
438-
* We should not have any allignment issues here, since sizeof()
450+
* We should not have any alignment issues here, since sizeof()
439451
* takes into account all paddings for us.
440452
*/
441453
ptrack_write_chunk(ptrack_tmp_fd, &crc, (char *) buf, writesz);
@@ -446,7 +458,7 @@ ptrackCheckpoint(void)
446458
}
447459
}
448460

449-
/* Write if anythig left */
461+
/* Write if anything left */
450462
if ((i + 1) % PTRACK_BUF_SIZE != 0)
451463
{
452464
size_t writesz = sizeof(pg_atomic_uint64) * j;
@@ -641,48 +653,56 @@ void
641653
ptrack_mark_block(RelFileNodeBackend smgr_rnode,
642654
ForkNumber forknum, BlockNumber blocknum)
643655
{
656+
PtBlockId bid;
644657
size_t hash;
658+
size_t slot1;
659+
size_t slot2;
645660
XLogRecPtr new_lsn;
646-
PtBlockId bid;
647661
/*
648662
* We use pg_atomic_uint64 here only for alignment purposes, because
649-
* pg_atomic_uint64 is forcely aligned on 8 bytes during the MSVC build.
663+
* pg_atomic_uint64 is forcedly aligned on 8 bytes during the MSVC build.
650664
*/
651665
pg_atomic_uint64 old_lsn;
652666
pg_atomic_uint64 old_init_lsn;
653667

654-
if (ptrack_map_size != 0 && (ptrack_map != NULL) &&
655-
smgr_rnode.backend == InvalidBackendId) /* do not track temporary
656-
* relations */
657-
{
658-
bid.relnode = smgr_rnode.node;
659-
bid.forknum = forknum;
660-
bid.blocknum = blocknum;
661-
hash = BID_HASH_FUNC(bid);
662-
663-
if (RecoveryInProgress())
664-
new_lsn = GetXLogReplayRecPtr(NULL);
665-
else
666-
new_lsn = GetXLogInsertRecPtr();
667-
668-
old_lsn.value = pg_atomic_read_u64(&ptrack_map->entries[hash]);
668+
if (ptrack_map_size == 0
669+
|| ptrack_map == NULL
670+
|| smgr_rnode.backend != InvalidBackendId) /* do not track temporary
671+
* relations */
672+
return;
669673

670-
/* Atomically assign new init LSN value */
671-
old_init_lsn.value = pg_atomic_read_u64(&ptrack_map->init_lsn);
674+
bid.relnode = smgr_rnode.node;
675+
bid.forknum = forknum;
676+
bid.blocknum = blocknum;
672677

673-
if (old_init_lsn.value == InvalidXLogRecPtr)
674-
{
675-
elog(DEBUG1, "ptrack_mark_block: init_lsn " UINT64_FORMAT " <- " UINT64_FORMAT, old_init_lsn.value, new_lsn);
678+
hash = BID_HASH_FUNC(bid);
679+
slot1 = hash % PtrackContentNblocks;
680+
slot2 = ((hash << 32) | (hash >> 32)) % PtrackContentNblocks;
676681

677-
while (old_init_lsn.value < new_lsn &&
678-
!pg_atomic_compare_exchange_u64(&ptrack_map->init_lsn, (uint64 *) &old_init_lsn.value, new_lsn));
679-
}
682+
if (RecoveryInProgress())
683+
new_lsn = GetXLogReplayRecPtr(NULL);
684+
else
685+
new_lsn = GetXLogInsertRecPtr();
680686

681-
elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT " <- " UINT64_FORMAT, hash, old_lsn.value, new_lsn);
687+
/* Atomically assign new init LSN value */
688+
old_init_lsn.value = pg_atomic_read_u64(&ptrack_map->init_lsn);
689+
if (old_init_lsn.value == InvalidXLogRecPtr)
690+
{
691+
elog(DEBUG1, "ptrack_mark_block: init_lsn " UINT64_FORMAT " <- " UINT64_FORMAT, old_init_lsn.value, new_lsn);
682692

683-
/* Atomically assign new LSN value */
684-
while (old_lsn.value < new_lsn &&
685-
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[hash], (uint64 *) &old_lsn.value, new_lsn));
686-
elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT, hash, pg_atomic_read_u64(&ptrack_map->entries[hash]));
693+
while (old_init_lsn.value < new_lsn &&
694+
!pg_atomic_compare_exchange_u64(&ptrack_map->init_lsn, (uint64 *) &old_init_lsn.value, new_lsn));
687695
}
696+
697+
/* Atomically assign new LSN value to the first slot */
698+
old_lsn.value = pg_atomic_read_u64(&ptrack_map->entries[slot1]);
699+
elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT " <- " UINT64_FORMAT, slot1, old_lsn.value, new_lsn);
700+
while (old_lsn.value < new_lsn &&
701+
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot1], (uint64 *) &old_lsn.value, new_lsn));
702+
elog(DEBUG3, "ptrack_mark_block: map[%zu]=" UINT64_FORMAT, hash, pg_atomic_read_u64(&ptrack_map->entries[slot1]));
703+
704+
/* And to the second */
705+
old_lsn.value = pg_atomic_read_u64(&ptrack_map->entries[slot2]);
706+
while (old_lsn.value < new_lsn &&
707+
!pg_atomic_compare_exchange_u64(&ptrack_map->entries[slot2], (uint64 *) &old_lsn.value, new_lsn));
688708
}

‎engine.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ typedef struct PtrackMapHdr
5050
{
5151
/*
5252
* Three magic bytes (+ \0) to be sure, that we are reading ptrack.map
53-
* with a right PtrackMapHdr strucutre.
53+
* with a right PtrackMapHdr structure.
5454
*/
5555
char magic[PTRACK_MAGIC_SIZE];
5656

@@ -72,7 +72,6 @@ typedef struct PtrackMapHdr
7272

7373
typedef PtrackMapHdr * PtrackMap;
7474

75-
/* TODO: check MAXALIGN usage below */
7675
/* Number of elements in ptrack map (LSN array) */
7776
#define PtrackContentNblocks \
7877
((ptrack_map_size - offsetof(PtrackMapHdr, entries) - sizeof(pg_crc32c)) / sizeof(pg_atomic_uint64))
@@ -84,9 +83,10 @@ typedef PtrackMapHdr * PtrackMap;
8483
/* CRC32 value offset in order to directly access it in the mmap'ed memory chunk */
8584
#define PtrackCrcOffset (PtrackActualSize - sizeof(pg_crc32c))
8685

87-
/* Map block address 'bid' to map slot */
86+
/* Block address 'bid' to hash. To get slot position in map should be divided
87+
* with '% PtrackContentNblocks' */
8888
#define BID_HASH_FUNC(bid) \
89-
(size_t)(DatumGetUInt64(hash_any_extended((unsigned char *)&bid, sizeof(bid), 0)) % PtrackContentNblocks)
89+
(size_t)(DatumGetUInt64(hash_any_extended((unsigned char *)&bid, sizeof(bid), 0)))
9090

9191
/*
9292
* Per process pointer to shared ptrack_map

‎ptrack--2.1--2.2.sql

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/* ptrack/ptrack--2.1--2.2.sql */
2+
3+
-- Complain if script is sourced in psql, rather than via ALTER EXTENSION
4+
\echo Use "ALTER EXTENSION ptrack UPDATE;" to load this file.\ quit
5+
6+
DROP FUNCTION ptrack_get_pagemapset(start_lsn pg_lsn);
7+
CREATE FUNCTION ptrack_get_pagemapset(start_lsn pg_lsn)
8+
RETURNS TABLE (path text,
9+
pagecount bigint,
10+
pagemap bytea)
11+
AS 'MODULE_PATHNAME'
12+
LANGUAGE C STRICT VOLATILE;
13+
14+
CREATE FUNCTION ptrack_get_change_stat(start_lsn pg_lsn)
15+
RETURNS TABLE (
16+
files bigint,
17+
pages numeric,
18+
"size, MB" numeric
19+
) AS
20+
$func$
21+
DECLARE
22+
block_size bigint;
23+
BEGIN
24+
block_size := (SELECT setting FROM pg_settings WHERE name = 'block_size');
25+
26+
RETURN QUERY
27+
SELECT changed_files,
28+
changed_pages,
29+
block_size * changed_pages / (1024.0 * 1024)
30+
FROM
31+
(SELECT count(path) AS changed_files,
32+
sum(pagecount) AS changed_pages
33+
FROM ptrack_get_pagemapset(start_lsn)) s;
34+
END
35+
$func$ LANGUAGE plpgsql;

‎ptrack.sql renamed to ‎ptrack--2.1.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
/* ptrack/ptrack--2.1.sql */
2+
13
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
24
\echo Use "CREATE EXTENSION ptrack" to load this file. \quit
35

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy