+#if PG_VERSION_NUM < 120000
+#include "access/htup_details.h"
+#endif
+#include "catalog/pg_tablespace.h"
+#include "catalog/pg_type.h"
#include "funcapi.h"
#include "miscadmin.h"
-#include "access/hash.h"
-#include "access/skey.h"
-#include "catalog/pg_type.h"
-#include "catalog/pg_tablespace.h"
+#include "nodes/pg_list.h"
+#include "port/pg_crc32c.h"
+#include "storage/copydir.h"
+#include "storage/ipc.h"
#include "storage/lmgr.h"
-#include "storage/ptrack.h"
+#if PG_VERSION_NUM >= 120000
+#include "storage/md.h"
+#endif
+#include "storage/smgr.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
+#include "utils/guc.h"
#include "utils/pg_lsn.h"
-#include "nodes/pg_list.h"
+
+#include "datapagemap.h"
+#include "ptrack.h"
+#include "engine.h"
PG_MODULE_MAGIC;
-void _PG_init(void);
-void _PG_fini(void);
+PtrackMap ptrack_map = NULL;
+uint64 ptrack_map_size = 0;
+int ptrack_map_size_tmp;
+
+static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
+static copydir_hook_type prev_copydir_hook = NULL;
+static mdwrite_hook_type prev_mdwrite_hook = NULL;
+static mdextend_hook_type prev_mdextend_hook = NULL;
+static ProcessSyncRequests_hook_type prev_ProcessSyncRequests_hook = NULL;
+#if PG_VERSION_NUM >= 170000
+static backup_checkpoint_request_hook_type prev_backup_checkpoint_request_hook = NULL;
+#endif
+
+void _PG_init(void);
+
+static void ptrack_shmem_startup_hook(void);
+static void ptrack_copydir_hook(const char *path);
+static void ptrack_mdwrite_hook(RelFileNodeBackend smgr_rnode,
+ ForkNumber forkno, BlockNumber blkno);
+static void ptrack_mdextend_hook(RelFileNodeBackend smgr_rnode,
+ ForkNumber forkno, BlockNumber blkno);
+static void ptrack_ProcessSyncRequests_hook(void);
+#if PG_VERSION_NUM >= 170000
+static void ptrack_backup_checkpoint_request_hook(void);
+#endif
static void ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid);
-static int ptrack_filelist_getnext(PtScanCtx *ctx);
+static int ptrack_filelist_getnext(PtScanCtx * ctx);
+#if PG_VERSION_NUM >= 150000
+static shmem_request_hook_type prev_shmem_request_hook = NULL;
+static void ptrack_shmem_request(void);
+#endif
/*
* Module load callback
@@ -60,85 +94,207 @@ static int ptrack_filelist_getnext(PtScanCtx *ctx);
void
_PG_init(void)
{
+ if (!process_shared_preload_libraries_in_progress)
+ elog(ERROR, "ptrack module must be initialized by Postmaster. "
+ "Put the following line to configuration file: "
+ "shared_preload_libraries='ptrack'");
+
+ /*
+ * Define (or redefine) custom GUC variables.
+ *
+ * XXX: for some reason assign_ptrack_map_size is called twice during the
+ * postmaster boot! First, it is always called with bootValue, so we use
+ * -1 as default value and no-op here. Next, it is called with the actual
+ * value from config.
+ */
+ DefineCustomIntVariable("ptrack.map_size",
+ "Sets the size of ptrack map in MB used for incremental backup (0 disabled).",
+ NULL,
+ &ptrack_map_size_tmp,
+ 0,
+#if SIZEOF_SIZE_T == 8
+ 0, 32 * 1024, /* limit to 32 GB */
+#else
+ 0, 256, /* limit to 256 MB */
+#endif
+ PGC_POSTMASTER,
+ GUC_UNIT_MB,
+ NULL,
+ assign_ptrack_map_size,
+ NULL);
+
+ /* Request server shared memory */
+ if (ptrack_map_size != 0)
+ {
+#if PG_VERSION_NUM >= 150000
+ prev_shmem_request_hook = shmem_request_hook;
+ shmem_request_hook = ptrack_shmem_request;
+#else
+ RequestAddinShmemSpace(PtrackActualSize);
+#endif
+ }
+ else
+ ptrackCleanFiles();
+
+ /* Install hooks */
+ prev_shmem_startup_hook = shmem_startup_hook;
+ shmem_startup_hook = ptrack_shmem_startup_hook;
+ prev_copydir_hook = copydir_hook;
+ copydir_hook = ptrack_copydir_hook;
+ prev_mdwrite_hook = mdwrite_hook;
+ mdwrite_hook = ptrack_mdwrite_hook;
+ prev_mdextend_hook = mdextend_hook;
+ mdextend_hook = ptrack_mdextend_hook;
+ prev_ProcessSyncRequests_hook = ProcessSyncRequests_hook;
+ ProcessSyncRequests_hook = ptrack_ProcessSyncRequests_hook;
+#if PG_VERSION_NUM >= 170000
+ prev_backup_checkpoint_request_hook = backup_checkpoint_request_hook;
+ backup_checkpoint_request_hook = ptrack_backup_checkpoint_request_hook;
+#endif
+}
+
+#if PG_VERSION_NUM >= 150000
+static void
+ptrack_shmem_request(void)
+{
+ if (prev_shmem_request_hook)
+ prev_shmem_request_hook();
+ RequestAddinShmemSpace(PtrackActualSize);
}
+#endif
/*
- * Module unload callback
+ * ptrack_shmem_startup hook: allocate or attach to shared memory.
*/
-void
-_PG_fini(void)
+static void
+ptrack_shmem_startup_hook(void)
{
+ bool map_found;
-}
+ if (prev_shmem_startup_hook)
+ prev_shmem_startup_hook();
-/********************************************************************/
-/* Datapage bitmapping structures and routines taken from pg_rewind */
-/* TODO: consider moving to another location */
-struct datapagemap
-{
- char *bitmap;
- int bitmapsize;
-};
-typedef struct datapagemap datapagemap_t;
+ /*
+ * Create or attach to the shared memory state
+ */
+ LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE);
-struct datapagemap_iterator
-{
- datapagemap_t *map;
- BlockNumber nextblkno;
-};
-typedef struct datapagemap_iterator datapagemap_iterator_t;
-static void datapagemap_add(datapagemap_t *map, BlockNumber blkno);
+ if (ptrack_map_size != 0)
+ {
+ ptrack_map = ShmemInitStruct("ptrack map",
+ PtrackActualSize,
+ &map_found);
+ if (!map_found)
+ {
+ ptrackMapInit();
+ elog(DEBUG1, "Shared memory for ptrack is ready");
+ }
+ }
+ else
+ {
+ ptrack_map = NULL;
+ }
+ LWLockRelease(AddinShmemInitLock);
+}
+
+/*
+ * Ptrack follow up for copydir() routine. It parses database OID
+ * and tablespace OID from path string. We do not need to recursively
+ * walk subdirs here, copydir() will do it for us if needed.
+ */
static void
-datapagemap_add(datapagemap_t *map, BlockNumber blkno)
+ptrack_copydir_hook(const char *path)
{
- int offset;
- int bitno;
+ Oid spcOid = InvalidOid;
+ Oid dbOid = InvalidOid;
+ int oidchars;
+ char oidbuf[OIDCHARS + 1];
- offset = blkno / 8;
- bitno = blkno % 8;
+ elog(DEBUG1, "ptrack_copydir_hook: path %s", path);
- /* enlarge or create bitmap if needed */
- if (map->bitmapsize <= offset)
+ if (strstr(path, "global/") == path)
+ spcOid = GLOBALTABLESPACE_OID;
+ else if (strstr(path, "base/") == path)
{
- int oldsize = map->bitmapsize;
- int newsize;
+ spcOid = DEFAULTTABLESPACE_OID;
+ oidchars = strspn(path + 5, "0123456789");
+ strncpy(oidbuf, path + 5, oidchars);
+ oidbuf[oidchars] = '\0';
+ dbOid = atooid(oidbuf);
+ }
+ else if (strstr(path, "pg_tblspc/") == path)
+ {
+ char *dbPos;
+
+ oidchars = strspn(path + 10, "0123456789");
+ strncpy(oidbuf, path + 10, oidchars);
+ oidbuf[oidchars] = '\0';
+ spcOid = atooid(oidbuf);
+
+ dbPos = strstr(path, TABLESPACE_VERSION_DIRECTORY) + strlen(TABLESPACE_VERSION_DIRECTORY) + 1;
+ oidchars = strspn(dbPos, "0123456789");
+ strncpy(oidbuf, dbPos, oidchars);
+ oidbuf[oidchars] = '\0';
+ dbOid = atooid(oidbuf);
+ }
- /*
- * The minimum to hold the new bit is offset + 1. But add some
- * headroom, so that we don't need to repeatedly enlarge the bitmap in
- * the common case that blocks are modified in order, from beginning
- * of a relation to the end.
- */
- newsize = offset + 1;
- newsize += 10;
+ elog(DEBUG1, "ptrack_copydir_hook: spcOid %u, dbOid %u", spcOid, dbOid);
- if (map->bitmap != NULL)
- map->bitmap = repalloc(map->bitmap, newsize);
- else
- map->bitmap = palloc(newsize);
+ ptrack_walkdir(path, spcOid, dbOid);
- /* zero out the newly allocated region */
- memset(&map->bitmap[oldsize], 0, newsize - oldsize);
+ if (prev_copydir_hook)
+ prev_copydir_hook(path);
+}
- map->bitmapsize = newsize;
- }
+static void
+ptrack_mdwrite_hook(RelFileNodeBackend smgr_rnode,
+ ForkNumber forknum, BlockNumber blocknum)
+{
+ ptrack_mark_block(smgr_rnode, forknum, blocknum);
+
+ if (prev_mdwrite_hook)
+ prev_mdwrite_hook(smgr_rnode, forknum, blocknum);
+}
+
+static void
+ptrack_mdextend_hook(RelFileNodeBackend smgr_rnode,
+ ForkNumber forknum, BlockNumber blocknum)
+{
+ ptrack_mark_block(smgr_rnode, forknum, blocknum);
+
+ if (prev_mdextend_hook)
+ prev_mdextend_hook(smgr_rnode, forknum, blocknum);
+}
- /* Set the bit */
- map->bitmap[offset] |= (1 << bitno);
+static void
+ptrack_ProcessSyncRequests_hook()
+{
+ ptrackCheckpoint();
+
+ if (prev_ProcessSyncRequests_hook)
+ prev_ProcessSyncRequests_hook();
}
-/********************************************************************/
+#if PG_VERSION_NUM >= 170000
+static void
+ptrack_backup_checkpoint_request_hook(void)
+{
+ ptrack_set_init_lsn();
+
+ if (prev_backup_checkpoint_request_hook)
+ prev_backup_checkpoint_request_hook();
+}
+#endif
/*
* Recursively walk through the path and add all data files to filelist.
*/
static void
ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
{
- DIR *dir;
+ DIR *dir;
struct dirent *de;
-
dir = AllocateDir(path);
while ((de = ReadDirExtended(dir, path, LOG)) != NULL)
@@ -160,47 +316,66 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
if (sret < 0)
{
- ereport(LOG,
+ ereport(WARNING,
(errcode_for_file_access(),
- errmsg("could not stat file \"%s\": %m", subpath)));
+ errmsg("ptrack: could not stat file \"%s\": %m", subpath)));
continue;
}
if (S_ISREG(fst.st_mode))
{
+ if (fst.st_size == 0)
+ {
+ elog(DEBUG3, "ptrack: skip empty file %s", subpath);
+
+ /* But try the next one */
+ continue;
+ }
+
/* Regular file inside database directory, otherwise skip it */
if (dbOid != InvalidOid || spcOid == GLOBALTABLESPACE_OID)
{
- int oidchars;
- char oidbuf[OIDCHARS + 1];
- char *segpath;
+#if PG_VERSION_NUM >= 170000
+ RelFileNumber relNumber;
+ unsigned segno;
+#else
+ int oidchars;
+ char oidbuf[OIDCHARS + 1];
+#endif
+ char *segpath;
PtrackFileList_i *pfl = palloc0(sizeof(PtrackFileList_i));
/*
* Check that filename seems to be a regular relation file.
*/
+#if PG_VERSION_NUM >= 170000
+ if (!parse_filename_for_nontemp_relation(de->d_name, &relNumber, &pfl->forknum, &segno))
+ continue;
+#else
if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, &pfl->forknum))
continue;
+#endif
+ /* Parse segno */
+ segpath = strstr(de->d_name, ".");
+ pfl->segno = segpath != NULL ? atoi(segpath + 1) : 0;
- /* Parse segno for main fork */
- if (pfl->forknum == MAIN_FORKNUM)
- {
- segpath = strstr(de->d_name, ".");
- pfl->segno = segpath != NULL ? atoi(segpath + 1) : 0;
- }
- else
- pfl->segno = 0;
-
+ /* Fill the pfl in */
+#if PG_VERSION_NUM >= 170000
+ nodeRel(pfl->relnode) = relNumber;
+#else
memcpy(oidbuf, de->d_name, oidchars);
oidbuf[oidchars] = '\0';
- pfl->relnode.relNode = atooid(oidbuf);
- pfl->relnode.dbNode = dbOid;
- pfl->relnode.spcNode = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid;
- pfl->path = GetRelationPath(dbOid, pfl->relnode.spcNode,
- pfl->relnode.relNode, InvalidBackendId, pfl->forknum);
+ nodeRel(pfl->relnode) = atooid(oidbuf);
+#endif
+ nodeDb(pfl->relnode) = dbOid;
+ nodeSpc(pfl->relnode) = spcOid == InvalidOid ? DEFAULTTABLESPACE_OID : spcOid;
+ pfl->path = GetRelationPath(dbOid, nodeSpc(pfl->relnode),
+ nodeRel(pfl->relnode), InvalidBackendId, pfl->forknum);
*filelist = lappend(*filelist, pfl);
- // elog(WARNING, "added file %s of rel %u to ptrack list", pfl->path, pfl->relnode.relNode);
+
+ elog(DEBUG3, "ptrack: added file %s of rel %u to file list",
+ pfl->path, nodeRel(pfl->relnode));
}
}
else if (S_ISDIR(fst.st_mode))
@@ -211,40 +386,55 @@ ptrack_gather_filelist(List **filelist, char *path, Oid spcOid, Oid dbOid)
else if (spcOid != InvalidOid && strcmp(de->d_name, TABLESPACE_VERSION_DIRECTORY) == 0)
ptrack_gather_filelist(filelist, subpath, spcOid, InvalidOid);
}
- // TODO: is it enough to properly check symlink support?
-#ifndef WIN32
+ /* TODO: is it enough to properly check symlink support? */
+#if !defined(WIN32) || (PG_VERSION_NUM >= 160000)
else if (S_ISLNK(fst.st_mode))
#else
else if (pgwin32_is_junction(subpath))
#endif
{
- /* We expect that symlinks with only digits in the name to be tablespaces */
+ /*
+ * We expect that symlinks with only digits in the name to be
+ * tablespaces
+ */
if (strspn(de->d_name + 1, "0123456789") == strlen(de->d_name + 1))
ptrack_gather_filelist(filelist, subpath, atooid(de->d_name), InvalidOid);
}
}
- FreeDir(dir); /* we ignore any error here */
+ FreeDir(dir); /* we ignore any error here */
}
static int
-ptrack_filelist_getnext(PtScanCtx *ctx)
+ptrack_filelist_getnext(PtScanCtx * ctx)
{
- PtrackFileList_i *pfl = NULL;
- ListCell *cell;
- char *fullpath;
- struct stat fst;
+ PtrackFileList_i *pfl = NULL;
+ ListCell *cell;
+ char *fullpath;
+ struct stat fst;
+ uint32 rel_st_size = 0;
+
+get_next:
/* No more file in the list */
if (list_length(ctx->filelist) == 0)
return -1;
+#ifdef foreach_current_index
+ /* Get first file from the head */
+ cell = list_tail(ctx->filelist);
+ pfl = (PtrackFileList_i *) lfirst(cell);
+
+ /* Remove this file from the list */
+ ctx->filelist = list_delete_last(ctx->filelist);
+#else
/* Get first file from the head */
cell = list_head(ctx->filelist);
pfl = (PtrackFileList_i *) lfirst(cell);
/* Remove this file from the list */
ctx->filelist = list_delete_first(ctx->filelist);
+#endif
if (pfl->segno > 0)
{
@@ -258,30 +448,40 @@ ptrack_filelist_getnext(PtScanCtx *ctx)
ctx->relpath = pfl->path;
}
- ctx->bid.relnode.spcNode = pfl->relnode.spcNode;
- ctx->bid.relnode.dbNode = pfl->relnode.dbNode;
- ctx->bid.relnode.relNode = pfl->relnode.relNode;
+ nodeSpc(ctx->bid.relnode) = nodeSpc(pfl->relnode);
+ nodeDb(ctx->bid.relnode) = nodeDb(pfl->relnode);
+ nodeRel(ctx->bid.relnode) = nodeRel(pfl->relnode);
ctx->bid.forknum = pfl->forknum;
ctx->bid.blocknum = 0;
if (stat(fullpath, &fst) != 0)
{
- elog(WARNING, "cannot stat file %s", fullpath);
+ elog(WARNING, "ptrack: cannot stat file %s", fullpath);
/* But try the next one */
- return ptrack_filelist_getnext(ctx);
+ goto get_next;
+ }
+
+ rel_st_size = fst.st_size;
+
+ if (rel_st_size == 0)
+ {
+ elog(DEBUG3, "ptrack: skip empty file %s", fullpath);
+
+ /* But try the next one */
+ goto get_next;
}
if (pfl->segno > 0)
{
- ctx->relsize = pfl->segno * RELSEG_SIZE + fst.st_size / BLCKSZ;
+ ctx->relsize = pfl->segno * RELSEG_SIZE + rel_st_size / BLCKSZ;
ctx->bid.blocknum = pfl->segno * RELSEG_SIZE;
}
else
/* Estimate relsize as size of first segment in blocks */
- ctx->relsize = fst.st_size / BLCKSZ;
+ ctx->relsize = rel_st_size / BLCKSZ;
- elog(DEBUG3, "got file %s with size %u from the ptrack list", pfl->path, ctx->relsize);
+ elog(DEBUG3, "ptrack: got file %s with size %u from the file list", pfl->path, ctx->relsize);
return 0;
}
@@ -299,83 +499,37 @@ ptrack_version(PG_FUNCTION_ARGS)
/*
* Function to get last ptrack map initialization LSN.
*/
-PG_FUNCTION_INFO_V1(pg_ptrack_control_lsn);
+PG_FUNCTION_INFO_V1(ptrack_init_lsn);
Datum
-pg_ptrack_control_lsn(PG_FUNCTION_ARGS)
+ptrack_init_lsn(PG_FUNCTION_ARGS)
{
if (ptrack_map != NULL)
- PG_RETURN_LSN(ptrack_map->init_lsn);
+ {
+ XLogRecPtr init_lsn = pg_atomic_read_u64(&ptrack_map->init_lsn);
+
+ PG_RETURN_LSN(init_lsn);
+ }
else
{
- elog(DEBUG1, "pg_ptrack_control_lsn(). no ptrack_map");
+ elog(WARNING, "ptrack is disabled");
PG_RETURN_LSN(InvalidXLogRecPtr);
}
}
-/*
- * Function to retrieve blocks via buffercache.
- */
-PG_FUNCTION_INFO_V1(pg_ptrack_get_block);
-Datum
-pg_ptrack_get_block(PG_FUNCTION_ARGS)
-{
- Oid tablespace_oid = PG_GETARG_OID(0);
- Oid db_oid = PG_GETARG_OID(1);
- Oid relfilenode = PG_GETARG_OID(2);
- BlockNumber blkno = PG_GETARG_UINT32(3);
- bytea *raw_page;
- char *raw_page_data;
- Buffer buf;
- RelFileNode rnode;
- BlockNumber nblocks;
- SMgrRelation smgr;
-
- rnode.dbNode = db_oid;
- rnode.spcNode = tablespace_oid;
- rnode.relNode = relfilenode;
-
- elog(DEBUG1, "pg_ptrack_get_block(%i, %i, %i, %u)",
- tablespace_oid, db_oid, relfilenode, blkno);
- smgr = smgropen(rnode, InvalidBackendId);
- nblocks = smgrnblocks(smgr, MAIN_FORKNUM);
-
- if (blkno >= nblocks)
- PG_RETURN_NULL();
-
- /* Initialize buffer to copy to */
- raw_page = (bytea *) palloc0(BLCKSZ + VARHDRSZ);
- SET_VARSIZE(raw_page, BLCKSZ + VARHDRSZ);
- raw_page_data = VARDATA(raw_page);
-
- buf = ReadBufferWithoutRelcache(rnode, MAIN_FORKNUM, blkno, RBM_NORMAL, NULL);
-
- if (buf == InvalidBuffer)
- elog(ERROR, "Block is not found in the buffer cache");
-
- LockBuffer(buf, BUFFER_LOCK_SHARE);
-
- memcpy(raw_page_data, BufferGetPage(buf), BLCKSZ);
-
- LockBuffer(buf, BUFFER_LOCK_UNLOCK);
- ReleaseBuffer(buf);
-
- PG_RETURN_BYTEA_P(raw_page);
-}
-
/*
* Return set of database blocks which were changed since specified LSN.
* This function may return false positives (blocks that have not been updated).
*/
-PG_FUNCTION_INFO_V1(pg_ptrack_get_pagemapset);
+PG_FUNCTION_INFO_V1(ptrack_get_pagemapset);
Datum
-pg_ptrack_get_pagemapset(PG_FUNCTION_ARGS)
+ptrack_get_pagemapset(PG_FUNCTION_ARGS)
{
- FuncCallContext *funcctx;
- PtScanCtx *ctx;
- MemoryContext oldcontext;
- XLogRecPtr update_lsn;
- datapagemap_t pagemap;
- char gather_path[MAXPGPATH];
+ PtScanCtx *ctx;
+ FuncCallContext *funcctx;
+ MemoryContext oldcontext;
+ datapagemap_t pagemap;
+ int64 pagecount = 0;
+ char gather_path[MAXPGPATH];
/* Exit immediately if there is no map */
if (ptrack_map == NULL)
@@ -384,6 +538,7 @@ pg_ptrack_get_pagemapset(PG_FUNCTION_ARGS)
if (SRF_IS_FIRSTCALL())
{
TupleDesc tupdesc;
+
funcctx = SRF_FIRSTCALL_INIT();
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
@@ -392,11 +547,15 @@ pg_ptrack_get_pagemapset(PG_FUNCTION_ARGS)
ctx->lsn = PG_GETARG_LSN(0);
ctx->filelist = NIL;
- // get_call_result_type(fcinfo, NULL, &funcctx->tuple_desc);
/* Make tuple descriptor */
- tupdesc = CreateTemplateTupleDesc(2);
+#if PG_VERSION_NUM >= 120000
+ tupdesc = CreateTemplateTupleDesc(3);
+#else
+ tupdesc = CreateTemplateTupleDesc(3, false);
+#endif
TupleDescInitEntry(tupdesc, (AttrNumber) 1, "path", TEXTOID, -1, 0);
- TupleDescInitEntry(tupdesc, (AttrNumber) 2, "pagemap", BYTEAOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "pagecount", INT8OID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "pagemap", BYTEAOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
funcctx->user_fctx = ctx;
@@ -432,17 +591,24 @@ pg_ptrack_get_pagemapset(PG_FUNCTION_ARGS)
while (true)
{
+ uint64 hash;
+ size_t slot1;
+ size_t slot2;
+ XLogRecPtr update_lsn1;
+ XLogRecPtr update_lsn2;
+
/* Stop traversal if there are no more segments */
- if (ctx->bid.blocknum > ctx->relsize)
+ if (ctx->bid.blocknum >= ctx->relsize)
{
/* We completed a segment and there is a bitmap to return */
if (pagemap.bitmap != NULL)
{
- Datum values[2];
- bool nulls[2] = {false};
- char pathname[MAXPGPATH];
- bytea *result = NULL;
- Size result_sz = pagemap.bitmapsize + VARHDRSZ;
+ Datum values[3];
+ bool nulls[3] = {false};
+ char pathname[MAXPGPATH];
+ bytea *result = NULL;
+ Size result_sz = pagemap.bitmapsize + VARHDRSZ;
+ HeapTuple htup = NULL;
/* Create a bytea copy of our bitmap */
result = (bytea *) palloc(result_sz);
@@ -452,32 +618,55 @@ pg_ptrack_get_pagemapset(PG_FUNCTION_ARGS)
strcpy(pathname, ctx->relpath);
values[0] = CStringGetTextDatum(pathname);
- values[1] = PointerGetDatum(result);
+ values[1] = Int64GetDatum(pagecount);
+ values[2] = PointerGetDatum(result);
pfree(pagemap.bitmap);
pagemap.bitmap = NULL;
pagemap.bitmapsize = 0;
+ pagecount = 0;
- SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(heap_form_tuple(funcctx->tuple_desc, values, nulls)));
- }
- else
- {
- /* We have just processed unchanged file, let's pick next */
- if (ptrack_filelist_getnext(ctx) < 0)
- SRF_RETURN_DONE(funcctx);
+ htup = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+ if (htup)
+ SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(htup));
}
+
+ if (ptrack_filelist_getnext(ctx) < 0)
+ SRF_RETURN_DONE(funcctx);
}
- update_lsn = pg_atomic_read_u64(&PtrackContent(ptrack_map)[BID_HASH_FUNC(ctx->bid)]);
+ hash = BID_HASH_FUNC(ctx->bid);
+ slot1 = (size_t)(hash % PtrackContentNblocks);
- if (update_lsn != InvalidXLogRecPtr)
- elog(DEBUG3, "update_lsn %X/%X of blckno %u of file %s",
- (uint32) (update_lsn >> 32), (uint32) update_lsn,
- ctx->bid.blocknum, ctx->relpath);
+ update_lsn1 = pg_atomic_read_u64(&ptrack_map->entries[slot1]);
- /* Block has been changed since specified LSN. Mark it in the bitmap */
- if (update_lsn >= ctx->lsn)
- datapagemap_add(&pagemap, ctx->bid.blocknum % ((BlockNumber) RELSEG_SIZE));
+#if USE_ASSERT_CHECKING
+ if (update_lsn1 != InvalidXLogRecPtr)
+ elog(DEBUG3, "ptrack: update_lsn1 %X/%X of blckno %u of file %s",
+ (uint32) (update_lsn1 >> 32), (uint32) update_lsn1,
+ ctx->bid.blocknum, ctx->relpath);
+#endif
+
+ /* Only probe the second slot if the first one is marked */
+ if (update_lsn1 >= ctx->lsn)
+ {
+ slot2 = (size_t)(((hash << 32) | (hash >> 32)) % PtrackContentNblocks);
+ update_lsn2 = pg_atomic_read_u64(&ptrack_map->entries[slot2]);
+
+#if USE_ASSERT_CHECKING
+ if (update_lsn2 != InvalidXLogRecPtr)
+ elog(DEBUG3, "ptrack: update_lsn2 %X/%X of blckno %u of file %s",
+ (uint32) (update_lsn1 >> 32), (uint32) update_lsn2,
+ ctx->bid.blocknum, ctx->relpath);
+#endif
+
+ /* Block has been changed since specified LSN. Mark it in the bitmap */
+ if (update_lsn2 >= ctx->lsn)
+ {
+ pagecount += 1;
+ datapagemap_add(&pagemap, ctx->bid.blocknum % ((BlockNumber) RELSEG_SIZE));
+ }
+ }
ctx->bid.blocknum += 1;
}
diff --git a/ptrack.control b/ptrack.control
index de4dbdf..7e3a2b7 100644
--- a/ptrack.control
+++ b/ptrack.control
@@ -1,5 +1,5 @@
# ptrack extension
-comment = 'public API for internal ptrack engine'
-default_version = '2.0'
+comment = 'block-level incremental backup engine'
+default_version = '2.4'
module_pathname = '$libdir/ptrack'
relocatable = true
diff --git a/ptrack.h b/ptrack.h
new file mode 100644
index 0000000..abeffb3
--- /dev/null
+++ b/ptrack.h
@@ -0,0 +1,88 @@
+/*-------------------------------------------------------------------------
+ *
+ * ptrack.h
+ * header for ptrack map for tracking updates of relation's pages
+ *
+ *
+ * Copyright (c) 2019-2022, Postgres Professional
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/storage/ptrack.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef PTRACK_H
+#define PTRACK_H
+
+#include "access/xlogdefs.h"
+#include "storage/block.h"
+#include "storage/buf.h"
+#if PG_VERSION_NUM >= 160000
+#include "storage/relfilelocator.h"
+#else
+#include "storage/relfilenode.h"
+#endif
+#include "storage/smgr.h"
+#include "utils/relcache.h"
+
+/* Ptrack version as a string */
+#define PTRACK_VERSION "2.4"
+/* Ptrack version as a number */
+#define PTRACK_VERSION_NUM 240
+/* Last ptrack version that changed map file format */
+#define PTRACK_MAP_FILE_VERSION_NUM 220
+
+#if PG_VERSION_NUM >= 160000
+#define RelFileNode RelFileLocator
+#define RelFileNodeBackend RelFileLocatorBackend
+#define nodeDb(node) (node).dbOid
+#define nodeSpc(node) (node).spcOid
+#define nodeRel(node) (node).relNumber
+#define nodeOf(ndbck) (ndbck).locator
+#else
+#define nodeDb(node) (node).dbNode
+#define nodeSpc(node) (node).spcNode
+#define nodeRel(node) (node).relNode
+#define nodeOf(ndbck) (ndbck).node
+#endif
+
+#if PG_VERSION_NUM >= 170000
+#define InvalidBackendId INVALID_PROC_NUMBER
+#endif
+
+/*
+ * Structure identifying block on the disk.
+ */
+typedef struct PtBlockId
+{
+ RelFileNode relnode;
+ ForkNumber forknum;
+ BlockNumber blocknum;
+} PtBlockId;
+
+/*
+ * Context for ptrack_get_pagemapset set returning function.
+ */
+typedef struct PtScanCtx
+{
+ XLogRecPtr lsn;
+ PtBlockId bid;
+ uint32 relsize;
+ char *relpath;
+ List *filelist;
+} PtScanCtx;
+
+/*
+ * List item type for ptrack data files list.
+ */
+typedef struct PtrackFileList_i
+{
+ RelFileNode relnode;
+ ForkNumber forknum;
+ int segno;
+ char *path;
+
+} PtrackFileList_i;
+
+#endif /* PTRACK_H */
diff --git a/t/001_basic.pl b/t/001_basic.pl
new file mode 100644
index 0000000..bdb1eca
--- /dev/null
+++ b/t/001_basic.pl
@@ -0,0 +1,205 @@
+#
+# Here we mostly do sanity checks and verify, that ptrack public API works
+# as expected. Data integrity after incremental backups taken via ptrack
+# is tested on the pg_probackup side.
+#
+
+use strict;
+use warnings;
+use Test::More;
+
+my $pg_15_modules;
+
+BEGIN
+{
+ $pg_15_modules = eval
+ {
+ require PostgreSQL::Test::Cluster;
+ require PostgreSQL::Test::Utils;
+ return 1;
+ };
+
+ unless (defined $pg_15_modules)
+ {
+ $pg_15_modules = 0;
+
+ require PostgresNode;
+ require TestLib;
+ }
+}
+
+plan tests => 23;
+
+note('PostgreSQL 15 modules are used: ' . ($pg_15_modules ? 'yes' : 'no'));
+
+my $node;
+my $res;
+my $res_stdout;
+my $res_stderr;
+
+# Create node.
+# Older versions of PostgreSQL modules use get_new_node function.
+# Newer use standard perl object constructor syntax.
+eval
+{
+ if ($pg_15_modules)
+ {
+ $node = PostgreSQL::Test::Cluster->new("node");
+ }
+ else
+ {
+ $node = PostgresNode::get_new_node("node");
+ }
+};
+
+$node->init;
+$node->start;
+
+# Could not load ptrack module after postmaster start
+($res, $res_stdout, $res_stderr) = $node->psql("postgres", "CREATE EXTENSION ptrack");
+is($res, 3, 'errors out without shared_preload_libraries = \'ptrack\'');
+like(
+ $res_stderr,
+ qr/ptrack module must be initialized by Postmaster/,
+ 'errors out without shared_preload_libraries = \'ptrack\'');
+
+# Load ptrack library
+$node->append_conf(
+ 'postgresql.conf', q{
+wal_level = 'minimal'
+shared_preload_libraries = 'ptrack'
+log_min_messages = debug1
+});
+$node->restart;
+
+$node->safe_psql("postgres", "CREATE EXTENSION ptrack");
+
+# Check some static functions
+$node->safe_psql("postgres", "SELECT ptrack_version()");
+
+# Could not use ptrack if disabled
+($res, $res_stdout, $res_stderr) = $node->psql("postgres", "SELECT ptrack_get_pagemapset('0/0')");
+is($res, 3, 'errors out if ptrack is disabled');
+like(
+ $res_stderr,
+ qr/ptrack is disabled/,
+ 'errors out if ptrack is disabled');
+($res, $res_stdout, $res_stderr) = $node->psql("postgres", "SELECT ptrack_init_lsn()");
+is($res, 0, 'only warning if ptrack is disabled');
+like(
+ $res_stdout,
+ qr/0\/0/,
+ 'should print init LSN 0/0 if disabled');
+like(
+ $res_stderr,
+ qr/ptrack is disabled/,
+ 'warning if ptrack is disabled');
+
+# Actually enable ptrack
+$node->append_conf(
+ 'postgresql.conf', q{
+ptrack.map_size = 13
+});
+$node->stop;
+$res = $node->start(fail_ok => 1);
+is($res, 0, 'could not start with wal_level = \'minimal\'');
+$node->append_conf(
+ 'postgresql.conf', q{
+wal_level = 'replica'
+});
+$node->start;
+
+# Do checkpoint (test ptrack hook)
+$node->safe_psql("postgres", "CHECKPOINT");
+
+# Remember pg_current_wal_flush_lsn() value
+my $flush_lsn = $node->safe_psql("postgres", "SELECT pg_current_wal_flush_lsn()");
+
+# Remember ptrack init_lsn
+my $init_lsn = $node->safe_psql("postgres", "SELECT ptrack_init_lsn()");
+unlike(
+ $init_lsn,
+ qr/0\/0/,
+ 'ptrack init LSN should not be 0/0 after CHECKPOINT');
+
+# Ptrack map should survive crash
+$node->stop('immediate');
+$node->start;
+$res_stdout = $node->safe_psql("postgres", "SELECT ptrack_init_lsn()");
+is($res_stdout, $init_lsn, 'ptrack init_lsn should be the same after crash recovery');
+
+# Do some stuff, which hits ptrack
+$node->safe_psql("postgres", "CREATE DATABASE ptrack_test");
+$node->safe_psql("postgres", "CREATE TABLE ptrack_test AS SELECT i AS id FROM generate_series(0, 1000) i");
+
+# Remember DB and relation oids
+my $db_oid = $node->safe_psql("postgres", "SELECT oid FROM pg_database WHERE datname = 'ptrack_test'");
+my $rel_oid = $node->safe_psql("postgres", "SELECT relfilenode FROM pg_class WHERE relname = 'ptrack_test'");
+
+# Data should survive clean restart
+$node->restart;
+$res_stdout = $node->safe_psql("postgres", "SELECT ptrack_get_pagemapset('$flush_lsn')");
+like(
+ $res_stdout,
+ qr/base\/$db_oid/,
+ 'ptrack pagemapset should contain new database oid');
+like(
+ $res_stdout,
+ qr/$rel_oid/,
+ 'ptrack pagemapset should contain new relation oid');
+
+# Check change stats
+$res_stdout = $node->safe_psql("postgres", "SELECT pages FROM ptrack_get_change_stat('$flush_lsn')");
+is($res_stdout > 0, 1, 'should be able to get aggregated stats of changes');
+
+# We should be able to change ptrack map size (but loose all changes)
+$node->append_conf(
+ 'postgresql.conf', q{
+ptrack.map_size = 14
+});
+$node->restart;
+
+$node->safe_psql("postgres", "CHECKPOINT");
+$res_stdout = $node->safe_psql("postgres", "SELECT ptrack_init_lsn()");
+unlike(
+ $res_stdout,
+ qr/0\/0/,
+ 'ptrack init LSN should not be 0/0 after CHECKPOINT');
+ok($res_stdout ne $init_lsn, 'ptrack init_lsn should not be the same after map resize');
+$res_stdout = $node->safe_psql("postgres", "SELECT ptrack_get_pagemapset('$flush_lsn')");
+unlike(
+ $res_stdout,
+ qr/base\/$db_oid/,
+ 'we should loose changes after ptrack map resize');
+
+# We should be able to turn off ptrack and clean up all files by stting ptrack.map_size = 0
+$node->append_conf(
+ 'postgresql.conf', q{
+ptrack.map_size = 0
+});
+$node->restart;
+
+# Check that we have lost everything
+ok(! -f $node->data_dir . "/global/ptrack.map", "ptrack.map should be cleaned up");
+ok(! -f $node->data_dir . "/global/ptrack.map.tmp", "ptrack.map.tmp should be cleaned up");
+
+($res, $res_stdout, $res_stderr) = $node->psql("postgres", "SELECT ptrack_get_pagemapset('0/0')");
+is($res, 3, 'errors out if ptrack is disabled');
+like(
+ $res_stderr,
+ qr/ptrack is disabled/,
+ 'errors out if ptrack is disabled');
+($res, $res_stdout, $res_stderr) = $node->psql("postgres", "SELECT ptrack_init_lsn()");
+is($res, 0, 'only warning if ptrack is disabled');
+like(
+ $res_stdout,
+ qr/0\/0/,
+ 'should print init LSN 0/0 if disabled');
+like(
+ $res_stderr,
+ qr/ptrack is disabled/,
+ 'warning if ptrack is disabled');
+
+$node->stop;
+
+done_testing;
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy