Skip to content

Commit 89e6e7a

Browse files
author
Nikita Glukhov
committed
Inline TOAST: compression
1 parent f061c82 commit 89e6e7a

File tree

8 files changed

+86
-68
lines changed

8 files changed

+86
-68
lines changed

‎src/backend/access/common/detoast.c

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ create_detoast_iterator(struct varlena *attr)
4949
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
5050
{
5151
iter->compressed = true;
52+
iter->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
5253

5354
/* prepare buffer to received decompressed data */
5455
iter->buf = create_toast_buffer(toast_pointer.va_rawsize, false);
55-
56-
/* initialize state for pglz_decompress_iterate() */
57-
iter->ctrl = 0;
58-
iter->ctrlc = INVALID_CTRLC;
59-
iter->len = 0;
60-
iter->off = 0;
6156
}
6257
else
6358
{
6459
iter->compressed = false;
60+
iter->compression_method = TOAST_INVALID_COMPRESSION_ID;
6561

6662
/* point the buffer directly at the raw data */
6763
iter->buf = fetch_iter->buf;
@@ -95,16 +91,14 @@ create_detoast_iterator(struct varlena *attr)
9591
iter->fetch_datum_iterator->buf = buf = create_toast_buffer(VARSIZE_ANY(attr), true);
9692
iter->fetch_datum_iterator->done = true;
9793
iter->compressed = true;
94+
iter->compression_method = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
9895

9996
memcpy((void *) buf->buf, attr, VARSIZE_ANY(attr));
10097
buf->limit = (char *) buf->capacity;
10198

10299
/* prepare buffer to received decompressed data */
103100
iter->buf = create_toast_buffer(TOAST_COMPRESS_EXTSIZE(attr) + VARHDRSZ, false);
104101

105-
/* initialize state for pglz_decompress_iterate() */
106-
iter->ctrl = 0;
107-
iter->ctrlc = INVALID_CTRLC;
108102
return iter;
109103
}
110104
else

‎src/backend/access/common/toast_compression.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ pglz_compress_datum(const struct varlena *value)
6464
len = pglz_compress(VARDATA_ANY(value),
6565
valsize,
6666
(char *) tmp + VARHDRSZ_COMPRESSED,
67+
NULL,
6768
NULL);
6869
if (len < 0)
6970
{

‎src/backend/access/common/toast_internals.c

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1135,28 +1135,12 @@ toast_extract_chunk_fields(Relation toastrel, TupleDesc toasttupDesc,
11351135
}
11361136
}
11371137

1138-
/* ----------
1139-
* create_fetch_datum_iterator -
1140-
*
1141-
* Initialize fetch datum iterator.
1142-
* ----------
1143-
*/
1144-
FetchDatumIterator
1145-
create_fetch_datum_iterator(struct varlena *attr)
1138+
static void
1139+
create_fetch_datum_iterator_scan(FetchDatumIterator iter)
11461140
{
11471141
int validIndex;
1148-
FetchDatumIterator iter;
1149-
1150-
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
1151-
elog(ERROR, "create_fetch_datum_iterator shouldn't be called for non-ondisk datums");
1152-
1153-
iter = (FetchDatumIterator) palloc0(sizeof(FetchDatumIteratorData));
1154-
1155-
/* Must copy to access aligned fields */
1156-
VARATT_EXTERNAL_GET_POINTER(iter->toast_pointer, attr);
11571142

1158-
iter->ressize = VARATT_EXTERNAL_GET_EXTSIZE(iter->toast_pointer);
1159-
iter->numchunks = ((iter->ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
1143+
MemoryContext oldcxt = MemoryContextSwitchTo(iter->mcxt);
11601144

11611145
/*
11621146
* Open the toast relation and its indexes
@@ -1189,6 +1173,33 @@ create_fetch_datum_iterator(struct varlena *attr)
11891173
iter->toastscan = systable_beginscan_ordered(iter->toastrel, iter->toastidxs[validIndex],
11901174
&iter->snapshot, 1, &iter->toastkey);
11911175

1176+
MemoryContextSwitchTo(oldcxt);
1177+
}
1178+
1179+
/* ----------
1180+
* create_fetch_datum_iterator -
1181+
*
1182+
* Initialize fetch datum iterator.
1183+
* ----------
1184+
*/
1185+
FetchDatumIterator
1186+
create_fetch_datum_iterator(struct varlena *attr)
1187+
{
1188+
FetchDatumIterator iter;
1189+
1190+
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
1191+
elog(ERROR, "create_fetch_datum_iterator shouldn't be called for non-ondisk datums");
1192+
1193+
iter = (FetchDatumIterator) palloc0(sizeof(FetchDatumIteratorData));
1194+
1195+
iter->mcxt = CurrentMemoryContext;
1196+
1197+
/* Must copy to access aligned fields */
1198+
VARATT_EXTERNAL_GET_POINTER(iter->toast_pointer, attr);
1199+
1200+
iter->ressize = VARATT_EXTERNAL_GET_EXTSIZE(iter->toast_pointer);
1201+
iter->numchunks = ((iter->ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
1202+
11921203
iter->buf = create_toast_buffer(iter->ressize + VARHDRSZ,
11931204
VARATT_EXTERNAL_IS_COMPRESSED(iter->toast_pointer));
11941205

@@ -1204,7 +1215,7 @@ free_fetch_datum_iterator(FetchDatumIterator iter)
12041215
if (iter == NULL)
12051216
return;
12061217

1207-
if (!iter->done)
1218+
if (!iter->done && iter->toastscan)
12081219
{
12091220
systable_endscan_ordered(iter->toastscan);
12101221
toast_close_indexes(iter->toastidxs, iter->num_indexes, AccessShareLock);
@@ -1236,6 +1247,9 @@ fetch_datum_iterate(FetchDatumIterator iter)
12361247

12371248
Assert(iter != NULL && !iter->done);
12381249

1250+
if (!iter->toastscan)
1251+
create_fetch_datum_iterator_scan(iter);
1252+
12391253
ttup = systable_getnext_ordered(iter->toastscan, ForwardScanDirection);
12401254
if (ttup == NULL)
12411255
{
@@ -1344,8 +1358,9 @@ ToastBuffer *
13441358
create_toast_buffer(int32 size, bool compressed)
13451359
{
13461360
ToastBuffer *buf = (ToastBuffer *) palloc0(sizeof(ToastBuffer));
1347-
buf->buf = (const char *) palloc0(size);
1348-
if (compressed) {
1361+
buf->buf = (const char *) palloc(size);
1362+
if (compressed)
1363+
{
13491364
SET_VARSIZE_COMPRESSED(buf->buf, size);
13501365
/*
13511366
* Note the constraint buf->position <= buf->limit may be broken
@@ -1375,6 +1390,7 @@ free_toast_buffer(ToastBuffer *buf)
13751390
pfree(buf);
13761391
}
13771392

1393+
#if 0
13781394
/* ----------
13791395
* pglz_decompress_iterate -
13801396
*
@@ -1392,7 +1408,7 @@ free_toast_buffer(ToastBuffer *buf)
13921408
*/
13931409
void
13941410
pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
1395-
DetoastIterator iter, unsigned char *destend)
1411+
DetoastIterator iter, const char *destend)
13961412
{
13971413
const unsigned char *sp;
13981414
const unsigned char *srcend;
@@ -1525,3 +1541,4 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
15251541
source->position = (char *) sp;
15261542
dest->limit = (char *) dp;
15271543
}
1544+
#endif

‎src/backend/access/transam/xloginsert.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -911,7 +911,7 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
911911
switch ((WalCompression) wal_compression)
912912
{
913913
case WAL_COMPRESSION_PGLZ:
914-
len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default);
914+
len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default, NULL);
915915
break;
916916

917917
case WAL_COMPRESSION_LZ4:

‎src/common/pg_lzcompress.c

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -507,9 +507,10 @@ pglz_find_match(int16 *hstart, const char *input, const char *end,
507507
*/
508508
int32
509509
pglz_compress(const char *source, int32 slen, char *dest,
510-
const PGLZ_Strategy *strategy)
510+
const PGLZ_Strategy *strategy, int32 *dlen)
511511
{
512512
unsigned char *bp = (unsigned char *) dest;
513+
unsigned char *bend;
513514
unsigned char *bstart = bp;
514515
int hist_next = 1;
515516
bool hist_recycle = false;
@@ -545,6 +546,16 @@ pglz_compress(const char *source, int32 slen, char *dest,
545546
slen > strategy->max_input_size)
546547
return -1;
547548

549+
if (dlen)
550+
{
551+
if (*dlen < 4)
552+
return -1;
553+
554+
bend = bstart + *dlen - 4;
555+
}
556+
else
557+
bend = bstart + PGLZ_MAX_OUTPUT(slen);
558+
548559
/*
549560
* Limit the match parameters to the supported range.
550561
*/
@@ -627,6 +638,9 @@ pglz_compress(const char *source, int32 slen, char *dest,
627638
if (!found_match && bp - bstart >= strategy->first_success_by)
628639
return -1;
629640

641+
if (bp > bend)
642+
break;
643+
630644
/*
631645
* Try to find a match in the history
632646
*/
@@ -671,15 +685,16 @@ pglz_compress(const char *source, int32 slen, char *dest,
671685
if (result_size >= result_max)
672686
return -1;
673687

688+
if (dlen)
689+
*dlen = dp - source;
690+
674691
/* success */
675692
return result_size;
676693
}
677694

678695
/* Opaque pglz decompression state */
679696
typedef struct pglz_state
680697
{
681-
const unsigned char *sp;
682-
unsigned char *dp;
683698
int32 len;
684699
int32 off;
685700
int ctrlc;
@@ -699,28 +714,23 @@ typedef struct pglz_state
699714
* ----------
700715
*/
701716
int32
702-
pglz_decompress_state(const char *source, int32 slen, char *dest,
717+
pglz_decompress_state(const char *source, int32 *slen, char *dest,
703718
int32 dlen, bool check_complete, bool last_cource_chunk,
704719
void **pstate)
705720
{
706721
pglz_state *state = pstate ? *pstate : NULL;
707-
const unsigned char *sp;
708-
const unsigned char *srcend;
709-
unsigned char *dp;
710-
unsigned char *destend;
722+
const unsigned char *sp = (const unsigned char *) source;
723+
const unsigned char *srcend = sp + *slen;
724+
unsigned char *dp = (unsigned char *) dest;
725+
unsigned char *destend = dp + dlen;
711726
unsigned char ctrl;
712727
int ctrlc;
713728
int32 len;
714729
int32 remlen;
715730
int32 off;
716731

717-
srcend = ((const unsigned char *) source) + slen;
718-
destend = ((unsigned char *) dest) + rawsize;
719-
720732
if (state)
721733
{
722-
sp = state->sp;
723-
dp = state->dp;
724734
ctrl = state->ctrl;
725735
ctrlc = state->ctrlc;
726736

@@ -742,7 +752,7 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
742752
if (dp >= destend)
743753
{
744754
state->len = remlen;
745-
state->dp = dp;
755+
*slen = 0;
746756
return (char *) dp - dest;
747757
}
748758

@@ -757,8 +767,6 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
757767
}
758768
else
759769
{
760-
sp = (const unsigned char *) source;
761-
dp = (unsigned char *) dest;
762770
ctrl = 0;
763771
ctrlc = 8;
764772
remlen = 0;
@@ -884,10 +892,10 @@ pglz_decompress_state(const char *source, int32 slen, char *dest,
884892

885893
state->ctrl = ctrl;
886894
state->ctrlc = ctrlc;
887-
state->sp = sp;
888-
state->dp = dp;
889895
state->len = remlen;
890896
state->off = off;
897+
898+
*slen = (const char *) sp - source;
891899
}
892900

893901
/*
@@ -948,5 +956,5 @@ int32
948956
pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize,
949957
bool check_complete)
950958
{
951-
return pglz_decompress_state(source, slen, dest, rawsize, check_complete, true, NULL);
959+
return pglz_decompress_state(source, &slen, dest, rawsize, check_complete, true, NULL);
952960
}

‎src/include/access/detoast.h

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ do { \
4747
#include "postgres.h"
4848
#include "access/genam.h"
4949
#include "access/toasterapi.h"
50+
#include "access/toast_compression.h"
5051

5152
/*
5253
* TOAST buffer is a producer consumer buffer.
@@ -78,10 +79,11 @@ typedef struct FetchDatumIteratorData
7879
ToastBuffer *buf;
7980
Relation toastrel;
8081
Relation *toastidxs;
82+
MemoryContext mcxt;
8183
SysScanDesc toastscan;
8284
ScanKeyData toastkey;
8385
SnapshotData snapshot;
84-
struct varatt_external toast_pointer;
86+
struct varatt_external toast_pointer;
8587
int32 ressize;
8688
int32 nextidx;
8789
int32 numchunks;
@@ -91,22 +93,13 @@ typedef struct FetchDatumIteratorData
9193

9294
typedef struct FetchDatumIteratorData *FetchDatumIterator;
9395

94-
/*
95-
* If "ctrlc" field in iterator is equal to INVALID_CTRLC, it means that
96-
* the field is invalid and need to read the control byte from the
97-
* source buffer in the next iteration, see pglz_decompress_iterate().
98-
*/
99-
#define INVALID_CTRLC 8
100-
10196
typedef struct DetoastIteratorData
10297
{
10398
ToastBuffer *buf;
10499
FetchDatumIterator fetch_datum_iterator;
105-
unsigned char ctrl;
106-
int ctrlc;
107100
int nrefs;
108-
int32 len;
109-
int32 off;
101+
void *decompression_state;
102+
ToastCompressionId compression_method;
110103
bool compressed; /* toast value is compressed? */
111104
bool done;
112105
} DetoastIteratorData;
@@ -118,8 +111,10 @@ extern void free_fetch_datum_iterator(FetchDatumIterator iter);
118111
extern void fetch_datum_iterate(FetchDatumIterator iter);
119112
extern ToastBuffer *create_toast_buffer(int32 size, bool compressed);
120113
extern void free_toast_buffer(ToastBuffer *buf);
114+
extern void toast_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
115+
DetoastIterator iter, const char *destend);
121116
extern void pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
122-
DetoastIterator iter, unsigned char *destend);
117+
DetoastIterator iter, char *destend);
123118

124119
/* ----------
125120
* create_detoast_iterator -
@@ -171,7 +166,7 @@ detoast_iterate(DetoastIterator detoast_iter, const char *destend)
171166
fetch_datum_iterate(fetch_iter);
172167

173168
if (detoast_iter->compressed)
174-
pglz_decompress_iterate(fetch_iter->buf, detoast_iter->buf, detoast_iter, (unsigned char *) destend);
169+
toast_decompress_iterate(fetch_iter->buf, detoast_iter->buf, detoast_iter, destend);
175170

176171
if (detoast_iter->buf->limit == detoast_iter->buf->capacity)
177172
detoast_iter->done = true;

‎src/include/access/toast_internals.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ typedef struct toast_compress_header
4040
#define TOAST_COMPRESS_METHOD(ptr) \
4141
(((toast_compress_header *) (ptr))->tcinfo >> VARLENA_EXTSIZE_BITS)
4242

43+
#define TOAST_COMPRESS_HDRSZ VARHDRSZ_COMPRESSED
44+
#define TOAST_COMPRESS_RAWDATA(attr) ((char *)(attr) + TOAST_COMPRESS_HDRSZ)
45+
4346
#define TOAST_COMPRESS_SET_SIZE_AND_COMPRESS_METHOD(ptr, len, cm_method) \
4447
do { \
4548
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \

‎src/include/common/pg_lzcompress.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,10 @@ extern PGDLLIMPORT const PGLZ_Strategy *const PGLZ_strategy_always;
8484
* ----------
8585
*/
8686
extern int32 pglz_compress(const char *source, int32 slen, char *dest,
87-
const PGLZ_Strategy *strategy);
87+
const PGLZ_Strategy *strategy, int32 *dlen);
8888
extern int32 pglz_decompress(const char *source, int32 slen, char *dest,
8989
int32 rawsize, bool check_complete);
90-
extern int32 pglz_decompress_state(const char *source, int32 slen,
90+
extern int32 pglz_decompress_state(const char *source, int32 *slen,
9191
char *dest, int32 dlen,
9292
bool check_complete, bool last_source_chunk,
9393
void **state);

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy