Skip to content

Commit da29fc6

Browse files
author
Nikita Glukhov
committed
Add logically chunked arrays to jsonb_toaster
1 parent fdf472a commit da29fc6

File tree

5 files changed

+343
-3
lines changed

5 files changed

+343
-3
lines changed

‎contrib/jsonb_toaster/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ MODULE_big = jsonb_toaster
44
OBJS = \
55
$(WIN32RES) \
66
jsonb_toaster.o \
7-
jsonb_toast_internals.o
7+
jsonb_toast_internals.o \
8+
jsonb_toast_array.o
89

910
EXTENSION = jsonb_toaster
1011
DATA = jsonb_toaster--1.0.sql

‎contrib/jsonb_toaster/expected/jsonb_toaster.out

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,3 +345,259 @@ select id, js->'c'->0->>'a' from test_jsonbz_arr order by id;
345345
19 | eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
346346
(20 rows)
347347

348+
create table test_jsonxa_arr (id int, js jsonb toaster jsonb_toaster);
349+
insert into test_jsonxa_arr
350+
select i, (select jsonb_agg(j) from generate_series(1, (2 ^ i)::int) j)
351+
from generate_series(7, 20) i;
352+
select id, pg_column_size(js) from test_jsonxa_arr order by id;
353+
id | pg_column_size
354+
----+----------------
355+
7 | 1544
356+
8 | 894
357+
9 | 1760
358+
10 | 108
359+
11 | 168
360+
12 | 288
361+
13 | 538
362+
14 | 1168
363+
15 | 111051
364+
16 | 222124
365+
17 | 444295
366+
18 | 888615
367+
19 | 1777262
368+
20 | 3554747
369+
(14 rows)
370+
371+
select id, (select count(*) from jsonb_array_elements(js)) from test_jsonxa_arr order by id;
372+
id | count
373+
----+---------
374+
7 | 128
375+
8 | 256
376+
9 | 512
377+
10 | 1024
378+
11 | 2048
379+
12 | 4096
380+
13 | 8192
381+
14 | 16384
382+
15 | 32768
383+
16 | 65536
384+
17 | 131072
385+
18 | 262144
386+
19 | 524288
387+
20 | 1048576
388+
(14 rows)
389+
390+
select id, js -> 100 from test_jsonxa_arr order by id;
391+
id | ?column?
392+
----+----------
393+
7 | 101
394+
8 | 101
395+
9 | 101
396+
10 | 101
397+
11 | 101
398+
12 | 101
399+
13 | 101
400+
14 | 101
401+
15 | 101
402+
16 | 101
403+
17 | 101
404+
18 | 101
405+
19 | 101
406+
20 | 101
407+
(14 rows)
408+
409+
select id, js -> 200 from test_jsonxa_arr order by id;
410+
id | ?column?
411+
----+----------
412+
7 |
413+
8 | 201
414+
9 | 201
415+
10 | 201
416+
11 | 201
417+
12 | 201
418+
13 | 201
419+
14 | 201
420+
15 | 201
421+
16 | 201
422+
17 | 201
423+
18 | 201
424+
19 | 201
425+
20 | 201
426+
(14 rows)
427+
428+
select id, js -> (jsonb_array_length(js) - 1) from test_jsonxa_arr order by id;
429+
id | ?column?
430+
----+----------
431+
7 | 128
432+
8 | 256
433+
9 | 512
434+
10 | 1024
435+
11 | 2048
436+
12 | 4096
437+
13 | 8192
438+
14 | 16384
439+
15 | 32768
440+
16 | 65536
441+
17 | 131072
442+
18 | 262144
443+
19 | 524288
444+
20 | 1048576
445+
(14 rows)
446+
447+
select id, json_query(js, '$[0 to 10]' with wrapper) from test_jsonxa_arr order by id;
448+
id | json_query
449+
----+-------------------------------------
450+
7 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
451+
8 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
452+
9 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
453+
10 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
454+
11 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
455+
12 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
456+
13 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
457+
14 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
458+
15 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
459+
16 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
460+
17 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
461+
18 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
462+
19 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
463+
20 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
464+
(14 rows)
465+
466+
update test_jsonxa_arr set js = jsonb_set(js, '{0}', '0');
467+
select id, (select count(*) from jsonb_array_elements(js)) from test_jsonxa_arr order by id;
468+
id | count
469+
----+---------
470+
7 | 128
471+
8 | 256
472+
9 | 512
473+
10 | 1024
474+
11 | 2048
475+
12 | 4096
476+
13 | 8192
477+
14 | 16384
478+
15 | 32768
479+
16 | 65536
480+
17 | 131072
481+
18 | 262144
482+
19 | 524288
483+
20 | 1048576
484+
(14 rows)
485+
486+
select id, js -> 0 from test_jsonxa_arr order by id;
487+
id | ?column?
488+
----+----------
489+
7 | 0
490+
8 | 0
491+
9 | 0
492+
10 | 0
493+
11 | 0
494+
12 | 0
495+
13 | 0
496+
14 | 0
497+
15 | 0
498+
16 | 0
499+
17 | 0
500+
18 | 0
501+
19 | 0
502+
20 | 0
503+
(14 rows)
504+
505+
select id, js -> 100 from test_jsonxa_arr order by id;
506+
id | ?column?
507+
----+----------
508+
7 | 101
509+
8 | 101
510+
9 | 101
511+
10 | 101
512+
11 | 101
513+
12 | 101
514+
13 | 101
515+
14 | 101
516+
15 | 101
517+
16 | 101
518+
17 | 101
519+
18 | 101
520+
19 | 101
521+
20 | 101
522+
(14 rows)
523+
524+
update test_jsonxa_arr set js = json_modify(js, set '$[0 to 3]' = '0');
525+
select id, json_query(js, '$[0 to 10]' with wrapper) from test_jsonxa_arr order by id;
526+
id | json_query
527+
----+---------------------------------------------
528+
7 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
529+
8 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
530+
9 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
531+
10 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
532+
11 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
533+
12 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
534+
13 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
535+
14 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
536+
15 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
537+
16 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
538+
17 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
539+
18 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
540+
19 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
541+
20 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
542+
(14 rows)
543+
544+
update test_jsonxa_arr set js = json_modify(js, insert '$[200 to 203]' = '0');
545+
select id, json_query(js, '$[200 to 210]' with wrapper) from test_jsonxa_arr order by id;
546+
id | json_query
547+
----+---------------------------------------------------------
548+
7 | ["0", "0", "0", "0"]
549+
8 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
550+
9 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
551+
10 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
552+
11 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
553+
12 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
554+
13 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
555+
14 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
556+
15 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
557+
16 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
558+
17 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
559+
18 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
560+
19 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
561+
20 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
562+
(14 rows)
563+
564+
update test_jsonxa_arr set js = json_modify(js, set '$[0 to 300]' = '0');
565+
select id, json_query(js, '$[290 to 310]' with wrapper) from test_jsonxa_arr order by id;
566+
id | json_query
567+
----+-----------------------------------------------------------------------------------------------------------
568+
7 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]
569+
8 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]
570+
9 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
571+
10 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
572+
11 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
573+
12 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
574+
13 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
575+
14 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
576+
15 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
577+
16 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
578+
17 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
579+
18 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
580+
19 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
581+
20 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
582+
(14 rows)
583+
584+
update test_jsonxa_arr set js = json_modify(js, insert '$[1000003 to 1000005]' = '0');
585+
select id, json_query(js, '$[1000000 to 1000010]' with wrapper) from test_jsonxa_arr order by id;
586+
id | json_query
587+
----+--------------------------------------------------------------------------------------
588+
7 | [null, null, null, "0", "0", "0"]
589+
8 | [null, null, null, "0", "0", "0"]
590+
9 | [null, null, null, "0", "0", "0"]
591+
10 | [null, null, null, "0", "0", "0"]
592+
11 | [null, null, null, "0", "0", "0"]
593+
12 | [null, null, null, "0", "0", "0"]
594+
13 | [null, null, null, "0", "0", "0"]
595+
14 | [null, null, null, "0", "0", "0"]
596+
15 | [null, null, null, "0", "0", "0"]
597+
16 | [null, null, null, "0", "0", "0"]
598+
17 | [null, null, null, "0", "0", "0"]
599+
18 | [null, null, null, "0", "0", "0"]
600+
19 | [null, null, null, "0", "0", "0"]
601+
20 | [999997, 999998, 999999, "0", 1000000, "0", 1000001, "0", 1000002, 1000003, 1000004]
602+
(14 rows)
603+

‎contrib/jsonb_toaster/jsonb_toast_internals.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -998,7 +998,9 @@ jsonx_create_fetch_datum_iterator(struct varlena *attr, Oid toasterid,
998998
iter->toasterid = toasterid;
999999
iter->chunk_tids_inline_size = inline_size;
10001000

1001-
if (inline_size <= 0 ||
1001+
if (type == JSONX_PLAIN_JSONB ||
1002+
type == JSONX_POINTER ||
1003+
type == JSONX_POINTER_COMPRESSED_CHUNKS ||
10021004
type == JSONX_POINTER_DIFF ||
10031005
type == JSONX_POINTER_DIFF_COMP)
10041006
{
@@ -1009,8 +1011,11 @@ jsonx_create_fetch_datum_iterator(struct varlena *attr, Oid toasterid,
10091011
type == JSONX_POINTER_COMPRESSED_CHUNKS ||
10101012
type == JSONX_POINTER_DIFF_COMP;
10111013
}
1012-
else
1014+
else if (type == JSONX_POINTER_DIRECT_TIDS_COMP ||
1015+
type == JSONX_POINTER_DIRECT_TIDS)
10131016
{
1017+
Assert(inline_size > 0);
1018+
10141019
iter->nchunk_tids = header & ~JSONX_POINTER_TYPE_MASK;
10151020
iter->compressed_chunks = false;
10161021

‎contrib/jsonb_toaster/jsonb_toaster.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#define JSONX_POINTER_COMPRESSED_CHUNKS 0x40000000
2525
#define JSONX_POINTER_DIFF 0x50000000
2626
#define JSONX_POINTER_DIFF_COMP 0x60000000
27+
#define JSONX_CHUCKED_ARRAY 0x70000000
2728

2829
#define JSONX_CUSTOM_PTR_HEADER_SIZE (INTALIGN(VARATT_CUSTOM_SIZE(0)) + sizeof(uint32))
2930

@@ -119,13 +120,33 @@ typedef struct JsonxPointerDiff
119120
char data[FLEXIBLE_ARRAY_MEMBER];
120121
} JsonxPointerDiff;
121122

123+
#define JSONXA_INLINE_CHUNK 0x80000000
124+
125+
typedef ItemPointerData JsonxArrayChunkPtr;
126+
typedef uint32 JsonxArrayChunkOffset;
127+
128+
typedef struct JsonxArray
129+
{
130+
int32 n_elems;
131+
int32 n_chunks;
132+
Oid toastrelid;
133+
JsonxArrayChunkOffset chunk_offsets[FLEXIBLE_ARRAY_MEMBER];
134+
/* JsonxArrayChunkPtr chunk_ptrs[FLEXIBLE_ARRAY_MEMBER]; */
135+
} JsonxArray;
136+
137+
#define JSONX_ARRAY_HDR_SIZE (JSONX_CUSTOM_PTR_HEADER_SIZE + offsetof(JsonxArray, chunk_offsets))
138+
139+
extern JsonContainerOps jsonxaContainerOps;
140+
extern void jsonxaInit(JsonContainerData *jc, Datum value);
141+
122142
extern Datum jsonx_toast_save_datum(Relation rel, Datum value,
123143
struct varlena *oldexternal,
124144
int options);
125145
extern Datum
126146
jsonx_toast_save_datum_ext(Relation rel, Oid toasterid, Datum value,
127147
struct varlena *oldexternal, int options,
128148
struct varlena **p_chunk_tids,
149+
ItemPointerData *chunk_tids,
129150
bool compress_chunks);
130151
extern void jsonx_toast_delete_datum(Datum value, bool is_speculative);
131152

@@ -145,6 +166,17 @@ jsonx_toast_make_pointer_diff(Oid toasterid, struct varatt_external *ptr,
145166
int32 diff_offset, int32 diff_len,
146167
const void *diff_data);
147168

169+
extern Datum jsonx_toast_array_chunks(Relation rel, Oid toastrelid,
170+
Oid toasterid, int options,
171+
int n_elems, int n_chunks,
172+
JsonxArrayChunkOffset *chunk_offsets,
173+
JsonxArrayChunkPtr *chunk_ptrs,
174+
Datum *chunk_jbs);
175+
176+
extern struct varlena *
177+
jsonx_toast_wrap_array_into_pointer(Oid toasterid, JsonxArray *array,
178+
int data_size);
179+
148180
extern struct varlena *
149181
jsonx_toast_compress_tids(struct varlena *chunk_tids, int max_size);
150182

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy