Allow usage of huge maintenance_work_mem for GIN build.

feodor · feodor · commit 30bb26b5e04c · 2015-09-02T20:08:58.000+03:00
Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem &gt; 4GB.

Robert Abraham &lt;robert.abraham86@googlemail.com&gt; with additions by me
diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c
@@ -14,6 +14,8 @@
 
 #include "postgres.h"
 
+#include <limits.h>
+
 #include "access/gin_private.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
 	 */
 	if (eo->count >= eo->maxcount)
 	{
+		if (eo->maxcount > INT_MAX)
+			ereport(ERROR,
+					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+					 errmsg("posting list is too long"),
+					 errhint("Reduce maintenance_work_mem")));
+
 		accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
 		eo->maxcount *= 2;
 		eo->list = (ItemPointerData *)
-			repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
+			repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
 		accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
 	}
 
diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c
@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
 		 */
 		if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
 			(GinPageHasFullRow(page) &&
-			 (accum.allocatedMemory >= maintenance_work_mem * 1024L)))
+			 (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
 		{
 			ItemPointerData *list;
 			uint32		nlist;
diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
 							   &htup->t_self);
 
 	/* If we've maxed out our available memory, dump everything to the index */
-	if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
+	if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
 	{
 		ItemPointerData *list;
 		Datum		key;
diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
 typedef struct
 {
 	GinState   *ginstate;
-	long		allocatedMemory;
+	Size		allocatedMemory;
 	GinEntryAccumulator *entryallocator;
 	uint32		eas_used;
 	RBTree	   *tree;

Original file line number	Diff line number	Diff line change
`@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,`
`814`	`814`	`*/`
`815`	`815`	`if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber \|\|`
`816`	`816`	`(GinPageHasFullRow(page) &&`
`817`		`- (accum.allocatedMemory >= maintenance_work_mem * 1024L)))`
	`817`	`+ (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))`
`818`	`818`	`{`
`819`	`819`	`ItemPointerData *list;`
`820`	`820`	`uint32 nlist;`
Original file line number	Diff line number	Diff line change
`@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,`
`281`	`281`	`&htup->t_self);`
`282`	`282`
`283`	`283`	`/* If we've maxed out our available memory, dump everything to the index */`
`284`		`- if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)`
	`284`	`+ if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)`
`285`	`285`	`{`
`286`	`286`	`ItemPointerData *list;`
`287`	`287`	`Datum key;`
Original file line number	Diff line number	Diff line change
`@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator`
`903`	`903`	`typedef struct`
`904`	`904`	`{`
`905`	`905`	`GinState *ginstate;`
`906`		`- long allocatedMemory;`
	`906`	`+ Size allocatedMemory;`
`907`	`907`	`GinEntryAccumulator *entryallocator;`
`908`	`908`	`uint32 eas_used;`
`909`	`909`	`RBTree *tree;`