Skip to content

Commit 0007490

Browse files
tglsfdctglsfdc
committed
Convert the arithmetic for shared memory size calculation from 'int'
to 'Size' (that is, size_t), and install overflow detection checks in it. This allows us to remove the former arbitrary restrictions on NBuffers etc. It won't make any difference in a 32-bit machine, but in a 64-bit machine you could theoretically have terabytes of shared buffers. (How efficiently we could manage 'em remains to be seen.) Similarly, num_temp_buffers, work_mem, and maintenance_work_mem can be set above 2Gb on a 64-bit machine. Original patch from Koichi Suzuki, additional work by moi.
1 parent 2299cea commit 0007490

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+773
-274
lines changed

‎configure

Lines changed: 416 additions & 0 deletions
Large diffs are not rendered by default.

‎configure.in

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
dnl Process this file with autoconf to produce a configure script.
2-
dnl $PostgreSQL: pgsql/configure.in,v 1.419 2005/08/17 20:20:10 tgl Exp $
2+
dnl $PostgreSQL: pgsql/configure.in,v 1.420 2005/08/20 23:26:06 tgl Exp $
33
dnl
44
dnl Developers, please strive to achieve this order:
55
dnl
@@ -1132,6 +1132,9 @@ fi
11321132
# Need a #define for the size of Datum (unsigned long)
11331133
AC_CHECK_SIZEOF([unsigned long])
11341134

1135+
# And one for the size of size_t (enables tweaks for > 32bit address space)
1136+
AC_CHECK_SIZEOF([size_t])
1137+
11351138
# Determine memory alignment requirements for the basic C data types.
11361139

11371140
PGAC_CHECK_ALIGNOF(short)

‎src/backend/access/transam/clog.c

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
* Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
2525
* Portions Copyright (c) 1994, Regents of the University of California
2626
*
27-
* $PostgreSQL: pgsql/src/backend/access/transam/clog.c,v 1.31 2005/06/30 00:00:50 tgl Exp $
27+
* $PostgreSQL: pgsql/src/backend/access/transam/clog.c,v 1.32 2005/08/20 23:26:08 tgl Exp $
2828
*
2929
*-------------------------------------------------------------------------
3030
*/
@@ -144,8 +144,7 @@ TransactionIdGetStatus(TransactionId xid)
144144
/*
145145
* Initialization of shared memory for CLOG
146146
*/
147-
148-
int
147+
Size
149148
CLOGShmemSize(void)
150149
{
151150
return SimpleLruShmemSize();

‎src/backend/access/transam/multixact.c

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
* Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
4343
* Portions Copyright (c) 1994, Regents of the University of California
4444
*
45-
* $PostgreSQL: pgsql/src/backend/access/transam/multixact.c,v 1.7 2005/08/20 01:29:27 ishii Exp $
45+
* $PostgreSQL: pgsql/src/backend/access/transam/multixact.c,v 1.8 2005/08/20 23:26:08 tgl Exp $
4646
*
4747
*-------------------------------------------------------------------------
4848
*/
@@ -1159,13 +1159,20 @@ AtEOXact_MultiXact(void)
11591159
* thus double memory. Also, reserve space for the shared MultiXactState
11601160
* struct and the per-backend MultiXactId arrays (two of those, too).
11611161
*/
1162-
int
1162+
Size
11631163
MultiXactShmemSize(void)
11641164
{
1165+
Size size;
1166+
11651167
#define SHARED_MULTIXACT_STATE_SIZE \
1166-
(sizeof(MultiXactStateData) + sizeof(MultiXactId) * 2 * MaxBackends)
1168+
add_size(sizeof(MultiXactStateData), \
1169+
mul_size(sizeof(MultiXactId) * 2, MaxBackends))
1170+
1171+
size = SHARED_MULTIXACT_STATE_SIZE;
1172+
size = add_size(size, SimpleLruShmemSize());
1173+
size = add_size(size, SimpleLruShmemSize());
11671174

1168-
return (SimpleLruShmemSize() * 2 + SHARED_MULTIXACT_STATE_SIZE);
1175+
return size;
11691176
}
11701177

11711178
void

‎src/backend/access/transam/slru.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
* Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
4949
* Portions Copyright (c) 1994, Regents of the University of California
5050
*
51-
* $PostgreSQL: pgsql/src/backend/access/transam/slru.c,v 1.26 2005/07/04 04:51:44 tgl Exp $
51+
* $PostgreSQL: pgsql/src/backend/access/transam/slru.c,v 1.27 2005/08/20 23:26:08 tgl Exp $
5252
*
5353
*-------------------------------------------------------------------------
5454
*/
@@ -141,9 +141,10 @@ static int SlruSelectLRUPage(SlruCtl ctl, int pageno);
141141
* Initialization of shared memory
142142
*/
143143

144-
int
144+
Size
145145
SimpleLruShmemSize(void)
146146
{
147+
/* we assume NUM_SLRU_BUFFERS isn't so large as to risk overflow */
147148
return BUFFERALIGN(sizeof(SlruSharedData)) + BLCKSZ * NUM_SLRU_BUFFERS;
148149
}
149150

‎src/backend/access/transam/subtrans.c

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
* Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
2323
* Portions Copyright (c) 1994, Regents of the University of California
2424
*
25-
* $PostgreSQL: pgsql/src/backend/access/transam/subtrans.c,v 1.9 2005/06/17 22:32:42 tgl Exp $
25+
* $PostgreSQL: pgsql/src/backend/access/transam/subtrans.c,v 1.10 2005/08/20 23:26:08 tgl Exp $
2626
*
2727
*-------------------------------------------------------------------------
2828
*/
@@ -161,8 +161,7 @@ SubTransGetTopmostTransaction(TransactionId xid)
161161
/*
162162
* Initialization of shared memory for SUBTRANS
163163
*/
164-
165-
int
164+
Size
166165
SUBTRANSShmemSize(void)
167166
{
168167
return SimpleLruShmemSize();

‎src/backend/access/transam/twophase.c

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* Portions Copyright (c) 1994, Regents of the University of California
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.9 2005/07/31 17:19:17 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.10 2005/08/20 23:26:10 tgl Exp $
1111
*
1212
* NOTES
1313
* Each global transaction is associated with a global transaction
@@ -152,13 +152,20 @@ static void ProcessRecords(char *bufptr, TransactionId xid,
152152
/*
153153
* Initialization of shared memory
154154
*/
155-
int
155+
Size
156156
TwoPhaseShmemSize(void)
157157
{
158+
Size size;
159+
158160
/* Need the fixed struct, the array of pointers, and the GTD structs */
159-
return MAXALIGN(offsetof(TwoPhaseStateData, prepXacts) +
160-
sizeof(GlobalTransaction) * max_prepared_xacts) +
161-
sizeof(GlobalTransactionData) * max_prepared_xacts;
161+
size = offsetof(TwoPhaseStateData, prepXacts);
162+
size = add_size(size, mul_size(max_prepared_xacts,
163+
sizeof(GlobalTransaction)));
164+
size = MAXALIGN(size);
165+
size = add_size(size, mul_size(max_prepared_xacts,
166+
sizeof(GlobalTransactionData)));
167+
168+
return size;
162169
}
163170

164171
void

‎src/backend/access/transam/xlog.c

Lines changed: 42 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
88
* Portions Copyright (c) 1994, Regents of the University of California
99
*
10-
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.215 2005/08/11 21:11:43 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.216 2005/08/20 23:26:10 tgl Exp $
1111
*
1212
*-------------------------------------------------------------------------
1313
*/
@@ -91,31 +91,6 @@
9191
#endif
9292
#endif
9393

94-
/*
95-
* Limitation of buffer-alignment for direct io depend on OS and filesystem,
96-
* but BLCKSZ is assumed to be enough for it.
97-
*/
98-
#ifdef O_DIRECT
99-
#define ALIGNOF_XLOG_BUFFER BLCKSZ
100-
#else
101-
#define ALIGNOF_XLOG_BUFFER MAXIMUM_ALIGNOF
102-
#endif
103-
104-
/*
105-
* Switch the alignment routine because ShmemAlloc() returns a max-aligned
106-
* buffer and ALIGNOF_XLOG_BUFFER may be greater than MAXIMUM_ALIGNOF.
107-
*/
108-
#if ALIGNOF_XLOG_BUFFER <= MAXIMUM_ALIGNOF
109-
#define XLOG_BUFFER_ALIGN(LEN) MAXALIGN((LEN))
110-
#else
111-
#define XLOG_BUFFER_ALIGN(LEN) ((LEN) + (ALIGNOF_XLOG_BUFFER))
112-
#endif
113-
/* assume sizeof(ptrdiff_t) == sizeof(void*) */
114-
#define POINTERALIGN(ALIGNVAL,PTR) \
115-
((char *)(((ptrdiff_t) (PTR) + (ALIGNVAL-1)) & ~((ptrdiff_t) (ALIGNVAL-1))))
116-
#define XLOG_BUFFER_POINTERALIGN(PTR) \
117-
POINTERALIGN((ALIGNOF_XLOG_BUFFER), (PTR))
118-
11994
#ifdef OPEN_DATASYNC_FLAG
12095
#define DEFAULT_SYNC_METHOD_STR "open_datasync"
12196
#define DEFAULT_SYNC_METHOD SYNC_METHOD_OPEN
@@ -135,6 +110,17 @@
135110
#endif
136111

137112

113+
/*
114+
* Limitation of buffer-alignment for direct IO depends on OS and filesystem,
115+
* but BLCKSZ is assumed to be enough for it.
116+
*/
117+
#ifdef O_DIRECT
118+
#define ALIGNOF_XLOG_BUFFER BLCKSZ
119+
#else
120+
#define ALIGNOF_XLOG_BUFFER ALIGNOF_BUFFER
121+
#endif
122+
123+
138124
/* File path names (all relative to $PGDATA) */
139125
#define BACKUP_LABEL_FILE "backup_label"
140126
#define RECOVERY_COMMAND_FILE "recovery.conf"
@@ -173,8 +159,6 @@ static int open_sync_bit = DEFAULT_SYNC_FLAGBIT;
173159

174160
#define XLOG_SYNC_BIT (enableFsync ? open_sync_bit : 0)
175161

176-
#define MinXLOGbuffers 4
177-
178162

179163
/*
180164
* ThisTimeLineID will be same in all backends --- it identifies current
@@ -3615,34 +3599,38 @@ UpdateControlFile(void)
36153599
/*
36163600
* Initialization of shared memory for XLOG
36173601
*/
3618-
3619-
int
3602+
Size
36203603
XLOGShmemSize(void)
36213604
{
3622-
if (XLOGbuffers < MinXLOGbuffers)
3623-
XLOGbuffers = MinXLOGbuffers;
3605+
Size size;
36243606

3625-
return XLOG_BUFFER_ALIGN(sizeof(XLogCtlData) + sizeof(XLogRecPtr) * XLOGbuffers)
3626-
+ BLCKSZ * XLOGbuffers +
3627-
MAXALIGN(sizeof(ControlFileData));
3607+
/* XLogCtl */
3608+
size = sizeof(XLogCtlData);
3609+
/* xlblocks array */
3610+
size = add_size(size, mul_size(sizeof(XLogRecPtr), XLOGbuffers));
3611+
/* extra alignment padding for XLOG I/O buffers */
3612+
size = add_size(size, ALIGNOF_XLOG_BUFFER);
3613+
/* and the buffers themselves */
3614+
size = add_size(size, mul_size(BLCKSZ, XLOGbuffers));
3615+
3616+
/*
3617+
* Note: we don't count ControlFileData, it comes out of the "slop
3618+
* factor" added by CreateSharedMemoryAndSemaphores. This lets us
3619+
* use this routine again below to compute the actual allocation size.
3620+
*/
3621+
3622+
return size;
36283623
}
36293624

36303625
void
36313626
XLOGShmemInit(void)
36323627
{
36333628
bool foundXLog,
36343629
foundCFile;
3635-
3636-
/* this must agree with space requested by XLOGShmemSize() */
3637-
if (XLOGbuffers < MinXLOGbuffers)
3638-
XLOGbuffers = MinXLOGbuffers;
3630+
char *allocptr;
36393631

36403632
XLogCtl = (XLogCtlData *)
3641-
ShmemInitStruct("XLOG Ctl",
3642-
XLOG_BUFFER_ALIGN(sizeof(XLogCtlData) +
3643-
sizeof(XLogRecPtr) * XLOGbuffers)
3644-
+ BLCKSZ * XLOGbuffers,
3645-
&foundXLog);
3633+
ShmemInitStruct("XLOG Ctl", XLOGShmemSize(), &foundXLog);
36463634
ControlFile = (ControlFileData *)
36473635
ShmemInitStruct("Control File", sizeof(ControlFileData), &foundCFile);
36483636

@@ -3660,17 +3648,16 @@ XLOGShmemInit(void)
36603648
* a multiple of the alignment for same, so no extra alignment padding
36613649
* is needed here.
36623650
*/
3663-
XLogCtl->xlblocks = (XLogRecPtr *)
3664-
(((char *) XLogCtl) + sizeof(XLogCtlData));
3651+
allocptr = ((char *) XLogCtl) + sizeof(XLogCtlData);
3652+
XLogCtl->xlblocks = (XLogRecPtr *) allocptr;
36653653
memset(XLogCtl->xlblocks, 0, sizeof(XLogRecPtr) * XLOGbuffers);
3654+
allocptr += sizeof(XLogRecPtr) * XLOGbuffers;
36663655

36673656
/*
3668-
* Here, on the other hand, we must MAXALIGN to ensure the page
3669-
* buffers have worst-case alignment.
3657+
* Align the start of the page buffers to an ALIGNOF_XLOG_BUFFER boundary.
36703658
*/
3671-
XLogCtl->pages = XLOG_BUFFER_POINTERALIGN(
3672-
((char *) XLogCtl)
3673-
+ sizeof(XLogCtlData) + sizeof(XLogRecPtr) * XLOGbuffers);
3659+
allocptr = (char *) TYPEALIGN(ALIGNOF_XLOG_BUFFER, allocptr);
3660+
XLogCtl->pages = allocptr;
36743661
memset(XLogCtl->pages, 0, BLCKSZ * XLOGbuffers);
36753662

36763663
/*
@@ -3728,8 +3715,9 @@ BootStrapXLOG(void)
37283715
/* First timeline ID is always 1 */
37293716
ThisTimeLineID = 1;
37303717

3731-
buffer = (char *) malloc(BLCKSZ + ALIGNOF_XLOG_BUFFER);
3732-
page = (XLogPageHeader) XLOG_BUFFER_POINTERALIGN(buffer);
3718+
/* page buffer must be aligned suitably for O_DIRECT */
3719+
buffer = (char *) palloc(BLCKSZ + ALIGNOF_XLOG_BUFFER);
3720+
page = (XLogPageHeader) TYPEALIGN(ALIGNOF_XLOG_BUFFER, buffer);
37333721
memset(page, 0, BLCKSZ);
37343722

37353723
/* Set up information for the initial checkpoint record */
@@ -3824,7 +3812,7 @@ BootStrapXLOG(void)
38243812
BootStrapSUBTRANS();
38253813
BootStrapMultiXact();
38263814

3827-
free(buffer);
3815+
pfree(buffer);
38283816
}
38293817

38303818
static char *

‎src/backend/commands/vacuumlazy.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
*
3232
*
3333
* IDENTIFICATION
34-
* $PostgreSQL: pgsql/src/backend/commands/vacuumlazy.c,v 1.56 2005/07/29 19:30:03 tgl Exp $
34+
* $PostgreSQL: pgsql/src/backend/commands/vacuumlazy.c,v 1.57 2005/08/20 23:26:13 tgl Exp $
3535
*
3636
*-------------------------------------------------------------------------
3737
*/
@@ -954,16 +954,16 @@ count_nondeletable_pages(Relation onerel, LVRelStats *vacrelstats)
954954
static void
955955
lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks)
956956
{
957-
int maxtuples;
957+
long maxtuples;
958958
int maxpages;
959959

960-
maxtuples = (int) ((maintenance_work_mem * 1024L) / sizeof(ItemPointerData));
960+
maxtuples = (maintenance_work_mem * 1024L) / sizeof(ItemPointerData);
961+
maxtuples = Min(maxtuples, INT_MAX);
961962
/* stay sane if small maintenance_work_mem */
962-
if (maxtuples < MAX_TUPLES_PER_PAGE)
963-
maxtuples = MAX_TUPLES_PER_PAGE;
963+
maxtuples = Max(maxtuples, MAX_TUPLES_PER_PAGE);
964964

965965
vacrelstats->num_dead_tuples = 0;
966-
vacrelstats->max_dead_tuples = maxtuples;
966+
vacrelstats->max_dead_tuples = (int) maxtuples;
967967
vacrelstats->dead_tuples = (ItemPointer)
968968
palloc(maxtuples * sizeof(ItemPointerData));
969969

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy