Skip to content

Commit e975123

Browse files
committed
Speed improvement for large object restore.
Mario Weilguni
1 parent 5d2fdf6 commit e975123

File tree

6 files changed

+64
-14
lines changed

6 files changed

+64
-14
lines changed

src/bin/pg_dump/pg_backup_archiver.c

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
*
1616
*
1717
* IDENTIFICATION
18-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.42 2002/02/11 00:18:20 tgl Exp $
18+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.43 2002/04/24 02:21:04 momjian Exp $
1919
*
2020
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
2121
*
@@ -819,6 +819,9 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
819819
AH->createdBlobXref = 1;
820820
}
821821

822+
/* Initialize the LO Buffer */
823+
AH->lo_buf_used = 0;
824+
822825
/*
823826
* Start long-running TXs if necessary
824827
*/
@@ -848,6 +851,19 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
848851
void
849852
EndRestoreBlob(ArchiveHandle *AH, Oid oid)
850853
{
854+
if(AH->lo_buf_used > 0) {
855+
/* Write remaining bytes from the LO buffer */
856+
int res;
857+
res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, AH->lo_buf_used);
858+
859+
ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = %d)\n",
860+
(int)AH->lo_buf_used, res);
861+
if (res != AH->lo_buf_used)
862+
die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
863+
res, AH->lo_buf_used);
864+
AH->lo_buf_used = 0;
865+
}
866+
851867
lo_close(AH->connection, AH->loFd);
852868
AH->writingBlob = 0;
853869

@@ -1228,14 +1244,27 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
12281244

12291245
if (AH->writingBlob)
12301246
{
1231-
res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
1232-
ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
1233-
(int) (size * nmemb), res);
1234-
if (res != size * nmemb)
1235-
die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
1236-
res, (int) (size * nmemb));
1247+
if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
1248+
/* Split LO buffer */
1249+
int remaining = AH->lo_buf_size - AH->lo_buf_used;
1250+
int slack = nmemb * size - remaining;
12371251

1238-
return res;
1252+
memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
1253+
res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
1254+
ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
1255+
AH->lo_buf_size, res);
1256+
if (res != AH->lo_buf_size)
1257+
die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
1258+
res, AH->lo_buf_size);
1259+
memcpy(AH->lo_buf, ptr + remaining, slack);
1260+
AH->lo_buf_used = slack;
1261+
} else {
1262+
/* LO Buffer is still large enough, buffer it */
1263+
memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
1264+
AH->lo_buf_used += size * nmemb;
1265+
}
1266+
1267+
return size * nmemb;
12391268
}
12401269
else if (AH->gzOut)
12411270
{

src/bin/pg_dump/pg_backup_archiver.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
*
1818
*
1919
* IDENTIFICATION
20-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.41 2001/11/05 17:46:30 momjian Exp $
20+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.42 2002/04/24 02:21:04 momjian Exp $
2121
*
2222
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
2323
* - Initial version.
@@ -41,6 +41,7 @@
4141
#include <errno.h>
4242

4343
#include "pqexpbuffer.h"
44+
#define LOBBUFSIZE 32768
4445

4546
#ifdef HAVE_LIBZ
4647
#include <zlib.h>
@@ -240,6 +241,9 @@ typedef struct _archiveHandle
240241

241242
RestoreOptions *ropt; /* Used to check restore options in
242243
* ahwrite etc */
244+
void *lo_buf;
245+
int lo_buf_used;
246+
int lo_buf_size;
243247
} ArchiveHandle;
244248

245249
typedef struct _tocEntry

src/bin/pg_dump/pg_backup_custom.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*
2020
*
2121
* IDENTIFICATION
22-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.17 2001/11/27 23:48:12 tgl Exp $
22+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.18 2002/04/24 02:21:04 momjian Exp $
2323
*
2424
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
2525
*
@@ -153,6 +153,12 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
153153
if (ctx->zp == NULL)
154154
die_horribly(AH, modulename, "out of memory\n");
155155

156+
/* Initialize LO buffering */
157+
AH->lo_buf_size = LOBBUFSIZE;
158+
AH->lo_buf = (void *)malloc(LOBBUFSIZE);
159+
if(AH->lo_buf == NULL)
160+
die_horribly(AH, modulename, "out of memory\n");
161+
156162
/*
157163
* zlibOutSize is the buffer size we tell zlib it can output to. We
158164
* actually allocate one extra byte because some routines want to

src/bin/pg_dump/pg_backup_files.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
*
2121
*
2222
* IDENTIFICATION
23-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.14 2001/10/25 05:49:52 momjian Exp $
23+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.15 2002/04/24 02:21:04 momjian Exp $
2424
*
2525
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
2626
*
@@ -113,6 +113,12 @@ InitArchiveFmt_Files(ArchiveHandle *AH)
113113
AH->formatData = (void *) ctx;
114114
ctx->filePos = 0;
115115

116+
/* Initialize LO buffering */
117+
AH->lo_buf_size = LOBBUFSIZE;
118+
AH->lo_buf = (void *)malloc(LOBBUFSIZE);
119+
if(AH->lo_buf == NULL)
120+
die_horribly(AH, modulename, "out of memory\n");
121+
116122
/*
117123
* Now open the TOC file
118124
*/

src/bin/pg_dump/pg_backup_null.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
*
1818
*
1919
* IDENTIFICATION
20-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.7 2001/06/27 21:21:37 petere Exp $
20+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.8 2002/04/24 02:21:04 momjian Exp $
2121
*
2222
* Modifications - 09-Jul-2000 - pjw@rhyme.com.au
2323
*
@@ -64,7 +64,6 @@ InitArchiveFmt_Null(ArchiveHandle *AH)
6464
*/
6565
if (AH->mode == archModeRead)
6666
die_horribly(AH, NULL, "this format cannot be read\n");
67-
6867
}
6968

7069
/*

src/bin/pg_dump/pg_backup_tar.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
*
1717
*
1818
* IDENTIFICATION
19-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.20 2001/10/28 06:25:58 momjian Exp $
19+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.21 2002/04/24 02:21:04 momjian Exp $
2020
*
2121
* Modifications - 28-Jun-2000 - pjw@rhyme.com.au
2222
*
@@ -157,6 +157,12 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
157157
ctx = (lclContext *) malloc(sizeof(lclContext));
158158
AH->formatData = (void *) ctx;
159159
ctx->filePos = 0;
160+
161+
/* Initialize LO buffering */
162+
AH->lo_buf_size = LOBBUFSIZE;
163+
AH->lo_buf = (void *)malloc(LOBBUFSIZE);
164+
if(AH->lo_buf == NULL)
165+
die_horribly(AH, modulename, "out of memory\n");
160166

161167
/*
162168
* Now open the TOC file

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy