Further hacking on performance of COPY OUT. It seems that fwrite()'s

tglsfdc · tglsfdc · commit 4d63e267742a · 2006-05-26T22:50:02.000Z
per-call overhead is quite significant, at least on Linux: whatever
it's doing is more than just shoving the bytes into a buffer.  Buffering
the data so we can call fwrite() just once per row seems to be a win.
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.265 2006/05/25 18:42:17 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.266 2006/05/26 22:50:02 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -95,7 +95,8 @@ typedef struct CopyStateData
 	/* low-level state data */
 	CopyDest	copy_dest;		/* type of copy source/destination */
 	FILE	   *copy_file;		/* used if copy_dest == COPY_FILE */
-	StringInfo	fe_msgbuf;		/* used if copy_dest == COPY_NEW_FE */
+	StringInfo	fe_msgbuf;		/* used for all dests during COPY TO, only
+								 * for dest == COPY_NEW_FE in COPY FROM */
 	bool		fe_copy;		/* true for all FE copy dests */
 	bool		fe_eof;			/* true if detected end of copy data */
 	EolType		eol_type;		/* EOL type of input */
@@ -287,7 +288,6 @@ SendCopyBegin(CopyState cstate)
 			pq_sendint(&buf, format, 2);		/* per-column formats */
 		pq_endmessage(&buf);
 		cstate->copy_dest = COPY_NEW_FE;
-		cstate->fe_msgbuf = makeStringInfo();
 	}
 	else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
 	{
@@ -364,23 +364,16 @@ SendCopyEnd(CopyState cstate)
 {
 	if (cstate->copy_dest == COPY_NEW_FE)
 	{
-		if (cstate->binary)
-		{
-			/* Need to flush out file trailer word */
-			CopySendEndOfRow(cstate);
-		}
-		else
-		{
-			/* Shouldn't have any unsent data */
-			Assert(cstate->fe_msgbuf->len == 0);
-		}
+		/* Shouldn't have any unsent data */
+		Assert(cstate->fe_msgbuf->len == 0);
 		/* Send Copy Done message */
 		pq_putemptymessage('c');
 	}
 	else
 	{
-		/* The FE/BE protocol uses \n as newline for all platforms */
-		CopySendData(cstate, "\\.\n", 3);
+		CopySendData(cstate, "\\.", 2);
+		/* Need to flush out the trailer (this also appends a newline) */
+		CopySendEndOfRow(cstate);
 		pq_endcopyout(false);
 	}
 }
@@ -390,53 +383,34 @@ SendCopyEnd(CopyState cstate)
  * CopySendString does the same for null-terminated strings
  * CopySendChar does the same for single characters
  * CopySendEndOfRow does the appropriate thing at end of each data row
+ *	(data is not actually flushed except by CopySendEndOfRow)
  *
  * NB: no data conversion is applied by these functions
  *----------
  */
 static void
 CopySendData(CopyState cstate, void *databuf, int datasize)
 {
-	switch (cstate->copy_dest)
-	{
-		case COPY_FILE:
-			fwrite(databuf, datasize, 1, cstate->copy_file);
-			if (ferror(cstate->copy_file))
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("could not write to COPY file: %m")));
-			break;
-		case COPY_OLD_FE:
-			if (pq_putbytes((char *) databuf, datasize))
-			{
-				/* no hope of recovering connection sync, so FATAL */
-				ereport(FATAL,
-						(errcode(ERRCODE_CONNECTION_FAILURE),
-						 errmsg("connection lost during COPY to stdout")));
-			}
-			break;
-		case COPY_NEW_FE:
-			appendBinaryStringInfo(cstate->fe_msgbuf,
-								   (char *) databuf, datasize);
-			break;
-	}
+	appendBinaryStringInfo(cstate->fe_msgbuf, (char *) databuf, datasize);
 }
 
 static void
 CopySendString(CopyState cstate, const char *str)
 {
-	CopySendData(cstate, (void *) str, strlen(str));
+	appendBinaryStringInfo(cstate->fe_msgbuf, str, strlen(str));
 }
 
 static void
 CopySendChar(CopyState cstate, char c)
 {
-	CopySendData(cstate, &c, 1);
+	appendStringInfoCharMacro(cstate->fe_msgbuf, c);
 }
 
 static void
 CopySendEndOfRow(CopyState cstate)
 {
+	StringInfo	fe_msgbuf = cstate->fe_msgbuf;
+
 	switch (cstate->copy_dest)
 	{
 		case COPY_FILE:
@@ -449,24 +423,40 @@ CopySendEndOfRow(CopyState cstate)
 				CopySendString(cstate, "\r\n");
 #endif
 			}
+
+			(void) fwrite(fe_msgbuf->data, fe_msgbuf->len,
+						  1, cstate->copy_file);
+			if (ferror(cstate->copy_file))
+				ereport(ERROR,
+						(errcode_for_file_access(),
+						 errmsg("could not write to COPY file: %m")));
 			break;
 		case COPY_OLD_FE:
 			/* The FE/BE protocol uses \n as newline for all platforms */
 			if (!cstate->binary)
 				CopySendChar(cstate, '\n');
+
+			if (pq_putbytes(fe_msgbuf->data, fe_msgbuf->len))
+			{
+				/* no hope of recovering connection sync, so FATAL */
+				ereport(FATAL,
+						(errcode(ERRCODE_CONNECTION_FAILURE),
+						 errmsg("connection lost during COPY to stdout")));
+			}
 			break;
 		case COPY_NEW_FE:
 			/* The FE/BE protocol uses \n as newline for all platforms */
 			if (!cstate->binary)
 				CopySendChar(cstate, '\n');
+
 			/* Dump the accumulated row as one CopyData message */
-			(void) pq_putmessage('d', cstate->fe_msgbuf->data,
-								 cstate->fe_msgbuf->len);
-			/* Reset fe_msgbuf to empty */
-			cstate->fe_msgbuf->len = 0;
-			cstate->fe_msgbuf->data[0] = '\0';
+			(void) pq_putmessage('d', fe_msgbuf->data, fe_msgbuf->len);
 			break;
 	}
+
+	/* Reset fe_msgbuf to empty */
+	fe_msgbuf->len = 0;
+	fe_msgbuf->data[0] = '\0';
 }
 
 /*
@@ -1237,6 +1227,9 @@ CopyTo(CopyState cstate)
 	attr_count = list_length(cstate->attnumlist);
 	null_print_client = cstate->null_print;		/* default */
 
+	/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
+	cstate->fe_msgbuf = makeStringInfo();
+
 	/* Get info about the columns we need to process. */
 	out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
 	force_quote = (bool *) palloc(num_phys_attrs * sizeof(bool));
@@ -1423,6 +1416,8 @@ CopyTo(CopyState cstate)
 	{
 		/* Generate trailer for a binary copy */
 		CopySendInt16(cstate, -1);
+		/* Need to flush out the trailer */
+		CopySendEndOfRow(cstate);
 	}
 
 	MemoryContextDelete(mycontext);
