Make WAL segment size configurable at initdb time.

For performance reasons a larger segment size than the default 16MB
can be useful. A larger segment size has two main benefits: Firstly,
in setups using archiving, it makes it easier to write scripts that
can keep up with higher amounts of WAL, secondly, the WAL has to be
written and synced to disk less frequently.

But at the same time large segment size are disadvantageous for
smaller databases. So far the segment size had to be configured at
compile time, often making it unrealistic to choose one fitting to a
particularly load. Therefore change it to a initdb time setting.

This includes a breaking changes to the xlogreader.h API, which now
requires the current segment size to be configured.  For that and
similar reasons a number of binaries had to be taught how to recognize
the current segment size.

Author: Beena Emerson, editorialized by Andres Freund
Reviewed-By: Andres Freund, David Steele, Kuntal Ghosh, Michael
    Paquier, Peter Eisentraut, Robert Hass, Tushar Ahuja
Discussion: https://postgr.es/m/CAOG9ApEAcQ--1ieKbhFzXSQPw_YLmepaa4hNdnY5+ZULpt81Mw@mail.gmail.com

1 files changed, 101 insertions, 14 deletions

diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c
index d7fa2a80c69..6aeca6e8f72 100644
--- a/contrib/pg_standby/pg_standby.c
+++ b/contrib/pg_standby/pg_standby.c
@@ -36,6 +36,8 @@
 
 const char *progname;
 
+int			WalSegSz = -1;
+
 /* Options and defaults */
 int			sleeptime = 5;		/* amount of time to sleep between file checks */
 int			waittime = -1;		/* how long we have been waiting, -1 no wait
@@ -100,6 +102,10 @@ int			nextWALFileType;
 
 struct stat stat_buf;
 
+static bool SetWALFileNameForCleanup(void);
+static bool SetWALSegSize(void);
+
+
 /* =====================================================================
  *
  *		  Customizable section
@@ -176,6 +182,35 @@ CustomizableNextWALFileReady(void)
 	if (stat(WALFilePath, &stat_buf) == 0)
 	{
 		/*
+		 * If we've not seen any WAL segments, we don't know the WAL segment
+		 * size, which we need. If it looks like a WAL segment, determine size
+		 * of segments for the cluster.
+		 */
+		if (WalSegSz == -1 && IsXLogFileName(nextWALFileName))
+		{
+			if (SetWALSegSize())
+			{
+				/*
+				 * Successfully determined WAL segment size. Can compute
+				 * cleanup cutoff now.
+				 */
+				need_cleanup = SetWALFileNameForCleanup();
+				if (debug)
+				{
+					fprintf(stderr,
+							_("WAL segment size:     %d \n"), WalSegSz);
+					fprintf(stderr, "Keep archive history: ");
+
+					if (need_cleanup)
+						fprintf(stderr, "%s and later\n",
+								exclusiveCleanupFileName);
+					else
+						fprintf(stderr, "no cleanup required\n");
+				}
+			}
+		}
+
+		/*
 		 * If it's a backup file, return immediately. If it's a regular file
 		 * return only if it's the right size already.
 		 */
@@ -184,7 +219,7 @@ CustomizableNextWALFileReady(void)
 			nextWALFileType = XLOG_BACKUP_LABEL;
 			return true;
 		}
-		else if (stat_buf.st_size == XLOG_SEG_SIZE)
+		else if (WalSegSz > 0 && stat_buf.st_size == WalSegSz)
 		{
 #ifdef WIN32
 
@@ -204,7 +239,7 @@ CustomizableNextWALFileReady(void)
 		/*
 		 * If still too small, wait until it is the correct size
 		 */
-		if (stat_buf.st_size > XLOG_SEG_SIZE)
+		if (WalSegSz > 0 && stat_buf.st_size > WalSegSz)
 		{
 			if (debug)
 			{
@@ -218,8 +253,6 @@ CustomizableNextWALFileReady(void)
 	return false;
 }
 
-#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
-
 static void
 CustomizableCleanupPriorWALFiles(void)
 {
@@ -315,6 +348,7 @@ SetWALFileNameForCleanup(void)
 	uint32		log_diff = 0,
 				seg_diff = 0;
 	bool		cleanup = false;
+	int			max_segments_per_logfile = (0xFFFFFFFF / WalSegSz);
 
 	if (restartWALFileName)
 	{
@@ -336,12 +370,12 @@ SetWALFileNameForCleanup(void)
 		sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
 		if (tli > 0 && seg > 0)
 		{
-			log_diff = keepfiles / MaxSegmentsPerLogFile;
-			seg_diff = keepfiles % MaxSegmentsPerLogFile;
+			log_diff = keepfiles / max_segments_per_logfile;
+			seg_diff = keepfiles % max_segments_per_logfile;
 			if (seg_diff > seg)
 			{
 				log_diff++;
-				seg = MaxSegmentsPerLogFile - (seg_diff - seg);
+				seg = max_segments_per_logfile - (seg_diff - seg);
 			}
 			else
 				seg -= seg_diff;
@@ -365,6 +399,66 @@ SetWALFileNameForCleanup(void)
 }
 
 /*
+ * Try to set the wal segment size from the WAL file specified by WALFilePath.
+ *
+ * Return true if size could be determined, false otherwise.
+ */
+static bool
+SetWALSegSize(void)
+{
+	bool		ret_val = false;
+	int			fd;
+	char	   *buf = (char *) malloc(XLOG_BLCKSZ);
+
+	Assert(WalSegSz == -1);
+
+	if ((fd = open(WALFilePath, O_RDWR, 0)) < 0)
+	{
+		fprintf(stderr, "%s: couldn't open WAL file \"%s\"\n",
+				progname, WALFilePath);
+		return false;
+	}
+	if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
+	{
+		XLogLongPageHeader longhdr = (XLogLongPageHeader) buf;
+
+		WalSegSz = longhdr->xlp_seg_size;
+
+		if (IsValidWalSegSize(WalSegSz))
+		{
+			/* successfully retrieved WAL segment size */
+			ret_val = true;
+		}
+		else
+			fprintf(stderr,
+					"%s: WAL segment size must be a power of two between 1MB and 1GB, but the WAL file header specifies %d bytes\n",
+					progname, WalSegSz);
+		close(fd);
+	}
+	else
+	{
+		/*
+		 * Don't complain loudly, this is to be expected for segments being
+		 * created.
+		 */
+		if (errno != 0)
+		{
+			if (debug)
+				fprintf(stderr, "could not read file \"%s\": %s",
+						WALFilePath, strerror(errno));
+		}
+		else
+		{
+			if (debug)
+				fprintf(stderr, "not enough data in file \"%s\"", WALFilePath);
+		}
+	}
+
+	fflush(stderr);
+	return ret_val;
+}
+
+/*
  * CheckForExternalTrigger()
  *
  *	  Is there a trigger file? Sets global 'Failover' variable to indicate
@@ -708,8 +802,6 @@ main(int argc, char **argv)
 
 	CustomizableInitialize();
 
-	need_cleanup = SetWALFileNameForCleanup();
-
 	if (debug)
 	{
 		fprintf(stderr, "Trigger file:         %s\n", triggerPath ? triggerPath : "<not set>");
@@ -721,11 +813,6 @@ main(int argc, char **argv)
 		fprintf(stderr, "Max wait interval:    %d %s\n",
 				maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
 		fprintf(stderr, "Command for restore:  %s\n", restoreCommand);
-		fprintf(stderr, "Keep archive history: ");
-		if (need_cleanup)
-			fprintf(stderr, "%s and later\n", exclusiveCleanupFileName);
-		else
-			fprintf(stderr, "no cleanup required\n");
 		fflush(stderr);
 	}