Add pg_rewind, for re-synchronizing a master server after failback.

Earlier versions of this tool were available (and still are) on github.

Thanks to Michael Paquier, Alvaro Herrera, Peter Eisentraut, Amit Kapila,
and Satoshi Nagayasu for review.

1 files changed, 261 insertions, 0 deletions

diff --git a/src/bin/pg_rewind/copy_fetch.c b/src/bin/pg_rewind/copy_fetch.c
new file mode 100644
index 00000000000..887fec9c9d0
--- /dev/null
+++ b/src/bin/pg_rewind/copy_fetch.c
@@ -0,0 +1,261 @@
+/*-------------------------------------------------------------------------
+ *
+ * copy_fetch.c
+ *	  Functions for using a data directory as the source.
+ *
+ * Portions Copyright (c) 2013-2015, PostgreSQL Global Development Group
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "datapagemap.h"
+#include "fetch.h"
+#include "file_ops.h"
+#include "filemap.h"
+#include "logging.h"
+#include "pg_rewind.h"
+
+#include "catalog/catalog.h"
+
+static void recurse_dir(const char *datadir, const char *path,
+			process_file_callback_t callback);
+
+static void execute_pagemap(datapagemap_t *pagemap, const char *path);
+
+/*
+ * Traverse through all files in a data directory, calling 'callback'
+ * for each file.
+ */
+void
+traverse_datadir(const char *datadir, process_file_callback_t callback)
+{
+	recurse_dir(datadir, NULL, callback);
+}
+
+/*
+ * recursive part of traverse_datadir
+ */
+static void
+recurse_dir(const char *datadir, const char *parentpath,
+			process_file_callback_t callback)
+{
+	DIR		   *xldir;
+	struct dirent *xlde;
+	char		fullparentpath[MAXPGPATH];
+
+	if (parentpath)
+		snprintf(fullparentpath, MAXPGPATH, "%s/%s", datadir, parentpath);
+	else
+		snprintf(fullparentpath, MAXPGPATH, "%s", datadir);
+
+	xldir = opendir(fullparentpath);
+	if (xldir == NULL)
+		pg_fatal("could not open directory \"%s\": %s\n",
+				 fullparentpath, strerror(errno));
+
+	while (errno = 0, (xlde = readdir(xldir)) != NULL)
+	{
+		struct stat fst;
+		char		fullpath[MAXPGPATH];
+		char		path[MAXPGPATH];
+
+		if (strcmp(xlde->d_name, ".") == 0 ||
+			strcmp(xlde->d_name, "..") == 0)
+			continue;
+
+		snprintf(fullpath, MAXPGPATH, "%s/%s", fullparentpath, xlde->d_name);
+
+		if (lstat(fullpath, &fst) < 0)
+		{
+			pg_log(PG_WARNING, "could not stat file \"%s\": %s",
+				   fullpath, strerror(errno));
+
+			/*
+			 * This is ok, if the new master is running and the file was just
+			 * removed. If it was a data file, there should be a WAL record of
+			 * the removal. If it was something else, it couldn't have been
+			 * critical anyway.
+			 *
+			 * TODO: But complain if we're processing the target dir!
+			 */
+		}
+
+		if (parentpath)
+			snprintf(path, MAXPGPATH, "%s/%s", parentpath, xlde->d_name);
+		else
+			snprintf(path, MAXPGPATH, "%s", xlde->d_name);
+
+		if (S_ISREG(fst.st_mode))
+			callback(path, FILE_TYPE_REGULAR, fst.st_size, NULL);
+		else if (S_ISDIR(fst.st_mode))
+		{
+			callback(path, FILE_TYPE_DIRECTORY, 0, NULL);
+			/* recurse to handle subdirectories */
+			recurse_dir(datadir, path, callback);
+		}
+#ifndef WIN32
+		else if (S_ISLNK(fst.st_mode))
+#else
+		else if (pgwin32_is_junction(fullpath))
+#endif
+		{
+#if defined(HAVE_READLINK) || defined(WIN32)
+			char		link_target[MAXPGPATH];
+			ssize_t		len;
+
+			len = readlink(fullpath, link_target, sizeof(link_target) - 1);
+			if (len == -1)
+				pg_fatal("readlink() failed on \"%s\": %s\n",
+						 fullpath, strerror(errno));
+
+			if (len == sizeof(link_target) - 1)
+			{
+				/* path was truncated */
+				pg_fatal("symbolic link \"%s\" target path too long\n",
+						 fullpath);
+			}
+
+			callback(path, FILE_TYPE_SYMLINK, 0, link_target);
+
+			/*
+			 * If it's a symlink within pg_tblspc, we need to recurse into it,
+			 * to process all the tablespaces.
+			 */
+			if (strcmp(parentpath, "pg_tblspc") == 0)
+				recurse_dir(datadir, path, callback);
+#else
+			pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n",
+					 fullpath);
+#endif   /* HAVE_READLINK */
+		}
+	}
+
+	if (errno)
+		pg_fatal("could not read directory \"%s\": %s\n",
+				 fullparentpath, strerror(errno));
+
+	if (closedir(xldir))
+		pg_fatal("could not close archive location \"%s\": %s\n",
+				 fullparentpath, strerror(errno));
+}
+
+/*
+ * Copy a file from source to target, between 'begin' and 'end' offsets.
+ *
+ * If 'trunc' is true, any existing file with the same name is truncated.
+ */
+static void
+copy_file_range(const char *path, off_t begin, off_t end, bool trunc)
+{
+	char		buf[BLCKSZ];
+	char		srcpath[MAXPGPATH];
+	int			srcfd;
+
+	snprintf(srcpath, sizeof(srcpath), "%s/%s", datadir_source, path);
+
+	srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0);
+	if (srcfd < 0)
+		pg_fatal("could not open source file \"%s\": %s\n",
+				 srcpath, strerror(errno));
+
+	if (lseek(srcfd, begin, SEEK_SET) == -1)
+		pg_fatal("could not seek in source file: %s\n", strerror(errno));
+
+	open_target_file(path, trunc);
+
+	while (end - begin > 0)
+	{
+		int			readlen;
+		int			len;
+
+		if (end - begin > sizeof(buf))
+			len = sizeof(buf);
+		else
+			len = end - begin;
+
+		readlen = read(srcfd, buf, len);
+
+		if (readlen < 0)
+			pg_fatal("could not read file \"%s\": %s\n",
+					 srcpath, strerror(errno));
+		else if (readlen == 0)
+			pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath);
+
+		write_target_range(buf, begin, readlen);
+		begin += readlen;
+	}
+
+	if (close(srcfd) != 0)
+		pg_fatal("error closing file \"%s\": %s\n", srcpath, strerror(errno));
+}
+
+/*
+ * Copy all relation data files from datadir_source to datadir_target, which
+ * are marked in the given data page map.
+ */
+void
+copy_executeFileMap(filemap_t *map)
+{
+	file_entry_t *entry;
+	int			i;
+
+	for (i = 0; i < map->narray; i++)
+	{
+		entry = map->array[i];
+		execute_pagemap(&entry->pagemap, entry->path);
+
+		switch (entry->action)
+		{
+			case FILE_ACTION_NONE:
+				/* ok, do nothing.. */
+				break;
+
+			case FILE_ACTION_COPY:
+				copy_file_range(entry->path, 0, entry->newsize, true);
+				break;
+
+			case FILE_ACTION_TRUNCATE:
+				truncate_target_file(entry->path, entry->newsize);
+				break;
+
+			case FILE_ACTION_COPY_TAIL:
+				copy_file_range(entry->path, entry->oldsize, entry->newsize, false);
+				break;
+
+			case FILE_ACTION_CREATE:
+				create_target(entry);
+				break;
+
+			case FILE_ACTION_REMOVE:
+				remove_target(entry);
+				break;
+		}
+	}
+
+	close_target_file();
+}
+
+static void
+execute_pagemap(datapagemap_t *pagemap, const char *path)
+{
+	datapagemap_iterator_t *iter;
+	BlockNumber blkno;
+	off_t		offset;
+
+	iter = datapagemap_iterate(pagemap);
+	while (datapagemap_next(iter, &blkno))
+	{
+		offset = blkno * BLCKSZ;
+		copy_file_range(path, offset, offset + BLCKSZ, false);
+		/* Ok, this block has now been copied from new data dir to old */
+	}
+	free(iter);
+}