summaryrefslogtreecommitdiff
path: root/src/bin
diff options
context:
space:
mode:
Diffstat (limited to 'src/bin')
-rw-r--r--src/bin/pg_controldata/pg_controldata.c2
-rw-r--r--src/bin/pg_resetwal/pg_resetwal.c38
-rw-r--r--src/bin/pg_resetwal/t/001_basic.pl2
-rw-r--r--src/bin/pg_upgrade/Makefile3
-rw-r--r--src/bin/pg_upgrade/meson.build4
-rw-r--r--src/bin/pg_upgrade/multixact_read_v18.c340
-rw-r--r--src/bin/pg_upgrade/multixact_read_v18.h37
-rw-r--r--src/bin/pg_upgrade/multixact_rewrite.c191
-rw-r--r--src/bin/pg_upgrade/pg_upgrade.c84
-rw-r--r--src/bin/pg_upgrade/pg_upgrade.h12
-rw-r--r--src/bin/pg_upgrade/slru_io.c268
-rw-r--r--src/bin/pg_upgrade/slru_io.h52
-rw-r--r--src/bin/pg_upgrade/t/007_multixact_conversion.pl427
13 files changed, 1427 insertions, 33 deletions
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 30ad46912e1..a4060309ae0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -271,7 +271,7 @@ main(int argc, char *argv[])
ControlFile->checkPointCopy.nextOid);
printf(_("Latest checkpoint's NextMultiXactId: %u\n"),
ControlFile->checkPointCopy.nextMulti);
- printf(_("Latest checkpoint's NextMultiOffset: %u\n"),
+ printf(_("Latest checkpoint's NextMultiOffset: %" PRIu64 "\n"),
ControlFile->checkPointCopy.nextMultiOffset);
printf(_("Latest checkpoint's oldestXID: %u\n"),
ControlFile->checkPointCopy.oldestXid);
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 07c95f9ab80..56012d5f4c4 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -115,6 +115,7 @@ static void KillExistingWALSummaries(void);
static void WriteEmptyXLOG(void);
static void usage(void);
static uint32 strtouint32_strict(const char *restrict s, char **restrict endptr, int base);
+static uint64 strtouint64_strict(const char *restrict s, char **restrict endptr, int base);
int
@@ -293,7 +294,7 @@ main(int argc, char *argv[])
case 'O':
errno = 0;
- next_mxoff_val = strtouint32_strict(optarg, &endptr, 0);
+ next_mxoff_val = strtouint64_strict(optarg, &endptr, 0);
if (endptr == optarg || *endptr != '\0' || errno != 0)
{
pg_log_error("invalid argument for option %s", "-O");
@@ -772,7 +773,7 @@ PrintControlValues(bool guessed)
ControlFile.checkPointCopy.nextOid);
printf(_("Latest checkpoint's NextMultiXactId: %u\n"),
ControlFile.checkPointCopy.nextMulti);
- printf(_("Latest checkpoint's NextMultiOffset: %u\n"),
+ printf(_("Latest checkpoint's NextMultiOffset: %" PRIu64 "\n"),
ControlFile.checkPointCopy.nextMultiOffset);
printf(_("Latest checkpoint's oldestXID: %u\n"),
ControlFile.checkPointCopy.oldestXid);
@@ -848,7 +849,7 @@ PrintNewControlValues(void)
if (next_mxoff_given)
{
- printf(_("NextMultiOffset: %u\n"),
+ printf(_("NextMultiOffset: %" PRIu64 "\n"),
ControlFile.checkPointCopy.nextMultiOffset);
}
@@ -1276,3 +1277,34 @@ strtouint32_strict(const char *restrict s, char **restrict endptr, int base)
return (uint32) val;
}
+
+/*
+ * strtouint64_strict -- like strtou64(), but doesn't accept negative values
+ */
+static uint64
+strtouint64_strict(const char *restrict s, char **restrict endptr, int base)
+{
+ uint64 val;
+ bool is_neg;
+
+ /* skip leading whitespace */
+ while (isspace((unsigned char) *s))
+ s++;
+
+ /*
+ * Is it negative? We still call strtou64() if it was, to set 'endptr'.
+ * (The current callers don't care though.)
+ */
+ is_neg = (*s == '-');
+
+ val = strtou64(s, endptr, base);
+
+ /* reject if it was negative */
+ if (errno == 0 && is_neg)
+ {
+ errno = ERANGE;
+ val = 0;
+ }
+
+ return val;
+}
diff --git a/src/bin/pg_resetwal/t/001_basic.pl b/src/bin/pg_resetwal/t/001_basic.pl
index 8717b144bc0..8bab9add74f 100644
--- a/src/bin/pg_resetwal/t/001_basic.pl
+++ b/src/bin/pg_resetwal/t/001_basic.pl
@@ -237,7 +237,7 @@ push @cmd,
sprintf("%d,%d", hex($files[0]) == 0 ? 3 : hex($files[0]), hex($files[-1]));
@files = get_slru_files('pg_multixact/offsets');
-$mult = 32 * $blcksz / 4;
+$mult = 32 * $blcksz / 8;
# --multixact-ids argument is "new,old"
push @cmd,
'--multixact-ids' => sprintf("%d,%d",
diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile
index 69fcf593cae..726df4b7525 100644
--- a/src/bin/pg_upgrade/Makefile
+++ b/src/bin/pg_upgrade/Makefile
@@ -18,11 +18,14 @@ OBJS = \
file.o \
function.o \
info.o \
+ multixact_read_v18.o \
+ multixact_rewrite.o \
option.o \
parallel.o \
pg_upgrade.o \
relfilenumber.o \
server.o \
+ slru_io.o \
tablespace.o \
task.o \
util.o \
diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build
index ac992f0d14b..41f1126206b 100644
--- a/src/bin/pg_upgrade/meson.build
+++ b/src/bin/pg_upgrade/meson.build
@@ -8,11 +8,14 @@ pg_upgrade_sources = files(
'file.c',
'function.c',
'info.c',
+ 'multixact_read_v18.c',
+ 'multixact_rewrite.c',
'option.c',
'parallel.c',
'pg_upgrade.c',
'relfilenumber.c',
'server.c',
+ 'slru_io.c',
'tablespace.c',
'task.c',
'util.c',
@@ -47,6 +50,7 @@ tests += {
't/004_subscription.pl',
't/005_char_signedness.pl',
't/006_transfer_modes.pl',
+ 't/007_multixact_conversion.pl',
],
'test_kwargs': {'priority': 40}, # pg_upgrade tests are slow
},
diff --git a/src/bin/pg_upgrade/multixact_read_v18.c b/src/bin/pg_upgrade/multixact_read_v18.c
new file mode 100644
index 00000000000..e7496a73e0e
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_read_v18.c
@@ -0,0 +1,340 @@
+/*
+ * multixact_read_v18.c
+ *
+ * Functions to read multixact SLRUs from clusters of PostgreSQL version 18
+ * and older. In version 19, the multixid offsets were expanded from 32 to 64
+ * bits.
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ * src/bin/pg_upgrade/multixact_read_v18.c
+ */
+
+#include "postgres_fe.h"
+
+#include "multixact_read_v18.h"
+#include "pg_upgrade.h"
+
+/*
+ * NOTE: below are a bunch of definitions that are copy-pasted from
+ * multixact.c from version 18. It's important that this file doesn't
+ * #include the new definitions with same names from "multixact_internal.h"!
+ *
+ * To further avoid confusion in the functions exposed outside this source
+ * file, we use MultiXactOffset32 to represent the old-style 32-bit multixid
+ * offsets. The new 64-bit MultiXactOffset should not be used anywhere in
+ * this file.
+ */
+#ifdef MULTIXACT_INTERNAL_H
+#error multixact_internal.h should not be included in multixact_read_v18.c
+#endif
+#define MultiXactOffset should_not_be_used
+
+/* We need four bytes per offset and 8 bytes per base for each page. */
+#define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(MultiXactOffset32))
+
+static inline int64
+MultiXactIdToOffsetPage(MultiXactId multi)
+{
+ return multi / MULTIXACT_OFFSETS_PER_PAGE;
+}
+
+static inline int
+MultiXactIdToOffsetEntry(MultiXactId multi)
+{
+ return multi % MULTIXACT_OFFSETS_PER_PAGE;
+}
+
+/*
+ * The situation for members is a bit more complex: we store one byte of
+ * additional flag bits for each TransactionId. To do this without getting
+ * into alignment issues, we store four bytes of flags, and then the
+ * corresponding 4 Xids. Each such 5-word (20-byte) set we call a "group", and
+ * are stored as a whole in pages. Thus, with 8kB BLCKSZ, we keep 409 groups
+ * per page. This wastes 12 bytes per page, but that's OK -- simplicity (and
+ * performance) trumps space efficiency here.
+ *
+ * Note that the "offset" macros work with byte offset, not array indexes, so
+ * arithmetic must be done using "char *" pointers.
+ */
+/* We need eight bits per xact, so one xact fits in a byte */
+#define MXACT_MEMBER_BITS_PER_XACT 8
+#define MXACT_MEMBER_FLAGS_PER_BYTE 1
+#define MXACT_MEMBER_XACT_BITMASK ((1 << MXACT_MEMBER_BITS_PER_XACT) - 1)
+
+/* how many full bytes of flags are there in a group? */
+#define MULTIXACT_FLAGBYTES_PER_GROUP 4
+#define MULTIXACT_MEMBERS_PER_MEMBERGROUP \
+ (MULTIXACT_FLAGBYTES_PER_GROUP * MXACT_MEMBER_FLAGS_PER_BYTE)
+/* size in bytes of a complete group */
+#define MULTIXACT_MEMBERGROUP_SIZE \
+ (sizeof(TransactionId) * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP)
+#define MULTIXACT_MEMBERGROUPS_PER_PAGE (BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE)
+#define MULTIXACT_MEMBERS_PER_PAGE \
+ (MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP)
+
+/* page in which a member is to be found */
+static inline int64
+MXOffsetToMemberPage(MultiXactOffset32 offset)
+{
+ return offset / MULTIXACT_MEMBERS_PER_PAGE;
+}
+
+/* Location (byte offset within page) of flag word for a given member */
+static inline int
+MXOffsetToFlagsOffset(MultiXactOffset32 offset)
+{
+ MultiXactOffset32 group = offset / MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+ int grouponpg = group % MULTIXACT_MEMBERGROUPS_PER_PAGE;
+ int byteoff = grouponpg * MULTIXACT_MEMBERGROUP_SIZE;
+
+ return byteoff;
+}
+
+/* Location (byte offset within page) of TransactionId of given member */
+static inline int
+MXOffsetToMemberOffset(MultiXactOffset32 offset)
+{
+ int member_in_group = offset % MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+
+ return MXOffsetToFlagsOffset(offset) +
+ MULTIXACT_FLAGBYTES_PER_GROUP +
+ member_in_group * sizeof(TransactionId);
+}
+
+static inline int
+MXOffsetToFlagsBitShift(MultiXactOffset32 offset)
+{
+ int member_in_group = offset % MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+ int bshift = member_in_group * MXACT_MEMBER_BITS_PER_XACT;
+
+ return bshift;
+}
+
+/*
+ * Construct reader of old multixacts.
+ *
+ * Returns the malloced memory used by the all other calls in this module.
+ */
+OldMultiXactReader *
+AllocOldMultiXactRead(char *pgdata, MultiXactId nextMulti,
+ MultiXactOffset32 nextOffset)
+{
+ OldMultiXactReader *state = state = pg_malloc(sizeof(*state));
+ char dir[MAXPGPATH] = {0};
+
+ state->nextMXact = nextMulti;
+ state->nextOffset = nextOffset;
+
+ pg_sprintf(dir, "%s/pg_multixact/offsets", pgdata);
+ state->offset = AllocSlruRead(dir, false);
+
+ pg_sprintf(dir, "%s/pg_multixact/members", pgdata);
+ state->members = AllocSlruRead(dir, false);
+
+ return state;
+}
+
+/*
+ * This is a simplified version of the GetMultiXactIdMembers() server
+ * function:
+ *
+ * - Only return the updating member, if any. Upgrade only cares about the
+ * updaters. If there is no updating member, return somewhat arbitrarily
+ * the first locking-only member, because we don't have any way to represent
+ * "no members".
+ *
+ * - Because there's no concurrent activity, we don't need to worry about
+ * locking and some corner cases.
+ *
+ * - Don't bail out on invalid entries. If the server crashes, it can leave
+ * invalid or half-written entries on disk. Such multixids won't appear
+ * anywhere else on disk, so the server will never try to read them. During
+ * upgrade, however, we scan through all multixids in order, and will
+ * encounter such invalid but unreferenced multixids too.
+ *
+ * Returns true on success, false if the multixact was invalid.
+ */
+bool
+GetOldMultiXactIdSingleMember(OldMultiXactReader *state, MultiXactId multi,
+ MultiXactMember *member)
+{
+ MultiXactId nextMXact,
+ nextOffset,
+ tmpMXact;
+ int64 pageno,
+ prev_pageno;
+ int entryno,
+ length;
+ char *buf;
+ MultiXactOffset32 *offptr,
+ offset;
+ MultiXactOffset32 nextMXOffset;
+ TransactionId result_xid = InvalidTransactionId;
+ MultiXactStatus result_status = 0;
+
+ nextMXact = state->nextMXact;
+ nextOffset = state->nextOffset;
+
+ /*
+ * Comment copied from GetMultiXactIdMembers in PostgreSQL v18
+ * multixact.c:
+ *
+ * Find out the offset at which we need to start reading MultiXactMembers
+ * and the number of members in the multixact. We determine the latter as
+ * the difference between this multixact's starting offset and the next
+ * one's. However, there are some corner cases to worry about:
+ *
+ * 1. This multixact may be the latest one created, in which case there is
+ * no next one to look at. The next multixact's offset should be set
+ * already, as we set it in RecordNewMultiXact(), but we used to not do
+ * that in older minor versions. To cope with that case, if this
+ * multixact is the latest one created, use the nextOffset value we read
+ * above as the endpoint.
+ *
+ * 2. Because GetNewMultiXactId skips over offset zero, to reserve zero
+ * for to mean "unset", there is an ambiguity near the point of offset
+ * wraparound. If we see next multixact's offset is one, is that our
+ * multixact's actual endpoint, or did it end at zero with a subsequent
+ * increment? We handle this using the knowledge that if the zero'th
+ * member slot wasn't filled, it'll contain zero, and zero isn't a valid
+ * transaction ID so it can't be a multixact member. Therefore, if we
+ * read a zero from the members array, just ignore it.
+ */
+
+ pageno = MultiXactIdToOffsetPage(multi);
+ entryno = MultiXactIdToOffsetEntry(multi);
+
+ buf = SlruReadSwitchPage(state->offset, pageno);
+ offptr = (MultiXactOffset32 *) buf;
+ offptr += entryno;
+ offset = *offptr;
+
+ if (offset == 0)
+ {
+ /* Invalid entry */
+ return false;
+ }
+
+ /*
+ * Use the same increment rule as GetNewMultiXactId(), that is, don't
+ * handle wraparound explicitly until needed.
+ */
+ tmpMXact = multi + 1;
+
+ if (nextMXact == tmpMXact)
+ {
+ /* Corner case 1: there is no next multixact */
+ nextMXOffset = nextOffset;
+ }
+ else
+ {
+ /* handle wraparound if needed */
+ if (tmpMXact < FirstMultiXactId)
+ tmpMXact = FirstMultiXactId;
+
+ prev_pageno = pageno;
+
+ pageno = MultiXactIdToOffsetPage(tmpMXact);
+ entryno = MultiXactIdToOffsetEntry(tmpMXact);
+
+ if (pageno != prev_pageno)
+ buf = SlruReadSwitchPage(state->offset, pageno);
+
+ offptr = (MultiXactOffset32 *) buf;
+ offptr += entryno;
+ nextMXOffset = *offptr;
+ }
+
+ if (nextMXOffset == 0)
+ {
+ /* Invalid entry */
+ return false;
+ }
+ length = nextMXOffset - offset;
+
+ /* read the members */
+ prev_pageno = -1;
+ for (int i = 0; i < length; i++, offset++)
+ {
+ TransactionId *xactptr;
+ uint32 *flagsptr;
+ int flagsoff;
+ int bshift;
+ int memberoff;
+ MultiXactStatus status;
+
+ pageno = MXOffsetToMemberPage(offset);
+ memberoff = MXOffsetToMemberOffset(offset);
+
+ if (pageno != prev_pageno)
+ {
+ buf = SlruReadSwitchPage(state->members, pageno);
+ prev_pageno = pageno;
+ }
+
+ xactptr = (TransactionId *) (buf + memberoff);
+ if (!TransactionIdIsValid(*xactptr))
+ {
+ /*
+ * Corner case 2: we are looking at unused slot zero
+ */
+ if (offset == 0)
+ continue;
+
+ /*
+ * Otherwise this is an invalid entry that should not be
+ * referenced from anywhere in the heap. We could return 'false'
+ * here, but we prefer to continue reading the members and
+ * converting them the best we can, to preserve evidence in case
+ * this is corruption that should not happen.
+ */
+ }
+
+ flagsoff = MXOffsetToFlagsOffset(offset);
+ bshift = MXOffsetToFlagsBitShift(offset);
+ flagsptr = (uint32 *) (buf + flagsoff);
+
+ status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK;
+
+ /*
+ * Remember the updating XID among the members, or first locking XID
+ * if no updating XID.
+ */
+ if (ISUPDATE_from_mxstatus(status))
+ {
+ /* sanity check */
+ if (ISUPDATE_from_mxstatus(result_status))
+ {
+ /*
+ * We don't expect to see more than one updating member, even
+ * if the server had crashed.
+ */
+ pg_fatal("multixact %u has more than one updating member",
+ multi);
+ }
+ result_xid = *xactptr;
+ result_status = status;
+ }
+ else if (!TransactionIdIsValid(result_xid))
+ {
+ result_xid = *xactptr;
+ result_status = status;
+ }
+ }
+
+ member->xid = result_xid;
+ member->status = result_status;
+ return true;
+}
+
+/*
+ * Frees the malloced reader.
+ */
+void
+FreeOldMultiXactReader(OldMultiXactReader *state)
+{
+ FreeSlruRead(state->offset);
+ FreeSlruRead(state->members);
+
+ pfree(state);
+}
diff --git a/src/bin/pg_upgrade/multixact_read_v18.h b/src/bin/pg_upgrade/multixact_read_v18.h
new file mode 100644
index 00000000000..6ef485b53e1
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_read_v18.h
@@ -0,0 +1,37 @@
+/*
+ * multixact_read_v18.h
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ * src/bin/pg_upgrade/multixact_read_v18.h
+ */
+#ifndef MULTIXACT_READ_V18_H
+#define MULTIXACT_READ_V18_H
+
+#include "access/multixact.h"
+#include "slru_io.h"
+
+/*
+ * MultiXactOffset changed from uint32 to uint64 between versions 18 and 19.
+ * MultiXactOffset32 is used to represent a 32-bit offset from the old
+ * cluster.
+ */
+typedef uint32 MultiXactOffset32;
+
+typedef struct OldMultiXactReader
+{
+ MultiXactId nextMXact;
+ MultiXactOffset32 nextOffset;
+
+ SlruSegState *offset;
+ SlruSegState *members;
+} OldMultiXactReader;
+
+extern OldMultiXactReader *AllocOldMultiXactRead(char *pgdata,
+ MultiXactId nextMulti,
+ MultiXactOffset32 nextOffset);
+extern bool GetOldMultiXactIdSingleMember(OldMultiXactReader *state,
+ MultiXactId multi,
+ MultiXactMember *member);
+extern void FreeOldMultiXactReader(OldMultiXactReader *reader);
+
+#endif /* MULTIXACT_READ_V18_H */
diff --git a/src/bin/pg_upgrade/multixact_rewrite.c b/src/bin/pg_upgrade/multixact_rewrite.c
new file mode 100644
index 00000000000..4e56922d83f
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_rewrite.c
@@ -0,0 +1,191 @@
+/*
+ * multixact_rewrite.c
+ *
+ * Functions to convert multixact SLRUs from the pre-v19 format to the current
+ * format with 64-bit MultiXactOffsets.
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ * src/bin/pg_upgrade/multixact_rewrite.c
+ */
+
+#include "postgres_fe.h"
+
+#include "access/multixact_internal.h"
+#include "multixact_read_v18.h"
+#include "pg_upgrade.h"
+
+static void RecordMultiXactOffset(SlruSegState *offsets_writer, MultiXactId multi,
+ MultiXactOffset offset);
+static void RecordMultiXactMembers(SlruSegState *members_writer,
+ MultiXactOffset offset,
+ int nmembers, MultiXactMember *members);
+
+/*
+ * Convert pg_multixact/offset and /members from the old pre-v19 format with
+ * 32-bit offsets to the current format.
+ *
+ * Multixids in the range [from_multi, to_multi) are read from the old
+ * cluster, and written in the new format. An important edge case is that if
+ * from_multi == to_multi, this initializes the new pg_multixact files in the
+ * new format without trying to open any old files. (We rely on that when
+ * upgrading from PostgreSQL version 9.2 or below.)
+ *
+ * Returns the new nextOffset value; the caller should set it in the new
+ * control file. The new members always start from offset 1, regardless of
+ * the offset range used in the old cluster.
+ */
+MultiXactOffset
+rewrite_multixacts(MultiXactId from_multi, MultiXactId to_multi)
+{
+ MultiXactOffset next_offset;
+ SlruSegState *offsets_writer;
+ SlruSegState *members_writer;
+ char dir[MAXPGPATH] = {0};
+ bool prev_multixid_valid = false;
+
+ /*
+ * The range of valid multi XIDs is unchanged by the conversion (they are
+ * referenced from the heap tables), but the members SLRU is rewritten to
+ * start from offset 1.
+ */
+ next_offset = 1;
+
+ /* Prepare to write the new SLRU files */
+ pg_sprintf(dir, "%s/pg_multixact/offsets", new_cluster.pgdata);
+ offsets_writer = AllocSlruWrite(dir, false);
+ SlruWriteSwitchPage(offsets_writer, MultiXactIdToOffsetPage(from_multi));
+
+ pg_sprintf(dir, "%s/pg_multixact/members", new_cluster.pgdata);
+ members_writer = AllocSlruWrite(dir, true /* use long segment names */ );
+ SlruWriteSwitchPage(members_writer, MXOffsetToMemberPage(next_offset));
+
+ /*
+ * Convert old multixids, if needed, by reading them one-by-one from the
+ * old cluster.
+ */
+ if (to_multi != from_multi)
+ {
+ OldMultiXactReader *old_reader;
+
+ old_reader = AllocOldMultiXactRead(old_cluster.pgdata,
+ old_cluster.controldata.chkpnt_nxtmulti,
+ old_cluster.controldata.chkpnt_nxtmxoff);
+
+ for (MultiXactId multi = from_multi; multi != to_multi;)
+ {
+ MultiXactMember member;
+ bool multixid_valid;
+
+ /*
+ * Read this multixid's members.
+ *
+ * Locking-only XIDs that may be part of multi-xids don't matter
+ * after upgrade, as there can be no transactions running across
+ * upgrade. So as a small optimization, we only read one member
+ * from each multixid: the one updating one, or if there was no
+ * update, arbitrarily the first locking xid.
+ */
+ multixid_valid = GetOldMultiXactIdSingleMember(old_reader, multi, &member);
+
+ /*
+ * Write the new offset to pg_multixact/offsets.
+ *
+ * Even if this multixid is invalid, we still need to write its
+ * offset if the *previous* multixid was valid. That's because
+ * when reading a multixid, the number of members is calculated
+ * from the difference between the two offsets.
+ */
+ RecordMultiXactOffset(offsets_writer, multi,
+ (multixid_valid || prev_multixid_valid) ? next_offset : 0);
+
+ /* Write the members */
+ if (multixid_valid)
+ {
+ RecordMultiXactMembers(members_writer, next_offset, 1, &member);
+ next_offset += 1;
+ }
+
+ /* Advance to next multixid, handling wraparound */
+ multi++;
+ if (multi < FirstMultiXactId)
+ multi = FirstMultiXactId;
+ prev_multixid_valid = multixid_valid;
+ }
+
+ FreeOldMultiXactReader(old_reader);
+ }
+
+ /* Write the final 'next' offset to the last SLRU page */
+ RecordMultiXactOffset(offsets_writer, to_multi,
+ prev_multixid_valid ? next_offset : 0);
+
+ /* Flush the last SLRU pages */
+ FreeSlruWrite(offsets_writer);
+ FreeSlruWrite(members_writer);
+
+ return next_offset;
+}
+
+
+/*
+ * Write one offset to the offset SLRU
+ */
+static void
+RecordMultiXactOffset(SlruSegState *offsets_writer, MultiXactId multi,
+ MultiXactOffset offset)
+{
+ int64 pageno;
+ int entryno;
+ char *buf;
+ MultiXactOffset *offptr;
+
+ pageno = MultiXactIdToOffsetPage(multi);
+ entryno = MultiXactIdToOffsetEntry(multi);
+
+ buf = SlruWriteSwitchPage(offsets_writer, pageno);
+ offptr = (MultiXactOffset *) buf;
+ offptr[entryno] = offset;
+}
+
+/*
+ * Write the members for one multixid in the members SLRU
+ *
+ * (Currently, this is only ever called with nmembers == 1)
+ */
+static void
+RecordMultiXactMembers(SlruSegState *members_writer,
+ MultiXactOffset offset,
+ int nmembers, MultiXactMember *members)
+{
+ for (int i = 0; i < nmembers; i++, offset++)
+ {
+ int64 pageno;
+ char *buf;
+ TransactionId *memberptr;
+ uint32 *flagsptr;
+ uint32 flagsval;
+ int bshift;
+ int flagsoff;
+ int memberoff;
+
+ Assert(members[i].status <= MultiXactStatusUpdate);
+
+ pageno = MXOffsetToMemberPage(offset);
+ memberoff = MXOffsetToMemberOffset(offset);
+ flagsoff = MXOffsetToFlagsOffset(offset);
+ bshift = MXOffsetToFlagsBitShift(offset);
+
+ buf = SlruWriteSwitchPage(members_writer, pageno);
+
+ memberptr = (TransactionId *) (buf + memberoff);
+
+ *memberptr = members[i].xid;
+
+ flagsptr = (uint32 *) (buf + flagsoff);
+
+ flagsval = *flagsptr;
+ flagsval &= ~(((1 << MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift);
+ flagsval |= (members[i].status << bshift);
+ *flagsptr = flagsval;
+ }
+}
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index 490e98fa26f..47119222655 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -43,6 +43,7 @@
#include <time.h>
+#include "access/multixact.h"
#include "catalog/pg_class_d.h"
#include "common/file_perm.h"
#include "common/logging.h"
@@ -807,15 +808,15 @@ copy_xact_xlog_xid(void)
new_cluster.pgdata);
check_ok();
- /*
- * If the old server is before the MULTIXACT_FORMATCHANGE_CAT_VER change
- * (see pg_upgrade.h) and the new server is after, then we don't copy
- * pg_multixact files, but we need to reset pg_control so that the new
- * server doesn't attempt to read multis older than the cutoff value.
- */
- if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER &&
- new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
+ /* Copy or convert pg_multixact files */
+ Assert(new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER);
+ Assert(new_cluster.controldata.cat_ver >= MULTIXACTOFFSET_FORMATCHANGE_CAT_VER);
+ if (old_cluster.controldata.cat_ver >= MULTIXACTOFFSET_FORMATCHANGE_CAT_VER)
{
+ /* No change in multixact format, just copy the files */
+ MultiXactId new_nxtmulti = old_cluster.controldata.chkpnt_nxtmulti;
+ MultiXactOffset new_nxtmxoff = old_cluster.controldata.chkpnt_nxtmxoff;
+
copy_subdir_files("pg_multixact/offsets", "pg_multixact/offsets");
copy_subdir_files("pg_multixact/members", "pg_multixact/members");
@@ -826,38 +827,67 @@ copy_xact_xlog_xid(void)
* counters here and the oldest multi present on system.
*/
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_resetwal\" -O %u -m %u,%u \"%s\"",
- new_cluster.bindir,
- old_cluster.controldata.chkpnt_nxtmxoff,
- old_cluster.controldata.chkpnt_nxtmulti,
+ "\"%s/pg_resetwal\" -O %" PRIu64 " -m %u,%u \"%s\"",
+ new_cluster.bindir, new_nxtmxoff, new_nxtmulti,
old_cluster.controldata.chkpnt_oldstMulti,
new_cluster.pgdata);
check_ok();
}
- else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
+ else
{
+ /* Conversion is needed */
+ MultiXactId nxtmulti;
+ MultiXactId oldstMulti;
+ MultiXactOffset nxtmxoff;
+
/*
- * Remove offsets/0000 file created by initdb that no longer matches
- * the new multi-xid value. "members" starts at zero so no need to
- * remove it.
+ * Determine the range of multixacts to convert.
*/
- remove_new_subdir("pg_multixact/offsets", false);
+ nxtmulti = old_cluster.controldata.chkpnt_nxtmulti;
+ if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
+ {
+ /* Versions 9.3 - 18: convert all multixids */
+ oldstMulti = old_cluster.controldata.chkpnt_oldstMulti;
+ }
+ else
+ {
+ /*
+ * In PostgreSQL 9.2 and below, multitransactions were only used
+ * for row locking, and as such don't need to be preserved during
+ * upgrade. In that case, we utilize rewrite_multixacts() just to
+ * initialize new, empty files in the new format.
+ *
+ * It's important that the oldest multi is set to the latest value
+ * used by the old system, so that multixact.c returns the empty
+ * set for multis that might be present on disk.
+ */
+ oldstMulti = nxtmulti;
+ }
+ /* handle wraparound */
+ if (nxtmulti < FirstMultiXactId)
+ nxtmulti = FirstMultiXactId;
+ if (oldstMulti < FirstMultiXactId)
+ oldstMulti = FirstMultiXactId;
- prep_status("Setting oldest multixact ID in new cluster");
+ /*
+ * Remove the files created by initdb in the new cluster.
+ * rewrite_multixacts() will create new ones.
+ */
+ remove_new_subdir("pg_multixact/members", false);
+ remove_new_subdir("pg_multixact/offsets", false);
/*
- * We don't preserve files in this case, but it's important that the
- * oldest multi is set to the latest value used by the old system, so
- * that multixact.c returns the empty set for multis that might be
- * present on disk. We set next multi to the value following that; it
- * might end up wrapped around (i.e. 0) if the old cluster had
- * next=MaxMultiXactId, but multixact.c can cope with that just fine.
+ * Create new pg_multixact files, converting old ones if needed.
*/
+ prep_status("Converting pg_multixact files");
+ nxtmxoff = rewrite_multixacts(oldstMulti, nxtmulti);
+ check_ok();
+
+ prep_status("Setting next multixact ID and offset for new cluster");
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_resetwal\" -m %u,%u \"%s\"",
+ "\"%s/pg_resetwal\" -O %" PRIu64 " -m %u,%u \"%s\"",
new_cluster.bindir,
- old_cluster.controldata.chkpnt_nxtmulti + 1,
- old_cluster.controldata.chkpnt_nxtmulti,
+ nxtmxoff, nxtmulti, oldstMulti,
new_cluster.pgdata);
check_ok();
}
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index e86336f4be9..be30dceed5c 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -115,6 +115,13 @@ extern char *output_files[];
#define MULTIXACT_FORMATCHANGE_CAT_VER 201301231
/*
+ * MultiXactOffset was changed from 32-bit to 64-bit in version 19, at this
+ * catalog version. pg_multixact files need to be converted when upgrading
+ * across this version.
+ */
+#define MULTIXACTOFFSET_FORMATCHANGE_CAT_VER 202512091
+
+/*
* large object chunk size added to pg_controldata,
* commit 5f93c37805e7485488480916b4585e098d3cc883
*/
@@ -235,7 +242,7 @@ typedef struct
uint32 chkpnt_nxtepoch;
uint32 chkpnt_nxtoid;
uint32 chkpnt_nxtmulti;
- uint32 chkpnt_nxtmxoff;
+ uint64 chkpnt_nxtmxoff;
uint32 chkpnt_oldstMulti;
uint32 chkpnt_oldstxid;
uint32 align;
@@ -499,6 +506,9 @@ void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster,
void report_extension_updates(ClusterInfo *cluster);
+/* multixact_rewrite.c */
+MultiXactOffset rewrite_multixacts(MultiXactId from_multi, MultiXactId to_multi);
+
/* parallel.c */
void parallel_exec_prog(const char *log_file, const char *opt_log_file,
const char *fmt,...) pg_attribute_printf(3, 4);
diff --git a/src/bin/pg_upgrade/slru_io.c b/src/bin/pg_upgrade/slru_io.c
new file mode 100644
index 00000000000..812a241fe62
--- /dev/null
+++ b/src/bin/pg_upgrade/slru_io.c
@@ -0,0 +1,268 @@
+/*
+ * slru_io.c
+ *
+ * Routines for reading and writing SLRU files during upgrade.
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ * src/bin/pg_upgrade/slru_io.c
+ */
+
+#include "postgres_fe.h"
+
+#include <fcntl.h>
+
+#include "common/fe_memutils.h"
+#include "common/file_perm.h"
+#include "common/file_utils.h"
+#include "pg_upgrade.h"
+#include "port/pg_iovec.h"
+#include "slru_io.h"
+
+static SlruSegState *AllocSlruSegState(const char *dir);
+static char *SlruFileName(SlruSegState *state, int64 segno);
+static void SlruFlush(SlruSegState *state);
+
+/* common parts of AllocSlruRead and AllocSlruWrite */
+static SlruSegState *
+AllocSlruSegState(const char *dir)
+{
+ SlruSegState *state = pg_malloc(sizeof(*state));
+
+ state->dir = pstrdup(dir);
+ state->fn = NULL;
+ state->fd = -1;
+ state->segno = -1;
+ state->pageno = 0;
+
+ /* state->writing and state->long_segment_names must be set by caller! */
+
+ return state;
+}
+
+/* similar to the backend function with the same name */
+static char *
+SlruFileName(SlruSegState *state, int64 segno)
+{
+ if (state->long_segment_names)
+ {
+ Assert(segno >= 0 && segno <= INT64CONST(0xFFFFFFFFFFFFFFF));
+ return psprintf("%s/%015" PRIX64, state->dir, segno);
+ }
+ else
+ {
+ Assert(segno >= 0 && segno <= INT64CONST(0xFFFFFF));
+ return psprintf("%s/%04X", state->dir, (unsigned int) segno);
+ }
+}
+
+/*
+ * Create SLRU reader for dir.
+ */
+SlruSegState *
+AllocSlruRead(const char *dir, bool long_segment_names)
+{
+ SlruSegState *state = AllocSlruSegState(dir);
+
+ state->writing = false;
+ state->long_segment_names = long_segment_names;
+
+ return state;
+}
+
+/*
+ * Read the given page into memory buffer.
+ *
+ * Reading can be done in random order.
+ *
+ * If the file containing 'pageno' does not exist, a fatal error is raised.
+ * If the file exists but is shorter than expected, the missing part is read
+ * as zeros and a warning is logged. That is reasonable behavior for current
+ * callers.
+ *
+ * This is the slow path of the inlineable SlruReadSwitchPage() function.
+ */
+char *
+SlruReadSwitchPageSlow(SlruSegState *state, uint64 pageno)
+{
+ int64 segno;
+ off_t offset;
+ ssize_t bytes_read;
+
+ Assert(!state->writing); /* read only mode */
+
+ if (state->segno != -1 && pageno == state->pageno)
+ return state->buf.data;
+
+ /* If the new page is on a different SLRU segment, open the new segment */
+ segno = pageno / SLRU_PAGES_PER_SEGMENT;
+ if (segno != state->segno)
+ {
+ if (state->segno != -1)
+ {
+ close(state->fd);
+ state->fd = -1;
+
+ pg_free(state->fn);
+ state->fn = NULL;
+
+ state->segno = -1;
+ }
+
+ state->fn = SlruFileName(state, segno);
+ if ((state->fd = open(state->fn, O_RDONLY | PG_BINARY, 0)) < 0)
+ pg_fatal("could not open file \"%s\": %m", state->fn);
+ state->segno = segno;
+ }
+
+ offset = (pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+ bytes_read = 0;
+ while (bytes_read < BLCKSZ)
+ {
+ ssize_t rc;
+
+ rc = pg_pread(state->fd,
+ &state->buf.data + bytes_read,
+ BLCKSZ - bytes_read,
+ offset + bytes_read);
+ if (rc < 0)
+ {
+ if (errno == EINTR)
+ continue;
+ pg_fatal("could not read file \"%s\": %m", state->fn);
+ }
+ if (rc == 0)
+ {
+ /* unexpected EOF */
+ pg_log(PG_WARNING, "unexpected EOF reading file \"%s\" at offset %zd, reading as zeros", state->fn,
+ offset + bytes_read);
+ memset(&state->buf.data + bytes_read, 0, BLCKSZ - bytes_read);
+ break;
+ }
+ bytes_read += rc;
+ }
+ state->pageno = pageno;
+
+ return state->buf.data;
+}
+
+/*
+ * Free the reader.
+ */
+void
+FreeSlruRead(SlruSegState *state)
+{
+ Assert(!state->writing); /* read only mode */
+
+ if (state->fd != -1)
+ close(state->fd);
+ pg_free(state);
+}
+
+/*
+ * Create SLRU writer for dir.
+ */
+SlruSegState *
+AllocSlruWrite(const char *dir, bool long_segment_names)
+{
+ SlruSegState *state = AllocSlruSegState(dir);
+
+ state->writing = true;
+ state->long_segment_names = long_segment_names;
+
+ return state;
+}
+
+/*
+ * Open the given page for writing.
+ *
+ * NOTE: This uses O_EXCL when stepping to a new segment, so this assumes that
+ * each segment is written in full before moving on to the next one. This
+ * limitation would be easy to lift if needed, but it fits the usage pattern
+ * of current callers.
+ *
+ * This is the slow path of the inlineable SlruWriteSwitchPage() function.
+ */
+char *
+SlruWriteSwitchPageSlow(SlruSegState *state, uint64 pageno)
+{
+ int64 segno;
+ off_t offset;
+
+ Assert(state->writing);
+
+ if (state->segno != -1 && pageno == state->pageno)
+ return state->buf.data;
+
+ segno = pageno / SLRU_PAGES_PER_SEGMENT;
+ offset = (pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+
+ SlruFlush(state);
+ memset(state->buf.data, 0, BLCKSZ);
+
+ if (segno != state->segno)
+ {
+ if (state->segno != -1)
+ {
+ close(state->fd);
+ state->fd = -1;
+
+ pg_free(state->fn);
+ state->fn = NULL;
+
+ state->segno = -1;
+ }
+
+ /* Create the segment */
+ state->fn = SlruFileName(state, segno);
+ if ((state->fd = open(state->fn, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
+ pg_file_create_mode)) < 0)
+ {
+ pg_fatal("could not create file \"%s\": %m", state->fn);
+ }
+
+ state->segno = segno;
+
+ if (offset > 0)
+ {
+ if (pg_pwrite_zeros(state->fd, offset, 0) < 0)
+ pg_fatal("could not write file \"%s\": %m", state->fn);
+ }
+ }
+
+ state->pageno = pageno;
+
+ return state->buf.data;
+}
+
+static void
+SlruFlush(SlruSegState *state)
+{
+ struct iovec iovec = {
+ .iov_base = &state->buf,
+ .iov_len = BLCKSZ,
+ };
+ off_t offset;
+
+ if (state->segno == -1)
+ return;
+
+ offset = (state->pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+
+ if (pg_pwritev_with_retry(state->fd, &iovec, 1, offset) < 0)
+ pg_fatal("could not write file \"%s\": %m", state->fn);
+}
+
+/*
+ * Free the writer.
+ */
+void
+FreeSlruWrite(SlruSegState *state)
+{
+ Assert(state->writing);
+
+ SlruFlush(state);
+
+ if (state->fd != -1)
+ close(state->fd);
+ pg_free(state);
+}
diff --git a/src/bin/pg_upgrade/slru_io.h b/src/bin/pg_upgrade/slru_io.h
new file mode 100644
index 00000000000..5c80a679b4d
--- /dev/null
+++ b/src/bin/pg_upgrade/slru_io.h
@@ -0,0 +1,52 @@
+/*
+ * slru_io.h
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ * src/bin/pg_upgrade/slru_io.h
+ */
+
+#ifndef SLRU_IO_H
+#define SLRU_IO_H
+
+/*
+ * State for reading or writing an SLRU, with a one page buffer.
+ */
+typedef struct SlruSegState
+{
+ bool writing;
+ bool long_segment_names;
+
+ char *dir;
+ char *fn;
+ int fd;
+ int64 segno;
+ uint64 pageno;
+
+ PGAlignedBlock buf;
+} SlruSegState;
+
+extern SlruSegState *AllocSlruRead(const char *dir, bool long_segment_names);
+extern char *SlruReadSwitchPageSlow(SlruSegState *state, uint64 pageno);
+extern void FreeSlruRead(SlruSegState *state);
+
+static inline char *
+SlruReadSwitchPage(SlruSegState *state, uint64 pageno)
+{
+ if (state->segno != -1 && pageno == state->pageno)
+ return state->buf.data;
+ return SlruReadSwitchPageSlow(state, pageno);
+}
+
+extern SlruSegState *AllocSlruWrite(const char *dir, bool long_segment_names);
+extern char *SlruWriteSwitchPageSlow(SlruSegState *state, uint64 pageno);
+extern void FreeSlruWrite(SlruSegState *state);
+
+static inline char *
+SlruWriteSwitchPage(SlruSegState *state, uint64 pageno)
+{
+ if (state->segno != -1 && pageno == state->pageno)
+ return state->buf.data;
+ return SlruWriteSwitchPageSlow(state, pageno);
+}
+
+#endif /* SLRU_IO_H */
diff --git a/src/bin/pg_upgrade/t/007_multixact_conversion.pl b/src/bin/pg_upgrade/t/007_multixact_conversion.pl
new file mode 100644
index 00000000000..443b93c7545
--- /dev/null
+++ b/src/bin/pg_upgrade/t/007_multixact_conversion.pl
@@ -0,0 +1,427 @@
+# Copyright (c) 2025, PostgreSQL Global Development Group
+
+# Version 19 expanded MultiXactOffset from 32 to 64 bits. Upgrading
+# across that requires rewriting the SLRU files to the new format.
+# This file contains tests for the conversion.
+#
+# To run, set 'oldinstall' ENV variable to point to a pre-v19
+# installation. If it's not set, or if it points to a v19 or above
+# installation, this still performs a very basic test, upgrading a
+# cluster with some multixacts. It's not very interesting, however,
+# because there's no conversion involved in that case.
+
+use strict;
+use warnings FATAL => 'all';
+
+use Math::BigInt;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Temp dir for a dumps.
+my $tempdir = PostgreSQL::Test::Utils::tempdir;
+
+# A workload that consumes multixids. The purpose of this is to
+# generate some multixids in the old cluster, so that we can test
+# upgrading them. The workload is a mix of KEY SHARE locking queries
+# and UPDATEs, and commits and aborts, to generate a mix of multixids
+# with different statuses. It consumes around 3000 multixids with
+# 30000 members. That's enough to span more than one multixids
+# 'offsets' page, and more than one 'members' segment.
+#
+# The workload leaves behind a table called 'mxofftest' containing a
+# small number of rows referencing some of the generated multixids.
+#
+# Because this function is used to generate test data on the old
+# installation, it needs to work with older PostgreSQL server
+# versions.
+#
+# The first argument is the cluster to connect to, the second argument
+# is a cluster using the new version. We need the 'psql' binary from
+# the new version, the new cluster is otherwise unused. (We need to
+# use the new 'psql' because some of the more advanced background psql
+# perl module features depend on a fairly recent psql version.)
+sub mxact_workload
+{
+ my $node = shift; # Cluster to connect to
+ my $binnode = shift; # Use the psql binary from this cluster
+
+ my $connstr = $node->connstr('postgres');
+
+ $node->start;
+ $node->safe_psql(
+ 'postgres', qq[
+ CREATE TABLE mxofftest (id INT PRIMARY KEY, n_updated INT)
+ WITH (AUTOVACUUM_ENABLED=FALSE);
+ INSERT INTO mxofftest SELECT G, 0 FROM GENERATE_SERIES(1, 50) G;
+ ]);
+
+ my $nclients = 20;
+ my $update_every = 13;
+ my $abort_every = 11;
+ my @connections = ();
+
+ # Silence the logging of the statements we run to avoid
+ # unnecessarily bloating the test logs. This runs before the
+ # upgrade we're testing, so the details should not be very
+ # interesting for debugging. But if needed, you can make it more
+ # verbose by setting this.
+ my $verbose = 0;
+
+ # Open multiple connections to the database. Start a transaction
+ # in each connection.
+ for (0 .. $nclients)
+ {
+ # Use the psql binary from the new installation. The
+ # BackgroundPsql functionality doesn't work with older psql
+ # versions.
+ my $conn = $binnode->background_psql('',
+ connstr => $node->connstr('postgres'));
+
+ $conn->query_safe("SET log_statement=none", verbose => $verbose)
+ unless $verbose;
+ $conn->query_safe("SET enable_seqscan=off", verbose => $verbose);
+ $conn->query_safe("BEGIN", verbose => $verbose);
+
+ push(@connections, $conn);
+ }
+
+ # Run queries using cycling through the connections in a
+ # round-robin fashion. We keep a transaction open in each
+ # connection at all times, and lock/update the rows. With 10
+ # connections, each SELECT FOR KEY SHARE query generates a new
+ # multixid, containing the 10 XIDs of all the transactions running
+ # at the time.
+ for (my $i = 0; $i < 3000; $i++)
+ {
+ my $conn = $connections[ $i % $nclients ];
+
+ my $sql = ($i % $abort_every == 0) ? "ABORT" : "COMMIT";
+ $conn->query_safe($sql, verbose => $verbose);
+
+ $conn->query_safe("BEGIN", verbose => $verbose);
+ if ($i % $update_every == 0)
+ {
+ $sql = qq[
+ UPDATE mxofftest SET n_updated = n_updated + 1 WHERE id = ${i} % 50;
+ ];
+ }
+ else
+ {
+ my $threshold = int($i / 3000 * 50);
+ $sql = qq[
+ select count(*) from (
+ SELECT * FROM mxofftest WHERE id >= $threshold FOR KEY SHARE
+ ) as x
+ ];
+ }
+ $conn->query_safe($sql, verbose => $verbose);
+ }
+
+ for my $conn (@connections)
+ {
+ $conn->quit();
+ }
+
+ $node->stop;
+ return;
+}
+
+# Return contents of the 'mxofftest' table, created by mxact_workload
+sub get_test_table_contents
+{
+ my ($node, $filename) = @_;
+
+ my $contents = $node->safe_psql('postgres',
+ "SELECT ctid, xmin, xmax, * FROM mxofftest");
+
+ my $path = $tempdir . '/' . $filename;
+ open(my $fh, '>', $path)
+ || die "could not open $path for writing $!";
+ print $fh $contents;
+ close($fh);
+
+ return $path;
+}
+
+# Return the members of all updating multixids in the given range
+sub get_updating_multixact_members
+{
+ my ($node, $from, $to, $filename) = @_;
+
+ my $path = $tempdir . '/' . $filename;
+ open(my $fh, '>', $path)
+ || die "could not open $path for writing $!";
+
+ if ($to >= $from)
+ {
+ my $res = $node->safe_psql(
+ 'postgres', qq[
+ SELECT multi, mode, xid
+ FROM generate_series($from, $to - 1) as multi,
+ pg_get_multixact_members(multi::text::xid)
+ WHERE mode not in ('keysh', 'sh');
+ ]);
+ print $fh $res;
+ }
+ else
+ {
+ # Multixids wrapped around. Split the query into two parts,
+ # before and after the wraparound.
+ my $res = $node->safe_psql(
+ 'postgres', qq[
+ SELECT multi, mode, xid
+ FROM generate_series($from, 4294967295) as multi,
+ pg_get_multixact_members(multi::text::xid)
+ WHERE mode not in ('keysh', 'sh');
+ ]);
+ print $fh $res;
+ $res = $node->safe_psql(
+ 'postgres', qq[
+ SELECT multi, mode, xid
+ FROM generate_series(1, $to - 1) as multi,
+ pg_get_multixact_members(multi::text::xid)
+ WHERE mode not in ('keysh', 'sh');
+ ]);
+ print $fh $res;
+ }
+
+ close($fh);
+ return $path;
+}
+
+# Read multixid related fields from the control file
+#
+# Note: This is used on both the old and the new installation, so the
+# command arguments and the output parsing used here must work with
+# all PostgreSQL versions supported by the test.
+sub read_multixid_fields
+{
+ my $node = shift;
+
+ my $pg_controldata_path = $node->installed_command('pg_controldata');
+ my ($stdout, $stderr) =
+ run_command([ $pg_controldata_path, $node->data_dir ]);
+ $stdout =~ /^Latest checkpoint's oldestMultiXid:\s*(.*)$/m
+ or die "could not read oldestMultiXid from pg_controldata";
+ my $oldest_multi_xid = $1;
+ $stdout =~ /^Latest checkpoint's NextMultiXactId:\s*(.*)$/m
+ or die "could not read NextMultiXactId from pg_controldata";
+ my $next_multi_xid = $1;
+ $stdout =~ /^Latest checkpoint's NextMultiOffset:\s*(.*)$/m
+ or die "could not read NextMultiOffset from pg_controldata";
+ my $next_multi_offset = $1;
+
+ return ($oldest_multi_xid, $next_multi_xid, $next_multi_offset);
+}
+
+# Reset a cluster's next multixid and mxoffset to given values.
+#
+# Note: This is used on the old insallation, so the command arguments
+# and the output parsing used here must work with all pre-v19
+# PostgreSQL versions supported by the test.
+sub reset_mxid_mxoffset_pre_v19
+{
+ my $node = shift;
+ my $mxid = shift;
+ my $mxoffset = shift;
+
+ my $pg_resetwal_path = $node->installed_command('pg_resetwal');
+ # Get block size
+ my ($out, $err) =
+ run_command([ $pg_resetwal_path, '--dry-run', $node->data_dir ]);
+ $out =~ /^Database block size: *(\d+)$/m or die;
+
+ # Verify that no multixids are currently in use. Resetting would
+ # destroy them. (A freshly initialized cluster has no multixids.)
+ $out =~ /^Latest checkpoint's NextMultiXactId: *(\d+)$/m or die;
+ my $next_mxid = $1;
+ $out =~ /^Latest checkpoint's oldestMultiXid: *(\d+)$/m or die;
+ my $oldest_mxid = $1;
+ die "cluster has some multixids in use" unless $next_mxid == $oldest_mxid;
+
+ # Extract a few other values from pg_resetwal --dry-run output
+ # that we need for the calculations below
+ $out =~ /^Database block size: *(\d+)$/m or die;
+ my $blcksz = $1;
+ # SLRU_PAGES_PER_SEGMENT is always 32 on pre-19 versions
+ my $slru_pages_per_segment = 32;
+
+ # Do the reset
+ my @cmd = (
+ $pg_resetwal_path,
+ '--pgdata' => $node->data_dir,
+ '--multixact-offset' => $mxoffset,
+ '--multixact-ids' => "$mxid,$mxid");
+ command_ok(\@cmd, 'reset multixids and offset');
+
+ # pg_resetwal just updates the control file. The cluster will
+ # refuse to start up, if the SLRU segments corresponding to the
+ # next multixid and offset does not exist. Create a segments that
+ # covers the given values, filled with zeros. But first remove
+ # any old segments.
+ unlink glob $node->data_dir . "/pg_multixact/offsets/*";
+ unlink glob $node->data_dir . "/pg_multixact/members/*";
+
+ # Initialize the 'offsets' SLRU file containing the new next multixid
+ # with zeros
+ #
+ # sizeof(MultiXactOffset) == 4 in PostgreSQL versions before 19
+ my $multixact_offsets_per_page = $blcksz / 4;
+ my $segno =
+ int($mxid / $multixact_offsets_per_page / $slru_pages_per_segment);
+ my $path =
+ sprintf('%s/pg_multixact/offsets/%04X', $node->data_dir, $segno);
+ open my $fh, ">", $path
+ or die "could not open \"$path\": $!";
+ binmode $fh;
+ my $bytes_per_seg = $slru_pages_per_segment * $blcksz;
+ syswrite($fh, "\0" x $bytes_per_seg) == $bytes_per_seg
+ or die "could not write to \"$path\": $!";
+ close $fh;
+
+ # Same for the 'members' SLRU
+ my $multixact_members_per_page = int($blcksz / 20) * 4;
+ $segno =
+ int($mxoffset / $multixact_members_per_page / $slru_pages_per_segment);
+ $path = sprintf "%s/pg_multixact/members/%04X", $node->data_dir, $segno;
+ open $fh, ">", $path
+ or die "could not open \"$path\": $!";
+ binmode $fh;
+ syswrite($fh, "\0" x $bytes_per_seg) == $bytes_per_seg
+ or die "could not write to \"$path\": $!";
+ close($fh);
+}
+
+# Main test workhorse routine. Dump data on old version, run
+# pg_upgrade, compare data after upgrade.
+sub upgrade_and_compare
+{
+ my $tag = shift;
+ my $oldnode = shift;
+ my $newnode = shift;
+
+ command_ok(
+ [
+ 'pg_upgrade', '--no-sync',
+ '--old-datadir' => $oldnode->data_dir,
+ '--new-datadir' => $newnode->data_dir,
+ '--old-bindir' => $oldnode->config_data('--bindir'),
+ '--new-bindir' => $newnode->config_data('--bindir'),
+ '--socketdir' => $newnode->host,
+ '--old-port' => $oldnode->port,
+ '--new-port' => $newnode->port,
+ ],
+ 'run of pg_upgrade for new instance');
+
+ # Dump contents of the test table, and the status of all updating
+ # multixids from the old cluster. (Locking-only multixids don't
+ # need to be preserved so we ignore those)
+ #
+ # Note: we do this *after* running pg_upgrade, to ensure that we
+ # don't set all the hint bits before upgrade by doing the SELECT
+ # on the table.
+ my ($multixids_start, $multixids_end, undef) =
+ read_multixid_fields($oldnode);
+ $oldnode->start;
+ my $old_table_contents =
+ get_test_table_contents($oldnode, "oldnode_${tag}_table_contents");
+ my $old_multixacts =
+ get_updating_multixact_members($oldnode, $multixids_start,
+ $multixids_end, "oldnode_${tag}_multixacts");
+ $oldnode->stop;
+
+ # Compare them with the upgraded cluster
+ $newnode->start;
+ my $new_table_contents =
+ get_test_table_contents($newnode, "newnode_${tag}_table_contents");
+ my $new_multixacts =
+ get_updating_multixact_members($newnode, $multixids_start,
+ $multixids_end, "newnode_${tag}_multixacts");
+ $newnode->stop;
+
+ compare_files($old_table_contents, $new_table_contents,
+ 'test table contents from original and upgraded clusters match');
+ compare_files($old_multixacts, $new_multixacts,
+ 'multixact members from original and upgraded clusters match');
+}
+
+my $old_version;
+
+# Basic scenario: Create a cluster using old installation, run
+# multixid-creating workload on it, then upgrade.
+#
+# This works even even if the old and new version is the same,
+# although it's not very interesting as the conversion routines only
+# run when upgrading from a pre-v19 cluster.
+{
+ my $tag = 'basic';
+ my $old =
+ PostgreSQL::Test::Cluster->new("${tag}_oldnode",
+ install_path => $ENV{oldinstall});
+ my $new = PostgreSQL::Test::Cluster->new("${tag}_newnode");
+
+ $old->init(extra => ['-k']);
+
+ $old_version = $old->pg_version;
+ note "old installation is version $old_version\n";
+
+ # Run the workload
+ my (undef, $start_mxid, $start_mxoff) = read_multixid_fields($old);
+ mxact_workload($old, $new);
+ my (undef, $finish_mxid, $finish_mxoff) = read_multixid_fields($old);
+
+ note "Testing upgrade, ${tag} scenario\n"
+ . " mxid from ${start_mxid} to ${finish_mxid}\n"
+ . " oldnode mxoff from ${start_mxoff} to ${finish_mxoff}\n";
+
+ $new->init;
+ upgrade_and_compare($tag, $old, $new);
+}
+
+# Wraparound scenario: This is the same as the basic scenario, but the
+# old cluster goes through multixid and offset wraparound.
+#
+# This requires the old installation to be version 18 or older,
+# because the hacks we use to reset the old cluster to a state just
+# before the wraparound rely on the pre-v19 file format. If the old
+# cluster is of v19 or above, multixact SLRU conversion is not needed
+# anyway.
+SKIP:
+{
+ skip
+ "skipping mxoffset conversion tests because upgrading from the old version does not require conversion"
+ if ($old_version >= '19devel');
+
+ my $tag = 'wraparound';
+ my $old =
+ PostgreSQL::Test::Cluster->new("${tag}_oldnode",
+ install_path => $ENV{oldinstall});
+ my $new = PostgreSQL::Test::Cluster->new("${tag}_newnode");
+
+ $old->init(extra => ['-k']);
+
+ # Reset the old cluster to just before multixid and 32-bit offset
+ # wraparound.
+ reset_mxid_mxoffset_pre_v19($old, 0xFFFFFA00, 0xFFFFEC00);
+
+ # Run the workload. This crosses multixid and offset wraparound.
+ my (undef, $start_mxid, $start_mxoff) = read_multixid_fields($old);
+ mxact_workload($old, $new);
+ my (undef, $finish_mxid, $finish_mxoff) = read_multixid_fields($old);
+
+ note "Testing upgrade, ${tag} scenario\n"
+ . " mxid from ${start_mxid} to ${finish_mxid}\n"
+ . " oldnode mxoff from ${start_mxoff} to ${finish_mxoff}\n";
+
+ # Verify that wraparounds happened.
+ cmp_ok($finish_mxid, '<', $start_mxid,
+ "multixid wrapped around in old cluster");
+ cmp_ok($finish_mxoff, '<', $start_mxoff,
+ "mxoff wrapped around in old cluster");
+
+ $new->init;
+ upgrade_and_compare($tag, $old, $new);
+}
+
+done_testing();
generated by cgit v1.2.3 (git 2.46.0) at 2026-01-24 17:51:35 +0000