For inplace update, send nontransactional invalidations.

The inplace update survives ROLLBACK.  The inval didn't, so another
backend's DDL could then update the row without incorporating the
inplace update.  In the test this fixes, a mix of CREATE INDEX and ALTER
TABLE resulted in a table with an index, yet relhasindex=f.  That is a
source of index corruption.  Back-patch to v12 (all supported versions).
The back branch versions don't change WAL, because those branches just
added end-of-recovery SIResetAll().  All branches change the ABI of
extern function PrepareToInvalidateCacheTuple().  No PGXN extension
calls that, and there's no apparent use case in extensions.

Reviewed by Nitin Motiani and (in earlier versions) Andres Freund.

Discussion: https://postgr.es/m/20240523000548.58.nmisch@google.com

1 files changed, 188 insertions, 71 deletions

diff --git a/src/backend/utils/cache/inval.c b/src/backend/utils/cache/inval.c
index e2b9976e931..f9570f4b1ce 100644
--- a/src/backend/utils/cache/inval.c
+++ b/src/backend/utils/cache/inval.c
@@ -99,6 +99,10 @@
  *	worth trying to avoid sending such inval traffic in the future, if those
  *	problems can be overcome cheaply.
  *
+ *	When making a nontransactional change to a cacheable object, we must
+ *	likewise send the invalidation immediately, before ending the change's
+ *	critical section.  This includes inplace heap updates, relmap, and smgr.
+ *
  *
  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
@@ -150,7 +154,7 @@ typedef struct InvalidationListHeader
 } InvalidationListHeader;
 
 /*----------------
- * Invalidation info is divided into two lists:
+ * Transactional invalidation info is divided into two lists:
  *	1) events so far in current command, not yet reflected to caches.
  *	2) events in previous commands of current transaction; these have
  *	   been reflected to local caches, and must be either broadcast to
@@ -166,26 +170,36 @@ typedef struct InvalidationListHeader
  *----------------
  */
 
-typedef struct TransInvalidationInfo
+/* fields common to both transactional and inplace invalidation */
+typedef struct InvalidationInfo
 {
-	/* Back link to parent transaction's info */
-	struct TransInvalidationInfo *parent;
-
-	/* Subtransaction nesting depth */
-	int			my_level;
-
 	/* head of current-command event list */
 	InvalidationListHeader CurrentCmdInvalidMsgs;
 
+	/* init file must be invalidated? */
+	bool		RelcacheInitFileInval;
+} InvalidationInfo;
+
+/* subclass adding fields specific to transactional invalidation */
+typedef struct TransInvalidationInfo
+{
+	/* Base class */
+	struct InvalidationInfo ii;
+
 	/* head of previous-commands event list */
 	InvalidationListHeader PriorCmdInvalidMsgs;
 
-	/* init file must be invalidated? */
-	bool		RelcacheInitFileInval;
+	/* Back link to parent transaction's info */
+	struct TransInvalidationInfo *parent;
+
+	/* Subtransaction nesting depth */
+	int			my_level;
 } TransInvalidationInfo;
 
 static TransInvalidationInfo *transInvalInfo = NULL;
 
+static InvalidationInfo *inplaceInvalInfo = NULL;
+
 static SharedInvalidationMessage *SharedInvalidMessagesArray;
 static int	numSharedInvalidMessagesArray;
 static int	maxSharedInvalidMessagesArray;
@@ -499,9 +513,12 @@ ProcessInvalidationMessagesMulti(InvalidationListHeader *hdr,
 static void
 RegisterCatcacheInvalidation(int cacheId,
 							 uint32 hashValue,
-							 Oid dbId)
+							 Oid dbId,
+							 void *context)
 {
-	AddCatcacheInvalidationMessage(&transInvalInfo->CurrentCmdInvalidMsgs,
+	InvalidationInfo *info = (InvalidationInfo *) context;
+
+	AddCatcacheInvalidationMessage(&info->CurrentCmdInvalidMsgs,
 								   cacheId, hashValue, dbId);
 }
 
@@ -511,10 +528,9 @@ RegisterCatcacheInvalidation(int cacheId,
  * Register an invalidation event for all catcache entries from a catalog.
  */
 static void
-RegisterCatalogInvalidation(Oid dbId, Oid catId)
+RegisterCatalogInvalidation(InvalidationInfo *info, Oid dbId, Oid catId)
 {
-	AddCatalogInvalidationMessage(&transInvalInfo->CurrentCmdInvalidMsgs,
-								  dbId, catId);
+	AddCatalogInvalidationMessage(&info->CurrentCmdInvalidMsgs, dbId, catId);
 }
 
 /*
@@ -523,10 +539,9 @@ RegisterCatalogInvalidation(Oid dbId, Oid catId)
  * As above, but register a relcache invalidation event.
  */
 static void
-RegisterRelcacheInvalidation(Oid dbId, Oid relId)
+RegisterRelcacheInvalidation(InvalidationInfo *info, Oid dbId, Oid relId)
 {
-	AddRelcacheInvalidationMessage(&transInvalInfo->CurrentCmdInvalidMsgs,
-								   dbId, relId);
+	AddRelcacheInvalidationMessage(&info->CurrentCmdInvalidMsgs, dbId, relId);
 
 	/*
 	 * Most of the time, relcache invalidation is associated with system
@@ -543,7 +558,7 @@ RegisterRelcacheInvalidation(Oid dbId, Oid relId)
 	 * as well.  Also zap when we are invalidating whole relcache.
 	 */
 	if (relId == InvalidOid || RelationIdIsInInitFile(relId))
-		transInvalInfo->RelcacheInitFileInval = true;
+		info->RelcacheInitFileInval = true;
 }
 
 /*
@@ -553,10 +568,9 @@ RegisterRelcacheInvalidation(Oid dbId, Oid relId)
  * Only needed for catalogs that don't have catcaches.
  */
 static void
-RegisterSnapshotInvalidation(Oid dbId, Oid relId)
+RegisterSnapshotInvalidation(InvalidationInfo *info, Oid dbId, Oid relId)
 {
-	AddSnapshotInvalidationMessage(&transInvalInfo->CurrentCmdInvalidMsgs,
-								   dbId, relId);
+	AddSnapshotInvalidationMessage(&info->CurrentCmdInvalidMsgs, dbId, relId);
 }
 
 /*
@@ -748,14 +762,18 @@ AcceptInvalidationMessages(void)
  * PrepareInvalidationState
  *		Initialize inval lists for the current (sub)transaction.
  */
-static void
+static InvalidationInfo *
 PrepareInvalidationState(void)
 {
 	TransInvalidationInfo *myInfo;
 
+	Assert(IsTransactionState());
+	/* Can't queue transactional message while collecting inplace messages. */
+	Assert(inplaceInvalInfo == NULL);
+
 	if (transInvalInfo != NULL &&
 		transInvalInfo->my_level == GetCurrentTransactionNestLevel())
-		return;
+		return (InvalidationInfo *) transInvalInfo;
 
 	myInfo = (TransInvalidationInfo *)
 		MemoryContextAllocZero(TopTransactionContext,
@@ -771,6 +789,29 @@ PrepareInvalidationState(void)
 		   myInfo->my_level > transInvalInfo->my_level);
 
 	transInvalInfo = myInfo;
+	return (InvalidationInfo *) myInfo;
+}
+
+/*
+ * PrepareInplaceInvalidationState
+ *		Initialize inval data for an inplace update.
+ *
+ * See previous function for more background.
+ */
+static InvalidationInfo *
+PrepareInplaceInvalidationState(void)
+{
+	InvalidationInfo *myInfo;
+
+	Assert(IsTransactionState());
+	/* limit of one inplace update under assembly */
+	Assert(inplaceInvalInfo == NULL);
+
+	/* gone after WAL insertion CritSection ends, so use current context */
+	myInfo = (InvalidationInfo *) palloc0(sizeof(InvalidationInfo));
+
+	inplaceInvalInfo = myInfo;
+	return myInfo;
 }
 
 /*
@@ -866,7 +907,7 @@ xactGetCommittedInvalidationMessages(SharedInvalidationMessage **msgs,
 	 * after we send the SI messages.  However, we need not do anything unless
 	 * we committed.
 	 */
-	*RelcacheInitFileInval = transInvalInfo->RelcacheInitFileInval;
+	*RelcacheInitFileInval = transInvalInfo->ii.RelcacheInitFileInval;
 
 	/*
 	 * Walk through TransInvalidationInfo to collect all the messages into a
@@ -878,7 +919,7 @@ xactGetCommittedInvalidationMessages(SharedInvalidationMessage **msgs,
 	 */
 	oldcontext = MemoryContextSwitchTo(CurTransactionContext);
 
-	ProcessInvalidationMessagesMulti(&transInvalInfo->CurrentCmdInvalidMsgs,
+	ProcessInvalidationMessagesMulti(&transInvalInfo->ii.CurrentCmdInvalidMsgs,
 									 MakeSharedInvalidMessagesArray);
 	ProcessInvalidationMessagesMulti(&transInvalInfo->PriorCmdInvalidMsgs,
 									 MakeSharedInvalidMessagesArray);
@@ -968,7 +1009,9 @@ ProcessCommittedInvalidationMessages(SharedInvalidationMessage *msgs,
 void
 AtEOXact_Inval(bool isCommit)
 {
-	/* Quick exit if no messages */
+	inplaceInvalInfo = NULL;
+
+	/* Quick exit if no transactional messages */
 	if (transInvalInfo == NULL)
 		return;
 
@@ -982,16 +1025,16 @@ AtEOXact_Inval(bool isCommit)
 		 * after we send the SI messages.  However, we need not do anything
 		 * unless we committed.
 		 */
-		if (transInvalInfo->RelcacheInitFileInval)
+		if (transInvalInfo->ii.RelcacheInitFileInval)
 			RelationCacheInitFilePreInvalidate();
 
 		AppendInvalidationMessages(&transInvalInfo->PriorCmdInvalidMsgs,
-								   &transInvalInfo->CurrentCmdInvalidMsgs);
+								   &transInvalInfo->ii.CurrentCmdInvalidMsgs);
 
 		ProcessInvalidationMessagesMulti(&transInvalInfo->PriorCmdInvalidMsgs,
 										 SendSharedInvalidMessages);
 
-		if (transInvalInfo->RelcacheInitFileInval)
+		if (transInvalInfo->ii.RelcacheInitFileInval)
 			RelationCacheInitFilePostInvalidate();
 	}
 	else
@@ -1007,6 +1050,45 @@ AtEOXact_Inval(bool isCommit)
 }
 
 /*
+ * PreInplace_Inval
+ *		Process queued-up invalidation before inplace update critical section.
+ *
+ * Tasks belong here if they are safe even if the inplace update does not
+ * complete.  Currently, this just unlinks a cache file, which can fail.  The
+ * sum of this and AtInplace_Inval() mirrors AtEOXact_Inval(isCommit=true).
+ */
+void
+PreInplace_Inval(void)
+{
+	Assert(CritSectionCount == 0);
+
+	if (inplaceInvalInfo && inplaceInvalInfo->RelcacheInitFileInval)
+		RelationCacheInitFilePreInvalidate();
+}
+
+/*
+ * AtInplace_Inval
+ *		Process queued-up invalidations after inplace update buffer mutation.
+ */
+void
+AtInplace_Inval(void)
+{
+	Assert(CritSectionCount > 0);
+
+	if (inplaceInvalInfo == NULL)
+		return;
+
+	ProcessInvalidationMessagesMulti(&inplaceInvalInfo->CurrentCmdInvalidMsgs,
+									 SendSharedInvalidMessages);
+
+	if (inplaceInvalInfo->RelcacheInitFileInval)
+		RelationCacheInitFilePostInvalidate();
+
+	inplaceInvalInfo = NULL;
+	/* inplace doesn't use SharedInvalidMessagesArray */
+}
+
+/*
  * AtEOSubXact_Inval
  *		Process queued-up invalidation messages at end of subtransaction.
  *
@@ -1028,9 +1110,20 @@ void
 AtEOSubXact_Inval(bool isCommit)
 {
 	int			my_level;
-	TransInvalidationInfo *myInfo = transInvalInfo;
+	TransInvalidationInfo *myInfo;
 
-	/* Quick exit if no messages. */
+	/*
+	 * Successful inplace update must clear this, but we clear it on abort.
+	 * Inplace updates allocate this in CurrentMemoryContext, which has
+	 * lifespan <= subtransaction lifespan.  Hence, don't free it explicitly.
+	 */
+	if (isCommit)
+		Assert(inplaceInvalInfo == NULL);
+	else
+		inplaceInvalInfo = NULL;
+
+	/* Quick exit if no transactional messages. */
+	myInfo = transInvalInfo;
 	if (myInfo == NULL)
 		return;
 
@@ -1064,8 +1157,8 @@ AtEOSubXact_Inval(bool isCommit)
 								   &myInfo->PriorCmdInvalidMsgs);
 
 		/* Pending relcache inval becomes parent's problem too */
-		if (myInfo->RelcacheInitFileInval)
-			myInfo->parent->RelcacheInitFileInval = true;
+		if (myInfo->ii.RelcacheInitFileInval)
+			myInfo->parent->ii.RelcacheInitFileInval = true;
 
 		/* Pop the transaction state stack */
 		transInvalInfo = myInfo->parent;
@@ -1112,29 +1205,24 @@ CommandEndInvalidationMessages(void)
 	if (transInvalInfo == NULL)
 		return;
 
-	ProcessInvalidationMessages(&transInvalInfo->CurrentCmdInvalidMsgs,
+	ProcessInvalidationMessages(&transInvalInfo->ii.CurrentCmdInvalidMsgs,
 								LocalExecuteInvalidationMessage);
 	AppendInvalidationMessages(&transInvalInfo->PriorCmdInvalidMsgs,
-							   &transInvalInfo->CurrentCmdInvalidMsgs);
+							   &transInvalInfo->ii.CurrentCmdInvalidMsgs);
 }
 
 
 /*
- * CacheInvalidateHeapTuple
- *		Register the given tuple for invalidation at end of command
- *		(ie, current command is creating or outdating this tuple).
- *		Also, detect whether a relcache invalidation is implied.
- *
- * For an insert or delete, tuple is the target tuple and newtuple is NULL.
- * For an update, we are called just once, with tuple being the old tuple
- * version and newtuple the new version.  This allows avoidance of duplicate
- * effort during an update.
+ * CacheInvalidateHeapTupleCommon
+ *		Common logic for end-of-command and inplace variants.
  */
-void
-CacheInvalidateHeapTuple(Relation relation,
-						 HeapTuple tuple,
-						 HeapTuple newtuple)
+static void
+CacheInvalidateHeapTupleCommon(Relation relation,
+							   HeapTuple tuple,
+							   HeapTuple newtuple,
+							   InvalidationInfo *(*prepare_callback) (void))
 {
+	InvalidationInfo *info;
 	Oid			tupleRelId;
 	Oid			databaseId;
 	Oid			relationId;
@@ -1158,11 +1246,8 @@ CacheInvalidateHeapTuple(Relation relation,
 	if (IsToastRelation(relation))
 		return;
 
-	/*
-	 * If we're not prepared to queue invalidation messages for this
-	 * subtransaction level, get ready now.
-	 */
-	PrepareInvalidationState();
+	/* Allocate any required resources. */
+	info = prepare_callback();
 
 	/*
 	 * First let the catcache do its thing
@@ -1171,11 +1256,12 @@ CacheInvalidateHeapTuple(Relation relation,
 	if (RelationInvalidatesSnapshotsOnly(tupleRelId))
 	{
 		databaseId = IsSharedRelation(tupleRelId) ? InvalidOid : MyDatabaseId;
-		RegisterSnapshotInvalidation(databaseId, tupleRelId);
+		RegisterSnapshotInvalidation(info, databaseId, tupleRelId);
 	}
 	else
 		PrepareToInvalidateCacheTuple(relation, tuple, newtuple,
-									  RegisterCatcacheInvalidation);
+									  RegisterCatcacheInvalidation,
+									  (void *) info);
 
 	/*
 	 * Now, is this tuple one of the primary definers of a relcache entry? See
@@ -1248,7 +1334,44 @@ CacheInvalidateHeapTuple(Relation relation,
 	/*
 	 * Yes.  We need to register a relcache invalidation event.
 	 */
-	RegisterRelcacheInvalidation(databaseId, relationId);
+	RegisterRelcacheInvalidation(info, databaseId, relationId);
+}
+
+/*
+ * CacheInvalidateHeapTuple
+ *		Register the given tuple for invalidation at end of command
+ *		(ie, current command is creating or outdating this tuple) and end of
+ *		transaction.  Also, detect whether a relcache invalidation is implied.
+ *
+ * For an insert or delete, tuple is the target tuple and newtuple is NULL.
+ * For an update, we are called just once, with tuple being the old tuple
+ * version and newtuple the new version.  This allows avoidance of duplicate
+ * effort during an update.
+ */
+void
+CacheInvalidateHeapTuple(Relation relation,
+						 HeapTuple tuple,
+						 HeapTuple newtuple)
+{
+	CacheInvalidateHeapTupleCommon(relation, tuple, newtuple,
+								   PrepareInvalidationState);
+}
+
+/*
+ * CacheInvalidateHeapTupleInplace
+ *		Register the given tuple for nontransactional invalidation pertaining
+ *		to an inplace update.  Also, detect whether a relcache invalidation is
+ *		implied.
+ *
+ * Like CacheInvalidateHeapTuple(), but for inplace updates.
+ */
+void
+CacheInvalidateHeapTupleInplace(Relation relation,
+								HeapTuple tuple,
+								HeapTuple newtuple)
+{
+	CacheInvalidateHeapTupleCommon(relation, tuple, newtuple,
+								   PrepareInplaceInvalidationState);
 }
 
 /*
@@ -1267,14 +1390,13 @@ CacheInvalidateCatalog(Oid catalogId)
 {
 	Oid			databaseId;
 
-	PrepareInvalidationState();
-
 	if (IsSharedRelation(catalogId))
 		databaseId = InvalidOid;
 	else
 		databaseId = MyDatabaseId;
 
-	RegisterCatalogInvalidation(databaseId, catalogId);
+	RegisterCatalogInvalidation(PrepareInvalidationState(),
+								databaseId, catalogId);
 }
 
 /*
@@ -1292,15 +1414,14 @@ CacheInvalidateRelcache(Relation relation)
 	Oid			databaseId;
 	Oid			relationId;
 
-	PrepareInvalidationState();
-
 	relationId = RelationGetRelid(relation);
 	if (relation->rd_rel->relisshared)
 		databaseId = InvalidOid;
 	else
 		databaseId = MyDatabaseId;
 
-	RegisterRelcacheInvalidation(databaseId, relationId);
+	RegisterRelcacheInvalidation(PrepareInvalidationState(),
+								 databaseId, relationId);
 }
 
 /*
@@ -1313,9 +1434,8 @@ CacheInvalidateRelcache(Relation relation)
 void
 CacheInvalidateRelcacheAll(void)
 {
-	PrepareInvalidationState();
-
-	RegisterRelcacheInvalidation(InvalidOid, InvalidOid);
+	RegisterRelcacheInvalidation(PrepareInvalidationState(),
+								 InvalidOid, InvalidOid);
 }
 
 /*
@@ -1329,14 +1449,13 @@ CacheInvalidateRelcacheByTuple(HeapTuple classTuple)
 	Oid			databaseId;
 	Oid			relationId;
 
-	PrepareInvalidationState();
-
 	relationId = classtup->oid;
 	if (classtup->relisshared)
 		databaseId = InvalidOid;
 	else
 		databaseId = MyDatabaseId;
-	RegisterRelcacheInvalidation(databaseId, relationId);
+	RegisterRelcacheInvalidation(PrepareInvalidationState(),
+								 databaseId, relationId);
 }
 
 /*
@@ -1350,8 +1469,6 @@ CacheInvalidateRelcacheByRelid(Oid relid)
 {
 	HeapTuple	tup;
 
-	PrepareInvalidationState();
-
 	tup = SearchSysCache1(RELOID, ObjectIdGetDatum(relid));
 	if (!HeapTupleIsValid(tup))
 		elog(ERROR, "cache lookup failed for relation %u", relid);