Add infrastructure to support EphemeralNamedRelation references.

A QueryEnvironment concept is added, which allows new types of
objects to be passed into queries from parsing on through
execution.  At this point, the only thing implemented is a
collection of EphemeralNamedRelation objects -- relations which
can be referenced by name in queries, but do not exist in the
catalogs.  The only type of ENR implemented is NamedTuplestore, but
provision is made to add more types fairly easily.

An ENR can carry its own TupleDesc or reference a relation in the
catalogs by relid.

Although these features can be used without SPI, convenience
functions are added to SPI so that ENRs can easily be used by code
run through SPI.

The initial use of all this is going to be transition tables in
AFTER triggers, but that will be added to each PL as a separate
commit.

An incidental effect of this patch is to produce a more informative
error message if an attempt is made to modify the contents of a CTE
from a referencing DML statement.  No tests previously covered that
possibility, so one is added.

Kevin Grittner and Thomas Munro
Reviewed by Heikki Linnakangas, David Fetter, and Thomas Munro
with valuable comments and suggestions from many others

9 files changed, 341 insertions, 13 deletions

diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile
index d1c1324399a..083b20f3fee 100644
--- a/src/backend/executor/Makefile
+++ b/src/backend/executor/Makefile
@@ -25,7 +25,8 @@ OBJS = execAmi.o execCurrent.o execExpr.o execExprInterp.o \
        nodeMaterial.o nodeMergeAppend.o nodeMergejoin.o nodeModifyTable.o \
        nodeNestloop.o nodeProjectSet.o nodeRecursiveunion.o nodeResult.o \
        nodeSamplescan.o nodeSeqscan.o nodeSetOp.o nodeSort.o nodeUnique.o \
-       nodeValuesscan.o nodeCtescan.o nodeWorktablescan.o \
+       nodeValuesscan.o \
+       nodeCtescan.o nodeNamedtuplestorescan.o nodeWorktablescan.o \
        nodeGroup.o nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o \
        nodeForeignscan.o nodeWindowAgg.o tstoreReceiver.o tqueue.o spi.o \
        nodeTableFuncscan.o
diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c
index 5d59f95a916..7e85c66da38 100644
--- a/src/backend/executor/execAmi.c
+++ b/src/backend/executor/execAmi.c
@@ -38,6 +38,7 @@
 #include "executor/nodeMergeAppend.h"
 #include "executor/nodeMergejoin.h"
 #include "executor/nodeModifyTable.h"
+#include "executor/nodeNamedtuplestorescan.h"
 #include "executor/nodeNestloop.h"
 #include "executor/nodeProjectSet.h"
 #include "executor/nodeRecursiveunion.h"
@@ -211,6 +212,10 @@ ExecReScan(PlanState *node)
 			ExecReScanCteScan((CteScanState *) node);
 			break;
 
+		case T_NamedTuplestoreScanState:
+			ExecReScanNamedTuplestoreScan((NamedTuplestoreScanState *) node);
+			break;
+
 		case T_WorkTableScanState:
 			ExecReScanWorkTableScan((WorkTableScanState *) node);
 			break;
@@ -571,6 +576,7 @@ ExecMaterializesOutput(NodeTag plantype)
 		case T_FunctionScan:
 		case T_TableFuncScan:
 		case T_CteScan:
+		case T_NamedTuplestoreScan:
 		case T_WorkTableScan:
 		case T_Sort:
 			return true;
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index f2995f2e7ba..920b12072fb 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -199,6 +199,11 @@ standard_ExecutorStart(QueryDesc *queryDesc, int eflags)
 	estate->es_sourceText = queryDesc->sourceText;
 
 	/*
+	 * Fill in the query environment, if any, from queryDesc.
+	 */
+	estate->es_queryEnv = queryDesc->queryEnv;
+
+	/*
 	 * If non-read-only query, set the command ID to mark output tuples with
 	 */
 	switch (queryDesc->operation)
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index b91b663c46f..469a32c7b0d 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -710,7 +710,7 @@ ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver,
 	return CreateQueryDesc(pstmt,
 						   queryString,
 						   GetActiveSnapshot(), InvalidSnapshot,
-						   receiver, paramLI, instrument_options);
+						   receiver, paramLI, NULL, instrument_options);
 }
 
 /*
diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c
index 80c77addb8e..486ddf17628 100644
--- a/src/backend/executor/execProcnode.c
+++ b/src/backend/executor/execProcnode.c
@@ -101,6 +101,7 @@
 #include "executor/nodeMergeAppend.h"
 #include "executor/nodeMergejoin.h"
 #include "executor/nodeModifyTable.h"
+#include "executor/nodeNamedtuplestorescan.h"
 #include "executor/nodeNestloop.h"
 #include "executor/nodeProjectSet.h"
 #include "executor/nodeRecursiveunion.h"
@@ -256,6 +257,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
 												   estate, eflags);
 			break;
 
+		case T_NamedTuplestoreScan:
+			result = (PlanState *) ExecInitNamedTuplestoreScan((NamedTuplestoreScan *) node,
+												   estate, eflags);
+			break;
+
 		case T_WorkTableScan:
 			result = (PlanState *) ExecInitWorkTableScan((WorkTableScan *) node,
 														 estate, eflags);
@@ -483,6 +489,10 @@ ExecProcNode(PlanState *node)
 			result = ExecCteScan((CteScanState *) node);
 			break;
 
+		case T_NamedTuplestoreScanState:
+			result = ExecNamedTuplestoreScan((NamedTuplestoreScanState *) node);
+			break;
+
 		case T_WorkTableScanState:
 			result = ExecWorkTableScan((WorkTableScanState *) node);
 			break;
@@ -751,6 +761,10 @@ ExecEndNode(PlanState *node)
 			ExecEndCteScan((CteScanState *) node);
 			break;
 
+		case T_NamedTuplestoreScanState:
+			ExecEndNamedTuplestoreScan((NamedTuplestoreScanState *) node);
+			break;
+
 		case T_WorkTableScanState:
 			ExecEndWorkTableScan((WorkTableScanState *) node);
 			break;
diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c
index 2613ffbb718..ce7b0642172 100644
--- a/src/backend/executor/execUtils.c
+++ b/src/backend/executor/execUtils.c
@@ -120,6 +120,8 @@ CreateExecutorState(void)
 	estate->es_param_list_info = NULL;
 	estate->es_param_exec_vals = NULL;
 
+	estate->es_queryEnv = NULL;
+
 	estate->es_query_cxt = qcontext;
 
 	estate->es_tupleTable = NIL;
diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c
index 3e4b0191c7e..3cadf953040 100644
--- a/src/backend/executor/functions.c
+++ b/src/backend/executor/functions.c
@@ -713,7 +713,8 @@ init_sql_fcache(FmgrInfo *finfo, Oid collation, bool lazyEvalOK)
 		queryTree_sublist = pg_analyze_and_rewrite_params(parsetree,
 														  fcache->src,
 									   (ParserSetupHook) sql_fn_parser_setup,
-														  fcache->pinfo);
+														  fcache->pinfo,
+														  NULL);
 		queryTree_list = lappend(queryTree_list, queryTree_sublist);
 		flat_query_list = list_concat(flat_query_list,
 									  list_copy(queryTree_sublist));
@@ -809,7 +810,9 @@ postquel_start(execution_state *es, SQLFunctionCachePtr fcache)
 							 GetActiveSnapshot(),
 							 InvalidSnapshot,
 							 dest,
-							 fcache->paramLI, 0);
+							 fcache->paramLI,
+							 es->qd ? es->qd->queryEnv : NULL,
+							 0);
 
 	/* Utility commands don't need Executor. */
 	if (es->qd->operation != CMD_UTILITY)
@@ -846,6 +849,7 @@ postquel_getnext(execution_state *es, SQLFunctionCachePtr fcache)
 					   fcache->src,
 					   PROCESS_UTILITY_QUERY,
 					   es->qd->params,
+					   es->qd->queryEnv,
 					   es->qd->dest,
 					   NULL);
 		result = true;			/* never stops early */
diff --git a/src/backend/executor/nodeNamedtuplestorescan.c b/src/backend/executor/nodeNamedtuplestorescan.c
new file mode 100644
index 00000000000..917b05197a4
--- /dev/null
+++ b/src/backend/executor/nodeNamedtuplestorescan.c
@@ -0,0 +1,198 @@
+/*-------------------------------------------------------------------------
+ *
+ * nodeNamedtuplestorescan.c
+ *	  routines to handle NamedTuplestoreScan nodes.
+ *
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/executor/nodeNamedtuplestorescan.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "executor/execdebug.h"
+#include "executor/nodeNamedtuplestorescan.h"
+#include "miscadmin.h"
+#include "utils/queryenvironment.h"
+
+static TupleTableSlot *NamedTuplestoreScanNext(NamedTuplestoreScanState *node);
+
+/* ----------------------------------------------------------------
+ *		NamedTuplestoreScanNext
+ *
+ *		This is a workhorse for ExecNamedTuplestoreScan
+ * ----------------------------------------------------------------
+ */
+static TupleTableSlot *
+NamedTuplestoreScanNext(NamedTuplestoreScanState *node)
+{
+	TupleTableSlot *slot;
+
+	/* We intentionally do not support backward scan. */
+	Assert(ScanDirectionIsForward(node->ss.ps.state->es_direction));
+
+	/*
+	 * Get the next tuple from tuplestore. Return NULL if no more tuples.
+	 */
+	slot = node->ss.ss_ScanTupleSlot;
+	(void) tuplestore_gettupleslot(node->relation, true, false, slot);
+	return slot;
+}
+
+/*
+ * NamedTuplestoreScanRecheck -- access method routine to recheck a tuple in
+ * EvalPlanQual
+ */
+static bool
+NamedTuplestoreScanRecheck(NamedTuplestoreScanState *node, TupleTableSlot *slot)
+{
+	/* nothing to check */
+	return true;
+}
+
+/* ----------------------------------------------------------------
+ *		ExecNamedTuplestoreScan(node)
+ *
+ *		Scans the CTE sequentially and returns the next qualifying tuple.
+ *		We call the ExecScan() routine and pass it the appropriate
+ *		access method functions.
+ * ----------------------------------------------------------------
+ */
+TupleTableSlot *
+ExecNamedTuplestoreScan(NamedTuplestoreScanState *node)
+{
+	return ExecScan(&node->ss,
+					(ExecScanAccessMtd) NamedTuplestoreScanNext,
+					(ExecScanRecheckMtd) NamedTuplestoreScanRecheck);
+}
+
+
+/* ----------------------------------------------------------------
+ *		ExecInitNamedTuplestoreScan
+ * ----------------------------------------------------------------
+ */
+NamedTuplestoreScanState *
+ExecInitNamedTuplestoreScan(NamedTuplestoreScan *node, EState *estate, int eflags)
+{
+	NamedTuplestoreScanState *scanstate;
+	EphemeralNamedRelation enr;
+
+	/* check for unsupported flags */
+	Assert(!(eflags & (EXEC_FLAG_BACKWARD | EXEC_FLAG_MARK)));
+
+	/*
+	 * NamedTuplestoreScan should not have any children.
+	 */
+	Assert(outerPlan(node) == NULL);
+	Assert(innerPlan(node) == NULL);
+
+	/*
+	 * create new NamedTuplestoreScanState for node
+	 */
+	scanstate = makeNode(NamedTuplestoreScanState);
+	scanstate->ss.ps.plan = (Plan *) node;
+	scanstate->ss.ps.state = estate;
+
+	enr = get_ENR(estate->es_queryEnv, node->enrname);
+	if (!enr)
+		elog(ERROR, "executor could not find named tuplestore \"%s\"",
+			 node->enrname);
+
+	Assert(enr->reldata);
+	scanstate->relation = (Tuplestorestate *) enr->reldata;
+	scanstate->tupdesc = ENRMetadataGetTupDesc(&(enr->md));
+	scanstate->readptr =
+		tuplestore_alloc_read_pointer(scanstate->relation, 0);
+
+	/*
+	 * The new read pointer copies its position from read pointer 0, which
+	 * could be anywhere, so explicitly rewind it.
+	 */
+	tuplestore_rescan(scanstate->relation);
+
+	/*
+	 * XXX: Should we add a function to free that read pointer when done?
+	 * This was attempted, but it did not improve performance or memory usage
+	 * in any tested cases.
+	 */
+
+	/*
+	 * Miscellaneous initialization
+	 *
+	 * create expression context for node
+	 */
+	ExecAssignExprContext(estate, &scanstate->ss.ps);
+
+	/*
+	 * initialize child expressions
+	 */
+	scanstate->ss.ps.qual =
+		ExecInitQual(node->scan.plan.qual, (PlanState *) scanstate);
+
+	/*
+	 * tuple table initialization
+	 */
+	ExecInitResultTupleSlot(estate, &scanstate->ss.ps);
+	ExecInitScanTupleSlot(estate, &scanstate->ss);
+
+	/*
+	 * The scan tuple type is specified for the tuplestore.
+	 */
+	ExecAssignScanType(&scanstate->ss, scanstate->tupdesc);
+
+	/*
+	 * Initialize result tuple type and projection info.
+	 */
+	ExecAssignResultTypeFromTL(&scanstate->ss.ps);
+	ExecAssignScanProjectionInfo(&scanstate->ss);
+
+	return scanstate;
+}
+
+/* ----------------------------------------------------------------
+ *		ExecEndNamedTuplestoreScan
+ *
+ *		frees any storage allocated through C routines.
+ * ----------------------------------------------------------------
+ */
+void
+ExecEndNamedTuplestoreScan(NamedTuplestoreScanState *node)
+{
+	/*
+	 * Free exprcontext
+	 */
+	ExecFreeExprContext(&node->ss.ps);
+
+	/*
+	 * clean out the tuple table
+	 */
+	ExecClearTuple(node->ss.ps.ps_ResultTupleSlot);
+	ExecClearTuple(node->ss.ss_ScanTupleSlot);
+}
+
+/* ----------------------------------------------------------------
+ *		ExecReScanNamedTuplestoreScan
+ *
+ *		Rescans the relation.
+ * ----------------------------------------------------------------
+ */
+void
+ExecReScanNamedTuplestoreScan(NamedTuplestoreScanState *node)
+{
+	Tuplestorestate *tuplestorestate = node->relation;
+
+	ExecClearTuple(node->ss.ps.ps_ResultTupleSlot);
+
+	ExecScanReScan(&node->ss);
+
+	/*
+	 * Rewind my own pointer.
+	 */
+	tuplestore_select_read_pointer(tuplestorestate, node->readptr);
+	tuplestore_rescan(tuplestorestate);
+}
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index eeaa4805e4d..54c022d0132 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -122,6 +122,7 @@ SPI_connect(void)
 	_SPI_current->procCxt = NULL;		/* in case we fail to create 'em */
 	_SPI_current->execCxt = NULL;
 	_SPI_current->connectSubid = GetCurrentSubTransactionId();
+	_SPI_current->queryEnv = NULL;
 
 	/*
 	 * Create memory contexts for this procedure
@@ -1193,7 +1194,7 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan,
 	 */
 
 	/* Replan if needed, and increment plan refcount for portal */
-	cplan = GetCachedPlan(plansource, paramLI, false);
+	cplan = GetCachedPlan(plansource, paramLI, false, _SPI_current->queryEnv);
 	stmt_list = cplan->stmt_list;
 
 	/* Pop the error context stack */
@@ -1532,6 +1533,10 @@ SPI_result_code_string(int code)
 			return "SPI_ERROR_NOOUTFUNC";
 		case SPI_ERROR_TYPUNKNOWN:
 			return "SPI_ERROR_TYPUNKNOWN";
+		case SPI_ERROR_REL_DUPLICATE:
+			return "SPI_ERROR_REL_DUPLICATE";
+		case SPI_ERROR_REL_NOT_FOUND:
+			return "SPI_ERROR_REL_NOT_FOUND";
 		case SPI_OK_CONNECT:
 			return "SPI_OK_CONNECT";
 		case SPI_OK_FINISH:
@@ -1560,6 +1565,10 @@ SPI_result_code_string(int code)
 			return "SPI_OK_UPDATE_RETURNING";
 		case SPI_OK_REWRITTEN:
 			return "SPI_OK_REWRITTEN";
+		case SPI_OK_REL_REGISTER:
+			return "SPI_OK_REL_REGISTER";
+		case SPI_OK_REL_UNREGISTER:
+			return "SPI_OK_REL_UNREGISTER";
 	}
 	/* Unrecognized code ... return something useful ... */
 	sprintf(buf, "Unrecognized SPI code %d", code);
@@ -1615,7 +1624,8 @@ SPI_plan_get_cached_plan(SPIPlanPtr plan)
 	error_context_stack = &spierrcontext;
 
 	/* Get the generic plan for the query */
-	cplan = GetCachedPlan(plansource, NULL, plan->saved);
+	cplan = GetCachedPlan(plansource, NULL, plan->saved,
+						  _SPI_current->queryEnv);
 	Assert(cplan == plansource->gplan);
 
 	/* Pop the error context stack */
@@ -1767,7 +1777,8 @@ _SPI_prepare_plan(const char *src, SPIPlanPtr plan)
 		 */
 		plansource = CreateCachedPlan(parsetree,
 									  src,
-									  CreateCommandTag(parsetree->stmt));
+									  CreateCommandTag(parsetree->stmt),
+									  _SPI_current->queryEnv);
 
 		/*
 		 * Parameter datatypes are driven by parserSetup hook if provided,
@@ -1779,14 +1790,16 @@ _SPI_prepare_plan(const char *src, SPIPlanPtr plan)
 			stmt_list = pg_analyze_and_rewrite_params(parsetree,
 													  src,
 													  plan->parserSetup,
-													  plan->parserSetupArg);
+													  plan->parserSetupArg,
+													  _SPI_current->queryEnv);
 		}
 		else
 		{
 			stmt_list = pg_analyze_and_rewrite(parsetree,
 											   src,
 											   plan->argtypes,
-											   plan->nargs);
+											   plan->nargs,
+											   _SPI_current->queryEnv);
 		}
 
 		/* Finish filling in the CachedPlanSource */
@@ -1975,14 +1988,16 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
 				stmt_list = pg_analyze_and_rewrite_params(parsetree,
 														  src,
 														  plan->parserSetup,
-													   plan->parserSetupArg);
+													   plan->parserSetupArg,
+													  _SPI_current->queryEnv);
 			}
 			else
 			{
 				stmt_list = pg_analyze_and_rewrite(parsetree,
 												   src,
 												   plan->argtypes,
-												   plan->nargs);
+												   plan->nargs,
+												   _SPI_current->queryEnv);
 			}
 
 			/* Finish filling in the CachedPlanSource */
@@ -2001,7 +2016,7 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
 		 * Replan if needed, and increment plan refcount.  If it's a saved
 		 * plan, the refcount must be backed by the CurrentResourceOwner.
 		 */
-		cplan = GetCachedPlan(plansource, paramLI, plan->saved);
+		cplan = GetCachedPlan(plansource, paramLI, plan->saved, _SPI_current->queryEnv);
 		stmt_list = cplan->stmt_list;
 
 		/*
@@ -2081,7 +2096,8 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
 										plansource->query_string,
 										snap, crosscheck_snapshot,
 										dest,
-										paramLI, 0);
+										paramLI, _SPI_current->queryEnv,
+										0);
 				res = _SPI_pquery(qdesc, fire_triggers,
 								  canSetTag ? tcount : 0);
 				FreeQueryDesc(qdesc);
@@ -2094,6 +2110,7 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
 							   plansource->query_string,
 							   PROCESS_UTILITY_QUERY,
 							   paramLI,
+							   _SPI_current->queryEnv,
 							   dest,
 							   completionTag);
 
@@ -2619,3 +2636,84 @@ _SPI_save_plan(SPIPlanPtr plan)
 
 	return newplan;
 }
+
+/*
+ * Internal lookup of ephemeral named relation by name.
+ */
+static EphemeralNamedRelation
+_SPI_find_ENR_by_name(const char *name)
+{
+	/* internal static function; any error is bug in SPI itself */
+	Assert(name != NULL);
+
+	/* fast exit if no tuplestores have been added */
+	if (_SPI_current->queryEnv == NULL)
+		return NULL;
+
+	return get_ENR(_SPI_current->queryEnv, name);
+}
+
+/*
+ * Register an ephemeral named relation for use by the planner and executor on
+ * subsequent calls using this SPI connection.
+ */
+int
+SPI_register_relation(EphemeralNamedRelation enr)
+{
+	EphemeralNamedRelation match;
+	int			res;
+
+	if (enr == NULL || enr->md.name == NULL)
+		return SPI_ERROR_ARGUMENT;
+
+	res = _SPI_begin_call(false);	/* keep current memory context */
+	if (res < 0)
+		return res;
+
+	match = _SPI_find_ENR_by_name(enr->md.name);
+	if (match)
+		res = SPI_ERROR_REL_DUPLICATE;
+	else
+	{
+		if (_SPI_current->queryEnv == NULL)
+			_SPI_current->queryEnv = create_queryEnv();
+
+		register_ENR(_SPI_current->queryEnv, enr);
+		res = SPI_OK_REL_REGISTER;
+	}
+
+	_SPI_end_call(false);
+
+	return res;
+}
+
+/*
+ * Unregister an ephemeral named relation by name.  This will probably be a
+ * rarely used function, since SPI_finish will clear it automatically.
+ */
+int
+SPI_unregister_relation(const char *name)
+{
+	EphemeralNamedRelation match;
+	int			res;
+
+	if (name == NULL)
+		return SPI_ERROR_ARGUMENT;
+
+	res = _SPI_begin_call(false);	/* keep current memory context */
+	if (res < 0)
+		return res;
+
+	match = _SPI_find_ENR_by_name(name);
+	if (match)
+	{
+		unregister_ENR(_SPI_current->queryEnv, match->md.name);
+		res = SPI_OK_REL_UNREGISTER;
+	}
+	else
+		res = SPI_ERROR_REL_NOT_FOUND;
+
+	_SPI_end_call(false);
+
+	return res;
+}