More fixes for planner's handling of LATERAL.

Re-allow subquery pullup for LATERAL subqueries, except when the subquery
is below an outer join and contains lateral references to relations outside
that outer join.  If we pull up in such a case, we risk introducing lateral
cross-references into outer joins' ON quals, which is something the code is
entirely unprepared to cope with right now; and I'm not sure it'll ever be
worth coping with.

Support lateral refs in VALUES (this seems to be the only additional path
type that needs such support as a consequence of re-allowing subquery
pullup).

Put in a slightly hacky fix for joinpath.c's refusal to consider
parameterized join paths even when there cannot be any unparameterized
ones.  This was causing "could not devise a query plan for the given query"
failures in queries involving more than two FROM items.

Put in an even more hacky fix for distribute_qual_to_rels() being unhappy
with join quals that contain references to rels outside their syntactic
scope; which is to say, disable that test altogether.  Need to think about
how to preserve some sort of debugging cross-check here, while not
expending more cycles than befits a debugging cross-check.

1 files changed, 160 insertions, 57 deletions

diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c
index 07b35f98bd1..f5deed4bc82 100644
--- a/src/backend/optimizer/prep/prepjointree.c
+++ b/src/backend/optimizer/prep/prepjointree.c
@@ -60,9 +60,14 @@ static Node *pull_up_sublinks_jointree_recurse(PlannerInfo *root, Node *jtnode,
 static Node *pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node,
 							  Node **jtlink1, Relids available_rels1,
 							  Node **jtlink2, Relids available_rels2);
+static Node *pull_up_subqueries_recurse(PlannerInfo *root, Node *jtnode,
+						   JoinExpr *lowest_outer_join,
+						   JoinExpr *lowest_nulling_outer_join,
+						   AppendRelInfo *containing_appendrel);
 static Node *pull_up_simple_subquery(PlannerInfo *root, Node *jtnode,
 						RangeTblEntry *rte,
 						JoinExpr *lowest_outer_join,
+						JoinExpr *lowest_nulling_outer_join,
 						AppendRelInfo *containing_appendrel);
 static Node *pull_up_simple_union_all(PlannerInfo *root, Node *jtnode,
 						 RangeTblEntry *rte);
@@ -71,14 +76,15 @@ static void pull_up_union_leaf_queries(Node *setOp, PlannerInfo *root,
 						   int childRToffset);
 static void make_setop_translation_list(Query *query, Index newvarno,
 							List **translated_vars);
-static bool is_simple_subquery(Query *subquery);
+static bool is_simple_subquery(Query *subquery, RangeTblEntry *rte,
+				   JoinExpr *lowest_outer_join);
 static bool is_simple_union_all(Query *subquery);
 static bool is_simple_union_all_recurse(Node *setOp, Query *setOpQuery,
 							List *colTypes);
 static bool is_safe_append_member(Query *subquery);
 static void replace_vars_in_jointree(Node *jtnode,
 						 pullup_replace_vars_context *context,
-						 JoinExpr *lowest_outer_join);
+						 JoinExpr *lowest_nulling_outer_join);
 static Node *pullup_replace_vars(Node *expr,
 					pullup_replace_vars_context *context);
 static Node *pullup_replace_vars_callback(Var *var,
@@ -585,10 +591,28 @@ inline_set_returning_functions(PlannerInfo *root)
  *		Also, subqueries that are simple UNION ALL structures can be
  *		converted into "append relations".
  *
+ * This recursively processes the jointree and returns a modified jointree.
+ */
+Node *
+pull_up_subqueries(PlannerInfo *root, Node *jtnode)
+{
+	/* Start off with no containing join nor appendrel */
+	return pull_up_subqueries_recurse(root, jtnode, NULL, NULL, NULL);
+}
+
+/*
+ * pull_up_subqueries_recurse
+ *		Recursive guts of pull_up_subqueries.
+ *
+ * If this jointree node is within either side of an outer join, then
+ * lowest_outer_join references the lowest such JoinExpr node; otherwise
+ * it is NULL.  We use this to constrain the effects of LATERAL subqueries.
+ *
  * If this jointree node is within the nullable side of an outer join, then
- * lowest_outer_join references the lowest such JoinExpr node; otherwise it
- * is NULL.  This forces use of the PlaceHolderVar mechanism for references
- * to non-nullable targetlist items, but only for references above that join.
+ * lowest_nulling_outer_join references the lowest such JoinExpr node;
+ * otherwise it is NULL.  This forces use of the PlaceHolderVar mechanism for
+ * references to non-nullable targetlist items, but only for references above
+ * that join.
  *
  * If we are looking at a member subquery of an append relation,
  * containing_appendrel describes that relation; else it is NULL.
@@ -603,15 +627,17 @@ inline_set_returning_functions(PlannerInfo *root)
  * subquery RangeTblRef entries will be replaced.  Also, we can't turn
  * pullup_replace_vars loose on the whole jointree, because it'll return a
  * mutated copy of the tree; we have to invoke it just on the quals, instead.
- * This behavior is what makes it reasonable to pass lowest_outer_join as a
- * pointer rather than some more-indirect way of identifying the lowest OJ.
- * Likewise, we don't replace append_rel_list members but only their
- * substructure, so the containing_appendrel reference is safe to use.
+ * This behavior is what makes it reasonable to pass lowest_outer_join and
+ * lowest_nulling_outer_join as pointers rather than some more-indirect way
+ * of identifying the lowest OJs.  Likewise, we don't replace append_rel_list
+ * members but only their substructure, so the containing_appendrel reference
+ * is safe to use.
  */
-Node *
-pull_up_subqueries(PlannerInfo *root, Node *jtnode,
-				   JoinExpr *lowest_outer_join,
-				   AppendRelInfo *containing_appendrel)
+static Node *
+pull_up_subqueries_recurse(PlannerInfo *root, Node *jtnode,
+						   JoinExpr *lowest_outer_join,
+						   JoinExpr *lowest_nulling_outer_join,
+						   AppendRelInfo *containing_appendrel)
 {
 	if (jtnode == NULL)
 		return NULL;
@@ -628,12 +654,12 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode,
 		 * unless is_safe_append_member says so.
 		 */
 		if (rte->rtekind == RTE_SUBQUERY &&
-			is_simple_subquery(rte->subquery) &&
-			!rte->security_barrier &&
+			is_simple_subquery(rte->subquery, rte, lowest_outer_join) &&
 			(containing_appendrel == NULL ||
 			 is_safe_append_member(rte->subquery)))
 			return pull_up_simple_subquery(root, jtnode, rte,
 										   lowest_outer_join,
+										   lowest_nulling_outer_join,
 										   containing_appendrel);
 
 		/*
@@ -658,8 +684,10 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode,
 
 		Assert(containing_appendrel == NULL);
 		foreach(l, f->fromlist)
-			lfirst(l) = pull_up_subqueries(root, lfirst(l),
-										   lowest_outer_join, NULL);
+			lfirst(l) = pull_up_subqueries_recurse(root, lfirst(l),
+												   lowest_outer_join,
+												   lowest_nulling_outer_join,
+												   NULL);
 	}
 	else if (IsA(jtnode, JoinExpr))
 	{
@@ -670,30 +698,46 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode,
 		switch (j->jointype)
 		{
 			case JOIN_INNER:
-				j->larg = pull_up_subqueries(root, j->larg,
-											 lowest_outer_join, NULL);
-				j->rarg = pull_up_subqueries(root, j->rarg,
-											 lowest_outer_join, NULL);
+				j->larg = pull_up_subqueries_recurse(root, j->larg,
+													 lowest_outer_join,
+													 lowest_nulling_outer_join,
+													 NULL);
+				j->rarg = pull_up_subqueries_recurse(root, j->rarg,
+													 lowest_outer_join,
+													 lowest_nulling_outer_join,
+													 NULL);
 				break;
 			case JOIN_LEFT:
 			case JOIN_SEMI:
 			case JOIN_ANTI:
-				j->larg = pull_up_subqueries(root, j->larg,
-											 lowest_outer_join, NULL);
-				j->rarg = pull_up_subqueries(root, j->rarg,
-											 j, NULL);
+				j->larg = pull_up_subqueries_recurse(root, j->larg,
+													 j,
+													 lowest_nulling_outer_join,
+													 NULL);
+				j->rarg = pull_up_subqueries_recurse(root, j->rarg,
+													 j,
+													 j,
+													 NULL);
 				break;
 			case JOIN_FULL:
-				j->larg = pull_up_subqueries(root, j->larg,
-											 j, NULL);
-				j->rarg = pull_up_subqueries(root, j->rarg,
-											 j, NULL);
+				j->larg = pull_up_subqueries_recurse(root, j->larg,
+													 j,
+													 j,
+													 NULL);
+				j->rarg = pull_up_subqueries_recurse(root, j->rarg,
+													 j,
+													 j,
+													 NULL);
 				break;
 			case JOIN_RIGHT:
-				j->larg = pull_up_subqueries(root, j->larg,
-											 j, NULL);
-				j->rarg = pull_up_subqueries(root, j->rarg,
-											 lowest_outer_join, NULL);
+				j->larg = pull_up_subqueries_recurse(root, j->larg,
+													 j,
+													 j,
+													 NULL);
+				j->rarg = pull_up_subqueries_recurse(root, j->rarg,
+													 j,
+													 lowest_nulling_outer_join,
+													 NULL);
 				break;
 			default:
 				elog(ERROR, "unrecognized join type: %d",
@@ -717,11 +761,12 @@ pull_up_subqueries(PlannerInfo *root, Node *jtnode,
  * all.
  *
  * rte is the RangeTblEntry referenced by jtnode.  Remaining parameters are
- * as for pull_up_subqueries.
+ * as for pull_up_subqueries_recurse.
  */
 static Node *
 pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 						JoinExpr *lowest_outer_join,
+						JoinExpr *lowest_nulling_outer_join,
 						AppendRelInfo *containing_appendrel)
 {
 	Query	   *parse = root->parse;
@@ -788,7 +833,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	 * handling an appendrel member.
 	 */
 	subquery->jointree = (FromExpr *)
-		pull_up_subqueries(subroot, (Node *) subquery->jointree, NULL, NULL);
+		pull_up_subqueries_recurse(subroot, (Node *) subquery->jointree,
+								   NULL, NULL, NULL);
 
 	/*
 	 * Now we must recheck whether the subquery is still simple enough to pull
@@ -796,10 +842,9 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	 *
 	 * We don't really need to recheck all the conditions involved, but it's
 	 * easier just to keep this "if" looking the same as the one in
-	 * pull_up_subqueries.
+	 * pull_up_subqueries_recurse.
 	 */
-	if (is_simple_subquery(subquery) &&
-		!rte->security_barrier &&
+	if (is_simple_subquery(subquery, rte, lowest_outer_join) &&
 		(containing_appendrel == NULL || is_safe_append_member(subquery)))
 	{
 		/* good to go */
@@ -846,7 +891,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	rvcontext.target_rte = rte;
 	rvcontext.outer_hasSubLinks = &parse->hasSubLinks;
 	rvcontext.varno = varno;
-	rvcontext.need_phvs = (lowest_outer_join != NULL ||
+	rvcontext.need_phvs = (lowest_nulling_outer_join != NULL ||
 						   containing_appendrel != NULL);
 	rvcontext.wrap_non_vars = (containing_appendrel != NULL);
 	/* initialize cache array with indexes 0 .. length(tlist) */
@@ -867,7 +912,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	parse->returningList = (List *)
 		pullup_replace_vars((Node *) parse->returningList, &rvcontext);
 	replace_vars_in_jointree((Node *) parse->jointree, &rvcontext,
-							 lowest_outer_join);
+							 lowest_nulling_outer_join);
 	Assert(parse->setOperations == NULL);
 	parse->havingQual = pullup_replace_vars(parse->havingQual, &rvcontext);
 
@@ -894,10 +939,10 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	 * Replace references in the joinaliasvars lists of join RTEs.
 	 *
 	 * You might think that we could avoid using PHVs for alias vars of joins
-	 * below lowest_outer_join, but that doesn't work because the alias vars
-	 * could be referenced above that join; we need the PHVs to be present in
-	 * such references after the alias vars get flattened.	(It might be worth
-	 * trying to be smarter here, someday.)
+	 * below lowest_nulling_outer_join, but that doesn't work because the
+	 * alias vars could be referenced above that join; we need the PHVs to be
+	 * present in such references after the alias vars get flattened.  (It
+	 * might be worth trying to be smarter here, someday.)
 	 */
 	foreach(lc, parse->rtable)
 	{
@@ -910,6 +955,38 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	}
 
 	/*
+	 * If the subquery had a LATERAL marker, propagate that to any of its
+	 * child RTEs that could possibly now contain lateral cross-references.
+	 * The children might or might not contain any actual lateral
+	 * cross-references, but we have to mark the pulled-up child RTEs so that
+	 * later planner stages will check for such.
+	 *
+	 * NB: although the parser only sets the lateral flag in subquery and
+	 * function RTEs, after this step it can also be set in VALUES RTEs.
+	 */
+	if (rte->lateral)
+	{
+		foreach(lc, subquery->rtable)
+		{
+			RangeTblEntry *child_rte = (RangeTblEntry *) lfirst(lc);
+
+			switch (child_rte->rtekind)
+			{
+				case RTE_SUBQUERY:
+				case RTE_FUNCTION:
+				case RTE_VALUES:
+					child_rte->lateral = true;
+					break;
+				case RTE_RELATION:
+				case RTE_JOIN:
+				case RTE_CTE:
+					/* these can't contain any lateral references */
+					break;
+			}
+		}
+	}
+
+	/*
 	 * Now append the adjusted rtable entries to upper query. (We hold off
 	 * until after fixing the upper rtable entries; no point in running that
 	 * code on the subquery ones too.)
@@ -1104,11 +1181,12 @@ pull_up_union_leaf_queries(Node *setOp, PlannerInfo *root, int parentRTindex,
 		 * must build the AppendRelInfo first, because this will modify it.)
 		 * Note that we can pass NULL for containing-join info even if we're
 		 * actually under an outer join, because the child's expressions
-		 * aren't going to propagate up above the join.
+		 * aren't going to propagate up to the join.
 		 */
 		rtr = makeNode(RangeTblRef);
 		rtr->rtindex = childRTindex;
-		(void) pull_up_subqueries(root, (Node *) rtr, NULL, appinfo);
+		(void) pull_up_subqueries_recurse(root, (Node *) rtr,
+										  NULL, NULL, appinfo);
 	}
 	else if (IsA(setOp, SetOperationStmt))
 	{
@@ -1158,9 +1236,15 @@ make_setop_translation_list(Query *query, Index newvarno,
  * is_simple_subquery
  *	  Check a subquery in the range table to see if it's simple enough
  *	  to pull up into the parent query.
+ *
+ * rte is the RTE_SUBQUERY RangeTblEntry that contained the subquery.
+ * (Note subquery is not necessarily equal to rte->subquery; it could be a
+ * processed copy of that.)
+ * lowest_outer_join is the lowest outer join above the subquery, or NULL.
  */
 static bool
-is_simple_subquery(Query *subquery)
+is_simple_subquery(Query *subquery, RangeTblEntry *rte,
+				   JoinExpr *lowest_outer_join)
 {
 	/*
 	 * Let's just make sure it's a valid subselect ...
@@ -1201,13 +1285,31 @@ is_simple_subquery(Query *subquery)
 		return false;
 
 	/*
-	 * Don't pull up a LATERAL subquery (hopefully, this is just a temporary
-	 * implementation restriction).
+	 * Don't pull up if the RTE represents a security-barrier view; we couldn't
+	 * prevent information leakage once the RTE's Vars are scattered about in
+	 * the upper query.
 	 */
-	if (contain_vars_of_level((Node *) subquery, 1))
+	if (rte->security_barrier)
 		return false;
 
 	/*
+	 * If the subquery is LATERAL, and we're below any outer join, and the
+	 * subquery contains lateral references to rels outside the outer join,
+	 * don't pull up.  Doing so would risk creating outer-join quals that
+	 * contain references to rels outside the outer join, which is a semantic
+	 * mess that doesn't seem worth addressing at the moment.
+	 */
+	if (rte->lateral && lowest_outer_join != NULL)
+	{
+		Relids	lvarnos = pull_varnos_of_level((Node *) subquery, 1);
+		Relids	jvarnos = get_relids_in_jointree((Node *) lowest_outer_join,
+												 true);
+
+		if (!bms_is_subset(lvarnos, jvarnos))
+			return false;
+	}
+
+	/*
 	 * Don't pull up a subquery that has any set-returning functions in its
 	 * targetlist.	Otherwise we might well wind up inserting set-returning
 	 * functions into places where they mustn't go, such as quals of higher
@@ -1358,13 +1460,13 @@ is_safe_append_member(Query *subquery)
  * expression in the jointree, without changing the jointree structure itself.
  * Ugly, but there's no other way...
  *
- * If we are at or below lowest_outer_join, we can suppress use of
+ * If we are at or below lowest_nulling_outer_join, we can suppress use of
  * PlaceHolderVars wrapped around the replacement expressions.
  */
 static void
 replace_vars_in_jointree(Node *jtnode,
 						 pullup_replace_vars_context *context,
-						 JoinExpr *lowest_outer_join)
+						 JoinExpr *lowest_nulling_outer_join)
 {
 	if (jtnode == NULL)
 		return;
@@ -1378,7 +1480,8 @@ replace_vars_in_jointree(Node *jtnode,
 		ListCell   *l;
 
 		foreach(l, f->fromlist)
-			replace_vars_in_jointree(lfirst(l), context, lowest_outer_join);
+			replace_vars_in_jointree(lfirst(l), context,
+									 lowest_nulling_outer_join);
 		f->quals = pullup_replace_vars(f->quals, context);
 	}
 	else if (IsA(jtnode, JoinExpr))
@@ -1386,14 +1489,14 @@ replace_vars_in_jointree(Node *jtnode,
 		JoinExpr   *j = (JoinExpr *) jtnode;
 		bool		save_need_phvs = context->need_phvs;
 
-		if (j == lowest_outer_join)
+		if (j == lowest_nulling_outer_join)
 		{
 			/* no more PHVs in or below this join */
 			context->need_phvs = false;
-			lowest_outer_join = NULL;
+			lowest_nulling_outer_join = NULL;
 		}
-		replace_vars_in_jointree(j->larg, context, lowest_outer_join);
-		replace_vars_in_jointree(j->rarg, context, lowest_outer_join);
+		replace_vars_in_jointree(j->larg, context, lowest_nulling_outer_join);
+		replace_vars_in_jointree(j->rarg, context, lowest_nulling_outer_join);
 		j->quals = pullup_replace_vars(j->quals, context);
 
 		/*