Fix rtree and contrib/rtree_gist search behavior for the 1-D box and

polygon operators (<<, &<, >>, &>).  Per ideas originally put forward
by andrew@supernews and later rediscovered by moi.  This patch just
fixes the existing opclasses, and does not add any new behavior as I
proposed earlier; that can be sorted out later.  In principle this
could be back-patched, since it changes only search behavior and not
system catalog entries nor rtree index contents.  I'm not currently
planning to do that, though, since I think it could use more testing.

1 files changed, 31 insertions, 13 deletions

diff --git a/src/backend/access/rtree/rtstrat.c b/src/backend/access/rtree/rtstrat.c
index d5f59fed6e6..c7104c9520c 100644
--- a/src/backend/access/rtree/rtstrat.c
+++ b/src/backend/access/rtree/rtstrat.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/rtree/rtstrat.c,v 1.25 2004/12/31 21:59:26 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/rtree/rtstrat.c,v 1.26 2005/06/24 00:18:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -28,20 +28,31 @@
  *	the leaf we search for equality.
  *
  *	This array maps leaf search operators to the internal search operators.
- *	We assume the normal ordering on operators:
- *
- *		left, left-or-overlap, overlap, right-or-overlap, right, same,
- *		contains, contained-by
  */
 static const StrategyNumber RTOperMap[RTNStrategies] = {
-	RTOverLeftStrategyNumber,
-	RTOverLeftStrategyNumber,
-	RTOverlapStrategyNumber,
-	RTOverRightStrategyNumber,
-	RTOverRightStrategyNumber,
-	RTContainsStrategyNumber,
-	RTContainsStrategyNumber,
-	RTOverlapStrategyNumber
+	RTOverRightStrategyNumber,	/* left */
+	RTRightStrategyNumber,		/* overleft */
+	RTOverlapStrategyNumber,	/* overlap */
+	RTLeftStrategyNumber,		/* overright */
+	RTOverLeftStrategyNumber,	/* right */
+	RTContainsStrategyNumber,	/* same */
+	RTContainsStrategyNumber,	/* contains */
+	RTOverlapStrategyNumber		/* contained-by */
+};
+
+/*
+ * We may need to negate the result of the selected operator.  (This could
+ * be avoided by expanding the set of operators required for an opclass.)
+ */
+static const bool RTNegateMap[RTNStrategies] = {
+	true,						/* left */
+	true,						/* overleft */
+	false,						/* overlap */
+	true,						/* overright */
+	true,						/* right */
+	false,						/* same */
+	false,						/* contains */
+	false						/* contained-by */
 };
 
 
@@ -51,3 +62,10 @@ RTMapToInternalOperator(StrategyNumber strat)
 	Assert(strat > 0 && strat <= RTNStrategies);
 	return RTOperMap[strat - 1];
 }
+
+bool
+RTMapToInternalNegate(StrategyNumber strat)
+{
+	Assert(strat > 0 && strat <= RTNStrategies);
+	return RTNegateMap[strat - 1];
+}