Avoid use of NOTICE to wait for snapshot invalidation

This idea (implemented in commits and bc32a12e0db2 and 9e8fa05d3412) of
using notices to detect that a session is sleeping was unreliable, so
simplify the concurrency controller session to just look at
pg_stat_activity for a process sleeping on the injection point we want
it to hit.  This change allows us to remove a secondary injection point
and the alternative expected output files.

Reproduced by Alexander Lakhin following a report in buildfarm member
skink (which runs the server under valgrind).

Author: Mihail Nikalayeu <mihailnikalayeu@gmail.com>
Reported-by: Alexander Lakhin <exclusion@gmail.com>
Reviewed-by: Álvaro Herrera <alvherre@kurilemu.de>
Discussion: https://postgr.es/m/3e302c96-cdd2-45ec-af84-03dbcdccde4a@gmail.com

7 files changed, 72 insertions, 261 deletions

diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c
index 434abbf6b6f..24f73a49d27 100644
--- a/src/backend/utils/time/snapmgr.c
+++ b/src/backend/utils/time/snapmgr.c
@@ -459,7 +459,6 @@ InvalidateCatalogSnapshot(void)
 		pairingheap_remove(&RegisteredSnapshots, &CatalogSnapshot->ph_node);
 		CatalogSnapshot = NULL;
 		SnapshotResetXmin();
-		INJECTION_POINT("pre-invalidate-catalog-snapshot-end", NULL);
 		INJECTION_POINT("invalidate-catalog-snapshot-end", NULL);
 	}
 }
diff --git a/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate.out b/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate.out
index 5a57cd5fa40..77e7d1a7815 100644
--- a/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate.out
+++ b/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate.out
@@ -1,6 +1,6 @@
 Parsed test spec with 5 sessions
 
-starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s5_noop s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
+starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
 injection_points_attach
 -----------------------
                        
@@ -37,16 +37,12 @@ case
     
 (1 row)
 
-step s5_noop: 
- <waiting ...>
 step s3_start_create_index: 
 	CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_special_duplicate ON test.tbl(abs(i)) WHERE i < 10000;
  <waiting ...>
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
 step s1_start_upsert: 
 	INSERT INTO test.tbl VALUES(13,now()) ON CONFLICT (abs(i)) WHERE i < 100 DO UPDATE SET updated_at = now();
  <waiting ...>
-step s5_noop: <... completed>
 step s4_wakeup_define_index_before_set_valid: 
 	SELECT injection_points_detach('define-index-before-set-valid');
 	SELECT injection_points_wakeup('define-index-before-set-valid');
@@ -65,6 +61,22 @@ step s2_start_upsert:
 	INSERT INTO test.tbl VALUES(13,now()) ON CONFLICT (abs(i)) WHERE i < 100 DO UPDATE SET updated_at = now();
  <waiting ...>
 step s5_wakeup_s1_from_invalidate_catalog_snapshot: 
+	DO $$
+		DECLARE
+			v_waiting_pid INTEGER;
+		BEGIN
+		LOOP
+			SELECT pid INTO v_waiting_pid
+			  FROM pg_stat_activity
+			 WHERE wait_event_type = 'InjectionPoint'
+				   AND wait_event = 'invalidate-catalog-snapshot-end'
+			 LIMIT 1;
+			EXIT WHEN v_waiting_pid IS NOT NULL;
+			PERFORM pg_sleep(100);
+		END LOOP;
+		END
+	$$;
+
 	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
 	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
 
@@ -106,7 +118,6 @@ injection_points_wakeup
                        
 (1 row)
 
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
 step s1_start_upsert: <... completed>
 step s2_start_upsert: <... completed>
 step s3_start_create_index: <... completed>
diff --git a/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate_1.out b/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate_1.out
deleted file mode 100644
index d453dd62617..00000000000
--- a/src/test/modules/injection_points/expected/index-concurrently-upsert-predicate_1.out
+++ /dev/null
@@ -1,116 +0,0 @@
-Parsed test spec with 5 sessions
-
-starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s5_noop s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-injection_points_attach
------------------------
-                       
-(1 row)
-
-injection_points_attach
------------------------
-                       
-(1 row)
-
-injection_points_attach
------------------------
-                       
-(1 row)
-
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_attach_invalidate_catalog_snapshot: 
-	SELECT injection_points_attach('invalidate-catalog-snapshot-end', 'wait');
- <waiting ...>
-step s4_wakeup_s1_setup: 
-	SELECT CASE WHEN
-			(SELECT pid FROM pg_stat_activity
-			      WHERE wait_event_type = 'InjectionPoint' AND
-			      wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
-			THEN injection_points_wakeup('invalidate-catalog-snapshot-end')
-		END;
-
-case
-----
-    
-(1 row)
-
-step s1_attach_invalidate_catalog_snapshot: <... completed>
-injection_points_attach
------------------------
-                       
-(1 row)
-
-step s5_noop: 
- <waiting ...>
-step s3_start_create_index: 
-	CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_special_duplicate ON test.tbl(abs(i)) WHERE i < 10000;
- <waiting ...>
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_start_upsert: 
-	INSERT INTO test.tbl VALUES(13,now()) ON CONFLICT (abs(i)) WHERE i < 100 DO UPDATE SET updated_at = now();
- <waiting ...>
-step s5_noop: <... completed>
-step s4_wakeup_define_index_before_set_valid: 
-	SELECT injection_points_detach('define-index-before-set-valid');
-	SELECT injection_points_wakeup('define-index-before-set-valid');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s2_start_upsert: 
-	INSERT INTO test.tbl VALUES(13,now()) ON CONFLICT (abs(i)) WHERE i < 100 DO UPDATE SET updated_at = now();
- <waiting ...>
-step s5_wakeup_s1_from_invalidate_catalog_snapshot: 
-	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
-	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s4_wakeup_s2: 
-	SELECT injection_points_detach('exec-insert-before-insert-speculative');
-	SELECT injection_points_wakeup('exec-insert-before-insert-speculative');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s4_wakeup_s1: 
-	SELECT injection_points_detach('check-exclusion-or-unique-constraint-no-conflict');
-	SELECT injection_points_wakeup('check-exclusion-or-unique-constraint-no-conflict');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_start_upsert: <... completed>
-step s2_start_upsert: <... completed>
-step s3_start_create_index: <... completed>
diff --git a/src/test/modules/injection_points/expected/index-concurrently-upsert.out b/src/test/modules/injection_points/expected/index-concurrently-upsert.out
index 97386a35bed..a2ef122625c 100644
--- a/src/test/modules/injection_points/expected/index-concurrently-upsert.out
+++ b/src/test/modules/injection_points/expected/index-concurrently-upsert.out
@@ -1,6 +1,6 @@
 Parsed test spec with 5 sessions
 
-starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s5_noop s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
+starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
 injection_points_attach
 -----------------------
                        
@@ -27,8 +27,8 @@ injection_points_attach
 step s4_wakeup_s1_setup: 
 	SELECT CASE WHEN
 			(SELECT pid FROM pg_stat_activity
-			      WHERE wait_event_type = 'InjectionPoint' AND
-			      wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
+				  WHERE wait_event_type = 'InjectionPoint' AND
+				  wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
 			THEN injection_points_wakeup('invalidate-catalog-snapshot-end')
 		END;
 
@@ -37,16 +37,12 @@ case
     
 (1 row)
 
-step s5_noop: 
- <waiting ...>
 step s3_start_create_index: 
 	CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_duplicate ON test.tbl(i);
  <waiting ...>
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
 step s1_start_upsert: 
-    INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
+	INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
  <waiting ...>
-step s5_noop: <... completed>
 step s4_wakeup_define_index_before_set_valid: 
 	SELECT injection_points_detach('define-index-before-set-valid');
 	SELECT injection_points_wakeup('define-index-before-set-valid');
@@ -65,6 +61,22 @@ step s2_start_upsert:
 	INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
  <waiting ...>
 step s5_wakeup_s1_from_invalidate_catalog_snapshot: 
+	DO $$
+		DECLARE
+			v_waiting_pid INTEGER;
+		BEGIN
+		LOOP
+			SELECT pid INTO v_waiting_pid
+			  FROM pg_stat_activity
+			 WHERE wait_event_type = 'InjectionPoint'
+				   AND wait_event = 'invalidate-catalog-snapshot-end'
+			 LIMIT 1;
+			EXIT WHEN v_waiting_pid IS NOT NULL;
+			PERFORM pg_sleep(100);
+		END LOOP;
+		END
+	$$;
+
 	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
 	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
 
@@ -106,7 +118,6 @@ injection_points_wakeup
                        
 (1 row)
 
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
 step s1_start_upsert: <... completed>
 step s2_start_upsert: <... completed>
 step s3_start_create_index: <... completed>
diff --git a/src/test/modules/injection_points/expected/index-concurrently-upsert_1.out b/src/test/modules/injection_points/expected/index-concurrently-upsert_1.out
deleted file mode 100644
index 4bd51b2b511..00000000000
--- a/src/test/modules/injection_points/expected/index-concurrently-upsert_1.out
+++ /dev/null
@@ -1,116 +0,0 @@
-Parsed test spec with 5 sessions
-
-starting permutation: s1_attach_invalidate_catalog_snapshot s4_wakeup_s1_setup s5_noop s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s5_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-injection_points_attach
------------------------
-                       
-(1 row)
-
-injection_points_attach
------------------------
-                       
-(1 row)
-
-injection_points_attach
------------------------
-                       
-(1 row)
-
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_attach_invalidate_catalog_snapshot: 
-	SELECT injection_points_attach('invalidate-catalog-snapshot-end', 'wait');
- <waiting ...>
-step s4_wakeup_s1_setup: 
-	SELECT CASE WHEN
-			(SELECT pid FROM pg_stat_activity
-			      WHERE wait_event_type = 'InjectionPoint' AND
-			      wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
-			THEN injection_points_wakeup('invalidate-catalog-snapshot-end')
-		END;
-
-case
-----
-    
-(1 row)
-
-step s1_attach_invalidate_catalog_snapshot: <... completed>
-injection_points_attach
------------------------
-                       
-(1 row)
-
-step s5_noop: 
- <waiting ...>
-step s3_start_create_index: 
-	CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_duplicate ON test.tbl(i);
- <waiting ...>
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_start_upsert: 
-    INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
- <waiting ...>
-step s5_noop: <... completed>
-step s4_wakeup_define_index_before_set_valid: 
-	SELECT injection_points_detach('define-index-before-set-valid');
-	SELECT injection_points_wakeup('define-index-before-set-valid');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s2_start_upsert: 
-	INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
- <waiting ...>
-step s5_wakeup_s1_from_invalidate_catalog_snapshot: 
-	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
-	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s4_wakeup_s2: 
-	SELECT injection_points_detach('exec-insert-before-insert-speculative');
-	SELECT injection_points_wakeup('exec-insert-before-insert-speculative');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-step s4_wakeup_s1: 
-	SELECT injection_points_detach('check-exclusion-or-unique-constraint-no-conflict');
-	SELECT injection_points_wakeup('check-exclusion-or-unique-constraint-no-conflict');
-
-injection_points_detach
------------------------
-                       
-(1 row)
-
-injection_points_wakeup
------------------------
-                       
-(1 row)
-
-s1: NOTICE:  notice triggered for injection point pre-invalidate-catalog-snapshot-end
-step s1_start_upsert: <... completed>
-step s2_start_upsert: <... completed>
-step s3_start_create_index: <... completed>
diff --git a/src/test/modules/injection_points/specs/index-concurrently-upsert-predicate.spec b/src/test/modules/injection_points/specs/index-concurrently-upsert-predicate.spec
index 1cbcaa6963f..d9b8d27fd1f 100644
--- a/src/test/modules/injection_points/specs/index-concurrently-upsert-predicate.spec
+++ b/src/test/modules/injection_points/specs/index-concurrently-upsert-predicate.spec
@@ -27,7 +27,6 @@ setup
 {
 	SELECT injection_points_set_local();
 	SELECT injection_points_attach('check-exclusion-or-unique-constraint-no-conflict', 'wait');
-	SELECT injection_points_attach('pre-invalidate-catalog-snapshot-end', 'notice');
 }
 step s1_attach_invalidate_catalog_snapshot
 {
@@ -91,11 +90,24 @@ step s4_wakeup_define_index_before_set_valid
 }
 
 session s5
-step s5_noop
-{
-}
 step s5_wakeup_s1_from_invalidate_catalog_snapshot
 {
+	DO $$
+		DECLARE
+			v_waiting_pid INTEGER;
+		BEGIN
+		LOOP
+			SELECT pid INTO v_waiting_pid
+			  FROM pg_stat_activity
+			 WHERE wait_event_type = 'InjectionPoint'
+				   AND wait_event = 'invalidate-catalog-snapshot-end'
+			 LIMIT 1;
+			EXIT WHEN v_waiting_pid IS NOT NULL;
+			PERFORM pg_sleep(100);
+		END LOOP;
+		END
+	$$;
+
 	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
 	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
 }
@@ -103,7 +115,6 @@ step s5_wakeup_s1_from_invalidate_catalog_snapshot
 permutation
 	s1_attach_invalidate_catalog_snapshot
 	s4_wakeup_s1_setup
-	s5_noop(s1_start_upsert notices 1)
 	s3_start_create_index(s1_start_upsert, s2_start_upsert)
 	s1_start_upsert
 	s4_wakeup_define_index_before_set_valid
diff --git a/src/test/modules/injection_points/specs/index-concurrently-upsert.spec b/src/test/modules/injection_points/specs/index-concurrently-upsert.spec
index 2a6d888dcea..6e08af74a93 100644
--- a/src/test/modules/injection_points/specs/index-concurrently-upsert.spec
+++ b/src/test/modules/injection_points/specs/index-concurrently-upsert.spec
@@ -26,7 +26,6 @@ setup
 {
 	SELECT injection_points_set_local();
 	SELECT injection_points_attach('check-exclusion-or-unique-constraint-no-conflict', 'wait');
-	SELECT injection_points_attach('pre-invalidate-catalog-snapshot-end', 'notice');
 }
 step s1_attach_invalidate_catalog_snapshot
 {
@@ -34,7 +33,7 @@ step s1_attach_invalidate_catalog_snapshot
 }
 step s1_start_upsert
 {
-    INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
+	INSERT INTO test.tbl VALUES (13,now()) ON CONFLICT (i) DO UPDATE SET updated_at = now();
 }
 
 session s2
@@ -68,8 +67,8 @@ step s4_wakeup_s1_setup
 {
 	SELECT CASE WHEN
 			(SELECT pid FROM pg_stat_activity
-			      WHERE wait_event_type = 'InjectionPoint' AND
-			      wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
+				  WHERE wait_event_type = 'InjectionPoint' AND
+				  wait_event = 'invalidate-catalog-snapshot-end') IS NOT NULL
 			THEN injection_points_wakeup('invalidate-catalog-snapshot-end')
 		END;
 }
@@ -90,11 +89,24 @@ step s4_wakeup_define_index_before_set_valid
 }
 
 session s5
-step s5_noop
-{
-}
 step s5_wakeup_s1_from_invalidate_catalog_snapshot
 {
+	DO $$
+		DECLARE
+			v_waiting_pid INTEGER;
+		BEGIN
+		LOOP
+			SELECT pid INTO v_waiting_pid
+			  FROM pg_stat_activity
+			 WHERE wait_event_type = 'InjectionPoint'
+				   AND wait_event = 'invalidate-catalog-snapshot-end'
+			 LIMIT 1;
+			EXIT WHEN v_waiting_pid IS NOT NULL;
+			PERFORM pg_sleep(100);
+		END LOOP;
+		END
+	$$;
+
 	SELECT injection_points_detach('invalidate-catalog-snapshot-end');
 	SELECT injection_points_wakeup('invalidate-catalog-snapshot-end');
 }
@@ -102,7 +114,6 @@ step s5_wakeup_s1_from_invalidate_catalog_snapshot
 permutation
 	s1_attach_invalidate_catalog_snapshot
 	s4_wakeup_s1_setup
-	s5_noop(s1_start_upsert notices 1)
 	s3_start_create_index(s1_start_upsert, s2_start_upsert)
 	s1_start_upsert
 	s4_wakeup_define_index_before_set_valid