Create a multiplexing structure for signals to Postgres child processes.

This patch gets us out from under the Unix limitation of two user-defined
signal types.  We already had done something similar for signals directed to
the postmaster process; this adds multiplexing for signals directed to
backends and auxiliary processes (so long as they're connected to shared
memory).

As proof of concept, replace the former usage of SIGUSR1 and SIGUSR2
for backends with use of the multiplexing mechanism.  There are still some
hard-wired definitions of SIGUSR1 and SIGUSR2 for other process types,
but getting rid of those doesn't seem interesting at the moment.

Fujii Masao

5 files changed, 293 insertions, 28 deletions

diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile
index cc74f8812bb..20ac1e75e45 100644
--- a/src/backend/storage/ipc/Makefile
+++ b/src/backend/storage/ipc/Makefile
@@ -1,7 +1,7 @@
 #
 # Makefile for storage/ipc
 #
-# $PostgreSQL: pgsql/src/backend/storage/ipc/Makefile,v 1.21 2008/02/19 10:30:08 petere Exp $
+# $PostgreSQL: pgsql/src/backend/storage/ipc/Makefile,v 1.22 2009/07/31 20:26:23 tgl Exp $
 #
 
 subdir = src/backend/storage/ipc
@@ -15,7 +15,7 @@ override CFLAGS+= -fno-inline
 endif
 endif
 
-OBJS = ipc.o ipci.o pmsignal.o procarray.o shmem.o shmqueue.o \
+OBJS = ipc.o ipci.o pmsignal.o procarray.o procsignal.o shmem.o shmqueue.o \
 	sinval.o sinvaladt.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 281215d2189..bd4f168bfb0 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.100 2009/05/05 19:59:00 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.101 2009/07/31 20:26:23 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -30,6 +30,7 @@
 #include "storage/pg_shmem.h"
 #include "storage/pmsignal.h"
 #include "storage/procarray.h"
+#include "storage/procsignal.h"
 #include "storage/sinvaladt.h"
 #include "storage/spin.h"
 
@@ -112,6 +113,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, BackendStatusShmemSize());
 		size = add_size(size, SInvalShmemSize());
 		size = add_size(size, PMSignalShmemSize());
+		size = add_size(size, ProcSignalShmemSize());
 		size = add_size(size, BgWriterShmemSize());
 		size = add_size(size, AutoVacuumShmemSize());
 		size = add_size(size, BTreeShmemSize());
@@ -208,6 +210,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	 * Set up interprocess signaling mechanisms
 	 */
 	PMSignalShmemInit();
+	ProcSignalShmemInit();
 	BgWriterShmemInit();
 	AutoVacuumShmemInit();
 
diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c
new file mode 100644
index 00000000000..9da2c5201e7
--- /dev/null
+++ b/src/backend/storage/ipc/procsignal.c
@@ -0,0 +1,262 @@
+/*-------------------------------------------------------------------------
+ *
+ * procsignal.c
+ *	  Routines for interprocess signalling
+ *
+ *
+ * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/procsignal.c,v 1.1 2009/07/31 20:26:23 tgl Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "bootstrap/bootstrap.h"
+#include "commands/async.h"
+#include "miscadmin.h"
+#include "storage/ipc.h"
+#include "storage/procsignal.h"
+#include "storage/shmem.h"
+#include "storage/sinval.h"
+
+
+/*
+ * The SIGUSR1 signal is multiplexed to support signalling multiple event
+ * types. The specific reason is communicated via flags in shared memory.
+ * We keep a boolean flag for each possible "reason", so that different
+ * reasons can be signaled to a process concurrently.  (However, if the same
+ * reason is signaled more than once nearly simultaneously, the process may
+ * observe it only once.)
+ *
+ * Each process that wants to receive signals registers its process ID 
+ * in the ProcSignalSlots array. The array is indexed by backend ID to make
+ * slot allocation simple, and to avoid having to search the array when you
+ * know the backend ID of the process you're signalling.  (We do support
+ * signalling without backend ID, but it's a bit less efficient.)
+ * 
+ * The flags are actually declared as "volatile sig_atomic_t" for maximum
+ * portability.  This should ensure that loads and stores of the flag
+ * values are atomic, allowing us to dispense with any explicit locking.
+ */
+typedef struct
+{
+	pid_t		pss_pid;
+	sig_atomic_t pss_signalFlags[NUM_PROCSIGNALS];
+} ProcSignalSlot;
+
+/*
+ * We reserve a slot for each possible BackendId, plus one for each
+ * possible auxiliary process type.  (This scheme assumes there is not
+ * more than one of any auxiliary process type at a time.)
+ */
+#define NumProcSignalSlots  (MaxBackends + NUM_AUXPROCTYPES)
+
+static ProcSignalSlot *ProcSignalSlots = NULL;
+static volatile ProcSignalSlot *MyProcSignalSlot = NULL;
+
+static bool CheckProcSignal(ProcSignalReason reason);
+static void CleanupProcSignalState(int status, Datum arg);
+
+/*
+ * ProcSignalShmemInit
+ *		Compute space needed for procsignal's shared memory
+ */
+Size
+ProcSignalShmemSize(void)
+{
+	return NumProcSignalSlots * sizeof(ProcSignalSlot);
+}
+
+/*
+ * ProcSignalShmemInit
+ *		Allocate and initialize procsignal's shared memory
+ */
+void
+ProcSignalShmemInit(void)
+{
+	Size		size = ProcSignalShmemSize();
+	bool		found;
+
+	ProcSignalSlots = (ProcSignalSlot *)
+		ShmemInitStruct("ProcSignalSlots", size, &found);
+
+	/* If we're first, set everything to zeroes */
+	if (!found)
+		MemSet(ProcSignalSlots, 0, size);
+}
+
+/*
+ * ProcSignalInit
+ *		Register the current process in the procsignal array
+ *
+ * The passed index should be my BackendId if the process has one,
+ * or MaxBackends + aux process type if not.
+ */
+void
+ProcSignalInit(int pss_idx)
+{
+	volatile ProcSignalSlot *slot;
+
+	Assert(pss_idx >= 1 && pss_idx <= NumProcSignalSlots);
+
+	slot = &ProcSignalSlots[pss_idx - 1];
+
+	/* sanity check */
+	if (slot->pss_pid != 0)
+		elog(LOG, "process %d taking over ProcSignal slot %d, but it's not empty",
+			 MyProcPid, pss_idx);
+
+	/* Clear out any leftover signal reasons */
+	MemSet(slot->pss_signalFlags, 0, NUM_PROCSIGNALS * sizeof(sig_atomic_t));
+
+	/* Mark slot with my PID */
+	slot->pss_pid = MyProcPid;
+
+	/* Remember slot location for CheckProcSignal */
+	MyProcSignalSlot = slot;
+
+	/* Set up to release the slot on process exit */
+	on_shmem_exit(CleanupProcSignalState, Int32GetDatum(pss_idx));
+}
+
+/*
+ * CleanupProcSignalState
+ *		Remove current process from ProcSignalSlots
+ *
+ * This function is called via on_shmem_exit() during backend shutdown.
+ */
+static void
+CleanupProcSignalState(int status, Datum arg)
+{
+	int			pss_idx = DatumGetInt32(arg);
+	volatile ProcSignalSlot *slot;
+
+	slot = &ProcSignalSlots[pss_idx - 1];
+	Assert(slot == MyProcSignalSlot);
+
+	/* sanity check */
+	if (slot->pss_pid != MyProcPid)
+	{
+		/*
+		 * don't ERROR here. We're exiting anyway, and don't want to
+		 * get into infinite loop trying to exit
+		 */
+		elog(LOG, "process %d releasing ProcSignal slot %d, but it contains %d",
+			 MyProcPid, pss_idx, (int) slot->pss_pid);
+		return;					/* XXX better to zero the slot anyway? */
+	}
+
+	slot->pss_pid = 0;
+}
+
+/*
+ * SendProcSignal
+ *		Send a signal to a Postgres process
+ *
+ * Providing backendId is optional, but it will speed up the operation.
+ *
+ * On success (a signal was sent), zero is returned.
+ * On error, -1 is returned, and errno is set (typically to ESRCH or EPERM).
+ *
+ * Not to be confused with ProcSendSignal
+ */
+int
+SendProcSignal(pid_t pid, ProcSignalReason reason, BackendId backendId)
+{
+	volatile ProcSignalSlot *slot;
+
+	if (backendId != InvalidBackendId)
+	{
+		slot = &ProcSignalSlots[backendId - 1];
+
+		/*
+		 * Note: Since there's no locking, it's possible that the target
+		 * process detaches from shared memory and exits right after this
+		 * test, before we set the flag and send signal. And the signal slot
+		 * might even be recycled by a new process, so it's remotely possible
+		 * that we set a flag for a wrong process. That's OK, all the signals
+		 * are such that no harm is done if they're mistakenly fired.
+		 */
+		if (slot->pss_pid == pid)
+		{
+			/* Atomically set the proper flag */
+			slot->pss_signalFlags[reason] = true;
+			/* Send signal */
+			return kill(pid, SIGUSR1);
+		}
+	}
+	else
+	{
+		/*
+		 * BackendId not provided, so search the array using pid.  We search
+		 * the array back to front so as to reduce search overhead.  Passing
+		 * InvalidBackendId means that the target is most likely an auxiliary
+		 * process, which will have a slot near the end of the array.
+		 */
+		int		i;
+
+		for (i = NumProcSignalSlots - 1; i >= 0; i--)
+		{
+			slot = &ProcSignalSlots[i];
+
+			if (slot->pss_pid == pid)
+			{
+				/* the above note about race conditions applies here too */
+
+				/* Atomically set the proper flag */
+				slot->pss_signalFlags[reason] = true;
+				/* Send signal */
+				return kill(pid, SIGUSR1);
+			}
+		}
+	}
+
+	errno = ESRCH;
+	return -1;
+}
+
+/*
+ * CheckProcSignal - check to see if a particular reason has been
+ * signaled, and clear the signal flag.  Should be called after receiving
+ * SIGUSR1.
+ */
+static bool
+CheckProcSignal(ProcSignalReason reason)
+{
+	volatile ProcSignalSlot *slot = MyProcSignalSlot;
+
+	if (slot != NULL)
+	{
+		/* Careful here --- don't clear flag if we haven't seen it set */
+		if (slot->pss_signalFlags[reason])
+		{
+			slot->pss_signalFlags[reason] = false;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * procsignal_sigusr1_handler - handle SIGUSR1 signal.
+ */
+void
+procsignal_sigusr1_handler(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	if (CheckProcSignal(PROCSIG_CATCHUP_INTERRUPT))
+		HandleCatchupInterrupt();
+
+	if (CheckProcSignal(PROCSIG_NOTIFY_INTERRUPT))
+		HandleNotifyInterrupt();
+
+	errno = save_errno;
+}
diff --git a/src/backend/storage/ipc/sinval.c b/src/backend/storage/ipc/sinval.c
index 3f063989e0c..1b6f3b2fc44 100644
--- a/src/backend/storage/ipc/sinval.c
+++ b/src/backend/storage/ipc/sinval.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/sinval.c,v 1.90 2009/06/11 14:49:02 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/sinval.c,v 1.91 2009/07/31 20:26:23 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,8 +26,8 @@
  * Because backends sitting idle will not be reading sinval events, we
  * need a way to give an idle backend a swift kick in the rear and make
  * it catch up before the sinval queue overflows and forces it to go
- * through a cache reset exercise.	This is done by sending SIGUSR1
- * to any backend that gets too far behind.
+ * through a cache reset exercise.	This is done by sending
+ * PROCSIG_CATCHUP_INTERRUPT to any backend that gets too far behind.
  *
  * State for catchup events consists of two flags: one saying whether
  * the signal handler is currently allowed to call ProcessCatchupEvent
@@ -145,9 +145,9 @@ ReceiveSharedInvalidMessages(
 
 
 /*
- * CatchupInterruptHandler
+ * HandleCatchupInterrupt
  *
- * This is the signal handler for SIGUSR1.
+ * This is called when PROCSIG_CATCHUP_INTERRUPT is received.
  *
  * If we are idle (catchupInterruptEnabled is set), we can safely
  * invoke ProcessCatchupEvent directly.  Otherwise, just set a flag
@@ -157,13 +157,11 @@ ReceiveSharedInvalidMessages(
  * since there's no longer any reason to do anything.)
  */
 void
-CatchupInterruptHandler(SIGNAL_ARGS)
+HandleCatchupInterrupt(void)
 {
-	int			save_errno = errno;
-
 	/*
-	 * Note: this is a SIGNAL HANDLER.	You must be very wary what you do
-	 * here.
+	 * Note: this is called by a SIGNAL HANDLER. You must be very wary what
+	 * you do here.
 	 */
 
 	/* Don't joggle the elbow of proc_exit */
@@ -217,8 +215,6 @@ CatchupInterruptHandler(SIGNAL_ARGS)
 		 */
 		catchupInterruptOccurred = 1;
 	}
-
-	errno = save_errno;
 }
 
 /*
@@ -273,8 +269,8 @@ EnableCatchupInterrupt(void)
  * a frontend command.	Signal handler execution of catchup events
  * is disabled until the next EnableCatchupInterrupt call.
  *
- * The SIGUSR2 signal handler also needs to call this, so as to
- * prevent conflicts if one signal interrupts the other.  So we
+ * The PROCSIG_NOTIFY_INTERRUPT signal handler also needs to call this,
+ * so as to prevent conflicts if one signal interrupts the other.  So we
  * must return the previous state of the flag.
  */
 bool
@@ -290,18 +286,19 @@ DisableCatchupInterrupt(void)
 /*
  * ProcessCatchupEvent
  *
- * Respond to a catchup event (SIGUSR1) from another backend.
+ * Respond to a catchup event (PROCSIG_CATCHUP_INTERRUPT) from another
+ * backend.
  *
- * This is called either directly from the SIGUSR1 signal handler,
- * or the next time control reaches the outer idle loop (assuming
- * there's still anything to do by then).
+ * This is called either directly from the PROCSIG_CATCHUP_INTERRUPT
+ * signal handler, or the next time control reaches the outer idle loop
+ * (assuming there's still anything to do by then).
  */
 static void
 ProcessCatchupEvent(void)
 {
 	bool		notify_enabled;
 
-	/* Must prevent SIGUSR2 interrupt while I am running */
+	/* Must prevent notify interrupt while I am running */
 	notify_enabled = DisableNotifyInterrupt();
 
 	/*
diff --git a/src/backend/storage/ipc/sinvaladt.c b/src/backend/storage/ipc/sinvaladt.c
index fcd1e42a7f6..dfa0ad7b5eb 100644
--- a/src/backend/storage/ipc/sinvaladt.c
+++ b/src/backend/storage/ipc/sinvaladt.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.78 2009/06/11 14:49:02 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.79 2009/07/31 20:26:23 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -21,6 +21,7 @@
 #include "storage/backendid.h"
 #include "storage/ipc.h"
 #include "storage/proc.h"
+#include "storage/procsignal.h"
 #include "storage/shmem.h"
 #include "storage/sinvaladt.h"
 #include "storage/spin.h"
@@ -118,7 +119,7 @@
  * we exceed CLEANUP_MIN.  Should be a power of 2 for speed.
  *
  * SIG_THRESHOLD: the minimum number of messages a backend must have fallen
- * behind before we'll send it SIGUSR1.
+ * behind before we'll send it PROCSIG_CATCHUP_INTERRUPT.
  *
  * WRITE_QUANTUM: the max number of messages to push into the buffer per
  * iteration of SIInsertDataEntries.  Noncritical but should be less than
@@ -551,7 +552,7 @@ SIGetDataEntries(SharedInvalidationMessage *data, int datasize)
  * minFree is the minimum number of message slots to make free.
  *
  * Possible side effects of this routine include marking one or more
- * backends as "reset" in the array, and sending a catchup interrupt (SIGUSR1)
+ * backends as "reset" in the array, and sending PROCSIG_CATCHUP_INTERRUPT
  * to some backend that seems to be getting too far behind.  We signal at
  * most one backend at a time, for reasons explained at the top of the file.
  *
@@ -644,18 +645,20 @@ SICleanupQueue(bool callerHasWriteLock, int minFree)
 		segP->nextThreshold = (numMsgs / CLEANUP_QUANTUM + 1) * CLEANUP_QUANTUM;
 
 	/*
-	 * Lastly, signal anyone who needs a catchup interrupt.  Since kill()
-	 * might not be fast, we don't want to hold locks while executing it.
+	 * Lastly, signal anyone who needs a catchup interrupt.  Since
+	 * SendProcSignal() might not be fast, we don't want to hold locks while
+	 * executing it.
 	 */
 	if (needSig)
 	{
 		pid_t		his_pid = needSig->procPid;
+		BackendId	his_backendId = (needSig - &segP->procState[0]) + 1;
 
 		needSig->signaled = true;
 		LWLockRelease(SInvalReadLock);
 		LWLockRelease(SInvalWriteLock);
 		elog(DEBUG4, "sending sinval catchup signal to PID %d", (int) his_pid);
-		kill(his_pid, SIGUSR1);
+		SendProcSignal(his_pid, PROCSIG_CATCHUP_INTERRUPT, his_backendId);
 		if (callerHasWriteLock)
 			LWLockAcquire(SInvalWriteLock, LW_EXCLUSIVE);
 	}