[PATCH] improve OProfile on many-way systems

Anton prompted me to get this patch merged.  It changes the core buffer
sync algorithm of OProfile to avoid global locks wherever possible.  Anton
tested an earlier version of this patch with some success.  I've lightly
tested this applied against 2.6.8.1-mm3 on my two-way machine.

The changes also have the happy side-effect of losing less samples after
munmap operations, and removing the blind spot of tasks exiting inside the
kernel.

Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

4 files changed, 54 insertions, 29 deletions

diff --git a/kernel/exit.c b/kernel/exit.c
index 7e3a51e727e1..c6ceaaee8a2e 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -787,6 +787,8 @@ asmlinkage NORET_TYPE void do_exit(long code)
 {
 	struct task_struct *tsk = current;
 
+	profile_task_exit(tsk);
+
 	if (unlikely(in_interrupt()))
 		panic("Aiee, killing interrupt handler!");
 	if (unlikely(!tsk->pid))
@@ -803,8 +805,6 @@ asmlinkage NORET_TYPE void do_exit(long code)
 				current->comm, current->pid,
 				preempt_count());
 
-	profile_exit_task(tsk);
- 
 	if (unlikely(current->ptrace & PT_TRACE_EXIT)) {
 		current->ptrace_message = code;
 		ptrace_notify((PTRACE_EVENT_EXIT << 8) | SIGTRAP);
diff --git a/kernel/fork.c b/kernel/fork.c
index 201f7bc57719..6074fa10410a 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -36,6 +36,7 @@
 #include <linux/ptrace.h>
 #include <linux/mount.h>
 #include <linux/audit.h>
+#include <linux/profile.h>
 #include <linux/rmap.h>
 
 #include <asm/pgtable.h>
@@ -76,11 +77,12 @@ int nr_processes(void)
 static kmem_cache_t *task_struct_cachep;
 #endif
 
-static void free_task(struct task_struct *tsk)
+void free_task(struct task_struct *tsk)
 {
 	free_thread_info(tsk->thread_info);
 	free_task_struct(tsk);
 }
+EXPORT_SYMBOL(free_task);
 
 void __put_task_struct(struct task_struct *tsk)
 {
@@ -93,7 +95,9 @@ void __put_task_struct(struct task_struct *tsk)
 	security_task_free(tsk);
 	free_uid(tsk->user);
 	put_group_info(tsk->group_info);
-	free_task(tsk);
+
+	if (!profile_handoff_task(tsk))
+		free_task(tsk);
 }
 
 void fastcall add_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)
diff --git a/kernel/profile.c b/kernel/profile.c
index b0a71500e4d8..cab14764cea0 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -53,31 +53,54 @@ void __init profile_init(void)
 #ifdef CONFIG_PROFILING
  
 static DECLARE_RWSEM(profile_rwsem);
-static struct notifier_block * exit_task_notifier;
-static struct notifier_block * exit_mmap_notifier;
-static struct notifier_block * exec_unmap_notifier;
+static rwlock_t handoff_lock = RW_LOCK_UNLOCKED;
+static struct notifier_block * task_exit_notifier;
+static struct notifier_block * task_free_notifier;
+static struct notifier_block * munmap_notifier;
  
-void profile_exit_task(struct task_struct * task)
+void profile_task_exit(struct task_struct * task)
 {
 	down_read(&profile_rwsem);
-	notifier_call_chain(&exit_task_notifier, 0, task);
+	notifier_call_chain(&task_exit_notifier, 0, task);
 	up_read(&profile_rwsem);
 }
  
-void profile_exit_mmap(struct mm_struct * mm)
+int profile_handoff_task(struct task_struct * task)
 {
-	down_read(&profile_rwsem);
-	notifier_call_chain(&exit_mmap_notifier, 0, mm);
-	up_read(&profile_rwsem);
+	int ret;
+	read_lock(&handoff_lock);
+	ret = notifier_call_chain(&task_free_notifier, 0, task);
+	read_unlock(&handoff_lock);
+	return (ret == NOTIFY_OK) ? 1 : 0;
 }
 
-void profile_exec_unmap(struct mm_struct * mm)
+void profile_munmap(unsigned long addr)
 {
 	down_read(&profile_rwsem);
-	notifier_call_chain(&exec_unmap_notifier, 0, mm);
+	notifier_call_chain(&munmap_notifier, 0, (void *)addr);
 	up_read(&profile_rwsem);
 }
 
+int task_handoff_register(struct notifier_block * n)
+{
+	int err = -EINVAL;
+
+	write_lock(&handoff_lock);
+	err = notifier_chain_register(&task_free_notifier, n);
+	write_unlock(&handoff_lock);
+	return err;
+}
+
+int task_handoff_unregister(struct notifier_block * n)
+{
+	int err = -EINVAL;
+
+	write_lock(&handoff_lock);
+	err = notifier_chain_unregister(&task_free_notifier, n);
+	write_unlock(&handoff_lock);
+	return err;
+}
+
 int profile_event_register(enum profile_type type, struct notifier_block * n)
 {
 	int err = -EINVAL;
@@ -85,14 +108,11 @@ int profile_event_register(enum profile_type type, struct notifier_block * n)
 	down_write(&profile_rwsem);
  
 	switch (type) {
-		case EXIT_TASK:
-			err = notifier_chain_register(&exit_task_notifier, n);
+		case PROFILE_TASK_EXIT:
+			err = notifier_chain_register(&task_exit_notifier, n);
 			break;
-		case EXIT_MMAP:
-			err = notifier_chain_register(&exit_mmap_notifier, n);
-			break;
-		case EXEC_UNMAP:
-			err = notifier_chain_register(&exec_unmap_notifier, n);
+		case PROFILE_MUNMAP:
+			err = notifier_chain_register(&munmap_notifier, n);
 			break;
 	}
  
@@ -109,14 +129,11 @@ int profile_event_unregister(enum profile_type type, struct notifier_block * n)
 	down_write(&profile_rwsem);
  
 	switch (type) {
-		case EXIT_TASK:
-			err = notifier_chain_unregister(&exit_task_notifier, n);
-			break;
-		case EXIT_MMAP:
-			err = notifier_chain_unregister(&exit_mmap_notifier, n);
+		case PROFILE_TASK_EXIT:
+			err = notifier_chain_unregister(&task_exit_notifier, n);
 			break;
-		case EXEC_UNMAP:
-			err = notifier_chain_unregister(&exec_unmap_notifier, n);
+		case PROFILE_MUNMAP:
+			err = notifier_chain_unregister(&munmap_notifier, n);
 			break;
 	}
 
@@ -156,6 +173,8 @@ void profile_hook(struct pt_regs * regs)
 
 EXPORT_SYMBOL_GPL(register_profile_notifier);
 EXPORT_SYMBOL_GPL(unregister_profile_notifier);
+EXPORT_SYMBOL_GPL(task_handoff_register);
+EXPORT_SYMBOL_GPL(task_handoff_unregister);
 
 #endif /* CONFIG_PROFILING */
 
diff --git a/kernel/timer.c b/kernel/timer.c
index 79db45d06bac..8f76781b9691 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -233,6 +233,8 @@ void add_timer_on(struct timer_list *timer, int cpu)
 	spin_unlock_irqrestore(&base->lock, flags);
 }
 
+EXPORT_SYMBOL(add_timer_on);
+
 /***
  * mod_timer - modify a timer's timeout
  * @timer: the timer to be modified