[PATCH] percpu: convert global page accounting

Convert global page state accounting to use per-cpu storage

(I think this code remains a little buggy, btw.  Note how I do

	per_cpu(page_states, cpu).member += (delta);

This gets done at interrupt time and hence is assuming that
the "+=" operation on a ulong is atomic wrt interrupts on
all architectures. How do we feel about that assumption?)

4 files changed, 45 insertions, 6 deletions

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 939f16910233..c340b447a963 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -86,4 +86,6 @@ extern void FASTCALL(free_pages(unsigned long addr, unsigned int order));
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr),0)
 
+void page_alloc_init(void);
+
 #endif /* __LINUX_GFP_H */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 5c770f49787a..282902bb9816 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -5,6 +5,8 @@
 #ifndef PAGE_FLAGS_H
 #define PAGE_FLAGS_H
 
+#include <linux/percpu.h>
+
 /*
  * Various page->flags bits:
  *
@@ -73,7 +75,7 @@
  * Global page accounting.  One instance per CPU.  Only unsigned longs are
  * allowed.
  */
-extern struct page_state {
+struct page_state {
 	unsigned long nr_dirty;
 	unsigned long nr_writeback;
 	unsigned long nr_pagecache;
@@ -103,7 +105,9 @@ extern struct page_state {
 	unsigned long kswapd_steal;
 	unsigned long pageoutrun;
 	unsigned long allocstall;
-} ____cacheline_aligned_in_smp page_states[NR_CPUS];
+};
+
+DECLARE_PER_CPU(struct page_state, page_states);
 
 extern void get_page_state(struct page_state *ret);
 extern void get_full_page_state(struct page_state *ret);
@@ -111,7 +115,7 @@ extern void get_full_page_state(struct page_state *ret);
 #define mod_page_state(member, delta)					\
 	do {								\
 		int cpu = get_cpu();					\
-		page_states[cpu].member += (delta);			\
+		per_cpu(page_states, cpu).member += (delta);		\
 		put_cpu();						\
 	} while (0)
 
diff --git a/init/main.c b/init/main.c
index 97d88c50366b..d75bf0acb1eb 100644
--- a/init/main.c
+++ b/init/main.c
@@ -26,6 +26,7 @@
 #include <linux/hdreg.h>
 #include <linux/bootmem.h>
 #include <linux/tty.h>
+#include <linux/gfp.h>
 #include <linux/percpu.h>
 #include <linux/kernel_stat.h>
 #include <linux/security.h>
@@ -388,6 +389,7 @@ asmlinkage void __init start_kernel(void)
 	setup_arch(&command_line);
 	setup_per_cpu_areas();
 	build_all_zonelists();
+	page_alloc_init();
 	printk("Kernel command line: %s\n", saved_command_line);
 	parse_options(command_line);
 	trap_init();
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 848b1ed8f001..0e7425d56652 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -25,6 +25,7 @@
 #include <linux/pagevec.h>
 #include <linux/blkdev.h>
 #include <linux/slab.h>
+#include <linux/notifier.h>
 
 struct pglist_data *pgdat_list;
 unsigned long totalram_pages;
@@ -573,8 +574,8 @@ unsigned int nr_free_highpages (void)
  * The result is unavoidably approximate - it can change
  * during and after execution of this function.
  */
-struct page_state page_states[NR_CPUS] __cacheline_aligned;
-EXPORT_SYMBOL(page_states);
+DEFINE_PER_CPU(struct page_state, page_states) = {0};
+EXPORT_PER_CPU_SYMBOL(page_states);
 
 void __get_page_state(struct page_state *ret, int nr)
 {
@@ -587,7 +588,7 @@ void __get_page_state(struct page_state *ret, int nr)
 		if (!cpu_online(cpu))
 			continue;
 
-		in = (unsigned long *)(page_states + cpu);
+		in = (unsigned long *)&per_cpu(page_states, cpu);
 		out = (unsigned long *)ret;
 		for (off = 0; off < nr; off++)
 			*out++ += *in++;
@@ -1197,3 +1198,33 @@ struct seq_operations vmstat_op = {
 };
 
 #endif /* CONFIG_PROC_FS */
+
+static void __devinit init_page_alloc_cpu(int cpu)
+{
+	struct page_state *ps = &per_cpu(page_states, cpu);
+	memset(ps, 0, sizeof(*ps));
+}
+	
+static int __devinit page_alloc_cpu_notify(struct notifier_block *self, 
+				unsigned long action, void *hcpu)
+{
+	int cpu = (unsigned long)hcpu;
+	switch(action) {
+	case CPU_UP_PREPARE:
+		init_page_alloc_cpu(cpu);
+		break;
+	default:
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block __devinitdata page_alloc_nb = {
+	.notifier_call	= page_alloc_cpu_notify,
+};
+
+void __init page_alloc_init(void)
+{
+	init_page_alloc_cpu(smp_processor_id());
+	register_cpu_notifier(&page_alloc_nb);
+}