[PATCH] hashed b_wait

Implements hashed waitqueues for buffer_heads.  Drops twelve bytes from
struct buffer_head.

5 files changed, 73 insertions, 33 deletions

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 5e97c4e07899..7a2f950cf1fc 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -166,7 +166,6 @@ static int grow_buffers(struct stripe_head *sh, int num, int b_size, int priorit
 		if (!bh)
 			return 1;
 		memset(bh, 0, sizeof (struct buffer_head));
-		init_waitqueue_head(&bh->b_wait);
 		if ((page = alloc_page(priority)))
 			bh->b_data = page_address(page);
 		else {
diff --git a/fs/buffer.c b/fs/buffer.c
index 52abaa214a5e..273ea1b3e54b 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -19,6 +19,7 @@
  */
 
 #include <linux/config.h>
+#include <linux/kernel.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
@@ -31,6 +32,7 @@
 #include <linux/module.h>
 #include <linux/writeback.h>
 #include <linux/mempool.h>
+#include <linux/hash.h>
 #include <asm/bitops.h>
 
 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_inode_buffers)
@@ -39,6 +41,14 @@
 atomic_t buffermem_pages = ATOMIC_INIT(0);
 
 /*
+ * Hashed waitqueue_head's for wait_on_buffer()
+ */
+#define BH_WAIT_TABLE_ORDER	7
+static struct bh_wait_queue_head {
+	wait_queue_head_t wqh;
+} ____cacheline_aligned_in_smp bh_wait_queue_heads[1<<BH_WAIT_TABLE_ORDER];
+
+/*
  * Several of these buffer list functions are exported to filesystems,
  * so we do funny things with the spinlocking to support those
  * filesystems while still using inode->i_bufferlist_lock for
@@ -76,6 +86,35 @@ init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
 	bh->b_private = private;
 }
 
+/*
+ * Return the address of the waitqueue_head to be used for this
+ * buffer_head
+ */
+static wait_queue_head_t *bh_waitq_head(struct buffer_head *bh)
+{
+	return &bh_wait_queue_heads[hash_ptr(bh, BH_WAIT_TABLE_ORDER)].wqh;
+}
+
+/*
+ * Wait on a buffer until someone does a wakeup on it.  Needs
+ * lots of external locking.  ext3 uses this.  Fix it.
+ */
+void sleep_on_buffer(struct buffer_head *bh)
+{
+	wait_queue_head_t *wq = bh_waitq_head(bh);
+	sleep_on(wq);
+}
+EXPORT_SYMBOL(sleep_on_buffer);
+
+void wake_up_buffer(struct buffer_head *bh)
+{
+	wait_queue_head_t *wq = bh_waitq_head(bh);
+
+	if (waitqueue_active(wq))
+		wake_up_all(wq);
+}
+EXPORT_SYMBOL(wake_up_buffer);
+
 void unlock_buffer(struct buffer_head *bh)
 {
 	/*
@@ -89,8 +128,32 @@ void unlock_buffer(struct buffer_head *bh)
 
 	clear_buffer_locked(bh);
 	smp_mb__after_clear_bit();
-	if (waitqueue_active(&bh->b_wait))
-		wake_up(&bh->b_wait);
+	wake_up_buffer(bh);
+}
+
+/*
+ * Block until a buffer comes unlocked.  This doesn't stop it
+ * from becoming locked again - you have to lock it yourself
+ * if you want to preserve its state.
+ */
+void __wait_on_buffer(struct buffer_head * bh)
+{
+	wait_queue_head_t *wq = bh_waitq_head(bh);
+	struct task_struct *tsk = current;
+	DECLARE_WAITQUEUE(wait, tsk);
+
+	get_bh(bh);
+	add_wait_queue(wq, &wait);
+	do {
+		run_task_queue(&tq_disk);
+		set_task_state(tsk, TASK_UNINTERRUPTIBLE);
+		if (!buffer_locked(bh))
+			break;
+		schedule();
+	} while (buffer_locked(bh));
+	tsk->state = TASK_RUNNING;
+	remove_wait_queue(wq, &wait);
+	put_bh(bh);
 }
 
 static inline void
@@ -122,30 +185,6 @@ __clear_page_buffers(struct page *page)
 }
 
 /*
- * Block until a buffer comes unlocked.  This doesn't stop it
- * from becoming locked again - you have to lock it yourself
- * if you want to preserve its state.
- */
-void __wait_on_buffer(struct buffer_head * bh)
-{
-	struct task_struct *tsk = current;
-	DECLARE_WAITQUEUE(wait, tsk);
-
-	get_bh(bh);
-	add_wait_queue(&bh->b_wait, &wait);
-	do {
-		run_task_queue(&tq_disk);
-		set_task_state(tsk, TASK_UNINTERRUPTIBLE);
-		if (!buffer_locked(bh))
-			break;
-		schedule();
-	} while (buffer_locked(bh));
-	tsk->state = TASK_RUNNING;
-	remove_wait_queue(&bh->b_wait, &wait);
-	put_bh(bh);
-}
-
-/*
  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
  * unlock the buffer. This is what ll_rw_block uses too.
  */
@@ -2265,7 +2304,6 @@ static void init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long fla
 		memset(bh, 0, sizeof(*bh));
 		bh->b_blocknr = -1;
 		INIT_LIST_HEAD(&bh->b_inode_buffers);
-		init_waitqueue_head(&bh->b_wait);
 	}
 }
 
@@ -2284,9 +2322,13 @@ static void bh_mempool_free(void *element, void *pool_data)
 
 void __init buffer_init(void)
 {
+	int i;
+
 	bh_cachep = kmem_cache_create("buffer_head",
 			sizeof(struct buffer_head), 0,
 			SLAB_HWCACHE_ALIGN, init_buffer_head, NULL);
 	bh_mempool = mempool_create(MAX_UNUSED_BUFFERS, bh_mempool_alloc,
 				bh_mempool_free, NULL);
+	for (i = 0; i < ARRAY_SIZE(bh_wait_queue_heads); i++)
+		init_waitqueue_head(&bh_wait_queue_heads[i].wqh);
 }
diff --git a/fs/jbd/commit.c b/fs/jbd/commit.c
index 1e8307671dc9..3fb33ea0f3d5 100644
--- a/fs/jbd/commit.c
+++ b/fs/jbd/commit.c
@@ -530,7 +530,7 @@ start_journal_io:
 		journal_file_buffer(jh, commit_transaction, BJ_Forget);
 		/* Wake up any transactions which were waiting for this
 		   IO to complete */
-		wake_up(&bh->b_wait);
+		wake_up_buffer(bh);
 		JBUFFER_TRACE(jh, "brelse shadowed buffer");
 		__brelse(bh);
 	}
diff --git a/fs/jbd/transaction.c b/fs/jbd/transaction.c
index 64b5b0434bfa..d0a1518729e5 100644
--- a/fs/jbd/transaction.c
+++ b/fs/jbd/transaction.c
@@ -654,7 +654,7 @@ repeat:
 			spin_unlock(&journal_datalist_lock);
 			unlock_journal(journal);
 			/* commit wakes up all shadow buffers after IO */
-			sleep_on(&jh2bh(jh)->b_wait);
+			sleep_on_buffer(jh2bh(jh));
 			lock_journal(journal);
 			goto repeat;
 		}
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index daf9b1c6d7d1..20a586b730ca 100644
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -56,9 +56,6 @@ struct buffer_head {
 	char * b_data;			/* pointer to data block */
 	bh_end_io_t *b_end_io;		/* I/O completion */
  	void *b_private;		/* reserved for b_end_io */
-
-	wait_queue_head_t b_wait;
-
 	struct list_head     b_inode_buffers; /* list of inode dirty buffers */
 };
 
@@ -166,6 +163,8 @@ void invalidate_bdev(struct block_device *, int);
 void __invalidate_buffers(kdev_t dev, int);
 int sync_buffers(struct block_device *, int);
 void __wait_on_buffer(struct buffer_head *);
+void sleep_on_buffer(struct buffer_head *bh);
+void wake_up_buffer(struct buffer_head *bh);
 int fsync_dev(kdev_t);
 int fsync_bdev(struct block_device *);
 int fsync_super(struct super_block *);