[PATCH] fix swapcache packing in the radix tree

First some terminology: this patch introduces a kernel-wide `pgoff_t'
type.  It is the index of a page into the pagecache.  The thing at
page->index.  For most mappings it is also the offset of the page into
that mapping.  This type has a very distinct function in the kernel and
it needs a name.  I don't have any particular plans to go and migrate
everything so we can support 64-bit pagecache indices on x86, but this
would be the way to do it.

This patch improves the packing density of swapcache pages in the radix
tree.

A swapcache page is identified by the `swap type' (indexes the swap
device) and the `offset' (into that swap device).  These two numbers
are encoded into a `swp_entry_t' machine word in arch-specific code
because the resulting number is placed into pagetables in a form which
will generate a fault.

The kernel also need to generate a pgoff_t for that page to index it
into the swapper_space radix tree.  That pgoff_t is usually
bitwise-identical to the swp_entry_t.  That worked OK when the
pagecache was using a hash.  But with a radix tree, it produces
catastrophically bad results.

x86 (and many other architectures) place the `type' field into the
low-order bits of the swp_entry_t.  So *all* swapcache pages are
basically identical in the eight low-order bits.  This produces a very
sparse radix tree for swapcache.  I'm observing packing densities of 1%
to 2%: so the typical 128-slot radix tree node has only one or two
pages in it.

The end result is that the kernel needs to allocate approximately one
new radix-tree node for each page which is added to the swapcache.  So
no wonder we're having radix-tree node exhaustion during swapout!
(It's actually quite encouraging that the kernel works as well as it
does).

The patch changes the encoding of the swp_entry_t so that its
most-significant bits contain the `type' field and the
least-significant bits contain the `offset' field, right-aligned.

That is: the encoding in swp_entry_t is now arch-independent.  The new
file <linux/swapops.h> has conversion functions which convert the
swp_entry_t to and from its machine pte representation.

Packing density in the swapper_space mapping goes up to around 90%
(observed) and the kernel is tons happier under swap load.


An alternative approach would be to create new conversion functions
which convert an arch-specific swp_entry_t to and from a pgoff_t.  I
tried that.  It worked, but I liked it less.

20 files changed, 176 insertions, 92 deletions

diff --git a/include/asm-alpha/pgtable.h b/include/asm-alpha/pgtable.h
index ac1ecc3f9415..a972181fd540 100644
--- a/include/asm-alpha/pgtable.h
+++ b/include/asm-alpha/pgtable.h
@@ -340,11 +340,11 @@ extern inline void update_mmu_cache(struct vm_area_struct * vma,
 extern inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
 { pte_t pte; pte_val(pte) = (type << 32) | (offset << 40); return pte; }
 
-#define SWP_TYPE(x)			(((x).val >> 32) & 0xff)
-#define SWP_OFFSET(x)			((x).val >> 40)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)			(((x).val >> 32) & 0xff)
+#define __swp_offset(x)			((x).val >> 40)
+#define __swp_entry(type, offset)	((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #ifndef CONFIG_DISCONTIGMEM
 #define kern_addr_valid(addr)	(1)
diff --git a/include/asm-arm/pgtable.h b/include/asm-arm/pgtable.h
index dbbb85bd3995..62456af5582a 100644
--- a/include/asm-arm/pgtable.h
+++ b/include/asm-arm/pgtable.h
@@ -142,11 +142,11 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
  *
  * We support up to 32GB of swap on 4k machines
  */
-#define SWP_TYPE(x)		(((x).val >> 2) & 0x7f)
-#define SWP_OFFSET(x)		((x).val >> 9)
-#define SWP_ENTRY(type,offset)	((swp_entry_t) { ((type) << 2) | ((offset) << 9) })
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(swp)	((pte_t) { (swp).val })
+#define __swp_type(x)		(((x).val >> 2) & 0x7f)
+#define __swp_offset(x)		((x).val >> 9)
+#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << 2) | ((offset) << 9) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(swp)	((pte_t) { (swp).val })
 
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
 /* FIXME: this is not correct */
diff --git a/include/asm-cris/pgtable.h b/include/asm-cris/pgtable.h
index 6aa04ba4d4b1..57e391330752 100644
--- a/include/asm-cris/pgtable.h
+++ b/include/asm-cris/pgtable.h
@@ -500,11 +500,11 @@ static inline void update_mmu_cache(struct vm_area_struct * vma,
 /* Encode and de-code a swap entry (must be !pte_none(e) && !pte_present(e)) */
 /* Since the PAGE_PRESENT bit is bit 4, we can use the bits above */
 
-#define SWP_TYPE(x)                     (((x).val >> 5) & 0x7f)
-#define SWP_OFFSET(x)                   ((x).val >> 12)
-#define SWP_ENTRY(type, offset)         ((swp_entry_t) { ((type) << 5) | ((offset) << 12) })
-#define pte_to_swp_entry(pte)           ((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)             ((pte_t) { (x).val })
+#define __swp_type(x)			(((x).val >> 5) & 0x7f)
+#define __swp_offset(x)			((x).val >> 12)
+#define __swp_entry(type, offset)	((swp_entry_t) { ((type) << 5) | ((offset) << 12) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #define kern_addr_valid(addr)   (1)
 
diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index 645005ac5cfb..f48db2beeeba 100644
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -269,11 +269,11 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 #define update_mmu_cache(vma,address,pte) do { } while (0)
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE(x)			(((x).val >> 1) & 0x3f)
-#define SWP_OFFSET(x)			((x).val >> 8)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { (pte).pte_low })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)			(((x).val >> 1) & 0x3f)
+#define __swp_offset(x)			((x).val >> 8)
+#define __swp_entry(type, offset)	((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { (pte).pte_low })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h
index 7ccd4d9e6994..cc8c0a29b335 100644
--- a/include/asm-ia64/pgtable.h
+++ b/include/asm-ia64/pgtable.h
@@ -402,11 +402,11 @@ pte_same (pte_t a, pte_t b)
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 extern void paging_init (void);
 
-#define SWP_TYPE(entry)			(((entry).val >> 1) & 0xff)
-#define SWP_OFFSET(entry)		(((entry).val << 1) >> 10)
-#define SWP_ENTRY(type,offset)		((swp_entry_t) { ((type) << 1) | ((long) (offset) << 9) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(entry)		(((entry).val >> 1) & 0xff)
+#define __swp_offset(entry)		(((entry).val << 1) >> 10)
+#define __swp_entry(type,offset)	((swp_entry_t) { ((type) << 1) | ((long) (offset) << 9) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #define io_remap_page_range remap_page_range	/* XXX is this right? */
 
diff --git a/include/asm-m68k/pgtable.h b/include/asm-m68k/pgtable.h
index 6fb8eb88612f..597f08991de3 100644
--- a/include/asm-m68k/pgtable.h
+++ b/include/asm-m68k/pgtable.h
@@ -145,20 +145,20 @@ extern inline void update_mmu_cache(struct vm_area_struct * vma,
 
 #ifdef CONFIG_SUN3
 /* Macros to (de)construct the fake PTEs representing swap pages. */
-#define SWP_TYPE(x)                ((x).val & 0x7F)
-#define SWP_OFFSET(x)      (((x).val) >> 7)
-#define SWP_ENTRY(type,offset) ((swp_entry_t) { ((type) | ((offset) << 7)) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)		((x).val & 0x7F)
+#define __swp_offset(x)		(((x).val) >> 7)
+#define __swp_entry(type,offset) ((swp_entry_t) { ((type) | ((offset) << 7)) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #else
 
 /* Encode and de-code a swap entry (must be !pte_none(e) && !pte_present(e)) */
-#define SWP_TYPE(x)			(((x).val >> 1) & 0xff)
-#define SWP_OFFSET(x)			((x).val >> 10)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { ((type) << 1) | ((offset) << 10) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)		(((x).val >> 1) & 0xff)
+#define __swp_offset(x)		((x).val >> 10)
+#define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 10) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #endif /* CONFIG_SUN3 */
 
diff --git a/include/asm-mips/pgtable.h b/include/asm-mips/pgtable.h
index 240b1327f412..bec6e22366e5 100644
--- a/include/asm-mips/pgtable.h
+++ b/include/asm-mips/pgtable.h
@@ -493,12 +493,11 @@ extern void paging_init(void);
 extern void update_mmu_cache(struct vm_area_struct *vma,
 				unsigned long address, pte_t pte);
 
-#define SWP_TYPE(x)		(((x).val >> 1) & 0x3f)
-#define SWP_OFFSET(x)		((x).val >> 8)
-#define SWP_ENTRY(type,offset)	((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)	((pte_t) { (x).val })
-
+#define __swp_type(x)		(((x).val >> 1) & 0x3f)
+#define __swp_offset(x)		((x).val >> 8)
+#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #define kern_addr_valid(addr)	(1)
 
diff --git a/include/asm-mips64/pgtable.h b/include/asm-mips64/pgtable.h
index b8258f5733b4..d3118912a2a5 100644
--- a/include/asm-mips64/pgtable.h
+++ b/include/asm-mips64/pgtable.h
@@ -553,11 +553,11 @@ extern void (*update_mmu_cache)(struct vm_area_struct *vma,
 extern inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
 { pte_t pte; pte_val(pte) = (type << 32) | (offset << 40); return pte; }
 
-#define SWP_TYPE(x)		(((x).val >> 32) & 0xff)
-#define SWP_OFFSET(x)		((x).val >> 40)
-#define SWP_ENTRY(type,offset)	((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)	((pte_t) { (x).val })
+#define __swp_type(x)		(((x).val >> 32) & 0xff)
+#define __swp_offset(x)		((x).val >> 40)
+#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #ifndef CONFIG_DISCONTIGMEM
 #define kern_addr_valid(addr)	(1)
diff --git a/include/asm-parisc/pgtable.h b/include/asm-parisc/pgtable.h
index 2448948197e7..c1d676cf92ea 100644
--- a/include/asm-parisc/pgtable.h
+++ b/include/asm-parisc/pgtable.h
@@ -312,14 +312,14 @@ extern inline void update_mmu_cache(struct vm_area_struct * vma,
 
 /* Encode and de-code a swap entry */
 
-#define SWP_TYPE(x)                     ((x).val & 0x3f)
-#define SWP_OFFSET(x)                   ( (((x).val >> 6) &  0x7) | \
+#define __swp_type(x)                     ((x).val & 0x3f)
+#define __swp_offset(x)                   ( (((x).val >> 6) &  0x7) | \
 					  (((x).val >> 7) & ~0x7) )
-#define SWP_ENTRY(type, offset)         ((swp_entry_t) { (type) | \
+#define __swp_entry(type, offset)         ((swp_entry_t) { (type) | \
 					    ((offset &  0x7) << 6) | \
 					    ((offset & ~0x7) << 7) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #define module_map	vmalloc
 #define module_unmap	vfree
diff --git a/include/asm-ppc/pgtable.h b/include/asm-ppc/pgtable.h
index a0514543259c..c5bfc1332e5f 100644
--- a/include/asm-ppc/pgtable.h
+++ b/include/asm-ppc/pgtable.h
@@ -482,11 +482,11 @@ extern void add_hash_page(unsigned context, unsigned long va,
  * must not include the _PAGE_PRESENT bit, or the _PAGE_HASHPTE bit
  * (if used).  -- paulus
  */
-#define SWP_TYPE(entry)			((entry).val & 0x3f)
-#define SWP_OFFSET(entry)		((entry).val >> 6)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { (type) | ((offset) << 6) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> 2 })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val << 2 })
+#define __swp_type(entry)		((entry).val & 0x3f)
+#define __swp_offset(entry)		((entry).val >> 6)
+#define __swp_entry(type, offset)	((swp_entry_t) { (type) | ((offset) << 6) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> 2 })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val << 2 })
 
 /* CONFIG_APUS */
 /* For virtual address to physical address conversion */
diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 48d9f43a3e49..e546e3f0f88c 100644
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -359,11 +359,11 @@ extern void paging_init(void);
 extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE(entry)			(((entry).val >> 1) & 0x3f)
-#define SWP_OFFSET(entry)		((entry).val >> 8)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> PTE_SHIFT })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val << PTE_SHIFT })
+#define __swp_type(entry)		(((entry).val >> 1) & 0x3f)
+#define __swp_offset(entry)		((entry).val >> 8)
+#define __swp_entry(type, offset)	((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> PTE_SHIFT })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val << PTE_SHIFT })
 
 /*
  * kern_addr_valid is intended to indicate whether an address is a valid
diff --git a/include/asm-s390/pgtable.h b/include/asm-s390/pgtable.h
index 6e156879dcaf..fa9993ad8e9d 100644
--- a/include/asm-s390/pgtable.h
+++ b/include/asm-s390/pgtable.h
@@ -485,12 +485,12 @@ extern inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
 	return pte;
 }
 
-#define SWP_TYPE(entry)		(((entry).val >> 1) & 0x3f)
-#define SWP_OFFSET(entry)	(((entry).val >> 12) & 0x7FFFF )
-#define SWP_ENTRY(type,offset)	((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
+#define __swp_type(entry)	(((entry).val >> 1) & 0x3f)
+#define __swp_offset(entry)	(((entry).val >> 12) & 0x7FFFF )
+#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
 
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)	((pte_t) { (x).val })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/include/asm-s390x/pgtable.h b/include/asm-s390x/pgtable.h
index f057679c0665..a2729aa46c35 100644
--- a/include/asm-s390x/pgtable.h
+++ b/include/asm-s390x/pgtable.h
@@ -505,12 +505,12 @@ extern inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
 	return pte;
 }
 
-#define SWP_TYPE(entry)		(((entry).val >> 1) & 0x3f)
-#define SWP_OFFSET(entry)	((entry).val >> 12)
-#define SWP_ENTRY(type,offset)	((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
+#define __swp_type(entry)	(((entry).val >> 1) & 0x3f)
+#define __swp_offset(entry)	((entry).val >> 12)
+#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
 
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)	((pte_t) { (x).val })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/include/asm-sh/pgtable.h b/include/asm-sh/pgtable.h
index 46d9a217e18a..74034ac9c862 100644
--- a/include/asm-sh/pgtable.h
+++ b/include/asm-sh/pgtable.h
@@ -294,11 +294,11 @@ extern void update_mmu_cache(struct vm_area_struct * vma,
  * NOTE: We should set ZEROs at the position of _PAGE_PRESENT
  *       and _PAGE_PROTONOE bits
  */
-#define SWP_TYPE(x)		((x).val & 0xff)
-#define SWP_OFFSET(x)		((x).val >> 10)
-#define SWP_ENTRY(type, offset)	((swp_entry_t) { (type) | ((offset) << 10) })
-#define pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)	((pte_t) { (x).val })
+#define __swp_type(x)		((x).val & 0xff)
+#define __swp_offset(x)		((x).val >> 10)
+#define __swp_entry(type, offset) ((swp_entry_t) { (type) | ((offset) << 10) })
+#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 
 /*
  * Routines for update of PTE 
diff --git a/include/asm-sparc/pgtable.h b/include/asm-sparc/pgtable.h
index b73aaa20847e..775e8483859a 100644
--- a/include/asm-sparc/pgtable.h
+++ b/include/asm-sparc/pgtable.h
@@ -376,11 +376,11 @@ BTFIXUPDEF_CALL(void, update_mmu_cache, struct vm_area_struct *, unsigned long,
 extern int invalid_segment;
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE(x)			(((x).val >> 2) & 0x7f)
-#define SWP_OFFSET(x)			(((x).val >> 9) & 0x3ffff)
-#define SWP_ENTRY(type,offset)		((swp_entry_t) { (((type) & 0x7f) << 2) | (((offset) & 0x3ffff) << 9) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)			(((x).val >> 2) & 0x7f)
+#define __swp_offset(x)			(((x).val >> 9) & 0x3ffff)
+#define __swp_entry(type,offset)	((swp_entry_t) { (((type) & 0x7f) << 2) | (((offset) & 0x3ffff) << 9) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 struct ctx_list {
 	struct ctx_list *next;
diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index 73d5951edfb9..e5c086958155 100644
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -298,16 +298,16 @@ extern inline pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space)
 }
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE(entry)		(((entry).val >> PAGE_SHIFT) & 0xffUL)
-#define SWP_OFFSET(entry)	((entry).val >> (PAGE_SHIFT + 8UL))
-#define SWP_ENTRY(type, offset)	\
+#define __swp_type(entry)	(((entry).val >> PAGE_SHIFT) & 0xffUL)
+#define __swp_offset(entry)	((entry).val >> (PAGE_SHIFT + 8UL))
+#define __swp_entry(type, offset)	\
 	( (swp_entry_t) \
 	  { \
 		(((long)(type) << PAGE_SHIFT) | \
                  ((long)(offset) << (PAGE_SHIFT + 8UL))) \
 	  } )
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 extern unsigned long prom_virt_to_phys(unsigned long, int *);
 
diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h
index 4c151605c479..09d3254fa9a7 100644
--- a/include/asm-x86_64/pgtable.h
+++ b/include/asm-x86_64/pgtable.h
@@ -329,11 +329,11 @@ extern inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 #define update_mmu_cache(vma,address,pte) do { } while (0)
 
 /* Encode and de-code a swap entry */
-#define SWP_TYPE(x)			(((x).val >> 1) & 0x3f)
-#define SWP_OFFSET(x)			((x).val >> 8)
-#define SWP_ENTRY(type, offset)		((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
-#define pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
-#define swp_entry_to_pte(x)		((pte_t) { (x).val })
+#define __swp_type(x)			(((x).val >> 1) & 0x3f)
+#define __swp_offset(x)			((x).val >> 8)
+#define __swp_entry(type, offset)	((swp_entry_t) { ((type) << 1) | ((offset) << 8) })
+#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
+#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 7e20b3016c7f..d0160265e3c5 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -11,7 +11,16 @@
 #define SWAP_FLAG_PRIO_MASK	0x7fff
 #define SWAP_FLAG_PRIO_SHIFT	0
 
-#define MAX_SWAPFILES 32
+/*
+ * MAX_SWAPFILES defines the maximum number of swaptypes: things which can
+ * be swapped to.  The swap type and the offset into that swap type are
+ * encoded into pte's and into pgoff_t's in the swapcache.  Using five bits
+ * for the type means that the maximum number of swapcache pages is 27 bits
+ * on 32-bit-pgoff_t architectures.  And that assumes that the architecture packs
+ * the type/offset into the pte as 5/27 as well.
+ */
+#define MAX_SWAPFILES_SHIFT	5
+#define MAX_SWAPFILES		(1 << MAX_SWAPFILES_SHIFT)
 
 /*
  * Magic header for a swap area. The first part of the union is
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
new file mode 100644
index 000000000000..500f9960d939
--- /dev/null
+++ b/include/linux/swapops.h
@@ -0,0 +1,68 @@
+/*
+ * swapcache pages are stored in the swapper_space radix tree.  We want to
+ * get good packing density in that tree, so the index should be dense in
+ * the low-order bits.
+ *
+ * We arrange the `type' and `offset' fields so that `type' is at the five
+ * high-order bits of the smp_entry_t and `offset' is right-aligned in the
+ * remaining bits.
+ *
+ * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
+ */
+#define SWP_TYPE_SHIFT(e)	(sizeof(e.val) * 8 - MAX_SWAPFILES_SHIFT)
+#define SWP_OFFSET_MASK(e)	((1 << SWP_TYPE_SHIFT(e)) - 1)
+
+/*
+ * Store a type+offset into a swp_entry_t in an arch-independent format
+ */
+static inline swp_entry_t swp_entry(unsigned type, pgoff_t offset)
+{
+	swp_entry_t ret;
+
+	ret.val = (type << SWP_TYPE_SHIFT(ret)) |
+			(offset & SWP_OFFSET_MASK(ret));
+	return ret;
+}
+
+/*
+ * Extract the `type' field from a swp_entry_t.  The swp_entry_t is in
+ * arch-independent format
+ */
+static inline unsigned swp_type(swp_entry_t entry)
+{
+	return (entry.val >> SWP_TYPE_SHIFT(entry)) &
+			((1 << MAX_SWAPFILES_SHIFT) - 1);
+}
+
+/*
+ * Extract the `offset' field from a swp_entry_t.  The swp_entry_t is in
+ * arch-independent format
+ */
+static inline pgoff_t swp_offset(swp_entry_t entry)
+{
+	return entry.val & SWP_OFFSET_MASK(entry);
+}
+
+/*
+ * Convert the arch-dependent pte representation of a swp_entry_t into an
+ * arch-independent swp_entry_t.
+ */
+static inline swp_entry_t pte_to_swp_entry(pte_t pte)
+{
+	swp_entry_t arch_entry;
+
+	arch_entry = __pte_to_swp_entry(pte);
+	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
+}
+
+/*
+ * Convert the arch-independent representation of a swp_entry_t into the
+ * arch-dependent pte representation.
+ */
+static inline pte_t swp_entry_to_pte(swp_entry_t entry)
+{
+	swp_entry_t arch_entry;
+
+	arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
+	return __swp_entry_to_pte(arch_entry);
+}
diff --git a/include/linux/types.h b/include/linux/types.h
index 211461bc97c0..c102bcf8be83 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -124,6 +124,14 @@ typedef u64 sector_t;
 typedef unsigned long sector_t;
 #endif
 
+/*
+ * The type of an index into the pagecache.  Use a #define so asm/types.h
+ * can override it.
+ */
+#ifndef pgoff_t
+#define pgoff_t unsigned long
+#endif
+
 #endif /* __KERNEL_STRICT_NAMES */
 
 /*