[ARM] Inline PMD entry cache handling

The common case is building a kernel for one CPU type, and we are
able to allow GCC to optimise any the PMD entry cache handling
assembly which will never be used.

16 files changed, 57 insertions, 142 deletions

diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c
index e57c2ccc10cd..b8312f498f63 100644
--- a/arch/arm/mach-sa1100/assabet.c
+++ b/arch/arm/mach-sa1100/assabet.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/tty.h>
 #include <linux/module.h>
+#include <linux/mm.h>
 #include <linux/errno.h>
 #include <linux/serial_core.h>
 #include <linux/delay.h>
@@ -25,6 +26,7 @@
 #include <asm/setup.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
+#include <asm/tlbflush.h>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
diff --git a/arch/arm/mm/mm-armv.c b/arch/arm/mm/mm-armv.c
index fc7ea269281d..3017f7b1481f 100644
--- a/arch/arm/mm/mm-armv.c
+++ b/arch/arm/mm/mm-armv.c
@@ -247,7 +247,7 @@ alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pg
 		pmdval = __pa(ptep) | prot_l1;
 		pmdp[0] = __pmd(pmdval);
 		pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
-		cpu_flush_pmd(pmdp);
+		flush_pmd_entry(pmdp);
 	}
 	ptep = pte_offset_kernel(pmdp, virt);
 
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index adadfd9fb410..046d5ca42998 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -365,23 +365,6 @@ ENTRY(cpu_arm1020_set_pgd)
 	mov	pc, lr
         
 /*
- * cpu_arm1020_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_arm1020_flush_pmd)
-#ifndef CONFIG_CPU_DCACHE_DISABLE
-	mcr	p15, 0, r0, c7, c10, 4
-	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry  (drain is done by TLB fns)
-#endif
-	mcr	p15, 0, r0, c7, c10, 4		@ drain WB
-	mov	pc, lr
-
-/*
  * cpu_arm1020_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -511,7 +494,6 @@ arm1020_processor_functions:
 
 	/* pgtable */
 	.word	cpu_arm1020_set_pgd
-	.word	cpu_arm1020_flush_pmd
 	.word	cpu_arm1020_set_pte
 
 	.size	arm1020_processor_functions, . - arm1020_processor_functions
diff --git a/arch/arm/mm/proc-arm6_7.S b/arch/arm/mm/proc-arm6_7.S
index 76b6eed5366c..ef62add1cda6 100644
--- a/arch/arm/mm/proc-arm6_7.S
+++ b/arch/arm/mm/proc-arm6_7.S
@@ -234,17 +234,6 @@ ENTRY(cpu_arm7_set_pgd)
 		mov	pc, lr
 
 /*
- * Function: arm6_flush_pmd(pmdp)
- *
- * Params  : r0 = Address to set
- *
- * Purpose : Set a PMD and flush it out of any WB cache
- */
-ENTRY(cpu_arm6_flush_pmd)
-ENTRY(cpu_arm7_flush_pmd)
-		mov	pc, lr
-
-/*
  * Function: arm6_7_set_pte(pte_t *ptep, pte_t pte)
  * Params  : r0 = Address to set
  *	   : r1 = value to set
@@ -346,7 +335,6 @@ ENTRY(arm6_processor_functions)
 
 		/* pgtable */
 		.word	cpu_arm6_set_pgd
-		.word	cpu_arm6_flush_pmd
 		.word	cpu_arm6_set_pte
 
 		.size	arm6_processor_functions, . - arm6_processor_functions
@@ -380,7 +368,6 @@ ENTRY(arm7_processor_functions)
 
 		/* pgtable */
 		.word	cpu_arm7_set_pgd
-		.word	cpu_arm7_flush_pmd
 		.word	cpu_arm7_set_pte
 
 		.size	arm7_processor_functions, . - arm7_processor_functions
diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S
index 391dc601d84d..5fd31b10f2bb 100644
--- a/arch/arm/mm/proc-arm720.S
+++ b/arch/arm/mm/proc-arm720.S
@@ -115,16 +115,6 @@ ENTRY(cpu_arm720_set_pgd)
 		mov	pc, lr
 
 /*
- * Function: arm720_flush_pmd(pmdp)
- *
- * Params  : r0 = Address to set
- *
- * Purpose : Set a PMD and flush it out of any WB cache
- */
-ENTRY(cpu_arm720_flush_pmd)
-		mov	pc, lr
-
-/*
  * Function: arm720_set_pte(pte_t *ptep, pte_t pte)
  * Params  : r0 = Address to set
  *	   : r1 = value to set
@@ -216,7 +206,6 @@ ENTRY(arm720_processor_functions)
 
 		/* pgtable */
 		.word	cpu_arm720_set_pgd
-		.word	cpu_arm720_flush_pmd
 		.word	cpu_arm720_set_pte
 
 		.size	arm720_processor_functions, . - arm720_processor_functions
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index cbeaaf9a0853..2b6baae17ade 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -368,20 +368,6 @@ ENTRY(cpu_arm920_set_pgd)
 	mov	pc, lr
 
 /*
- * cpu_arm920_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_arm920_flush_pmd)
-	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry
-	mcr	p15, 0, r0, c7, c10, 4		@ drain WB
-	mov	pc, lr
-
-/*
  * cpu_arm920_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -499,7 +485,6 @@ arm920_processor_functions:
 
 	/* pgtable */
 	.word	cpu_arm920_set_pgd
-	.word	cpu_arm920_flush_pmd
 	.word	cpu_arm920_set_pte
 
 	.size	arm920_processor_functions, . - arm920_processor_functions
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index f115e476bc7b..a4cc0be63b63 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -369,20 +369,6 @@ ENTRY(cpu_arm922_set_pgd)
 	mov	pc, lr
 
 /*
- * cpu_arm922_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_arm922_flush_pmd)
-	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry
-	mcr	p15, 0, r0, c7, c10, 4		@ drain WB
-	mov	pc, lr
-
-/*
  * cpu_arm922_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -498,7 +484,6 @@ arm922_processor_functions:
 
 	/* pgtable */
 	.word	cpu_arm922_set_pgd
-	.word	cpu_arm922_flush_pmd
 	.word	cpu_arm922_set_pte
 
 	.size	arm922_processor_functions, . - arm922_processor_functions
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index a2cdcdd3d9f3..fdaf694e8151 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -350,22 +350,6 @@ ENTRY(cpu_arm926_set_pgd)
 	mov	pc, lr
 
 /*
- * cpu_arm926_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_arm926_flush_pmd)
-#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
-	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry
-#endif
-	mcr	p15, 0, r0, c7, c10, 4		@ drain WB
-	mov	pc, lr
-
-/*
  * cpu_arm926_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -496,7 +480,6 @@ arm926_processor_functions:
 
 	/* pgtable */
 	.word	cpu_arm926_set_pgd
-	.word	cpu_arm926_flush_pmd
 	.word	cpu_arm926_set_pte
 
 	.size	arm926_processor_functions, . - arm926_processor_functions
diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S
index 7242fe7bfe70..ffc12970010b 100644
--- a/arch/arm/mm/proc-sa110.S
+++ b/arch/arm/mm/proc-sa110.S
@@ -425,21 +425,6 @@ ENTRY(cpu_sa1100_set_pgd)
 	mov	pc, lr
 
 /*
- * cpu_sa110_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_sa110_flush_pmd)
-ENTRY(cpu_sa1100_flush_pmd)
-	mcr	p15, 0, r0, c7, c10, 1		@ clean D entry
-	mcr	p15, 0, r0, c7, c10, 4		@ drain WB
-	mov	pc, lr
-
-/*
  * cpu_sa110_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -540,7 +525,6 @@ ENTRY(sa110_processor_functions)
 
 	/* pgtable */
 	.word	cpu_sa110_set_pgd
-	.word	cpu_sa110_flush_pmd
 	.word	cpu_sa110_set_pte
 
 	.size	sa110_processor_functions, . - sa110_processor_functions
@@ -573,7 +557,6 @@ ENTRY(sa1100_processor_functions)
 
 	/* pgtable */
 	.word	cpu_sa1100_set_pgd
-	.word	cpu_sa1100_flush_pmd
 	.word	cpu_sa1100_set_pte
 
 	.size	sa1100_processor_functions, . - sa1100_processor_functions
diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c
index 47c5448e619f..697fb05c70c4 100644
--- a/arch/arm/mm/proc-syms.c
+++ b/arch/arm/mm/proc-syms.c
@@ -27,7 +27,6 @@ EXPORT_SYMBOL(cpu_dcache_invalidate_range);
 EXPORT_SYMBOL(cpu_icache_invalidate_range);
 EXPORT_SYMBOL(cpu_icache_invalidate_page);
 EXPORT_SYMBOL(cpu_set_pgd);
-EXPORT_SYMBOL(cpu_flush_pmd);
 EXPORT_SYMBOL(cpu_set_pte);
 #else
 EXPORT_SYMBOL(processor);
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index ba13ebc13e2e..71855af07d41 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -573,21 +573,6 @@ ENTRY(cpu_xscale_set_pgd)
 	cpwait_ret lr, ip
 
 /*
- * cpu_xscale_flush_pmd(pmdp)
- *
- * Set a level 1 translation table entry, and clean it out of
- * any caches such that the MMUs can load it correctly.
- *
- * pmdp: pointer to PMD entry
- */
-	.align	5
-ENTRY(cpu_xscale_flush_pmd)
-	mov	ip, #0
-	mcr	p15, 0, r0, c7, c10, 1		@ Clean D cache line
-	mcr	p15, 0, ip, c7, c10, 4		@ Drain Write (& Fill) Buffer
-	mov	pc, lr
-
-/*
  * cpu_xscale_set_pte(ptep, pte)
  *
  * Set a PTE and flush it out
@@ -719,7 +704,6 @@ ENTRY(xscale_processor_functions)
 
 	/* pgtable */
 	.word	cpu_xscale_set_pgd
-	.word	cpu_xscale_flush_pmd
 	.word	cpu_xscale_set_pte
 	.size	xscale_processor_functions, . - xscale_processor_functions
 
diff --git a/include/asm-arm/cpu-multi32.h b/include/asm-arm/cpu-multi32.h
index 0ed88c812944..59835af6fa00 100644
--- a/include/asm-arm/cpu-multi32.h
+++ b/include/asm-arm/cpu-multi32.h
@@ -94,10 +94,6 @@ extern struct processor {
 		 */
 		void (*set_pgd)(unsigned long pgd_phys, struct mm_struct *mm);
 		/*
-		 * Set a PMD (handling IMP bit 4)
-		 */
-		void (*flush_pmd)(pmd_t *pmdp);
-		/*
 		 * Set a PTE
 		 */
 		void (*set_pte)(pte_t *ptep, pte_t pte);
@@ -126,7 +122,6 @@ extern const struct processor sa110_processor_functions;
 #define cpu_icache_invalidate_page(vp)		processor.icache.invalidate_page(vp)
 
 #define cpu_set_pgd(pgd,mm)			processor.pgtable.set_pgd(pgd,mm)
-#define cpu_flush_pmd(pmdp)			processor.pgtable.flush_pmd(pmdp)
 #define cpu_set_pte(ptep, pte)			processor.pgtable.set_pte(ptep, pte)
 
 #define cpu_switch_mm(pgd,mm)			cpu_set_pgd(__virt_to_phys((unsigned long)(pgd)),mm)
diff --git a/include/asm-arm/cpu-single.h b/include/asm-arm/cpu-single.h
index 843e3a03db58..738b61dc2553 100644
--- a/include/asm-arm/cpu-single.h
+++ b/include/asm-arm/cpu-single.h
@@ -36,7 +36,6 @@
 #define cpu_icache_invalidate_range	__cpu_fn(CPU_NAME,_icache_invalidate_range)
 #define cpu_icache_invalidate_page	__cpu_fn(CPU_NAME,_icache_invalidate_page)
 #define cpu_set_pgd			__cpu_fn(CPU_NAME,_set_pgd)
-#define cpu_flush_pmd			__cpu_fn(CPU_NAME,_flush_pmd)
 #define cpu_set_pte			__cpu_fn(CPU_NAME,_set_pte)
 
 #ifndef __ASSEMBLY__
@@ -65,7 +64,6 @@ extern void cpu_icache_invalidate_range(unsigned long start, unsigned long end);
 extern void cpu_icache_invalidate_page(void *virt_page);
 
 extern void cpu_set_pgd(unsigned long pgd_phys, struct mm_struct *mm);
-extern void cpu_flush_pmd(pmd_t *pmdp);
 extern void cpu_set_pte(pte_t *ptep, pte_t pte);
 
 extern volatile void cpu_reset(unsigned long addr);
diff --git a/include/asm-arm/proc-armv/pgalloc.h b/include/asm-arm/proc-armv/pgalloc.h
index 3263c346ccba..0e65ab7362e4 100644
--- a/include/asm-arm/proc-armv/pgalloc.h
+++ b/include/asm-arm/proc-armv/pgalloc.h
@@ -6,6 +6,7 @@
  * Page table allocation/freeing primitives for 32-bit ARM processors.
  */
 #include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
 #include "pgtable.h"
 
 /*
@@ -92,7 +93,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 	pmdval = __pa(pte_ptr) | _PAGE_KERNEL_TABLE;
 	pmdp[0] = __pmd(pmdval);
 	pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
-	cpu_flush_pmd(pmdp);
+	flush_pmd_entry(pmdp);
 }
 
 static inline void
@@ -105,5 +106,5 @@ pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
 	pmdval = page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE;
 	pmdp[0] = __pmd(pmdval);
 	pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
-	cpu_flush_pmd(pmdp);
+	flush_pmd_entry(pmdp);
 }
diff --git a/include/asm-arm/proc-armv/pgtable.h b/include/asm-arm/proc-armv/pgtable.h
index 53f2b3da4d16..616d80d69b1f 100644
--- a/include/asm-arm/proc-armv/pgtable.h
+++ b/include/asm-arm/proc-armv/pgtable.h
@@ -51,6 +51,7 @@
 #define PMD_SECT_TEX(x)		((x) << 12)	/* v5 */
 
 #define PMD_SECT_UNCACHED	(0)
+#define PMD_SECT_BUFFERED	(PMD_SECT_BUFFERABLE)
 #define PMD_SECT_WT		(PMD_SECT_CACHEABLE)
 #define PMD_SECT_WB		(PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE)
 #define PMD_SECT_MINICACHE	(PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE)
@@ -120,14 +121,19 @@
 #define _PAGE_KERNEL_TABLE	(PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
 
 #define pmd_bad(pmd)		(pmd_val(pmd) & 2)
-#define set_pmd(pmdp,pmd)	do { *pmdp = pmd; cpu_flush_pmd(pmdp); } while (0)
 
-static inline void pmd_clear(pmd_t *pmdp)
-{
-	pmdp[0] = __pmd(0);
-	pmdp[1] = __pmd(0);
-	cpu_flush_pmd(pmdp);
-}
+#define set_pmd(pmdp,pmd)		\
+	do {				\
+		*pmdp = pmd;		\
+		flush_pmd_entry(pmdp);	\
+	} while (0)
+
+#define pmd_clear(pmdp)			\
+	do {				\
+		pmdp[0] = __pmd(0);	\
+		pmdp[1] = __pmd(0);	\
+		clean_pmd_entry(pmdp);	\
+	} while (0)
 
 static inline pte_t *pmd_page_kernel(pmd_t pmd)
 {
diff --git a/include/asm-arm/proc-armv/tlbflush.h b/include/asm-arm/proc-armv/tlbflush.h
index d063ede9c50f..278c0624c11f 100644
--- a/include/asm-arm/proc-armv/tlbflush.h
+++ b/include/asm-arm/proc-armv/tlbflush.h
@@ -20,6 +20,7 @@
 #define TLB_V4_D_FULL	(1 << 10)
 #define TLB_V4_I_FULL	(1 << 11)
 
+#define TLB_DCLEAN	(1 << 30)
 #define TLB_WB		(1 << 31)
 
 /*
@@ -65,7 +66,7 @@
 # define v4_always_flags	(-1UL)
 #endif
 
-#define v4wbi_tlb_flags	(TLB_WB | \
+#define v4wbi_tlb_flags	(TLB_WB | TLB_DCLEAN | \
 			 TLB_V4_I_FULL | TLB_V4_D_FULL | \
 			 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
 
@@ -84,7 +85,7 @@
 # define v4wbi_always_flags	(-1UL)
 #endif
 
-#define v4wb_tlb_flags	(TLB_WB | \
+#define v4wb_tlb_flags	(TLB_WB | TLB_DCLEAN | \
 			 TLB_V4_I_FULL | TLB_V4_D_FULL | \
 			 TLB_V4_D_PAGE)
 
@@ -287,6 +288,41 @@ static inline void flush_tlb_kernel_page(unsigned long kaddr)
 		asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
 }
 
+/*
+ *	flush_pmd_entry
+ *
+ *	Flush a PMD entry (word aligned, or double-word aligned) to
+ *	RAM if the TLB for the CPU we are running on requires this.
+ *	This is typically used when we are creating PMD entries.
+ *
+ *	clean_pmd_entry
+ *
+ *	Clean (but don't drain the write buffer) if the CPU requires
+ *	these operations.  This is typically used when we are removing
+ *	PMD entries.
+ */
+static inline void flush_pmd_entry(pmd_t *pmd)
+{
+	const unsigned int zero = 0;
+	const unsigned int __tlb_flag = __cpu_tlb_flags;
+
+	if (tlb_flag(TLB_DCLEAN))
+		asm("mcr%?	p15, 0, %0, c7, c10, 1	@ flush_pmd"
+			: : "r" (pmd));
+	if (tlb_flag(TLB_WB))
+		asm("mcr%?	p15, 0, %0, c7, c10, 4	@ flush_pmd"
+			: : "r" (zero));
+}
+
+static inline void clean_pmd_entry(pmd_t *pmd)
+{
+	const unsigned int __tlb_flag = __cpu_tlb_flags;
+
+	if (tlb_flag(TLB_DCLEAN))
+		asm("mcr%?	p15, 0, %0, c7, c10, 1	@ flush_pmd"
+			: : "r" (pmd));
+}
+
 #undef tlb_flag
 #undef always_tlb_flags
 #undef possible_tlb_flags