[ARM] Optimise set_pmd

Since we store two hardware pte tables contiguously, a common
operation is to set two pmd entries.  Rather than call set_pmd()
with the associated overhead twice, we set the two PMD entries,
and then call cpu_flush_pmd() to perform any CPU specific handling.

2 files changed, 16 insertions, 15 deletions

diff --git a/include/asm-arm/proc-armv/pgalloc.h b/include/asm-arm/proc-armv/pgalloc.h
index 53e760417601..4440be79d5ac 100644
--- a/include/asm-arm/proc-armv/pgalloc.h
+++ b/include/asm-arm/proc-armv/pgalloc.h
@@ -96,7 +96,7 @@ static inline void
 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 {
 	unsigned long pte_ptr = (unsigned long)ptep;
-	pmd_t pmd;
+	unsigned long pmdval;
 
 	BUG_ON(mm != &init_mm);
 
@@ -105,21 +105,21 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 	 * address of the PTE table
 	 */
 	pte_ptr -= PTRS_PER_PTE * sizeof(void *);
-	pmd_val(pmd) = __pa(pte_ptr) | _PAGE_KERNEL_TABLE;
-	set_pmd(pmdp, pmd);
-	pmd_val(pmd) += 256 * sizeof(pte_t);
-	set_pmd(pmdp + 1, pmd);
+	pmdval = __pa(pte_ptr) | _PAGE_KERNEL_TABLE;
+	pmdp[0] = __pmd(pmdval);
+	pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
+	cpu_flush_pmd(pmdp);
 }
 
 static inline void
 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
 {
-	pmd_t pmd;
+	unsigned long pmdval;
 
 	BUG_ON(mm == &init_mm);
 
-	pmd_val(pmd) = __pa(page_address(ptep)) | _PAGE_USER_TABLE;
-	set_pmd(pmdp, pmd);
-	pmd_val(pmd) += 256 * sizeof(pte_t);
-	set_pmd(pmdp + 1, pmd);
+	pmdval = page_to_pfn(ptep) << PAGE_SHIFT | _PAGE_USER_TABLE;
+	pmdp[0] = __pmd(pmdval);
+	pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
+	cpu_flush_pmd(pmdp);
 }
diff --git a/include/asm-arm/proc-armv/pgtable.h b/include/asm-arm/proc-armv/pgtable.h
index ea268a27e9a5..34c898876c5a 100644
--- a/include/asm-arm/proc-armv/pgtable.h
+++ b/include/asm-arm/proc-armv/pgtable.h
@@ -115,16 +115,17 @@
 
 #include <asm/proc/domain.h>
 
-#define _PAGE_USER_TABLE	(PMD_TYPE_TABLE | PMD_DOMAIN(DOMAIN_USER))
-#define _PAGE_KERNEL_TABLE	(PMD_TYPE_TABLE | PMD_DOMAIN(DOMAIN_KERNEL))
+#define _PAGE_USER_TABLE	(PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
+#define _PAGE_KERNEL_TABLE	(PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
 
 #define pmd_bad(pmd)		(pmd_val(pmd) & 2)
-#define set_pmd(pmdp,pmd)	cpu_set_pmd(pmdp, pmd)
+#define set_pmd(pmdp,pmd)	do { *pmdp = pmd; cpu_flush_pmd(pmdp); } while (0)
 
 static inline void pmd_clear(pmd_t *pmdp)
 {
-	set_pmd(pmdp, __pmd(0));
-	set_pmd(pmdp + 1, __pmd(0));
+	pmdp[0], __pmd(0));
+	pmdp[1], __pmd(0));
+	cpu_flush_pmd(pmdp);
 }
 
 static inline pte_t *pmd_page_kernel(pmd_t pmd)