ppc64: hashtable management rework for SMP scalability

get rid of global lock on hpte insert and remove, use a software bit
 for pSeries and the relevant interfaces for pSeries LPAR to avoid
 races.
preload hpte entries in update_mmu_cache.
keep cpu_vm_mask and use tlbiel when mm has only run locally
batch tlb flushes where possible.
add large page support in preparation for generic large page support.
Remove HPTENOIX, we always put slot information into linux ptes now.
Note: pSeries and pSeries LPAR so far, iSeries coming next.

3 files changed, 89 insertions, 66 deletions

diff --git a/include/asm-ppc64/machdep.h b/include/asm-ppc64/machdep.h
index 304670e2177b..8b66663f5345 100644
--- a/include/asm-ppc64/machdep.h
+++ b/include/asm-ppc64/machdep.h
@@ -24,26 +24,33 @@ struct machdep_calls {
 	/* High use functions in the first cachelines, low use functions
 	 * follow.  DRENG collect profile data.
 	 */
-	void            (*hpte_invalidate)(unsigned long slot);
-
-	void            (*hpte_updatepp)(long slot, 
+	void            (*hpte_invalidate)(unsigned long slot,
+					   unsigned long va,
+					   int large,
+					   int local);
+	long		(*hpte_updatepp)(unsigned long slot, 
 					 unsigned long newpp, 
-					 unsigned long va);
+					 unsigned long va,
+					 int large);
 	void            (*hpte_updateboltedpp)(unsigned long newpp, 
 					       unsigned long ea);
-	unsigned long	(*hpte_getword0)(unsigned long slot);
-
-	long		(*hpte_find)( unsigned long vpn );
-
-	long		(*hpte_selectslot)(unsigned long vpn); 
+	long		(*insert_hpte)(unsigned long hpte_group,
+				       unsigned long vpn,
+				       unsigned long prpn,
+				       int secondary, 
+				       unsigned long hpteflags, 
+				       int bolted,
+				       int large);
+	long		(*remove_hpte)(unsigned long hpte_group);
+	void		(*flush_hash_range)(unsigned long context,
+					    unsigned long number,
+					    int local);
+	void		(*make_pte)(void *htab, unsigned long va,
+				    unsigned long pa,
+				    int mode,
+				    unsigned long hash_mask,
+				    int large);
 
-	void		(*hpte_create_valid)(unsigned long slot,
-					     unsigned long vpn,
-					     unsigned long prpn,
-					     unsigned hash, 
-					     void * ptep,
-					     unsigned hpteflags, 
-					     unsigned bolted);
 	void		(*tce_build)(struct TceTable * tbl,
 				     long tcenum,
 				     unsigned long uaddr,
diff --git a/include/asm-ppc64/mmu.h b/include/asm-ppc64/mmu.h
index df830a68e927..a0e55d9d023a 100644
--- a/include/asm-ppc64/mmu.h
+++ b/include/asm-ppc64/mmu.h
@@ -77,7 +77,7 @@ typedef struct {
 	unsigned long resv0: 7; /* Padding to a 64b boundary */
 } slb_dword1;
 
-typedef struct _SLBE {
+typedef struct {
 	union {
 		unsigned long dword0;
 		slb_dword0    dw0;
@@ -107,26 +107,13 @@ typedef struct {
 	unsigned long avpn:57; /* vsid | api == avpn  */
 	unsigned long :     2; /* Software use */
 	unsigned long bolted: 1; /* HPTE is "bolted" */
-	unsigned long :     1; /* Software use */
+	unsigned long lock: 1; /* lock on pSeries SMP */
 	unsigned long l:    1; /* Virtual page is large (L=1) or 4 KB (L=0) */
 	unsigned long h:    1; /* Hash function identifier */
 	unsigned long v:    1; /* Valid (v=1) or invalid (v=0) */
 } Hpte_dword0;
 
 typedef struct {
-	unsigned long :     6; /* unused - padding */
-	unsigned long ac:   1; /* Address compare */
-	unsigned long r:    1; /* Referenced */
-	unsigned long c:    1; /* Changed */
-	unsigned long w:	1; /* Write-thru cache mode */
-	unsigned long i:	1; /* Cache inhibited */
-	unsigned long m:	1; /* Memory coherence required */
-	unsigned long g:	1; /* Guarded */
-	unsigned long n:	1; /* No-execute */
-	unsigned long pp:   2; /* Page protection bits 1:2 */
-} Hpte_flags;
-
-typedef struct {
 	unsigned long pp0:  1; /* Page protection bit 0 */
 	unsigned long :     1; /* Reserved */
 	unsigned long rpn: 50; /* Real page number */
@@ -134,12 +121,12 @@ typedef struct {
 	unsigned long ac:   1; /* Address compare */ 
 	unsigned long r:    1; /* Referenced */
 	unsigned long c:    1; /* Changed */
-	unsigned long w:	1; /* Write-thru cache mode */
-	unsigned long i:	1; /* Cache inhibited */
-	unsigned long m:	1; /* Memory coherence required */
-	unsigned long g:	1; /* Guarded */
-	unsigned long n:	1; /* No-execute */
-	unsigned long pp:	2; /* Page protection bits 1:2 */
+	unsigned long w:    1; /* Write-thru cache mode */
+	unsigned long i:    1; /* Cache inhibited */
+	unsigned long m:    1; /* Memory coherence required */
+	unsigned long g:    1; /* Guarded */
+	unsigned long n:    1; /* No-execute */
+	unsigned long pp:   2; /* Page protection bits 1:2 */
 } Hpte_dword1;
 
 typedef struct {
@@ -148,7 +135,7 @@ typedef struct {
 	unsigned long flags: 10;	/* HPTE flags */
 } Hpte_dword1_flags;
 
-typedef struct _HPTE {
+typedef struct {
 	union {
 		unsigned long dword0;
 		Hpte_dword0   dw0;
@@ -156,21 +143,8 @@ typedef struct _HPTE {
 
 	union {
 		unsigned long dword1;
-		struct {
-			unsigned long pp0:  1; /* Page protection bit 0 */
-			unsigned long ts:   1; /* Tag set bit */ 
-			unsigned long rpn: 50; /* Real page number */
-			unsigned long :     2; /* Unused */
-			unsigned long ac:   1; /* Address compare bit */
-			unsigned long r:    1; /* Referenced */
-			unsigned long c:    1; /* Changed */
-			unsigned long w:    1; /* Write-thru cache mode */
-			unsigned long i:    1; /* Cache inhibited */
-			unsigned long m:    1; /* Memory coherence */
-			unsigned long g:    1; /* Guarded */
-			unsigned long n:    1; /* No-execute page if N=1 */
-			unsigned long pp:   2; /* Page protection bit 1:2 */
-		} dw1;
+		Hpte_dword1 dw1;
+		Hpte_dword1_flags flags;
 	} dw1;
 } HPTE; 
 
@@ -204,6 +178,8 @@ void create_valid_hpte( unsigned long slot, unsigned long vpn,
 #define PT_SHIFT (12)			/* Page Table */
 #define PT_MASK  0x02FF
 
+#define LARGE_PAGE_SHIFT 24
+
 static inline unsigned long hpt_hash(unsigned long vpn, int large)
 {
 	unsigned long vsid;
@@ -220,20 +196,36 @@ static inline unsigned long hpt_hash(unsigned long vpn, int large)
 	return (vsid & 0x7fffffffff) ^ page;
 }
 
-#define PG_SHIFT (12)			/* Page Entry */
+static inline void _tlbie(unsigned long va, int large)
+{
+	asm volatile("ptesync": : :"memory");
+
+	if (large) {
+		asm volatile("clrldi	%0,%0,16\n\
+			      tlbie	%0,1" : : "r"(va) : "memory");
+	} else {
+		asm volatile("clrldi	%0,%0,16\n\
+			      tlbie	%0,0" : : "r"(va) : "memory");
+	}
 
-extern __inline__ void _tlbie( unsigned long va )
+	asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
+
+static inline void _tlbiel(unsigned long va, int large)
 {
-	__asm__ __volatile__ ( " \n\
-		clrldi	%0,%0,16 \n\
-		ptesync		 \n\
-		tlbie	%0	 \n\
-		eieio		 \n\
-		tlbsync		 \n\
-		ptesync"
-		: : "r" (va) : "memory" );
+	asm volatile("ptesync": : :"memory");
+
+	if (large) {
+		asm volatile("clrldi	%0,%0,16\n\
+			      tlbiel	%0,1" : : "r"(va) : "memory");
+	} else {
+		asm volatile("clrldi	%0,%0,16\n\
+			      tlbiel	%0,0" : : "r"(va) : "memory");
+	}
+
+	asm volatile("ptesync": : :"memory");
 }
- 
+
 #endif /* __ASSEMBLY__ */
 
 /* Block size masks */
diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 32668b8e59f1..c81ad14b6b5e 100644
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -7,6 +7,7 @@
  */
 
 #ifndef __ASSEMBLY__
+#include <linux/threads.h>
 #include <asm/processor.h>		/* For TASK_SIZE */
 #include <asm/mmu.h>
 #include <asm/page.h>
@@ -93,13 +94,15 @@
 #define _PAGE_WRITETHRU	0x040UL	/* W: cache write-through */
 #define _PAGE_DIRTY	0x080UL	/* C: page changed */
 #define _PAGE_ACCESSED	0x100UL	/* R: page referenced */
+#if 0
 #define _PAGE_HPTENOIX	0x200UL /* software: pte HPTE slot unknown */
+#endif
 #define _PAGE_HASHPTE	0x400UL	/* software: pte has an associated HPTE */
 #define _PAGE_EXEC	0x800UL	/* software: i-cache coherence required */
 #define _PAGE_SECONDARY 0x8000UL /* software: HPTE is in secondary group */
 #define _PAGE_GROUP_IX  0x7000UL /* software: HPTE index within group */
 /* Bits 0x7000 identify the index within an HPT Group */
-#define _PAGE_HPTEFLAGS (_PAGE_HASHPTE | _PAGE_HPTENOIX | _PAGE_SECONDARY | _PAGE_GROUP_IX)
+#define _PAGE_HPTEFLAGS (_PAGE_HASHPTE | _PAGE_SECONDARY | _PAGE_GROUP_IX)
 /* PAGE_MASK gives the right answer below, but only by accident */
 /* It should be preserving the high 48 bits and then specifically */
 /* preserving _PAGE_SECONDARY | _PAGE_GROUP_IX */
@@ -397,6 +400,7 @@ extern void paging_init(void);
  * as entries are faulted into the hash table by the low-level
  * data/instruction access exception handlers.
  */
+#if 0
 /*
  * We won't be able to use update_mmu_cache to update the 
  * hardware page table because we need to update the pte
@@ -404,9 +408,29 @@ extern void paging_init(void);
  * its value.
  */
 #define update_mmu_cache(vma, addr, pte)	do { } while (0)
+#else
+/*
+ * This gets called at the end of handling a page fault, when
+ * the kernel has put a new PTE into the page table for the process.
+ * We use it to put a corresponding HPTE into the hash table
+ * ahead of time, instead of waiting for the inevitable extra
+ * hash-table miss exception.
+ */
+extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
+#endif
 
 extern void flush_hash_segments(unsigned low_vsid, unsigned high_vsid);
-extern void flush_hash_page(unsigned long context, unsigned long ea, pte_t pte);
+extern void flush_hash_page(unsigned long context, unsigned long ea, pte_t pte,
+			    int local);
+void flush_hash_range(unsigned long context, unsigned long number, int local);
+
+/* TLB flush batching */
+#define MAX_BATCH_FLUSH 128
+struct tlb_batch_data {
+	pte_t pte;
+	unsigned long addr;
+};
+extern struct tlb_batch_data tlb_batch_array[NR_CPUS][MAX_BATCH_FLUSH];
 
 /* Encode and de-code a swap entry */
 #define SWP_TYPE(entry)			(((entry).val >> 1) & 0x3f)