Merge penguin.transmeta.com:/home/penguin/torvalds/repositories/kernel/tls-tree

into penguin.transmeta.com:/home/penguin/torvalds/repositories/kernel/linux

3 files changed, 76 insertions, 54 deletions

diff --git a/include/asm-i386/desc.h b/include/asm-i386/desc.h
index 197ffcc2dd2f..e1b2f75538f8 100644
--- a/include/asm-i386/desc.h
+++ b/include/asm-i386/desc.h
@@ -4,72 +4,59 @@
 #include <asm/ldt.h>
 
 /*
- * The layout of the GDT under Linux:
+ * The layout of the per-CPU GDT under Linux:
  *
  *   0 - null
- *   1 - not used
+ *   1 - Thread-Local Storage (TLS) segment
  *   2 - kernel code segment
  *   3 - kernel data segment
- *   4 - user code segment                  <-- new cacheline 
+ *   4 - user code segment		<==== new cacheline
  *   5 - user data segment
- *   6 - not used
- *   7 - not used
- *   8 - APM BIOS support                   <-- new cacheline 
+ *   6 - TSS
+ *   7 - LDT
+ *   8 - APM BIOS support		<==== new cacheline
  *   9 - APM BIOS support
  *  10 - APM BIOS support
  *  11 - APM BIOS support
- *  12 - PNPBIOS support
+ *  12 - PNPBIOS support		<==== new cacheline
  *  13 - PNPBIOS support
  *  14 - PNPBIOS support
  *  15 - PNPBIOS support
- *  16 - PNPBIOS support
+ *  16 - PNPBIOS support		<==== new cacheline
  *  17 - not used
  *  18 - not used
  *  19 - not used
+ */
+#define TLS_ENTRY 1
+#define TSS_ENTRY 6
+#define LDT_ENTRY 7
+/*
+ * The interrupt descriptor table has room for 256 idt's,
+ * the global descriptor table is dependent on the number
+ * of tasks we can have..
  *
- * The TSS+LDT descriptors are spread out a bit so that every CPU
- * has an exclusive cacheline for the per-CPU TSS and LDT:
- *
- *  20 - CPU#0 TSS                          <-- new cacheline 
- *  21 - CPU#0 LDT
- *  22 - not used 
- *  23 - not used 
- *  24 - CPU#1 TSS                          <-- new cacheline 
- *  25 - CPU#1 LDT
- *  26 - not used 
- *  27 - not used 
- *  ... NR_CPUS per-CPU TSS+LDT's if on SMP
- *
- * Entry into gdt where to find first TSS.
+ * We pad the GDT to cacheline boundary.
  */
-#define __FIRST_TSS_ENTRY 20
-#define __FIRST_LDT_ENTRY (__FIRST_TSS_ENTRY+1)
-
-#define __TSS(n) (((n)<<2) + __FIRST_TSS_ENTRY)
-#define __LDT(n) (((n)<<2) + __FIRST_LDT_ENTRY)
+#define IDT_ENTRIES 256
+#define GDT_ENTRIES 20
 
 #ifndef __ASSEMBLY__
 
 #include <asm/mmu.h>
 
-struct desc_struct {
-	unsigned long a,b;
-};
+#define GDT_SIZE (GDT_ENTRIES*sizeof(struct desc_struct))
 
-extern struct desc_struct gdt_table[];
-extern struct desc_struct *idt, *gdt;
+extern struct desc_struct cpu_gdt_table[NR_CPUS][GDT_ENTRIES];
 
 struct Xgt_desc_struct {
 	unsigned short size;
 	unsigned long address __attribute__((packed));
-};
+} __attribute__ ((packed));
 
-#define idt_descr (*(struct Xgt_desc_struct *)((char *)&idt - 2))
-#define gdt_descr (*(struct Xgt_desc_struct *)((char *)&gdt - 2))
+extern struct Xgt_desc_struct idt_descr, cpu_gdt_descr[NR_CPUS];
 
-#define load_TR(n) __asm__ __volatile__("ltr %%ax"::"a" (__TSS(n)<<3))
-
-#define __load_LDT(n) __asm__ __volatile__("lldt %%ax"::"a" (__LDT(n)<<3))
+#define load_TR_desc() __asm__ __volatile__("ltr %%ax"::"a" (TSS_ENTRY<<3))
+#define load_LDT_desc() __asm__ __volatile__("lldt %%ax"::"a" (LDT_ENTRY<<3))
 
 /*
  * This is the ldt that every process will get unless we need
@@ -77,14 +64,43 @@ struct Xgt_desc_struct {
  */
 extern struct desc_struct default_ldt[];
 extern void set_intr_gate(unsigned int irq, void * addr);
-extern void set_ldt_desc(unsigned int n, void *addr, unsigned int size);
-extern void set_tss_desc(unsigned int n, void *addr);
+
+#define _set_tssldt_desc(n,addr,limit,type) \
+__asm__ __volatile__ ("movw %w3,0(%2)\n\t" \
+	"movw %%ax,2(%2)\n\t" \
+	"rorl $16,%%eax\n\t" \
+	"movb %%al,4(%2)\n\t" \
+	"movb %4,5(%2)\n\t" \
+	"movb $0,6(%2)\n\t" \
+	"movb %%ah,7(%2)\n\t" \
+	"rorl $16,%%eax" \
+	: "=m"(*(n)) : "a" (addr), "r"(n), "ir"(limit), "i"(type))
+
+static inline void set_tss_desc(unsigned int cpu, void *addr)
+{
+	_set_tssldt_desc(&cpu_gdt_table[cpu][TSS_ENTRY], (int)addr, 235, 0x89);
+}
+
+static inline void set_ldt_desc(unsigned int cpu, void *addr, unsigned int size)
+{
+	_set_tssldt_desc(&cpu_gdt_table[cpu][LDT_ENTRY], (int)addr, ((size << 3)-1), 0x82);
+}
+
+#define TLS_FLAGS_MASK			0x00000007
+
+#define TLS_FLAG_LIMIT_IN_PAGES		0x00000001
+#define TLS_FLAG_WRITABLE		0x00000002
+#define TLS_FLAG_CLEAR			0x00000004
+
+static inline void load_TLS_desc(struct thread_struct *t, unsigned int cpu)
+{
+	cpu_gdt_table[cpu][TLS_ENTRY] = t->tls_desc;
+}
 
 static inline void clear_LDT(void)
 {
-	int cpu = smp_processor_id();
-	set_ldt_desc(cpu, &default_ldt[0], 5);
-	__load_LDT(cpu);
+	set_ldt_desc(smp_processor_id(), &default_ldt[0], 5);
+	load_LDT_desc();
 }
 
 /*
@@ -92,17 +108,16 @@ static inline void clear_LDT(void)
  */
 static inline void load_LDT (mm_context_t *pc)
 {
-	int cpu = smp_processor_id();
 	void *segments = pc->ldt;
 	int count = pc->size;
 
-	if (!count) {
+	if (likely(!count)) {
 		segments = &default_ldt[0];
 		count = 5;
 	}
 		
-	set_ldt_desc(cpu, segments, count);
-	__load_LDT(cpu);
+	set_ldt_desc(smp_processor_id(), segments, count);
+	load_LDT_desc();
 }
 
 #endif /* !__ASSEMBLY__ */
diff --git a/include/asm-i386/mmu_context.h b/include/asm-i386/mmu_context.h
index 417f2378659d..0da1f3a11983 100644
--- a/include/asm-i386/mmu_context.h
+++ b/include/asm-i386/mmu_context.h
@@ -17,7 +17,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
 
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk, unsigned cpu)
 {
-	if(cpu_tlbstate[cpu].state == TLBSTATE_OK)
+	if (cpu_tlbstate[cpu].state == TLBSTATE_OK)
 		cpu_tlbstate[cpu].state = TLBSTATE_LAZY;	
 }
 #else
@@ -40,18 +40,18 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, str
 		/* Re-load page tables */
 		load_cr3(next->pgd);
 
-		/* load_LDT, if either the previous or next thread
-		 * has a non-default LDT.
+		/*
+		 * load the LDT, if the LDT is different:
 		 */
-		if (next->context.size+prev->context.size)
+		if (unlikely(prev->context.ldt != next->context.ldt))
 			load_LDT(&next->context);
 	}
 #ifdef CONFIG_SMP
 	else {
 		cpu_tlbstate[cpu].state = TLBSTATE_OK;
-		if(cpu_tlbstate[cpu].active_mm != next)
+		if (cpu_tlbstate[cpu].active_mm != next)
 			BUG();
-		if(!test_and_set_bit(cpu, &next->cpu_vm_mask)) {
+		if (!test_and_set_bit(cpu, &next->cpu_vm_mask)) {
 			/* We were in lazy tlb mode and leave_mm disabled 
 			 * tlb flush IPI delivery. We must reload %cr3.
 			 */
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h
index e9e980d0c93a..9d407b2c088b 100644
--- a/include/asm-i386/processor.h
+++ b/include/asm-i386/processor.h
@@ -18,6 +18,10 @@
 #include <linux/config.h>
 #include <linux/threads.h>
 
+struct desc_struct {
+	unsigned long a,b;
+};
+
 /*
  * Default implementation of macro that returns current
  * instruction pointer ("program counter").
@@ -372,6 +376,9 @@ struct thread_struct {
 	unsigned long		v86flags, v86mask, v86mode, saved_esp0;
 /* IO permissions */
 	unsigned long	*ts_io_bitmap;
+/* TLS info and cached descriptor */
+	unsigned int tls_base, tls_limit, tls_flags;
+	struct desc_struct tls_desc;
 };
 
 #define INIT_THREAD  {						\
@@ -395,7 +402,7 @@ struct thread_struct {
 	0,0,0,0, /* esp,ebp,esi,edi */				\
 	0,0,0,0,0,0, /* es,cs,ss */				\
 	0,0,0,0,0,0, /* ds,fs,gs */				\
-	__LDT(0),0, /* ldt */					\
+	LDT_ENTRY,0, /* ldt */					\
 	0, INVALID_IO_BITMAP_OFFSET, /* tace, bitmap */		\
 	{~0, } /* ioperm */					\
 }