From d4361b6c4ee2dbd5eee1f6932b88532525bb50a7 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Sun, 18 Jan 2004 18:22:50 -0800 Subject: [PATCH] ppc64: Fix {pte,pmd}_free vs. hash_page race by relaying actual deallocation with RCU, from Ben Herrenschmidt From: Anton Blanchard Fix {pte,pmd}_free vs. hash_page race by relaying actual deallocation with RCU, from Ben Herrenschmidt --- include/asm-ppc64/pgalloc.h | 55 ++++++++++++++++++++++++++++++++++++++++----- include/asm-ppc64/tlb.h | 4 ++++ 2 files changed, 53 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/asm-ppc64/pgalloc.h b/include/asm-ppc64/pgalloc.h index 23b0cde55e4e..43c70d211c57 100644 --- a/include/asm-ppc64/pgalloc.h +++ b/include/asm-ppc64/pgalloc.h @@ -3,7 +3,10 @@ #include #include +#include +#include #include +#include extern kmem_cache_t *zero_cache; @@ -40,8 +43,6 @@ pmd_free(pmd_t *pmd) kmem_cache_free(zero_cache, pmd); } -#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd) - #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte) #define pmd_populate(mm, pmd, pte_page) \ pmd_populate_kernel(mm, pmd, page_address(pte_page)) @@ -62,15 +63,57 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address) return NULL; } - -static inline void -pte_free_kernel(pte_t *pte) + +static inline void pte_free_kernel(pte_t *pte) { kmem_cache_free(zero_cache, pte); } #define pte_free(pte_page) pte_free_kernel(page_address(pte_page)) -#define __pte_free_tlb(tlb, pte) pte_free(pte) + +struct pte_freelist_batch +{ + struct rcu_head rcu; + unsigned int index; + struct page * pages[0]; +}; + +#define PTE_FREELIST_SIZE ((PAGE_SIZE - sizeof(struct pte_freelist_batch) / \ + sizeof(struct page *))) + +extern void pte_free_now(struct page *ptepage); +extern void pte_free_submit(struct pte_freelist_batch *batch); + +DECLARE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); + +static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage) +{ + /* This is safe as we are holding page_table_lock */ + cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id()); + struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); + + if (atomic_read(&tlb->mm->mm_users) < 2 || + cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) { + pte_free(ptepage); + return; + } + + if (*batchp == NULL) { + *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC); + if (*batchp == NULL) { + pte_free_now(ptepage); + return; + } + (*batchp)->index = 0; + } + (*batchp)->pages[(*batchp)->index++] = ptepage; + if ((*batchp)->index == PTE_FREELIST_SIZE) { + pte_free_submit(*batchp); + *batchp = NULL; + } +} + +#define __pmd_free_tlb(tlb, pmd) __pte_free_tlb(tlb, virt_to_page(pmd)) #define check_pgt_cache() do { } while (0) diff --git a/include/asm-ppc64/tlb.h b/include/asm-ppc64/tlb.h index 0a63e2811eee..42c0122d2630 100644 --- a/include/asm-ppc64/tlb.h +++ b/include/asm-ppc64/tlb.h @@ -74,6 +74,8 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, batch->index = i; } +extern void pte_free_finish(void); + static inline void tlb_flush(struct mmu_gather *tlb) { int cpu = smp_processor_id(); @@ -86,6 +88,8 @@ static inline void tlb_flush(struct mmu_gather *tlb) flush_hash_range(tlb->mm->context, batch->index, local); batch->index = 0; + + pte_free_finish(); } #endif /* _PPC64_TLB_H */ -- cgit v1.2.3