summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/asm-alpha/tlb.h6
-rw-r--r--include/asm-arm/tlb.h6
-rw-r--r--include/asm-generic/tlb.h37
-rw-r--r--include/asm-i386/pgalloc.h4
-rw-r--r--include/asm-i386/tlb.h2
-rw-r--r--include/asm-ia64/pgalloc.h4
-rw-r--r--include/asm-ia64/tlb.h2
-rw-r--r--include/asm-m68k/motorola_pgalloc.h4
-rw-r--r--include/asm-m68k/sun3_pgalloc.h4
-rw-r--r--include/asm-m68k/tlb.h2
-rw-r--r--include/asm-ppc/pgalloc.h4
-rw-r--r--include/asm-ppc/tlb.h4
-rw-r--r--include/asm-ppc64/pgalloc.h4
-rw-r--r--include/asm-ppc64/tlb.h2
-rw-r--r--include/asm-s390/pgalloc.h4
-rw-r--r--include/asm-s390/tlb.h2
-rw-r--r--include/asm-s390x/pgalloc.h4
-rw-r--r--include/asm-s390x/tlb.h2
-rw-r--r--include/asm-sparc/pgalloc.h4
-rw-r--r--include/asm-sparc/tlb.h2
-rw-r--r--include/asm-sparc64/tlb.h6
-rw-r--r--include/asm-x86_64/pgalloc.h4
-rw-r--r--include/asm-x86_64/tlb.h2
23 files changed, 75 insertions, 40 deletions
diff --git a/include/asm-alpha/tlb.h b/include/asm-alpha/tlb.h
index 314b19ab4522..aa91335533e0 100644
--- a/include/asm-alpha/tlb.h
+++ b/include/asm-alpha/tlb.h
@@ -3,13 +3,13 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
#include <asm-generic/tlb.h>
-#define pte_free_tlb(tlb,pte) pte_free(pte)
-#define pmd_free_tlb(tlb,pmd) pmd_free(pmd)
+#define __pte_free_tlb(tlb,pte) pte_free(pte)
+#define __pmd_free_tlb(tlb,pmd) pmd_free(pmd)
#endif
diff --git a/include/asm-arm/tlb.h b/include/asm-arm/tlb.h
index a7c2473b92f5..318357c01183 100644
--- a/include/asm-arm/tlb.h
+++ b/include/asm-arm/tlb.h
@@ -11,11 +11,11 @@
#define tlb_end_vma(tlb,vma) \
flush_tlb_range(vma, vma->vm_start, vma->vm_end)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
#include <asm-generic/tlb.h>
-#define pmd_free_tlb(tlb, pmd) pmd_free(pmd)
-#define pte_free_tlb(tlb, pte) pte_free(pte)
+#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
+#define __pte_free_tlb(tlb, pte) pte_free(pte)
#endif
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 8ce15cf20dbf..e629251cb7a7 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -36,9 +36,12 @@
typedef struct free_pte_ctx {
struct mm_struct *mm;
unsigned int nr; /* set to ~0U means fast mode */
+ unsigned int need_flush;/* Really unmapped some ptes? */
unsigned int fullmm; /* non-zero means full mm flush */
unsigned long freed;
struct page * pages[FREE_PTE_NR];
+ unsigned long flushes;/* stats: count avoided flushes */
+ unsigned long avoided_flushes;
} mmu_gather_t;
/* Users of the generic TLB shootdown code must declare this storage space. */
@@ -66,6 +69,13 @@ static inline void tlb_flush_mmu(mmu_gather_t *tlb, unsigned long start, unsigne
{
unsigned long nr;
+ if (!tlb->need_flush) {
+ tlb->avoided_flushes++;
+ return;
+ }
+ tlb->need_flush = 0;
+ tlb->flushes++;
+
tlb_flush(tlb);
nr = tlb->nr;
if (!tlb_fast_mode(tlb)) {
@@ -103,6 +113,7 @@ static inline void tlb_finish_mmu(mmu_gather_t *tlb, unsigned long start, unsign
*/
static inline void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
{
+ tlb->need_flush = 1;
if (tlb_fast_mode(tlb)) {
free_page_and_swap_cache(page);
return;
@@ -112,5 +123,29 @@ static inline void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
tlb_flush_mmu(tlb, 0, 0);
}
-#endif /* _ASM_GENERIC__TLB_H */
+/**
+ * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
+ *
+ * Record the fact that pte's were really umapped in ->need_flush, so we can
+ * later optimise away the tlb invalidate. This helps when userspace is
+ * unmapping already-unmapped pages, which happens quite a lot.
+ */
+#define tlb_remove_tlb_entry(tlb, ptep, address) \
+ do { \
+ tlb->need_flush = 1; \
+ __tlb_remove_tlb_entry(tlb, ptep, address); \
+ } while (0)
+
+#define pte_free_tlb(tlb, ptep) \
+ do { \
+ tlb->need_flush = 1; \
+ __pte_free_tlb(tlb, ptep); \
+ } while (0)
+
+#define pmd_free_tlb(tlb, pmdp) \
+ do { \
+ tlb->need_flush = 1; \
+ __pmd_free_tlb(tlb, pmdp); \
+ } while (0)
+#endif /* _ASM_GENERIC__TLB_H */
diff --git a/include/asm-i386/pgalloc.h b/include/asm-i386/pgalloc.h
index f2d63db16cf5..7abd8f6a308b 100644
--- a/include/asm-i386/pgalloc.h
+++ b/include/asm-i386/pgalloc.h
@@ -37,7 +37,7 @@ static inline void pte_free(struct page *pte)
}
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
/*
* allocating and freeing a pmd is trivial: the 1-entry pmd is
@@ -47,7 +47,7 @@ static inline void pte_free(struct page *pte)
#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
-#define pmd_free_tlb(tlb,x) do { } while (0)
+#define __pmd_free_tlb(tlb,x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
#define check_pgt_cache() do { } while (0)
diff --git a/include/asm-i386/tlb.h b/include/asm-i386/tlb.h
index 0cd54638cc97..c006c5c92bea 100644
--- a/include/asm-i386/tlb.h
+++ b/include/asm-i386/tlb.h
@@ -7,7 +7,7 @@
*/
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
/*
* .. because we flush the whole mm when it
diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h
index 493406169337..2e6134af88bc 100644
--- a/include/asm-ia64/pgalloc.h
+++ b/include/asm-ia64/pgalloc.h
@@ -108,7 +108,7 @@ pmd_free (pmd_t *pmd)
++pgtable_cache_size;
}
-#define pmd_free_tlb(tlb, pmd) pmd_free(pmd)
+#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
static inline void
pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte)
@@ -154,7 +154,7 @@ pte_free_kernel (pte_t *pte)
free_page((unsigned long) pte);
}
-#define pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte))
extern void check_pgt_cache (void);
diff --git a/include/asm-ia64/tlb.h b/include/asm-ia64/tlb.h
index ea335eb1b553..9e8d54722f2c 100644
--- a/include/asm-ia64/tlb.h
+++ b/include/asm-ia64/tlb.h
@@ -172,7 +172,7 @@ tlb_finish_mmu (mmu_gather_t *tlb, unsigned long start, unsigned long end)
* PTE, not just those pointing to (normal) physical memory.
*/
static inline void
-tlb_remove_tlb_entry (mmu_gather_t *tlb, pte_t *ptep, unsigned long address)
+__tlb_remove_tlb_entry (mmu_gather_t *tlb, pte_t *ptep, unsigned long address)
{
if (tlb->start_addr == ~0UL)
tlb->start_addr = address;
diff --git a/include/asm-m68k/motorola_pgalloc.h b/include/asm-m68k/motorola_pgalloc.h
index 6dab444dea28..a84f40959851 100644
--- a/include/asm-m68k/motorola_pgalloc.h
+++ b/include/asm-m68k/motorola_pgalloc.h
@@ -55,7 +55,7 @@ static inline void pte_free(struct page *page)
__free_page(page);
}
-static inline void pte_free_tlb(mmu_gather_t *tlb, struct page *page)
+static inline void __pte_free_tlb(mmu_gather_t *tlb, struct page *page)
{
cache_page(kmap(page));
kunmap(page);
@@ -73,7 +73,7 @@ static inline int pmd_free(pmd_t *pmd)
return free_pointer_table(pmd);
}
-static inline int pmd_free_tlb(mmu_gather_t *tlb, pmd_t *pmd)
+static inline int __pmd_free_tlb(mmu_gather_t *tlb, pmd_t *pmd)
{
return free_pointer_table(pmd);
}
diff --git a/include/asm-m68k/sun3_pgalloc.h b/include/asm-m68k/sun3_pgalloc.h
index 75e735e8c12c..5229c070c7ff 100644
--- a/include/asm-m68k/sun3_pgalloc.h
+++ b/include/asm-m68k/sun3_pgalloc.h
@@ -31,7 +31,7 @@ static inline void pte_free(struct page *page)
__free_page(page);
}
-static inline void pte_free_tlb(mmu_gather_t *tlb, struct page *page)
+static inline void __pte_free_tlb(mmu_gather_t *tlb, struct page *page)
{
tlb_remove_page(tlb, page);
}
@@ -76,7 +76,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p
* inside the pgd, so has no extra memory associated with it.
*/
#define pmd_free(x) do { } while (0)
-#define pmd_free_tlb(tlb, x) do { } while (0)
+#define __pmd_free_tlb(tlb, x) do { } while (0)
static inline void pgd_free(pgd_t * pgd)
{
diff --git a/include/asm-m68k/tlb.h b/include/asm-m68k/tlb.h
index c90d49d13b74..1785cff73449 100644
--- a/include/asm-m68k/tlb.h
+++ b/include/asm-m68k/tlb.h
@@ -7,7 +7,7 @@
*/
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
/*
* .. because we flush the whole mm when it
diff --git a/include/asm-ppc/pgalloc.h b/include/asm-ppc/pgalloc.h
index a0ea3e813a59..29bc46775052 100644
--- a/include/asm-ppc/pgalloc.h
+++ b/include/asm-ppc/pgalloc.h
@@ -20,7 +20,7 @@ extern void pgd_free(pgd_t *pgd);
*/
#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
-#define pmd_free_tlb(tlb,x) do { } while (0)
+#define __pmd_free_tlb(tlb,x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
#define pmd_populate_kernel(mm, pmd, pte) \
@@ -33,7 +33,7 @@ extern struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr);
extern void pte_free_kernel(pte_t *pte);
extern void pte_free(struct page *pte);
-#define pte_free_tlb(tlb, pte) pte_free((pte))
+#define __pte_free_tlb(tlb, pte) pte_free((pte))
#define check_pgt_cache() do { } while (0)
diff --git a/include/asm-ppc/tlb.h b/include/asm-ppc/tlb.h
index fd7cd0b460cc..f347071d6886 100644
--- a/include/asm-ppc/tlb.h
+++ b/include/asm-ppc/tlb.h
@@ -34,7 +34,7 @@ extern void tlb_flush(struct free_pte_ctx *tlb);
extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
unsigned long address);
-static inline void tlb_remove_tlb_entry(mmu_gather_t *tlb, pte_t *ptep,
+static inline void __tlb_remove_tlb_entry(mmu_gather_t *tlb, pte_t *ptep,
unsigned long address)
{
if (pte_val(*ptep) & _PAGE_HASHPTE)
@@ -50,7 +50,7 @@ struct flush_tlb_arch { };
#define tlb_finish_arch(tlb) do { } while (0)
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
/* Get the generic bits... */
diff --git a/include/asm-ppc64/pgalloc.h b/include/asm-ppc64/pgalloc.h
index b5b445ca7fe3..ee905f6052a5 100644
--- a/include/asm-ppc64/pgalloc.h
+++ b/include/asm-ppc64/pgalloc.h
@@ -53,7 +53,7 @@ pmd_free(pmd_t *pmd)
free_page((unsigned long)pmd);
}
-#define pmd_free_tlb(tlb, pmd) pmd_free(pmd)
+#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte)
#define pmd_populate(mm, pmd, pte_page) \
@@ -88,7 +88,7 @@ pte_free_kernel(pte_t *pte)
}
#define pte_free(pte_page) pte_free_kernel(page_address(pte_page))
-#define pte_free_tlb(tlb, pte) pte_free(pte)
+#define __pte_free_tlb(tlb, pte) pte_free(pte)
#define check_pgt_cache() do { } while (0)
diff --git a/include/asm-ppc64/tlb.h b/include/asm-ppc64/tlb.h
index 7e879e466ff3..a60daa84fb7b 100644
--- a/include/asm-ppc64/tlb.h
+++ b/include/asm-ppc64/tlb.h
@@ -40,7 +40,7 @@ struct ppc64_tlb_batch {
extern struct ppc64_tlb_batch ppc64_tlb_batch[NR_CPUS];
-static inline void tlb_remove_tlb_entry(mmu_gather_t *tlb, pte_t *ptep,
+static inline void __tlb_remove_tlb_entry(mmu_gather_t *tlb, pte_t *ptep,
unsigned long address)
{
int cpu = smp_processor_id();
diff --git a/include/asm-s390/pgalloc.h b/include/asm-s390/pgalloc.h
index 048e296e799f..9ef3dd733514 100644
--- a/include/asm-s390/pgalloc.h
+++ b/include/asm-s390/pgalloc.h
@@ -49,7 +49,7 @@ static inline void pgd_free(pgd_t *pgd)
*/
#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(x) do { } while (0)
-#define pmd_free_tlb(tlb,x) do { } while (0)
+#define __pmd_free_tlb(tlb,x) do { } while (0)
#define pgd_populate(mm, pmd, pte) BUG()
static inline void
@@ -107,7 +107,7 @@ static inline void pte_free(struct page *pte)
__free_page(pte);
}
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
/*
* This establishes kernel virtual mappings (e.g., as a result of a
diff --git a/include/asm-s390/tlb.h b/include/asm-s390/tlb.h
index 91d6dd2fcf1c..51bd957b85bd 100644
--- a/include/asm-s390/tlb.h
+++ b/include/asm-s390/tlb.h
@@ -7,7 +7,7 @@
*/
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
/*
* .. because we flush the whole mm when it
diff --git a/include/asm-s390x/pgalloc.h b/include/asm-s390x/pgalloc.h
index 95c23a735226..282ec93f29d4 100644
--- a/include/asm-s390x/pgalloc.h
+++ b/include/asm-s390x/pgalloc.h
@@ -68,7 +68,7 @@ static inline void pmd_free (pmd_t *pmd)
free_pages((unsigned long) pmd, 2);
}
-#define pmd_free_tlb(tlb,pmd) pmd_free(pmd)
+#define __pmd_free_tlb(tlb,pmd) pmd_free(pmd)
static inline void
pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
@@ -123,7 +123,7 @@ static inline void pte_free(struct page *pte)
__free_page(pte);
}
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
/*
* This establishes kernel virtual mappings (e.g., as a result of a
diff --git a/include/asm-s390x/tlb.h b/include/asm-s390x/tlb.h
index 84dfa35034ae..d90e9b8ecbf2 100644
--- a/include/asm-s390x/tlb.h
+++ b/include/asm-s390x/tlb.h
@@ -7,7 +7,7 @@
*/
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
/*
* .. because we flush the whole mm when it
diff --git a/include/asm-sparc/pgalloc.h b/include/asm-sparc/pgalloc.h
index 09c9decddbc4..126800acd10d 100644
--- a/include/asm-sparc/pgalloc.h
+++ b/include/asm-sparc/pgalloc.h
@@ -47,7 +47,7 @@ BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *)
#define free_pmd_fast(pmd) BTFIXUP_CALL(free_pmd_fast)(pmd)
#define pmd_free(pmd) free_pmd_fast(pmd)
-#define pmd_free_tlb(tlb, pmd) pmd_free(pmd)
+#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *)
#define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE)
@@ -64,6 +64,6 @@ BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *)
BTFIXUPDEF_CALL(void, pte_free, struct page *)
#define pte_free(pte) BTFIXUP_CALL(pte_free)(pte)
-#define pte_free_tlb(tlb, pte) pte_free(pte)
+#define __pte_free_tlb(tlb, pte) pte_free(pte)
#endif /* _SPARC_PGALLOC_H */
diff --git a/include/asm-sparc/tlb.h b/include/asm-sparc/tlb.h
index c9344b8767b4..6d02d1ce53f3 100644
--- a/include/asm-sparc/tlb.h
+++ b/include/asm-sparc/tlb.h
@@ -11,7 +11,7 @@ do { \
flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
} while (0)
-#define tlb_remove_tlb_entry(tlb, pte, address) \
+#define __tlb_remove_tlb_entry(tlb, pte, address) \
do { } while (0)
#define tlb_flush(tlb) \
diff --git a/include/asm-sparc64/tlb.h b/include/asm-sparc64/tlb.h
index 2a7db43c6fd7..8b507a36cf95 100644
--- a/include/asm-sparc64/tlb.h
+++ b/include/asm-sparc64/tlb.h
@@ -16,12 +16,12 @@ do { if (!(tlb)->fullmm) \
flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
} while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) \
+#define __tlb_remove_tlb_entry(tlb, ptep, address) \
do { } while (0)
#include <asm-generic/tlb.h>
-#define pmd_free_tlb(tlb, pmd) pmd_free(pmd)
-#define pte_free_tlb(tlb, pte) pte_free(pte)
+#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
+#define __pte_free_tlb(tlb, pte) pte_free(pte)
#endif /* _SPARC64_TLB_H */
diff --git a/include/asm-x86_64/pgalloc.h b/include/asm-x86_64/pgalloc.h
index 5edbfacc7eb4..899f603e69fa 100644
--- a/include/asm-x86_64/pgalloc.h
+++ b/include/asm-x86_64/pgalloc.h
@@ -75,7 +75,7 @@ extern inline void pte_free(struct page *pte)
__free_page(pte);
}
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
-#define pmd_free_tlb(tlb,x) do { } while (0)
+#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define __pmd_free_tlb(tlb,x) do { } while (0)
#endif /* _X86_64_PGALLOC_H */
diff --git a/include/asm-x86_64/tlb.h b/include/asm-x86_64/tlb.h
index 9a549844a7ff..cd4c3c590a0e 100644
--- a/include/asm-x86_64/tlb.h
+++ b/include/asm-x86_64/tlb.h
@@ -4,7 +4,7 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
generated by cgit v1.2.3 (git 2.46.0) at 2026-01-29 19:41:29 +0000