From edcc56dc6a7c758c4862321fc2c3a9d5a1f4dc5e Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Mon, 23 Aug 2004 21:24:11 -0700 Subject: [PATCH] rmaplock: kill page_map_lock The pte_chains rmap used pte_chain_lock (bit_spin_lock on PG_chainlock) to lock its pte_chains. We kept this (as page_map_lock: bit_spin_lock on PG_maplock) when we moved to objrmap. But the file objrmap locks its vma tree with mapping->i_mmap_lock, and the anon objrmap locks its vma list with anon_vma->lock: so isn't the page_map_lock superfluous? Pretty much, yes. The mapcount was protected by it, and needs to become an atomic: starting at -1 like page _count, so nr_mapped can be tracked precisely up and down. The last page_remove_rmap can't clear anon page mapping any more, because of races with page_add_rmap; from which some BUG_ONs must go for the same reason, but they've served their purpose. vmscan decisions are naturally racy, little change there beyond removing page_map_lock/unlock. But to stabilize the file-backed page->mapping against truncation while acquiring i_mmap_lock, page_referenced_file now needs page lock to be held even for refill_inactive_zone. There's a similar issue in acquiring anon_vma->lock, where page lock doesn't help: which this patch pretends to handle, but actually it needs the next. Roughly 10% cut off lmbench fork numbers on my 2*HT*P4. Must confess my testing failed to show the races even while they were knowingly exposed: would benefit from testing on racier equipment. Signed-off-by: Hugh Dickins Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 22 ++++++++++++++++++---- include/linux/page-flags.h | 3 +-- include/linux/rmap.h | 13 +++---------- 3 files changed, 22 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index ff1aa78f9775..42dca234d166 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -201,10 +201,9 @@ struct page { page_flags_t flags; /* Atomic flags, some possibly * updated asynchronously */ atomic_t _count; /* Usage count, see below. */ - unsigned int mapcount; /* Count of ptes mapped in mms, + atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped - * & limit reverse map searches, - * protected by PG_maplock. + * & limit reverse map searches. */ unsigned long private; /* Mapping-private opaque data: * usually used for buffer_heads @@ -477,12 +476,27 @@ static inline pgoff_t page_index(struct page *page) return page->index; } +/* + * The atomic page->_mapcount, like _count, starts from -1: + * so that transitions both from it and to it can be tracked, + * using atomic_inc_and_test and atomic_add_negative(-1). + */ +static inline void reset_page_mapcount(struct page *page) +{ + atomic_set(&(page)->_mapcount, -1); +} + +static inline int page_mapcount(struct page *page) +{ + return atomic_read(&(page)->_mapcount) + 1; +} + /* * Return true if this page is mapped into pagetables. */ static inline int page_mapped(struct page *page) { - return page->mapcount != 0; + return atomic_read(&(page)->_mapcount) >= 0; } /* diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 0cc396d8b149..da59ba5931ac 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -69,12 +69,11 @@ #define PG_private 12 /* Has something at ->private */ #define PG_writeback 13 /* Page is under writeback */ #define PG_nosave 14 /* Used for system suspend/resume */ -#define PG_maplock 15 /* Lock bit for rmap to ptes */ +#define PG_compound 15 /* Part of a compound page */ #define PG_swapcache 16 /* Swap page: swp_entry_t in private */ #define PG_mappedtodisk 17 /* Has blocks allocated on-disk */ #define PG_reclaim 18 /* To be reclaimed asap */ -#define PG_compound 19 /* Part of a compound page */ /* diff --git a/include/linux/rmap.h b/include/linux/rmap.h index e3148341f476..d8aa006a5fe5 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -9,11 +9,6 @@ #include #include -#define page_map_lock(page) \ - bit_spin_lock(PG_maplock, (unsigned long *)&(page)->flags) -#define page_map_unlock(page) \ - bit_spin_unlock(PG_maplock, (unsigned long *)&(page)->flags) - /* * The anon_vma heads a list of private "related" vmas, to scan if * an anonymous page pointing to this anon_vma needs to be unmapped: @@ -87,15 +82,13 @@ void page_remove_rmap(struct page *); */ static inline void page_dup_rmap(struct page *page) { - page_map_lock(page); - page->mapcount++; - page_map_unlock(page); + atomic_inc(&page->_mapcount); } /* * Called from mm/vmscan.c to handle paging out */ -int page_referenced(struct page *); +int page_referenced(struct page *, int is_locked); int try_to_unmap(struct page *); #else /* !CONFIG_MMU */ @@ -104,7 +97,7 @@ int try_to_unmap(struct page *); #define anon_vma_prepare(vma) (0) #define anon_vma_link(vma) do {} while (0) -#define page_referenced(page) TestClearPageReferenced(page) +#define page_referenced(page,l) TestClearPageReferenced(page) #define try_to_unmap(page) SWAP_FAIL #endif /* CONFIG_MMU */ -- cgit v1.2.3