diff options
| -rw-r--r-- | arch/mips/kernel/irixelf.c | 11 | ||||
| -rw-r--r-- | arch/sparc64/kernel/binfmt_aout32.c | 12 | ||||
| -rw-r--r-- | arch/x86_64/ia32/ia32_aout.c | 12 | ||||
| -rw-r--r-- | arch/x86_64/mm/k8topology.c | 23 | ||||
| -rw-r--r-- | arch/x86_64/mm/numa.c | 10 | ||||
| -rw-r--r-- | arch/x86_64/mm/srat.c | 19 | ||||
| -rw-r--r-- | fs/binfmt_aout.c | 17 | ||||
| -rw-r--r-- | fs/binfmt_elf.c | 12 | ||||
| -rw-r--r-- | include/asm-x86_64/numa.h | 2 | ||||
| -rw-r--r-- | include/linux/nodemask.h | 9 | ||||
| -rw-r--r-- | mm/mmap.c | 54 |
11 files changed, 138 insertions, 43 deletions
diff --git a/arch/mips/kernel/irixelf.c b/arch/mips/kernel/irixelf.c index aee1f399e15b..43c254299e4a 100644 --- a/arch/mips/kernel/irixelf.c +++ b/arch/mips/kernel/irixelf.c @@ -127,7 +127,9 @@ static void set_brk(unsigned long start, unsigned long end) end = PAGE_ALIGN(end); if (end <= start) return; + down_write(¤t->mm->mmap_sem); do_brk(start, end - start); + up_write(¤t->mm->mmap_sem); } @@ -375,7 +377,9 @@ static unsigned int load_irix_interp(struct elfhdr * interp_elf_ex, /* Map the last of the bss segment */ if (last_bss > len) { + down_write(¤t->mm->mmap_sem); do_brk(len, (last_bss - len)); + up_write(¤t->mm->mmap_sem); } kfree(elf_phdata); @@ -562,7 +566,9 @@ void irix_map_prda_page (void) unsigned long v; struct prda *pp; + down_write(¤t->mm->mmap_sem); v = do_brk (PRDA_ADDRESS, PAGE_SIZE); + up_write(¤t->mm->mmap_sem); if (v < 0) return; @@ -852,8 +858,11 @@ static int load_irix_library(struct file *file) len = (elf_phdata->p_filesz + elf_phdata->p_vaddr+ 0xfff) & 0xfffff000; bss = elf_phdata->p_memsz + elf_phdata->p_vaddr; - if (bss > len) + if (bss > len) { + down_write(¤t->mm->mmap_sem); do_brk(len, bss-len); + up_write(¤t->mm->mmap_sem); + } kfree(elf_phdata); return 0; } diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c index c54d806e13d2..1afe88f6abc0 100644 --- a/arch/sparc64/kernel/binfmt_aout32.c +++ b/arch/sparc64/kernel/binfmt_aout32.c @@ -49,7 +49,9 @@ static void set_brk(unsigned long start, unsigned long end) end = PAGE_ALIGN(end); if (end <= start) return; + down_write(¤t->mm->mmap_sem); do_brk(start, end - start); + up_write(¤t->mm->mmap_sem); } /* @@ -246,10 +248,14 @@ static int load_aout32_binary(struct linux_binprm * bprm, struct pt_regs * regs) if (N_MAGIC(ex) == NMAGIC) { loff_t pos = fd_offset; /* Fuck me plenty... */ + down_write(¤t->mm->mmap_sem); error = do_brk(N_TXTADDR(ex), ex.a_text); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char __user *)N_TXTADDR(ex), ex.a_text, &pos); + down_write(¤t->mm->mmap_sem); error = do_brk(N_DATADDR(ex), ex.a_data); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char __user *)N_DATADDR(ex), ex.a_data, &pos); goto beyond_if; @@ -257,8 +263,10 @@ static int load_aout32_binary(struct linux_binprm * bprm, struct pt_regs * regs) if (N_MAGIC(ex) == OMAGIC) { loff_t pos = fd_offset; + down_write(¤t->mm->mmap_sem); do_brk(N_TXTADDR(ex) & PAGE_MASK, ex.a_text+ex.a_data + PAGE_SIZE - 1); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char __user *)N_TXTADDR(ex), ex.a_text+ex.a_data, &pos); } else { @@ -272,7 +280,9 @@ static int load_aout32_binary(struct linux_binprm * bprm, struct pt_regs * regs) if (!bprm->file->f_op->mmap) { loff_t pos = fd_offset; + down_write(¤t->mm->mmap_sem); do_brk(0, ex.a_text+ex.a_data); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char __user *)N_TXTADDR(ex), ex.a_text+ex.a_data, &pos); @@ -389,7 +399,9 @@ static int load_aout32_library(struct file *file) len = PAGE_ALIGN(ex.a_text + ex.a_data); bss = ex.a_text + ex.a_data + ex.a_bss; if (bss > len) { + down_write(¤t->mm->mmap_sem); error = do_brk(start_addr + len, bss - len); + up_write(¤t->mm->mmap_sem); retval = error; if (error != start_addr + len) goto out; diff --git a/arch/x86_64/ia32/ia32_aout.c b/arch/x86_64/ia32/ia32_aout.c index 6ac57fcf1340..2014f2113cd8 100644 --- a/arch/x86_64/ia32/ia32_aout.c +++ b/arch/x86_64/ia32/ia32_aout.c @@ -115,7 +115,9 @@ static void set_brk(unsigned long start, unsigned long end) end = PAGE_ALIGN(end); if (end <= start) return; + down_write(¤t->mm->mmap_sem); do_brk(start, end - start); + up_write(¤t->mm->mmap_sem); } #if CORE_DUMP @@ -325,7 +327,10 @@ static int load_aout_binary(struct linux_binprm * bprm, struct pt_regs * regs) pos = 32; map_size = ex.a_text+ex.a_data; + down_write(¤t->mm->mmap_sem); error = do_brk(text_addr & PAGE_MASK, map_size); + up_write(¤t->mm->mmap_sem); + if (error != (text_addr & PAGE_MASK)) { send_sig(SIGKILL, current, 0); return error; @@ -361,7 +366,9 @@ static int load_aout_binary(struct linux_binprm * bprm, struct pt_regs * regs) if (!bprm->file->f_op->mmap||((fd_offset & ~PAGE_MASK) != 0)) { loff_t pos = fd_offset; + down_write(¤t->mm->mmap_sem); do_brk(N_TXTADDR(ex), ex.a_text+ex.a_data); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file,(char *)N_TXTADDR(ex), ex.a_text+ex.a_data, &pos); flush_icache_range((unsigned long) N_TXTADDR(ex), @@ -469,8 +476,9 @@ static int load_aout_library(struct file *file) error_time = jiffies; } #endif - + down_write(¤t->mm->mmap_sem); do_brk(start_addr, ex.a_text + ex.a_data + ex.a_bss); + up_write(¤t->mm->mmap_sem); file->f_op->read(file, (char *)start_addr, ex.a_text + ex.a_data, &pos); @@ -494,7 +502,9 @@ static int load_aout_library(struct file *file) len = PAGE_ALIGN(ex.a_text + ex.a_data); bss = ex.a_text + ex.a_data + ex.a_bss; if (bss > len) { + down_write(¤t->mm->mmap_sem); error = do_brk(start_addr + len, bss - len); + up_write(¤t->mm->mmap_sem); retval = error; if (error != start_addr + len) goto out; diff --git a/arch/x86_64/mm/k8topology.c b/arch/x86_64/mm/k8topology.c index b3acd04b3b30..55a2e67e32a6 100644 --- a/arch/x86_64/mm/k8topology.c +++ b/arch/x86_64/mm/k8topology.c @@ -47,6 +47,10 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) int nodeid, i, nb; int found = 0; u32 reg; + unsigned numnodes; + nodemask_t nodes_parsed; + + nodes_clear(nodes_parsed); nb = find_northbridge(); if (nb < 0) @@ -55,10 +59,9 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) printk(KERN_INFO "Scanning NUMA topology in Northbridge %d\n", nb); reg = read_pci_config(0, nb, 0, 0x60); - for (i = 0; i <= ((reg >> 4) & 7); i++) - node_set_online(i); + numnodes = ((reg >> 4) & 7) + 1; - printk(KERN_INFO "Number of nodes %d (%x)\n", num_online_nodes(), reg); + printk(KERN_INFO "Number of nodes %d\n", numnodes); memset(&nodes,0,sizeof(nodes)); prevbase = 0; @@ -70,11 +73,11 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) nodeid = limit & 7; if ((base & 3) == 0) { - if (i < num_online_nodes()) + if (i < numnodes) printk("Skipping disabled node %d\n", i); continue; } - if (nodeid >= num_online_nodes()) { + if (nodeid >= numnodes) { printk("Ignoring excess node %d (%lx:%lx)\n", nodeid, base, limit); continue; @@ -90,7 +93,7 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) nodeid, (base>>8)&3, (limit>>8) & 3); return -1; } - if (node_online(nodeid)) { + if (node_isset(nodeid, nodes_parsed)) { printk(KERN_INFO "Node %d already present. Skipping\n", nodeid); continue; @@ -138,12 +141,14 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) nodes[nodeid].end = limit; prevbase = base; + + node_set(nodeid, nodes_parsed); } if (!found) return -1; - memnode_shift = compute_hash_shift(nodes); + memnode_shift = compute_hash_shift(nodes, numnodes); if (memnode_shift < 0) { printk(KERN_ERR "No NUMA node hash function found. Contact maintainer\n"); return -1; @@ -154,8 +159,8 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) if (nodes[i].start != nodes[i].end) { /* assume 1:1 NODE:CPU */ cpu_to_node[i] = i; - setup_node_bootmem(i, nodes[i].start, nodes[i].end); - } + setup_node_bootmem(i, nodes[i].start, nodes[i].end); + } } numa_init_array(); diff --git a/arch/x86_64/mm/numa.c b/arch/x86_64/mm/numa.c index b1a7db4f4d6d..ecc0c60989c0 100644 --- a/arch/x86_64/mm/numa.c +++ b/arch/x86_64/mm/numa.c @@ -34,9 +34,7 @@ cpumask_t node_to_cpumask[MAX_NUMNODES]; int numa_off __initdata; -unsigned long nodes_present; - -int __init compute_hash_shift(struct node *nodes) +int __init compute_hash_shift(struct node *nodes, int numnodes) { int i; int shift = 24; @@ -45,7 +43,7 @@ int __init compute_hash_shift(struct node *nodes) /* When in doubt use brute force. */ while (shift < 48) { memset(memnodemap,0xff,sizeof(*memnodemap) * NODEMAPSIZE); - for_each_online_node(i) { + for (i = 0; i < numnodes; i++) { if (nodes[i].start == nodes[i].end) continue; for (addr = nodes[i].start; @@ -197,7 +195,7 @@ static int numa_emulation(unsigned long start_pfn, unsigned long end_pfn) (nodes[i].end - nodes[i].start) >> 20); node_set_online(i); } - memnode_shift = compute_hash_shift(nodes); + memnode_shift = compute_hash_shift(nodes, numa_fake); if (memnode_shift < 0) { memnode_shift = 0; printk(KERN_ERR "No NUMA hash function found. Emulation disabled.\n"); @@ -274,7 +272,7 @@ void __init paging_init(void) /* [numa=off] */ __init int numa_setup(char *opt) { - if (!strcmp(opt,"off")) + if (!strncmp(opt,"off",3)) numa_off = 1; #ifdef CONFIG_NUMA_EMU if(!strncmp(opt, "fake=", 5)) { diff --git a/arch/x86_64/mm/srat.c b/arch/x86_64/mm/srat.c index 1db7483a0564..3bd04832f041 100644 --- a/arch/x86_64/mm/srat.c +++ b/arch/x86_64/mm/srat.c @@ -20,17 +20,20 @@ static struct acpi_table_slit *acpi_slit; -static DECLARE_BITMAP(nodes_parsed, MAX_NUMNODES) __initdata; +static nodemask_t nodes_parsed __initdata; +static nodemask_t nodes_found __initdata; static struct node nodes[MAX_NUMNODES] __initdata; static __u8 pxm2node[256] __initdata = { [0 ... 255] = 0xff }; static __init int setup_node(int pxm) { - if (pxm2node[pxm] == 0xff) { - if (num_online_nodes() >= MAX_NUMNODES) + unsigned node = pxm2node[pxm]; + if (node == 0xff) { + if (nodes_weight(nodes_found) >= MAX_NUMNODES) return -1; - pxm2node[pxm] = num_online_nodes(); - node_set_online(num_online_nodes()); + node = first_unset_node(nodes_found); + node_set(node, nodes_found); + pxm2node[pxm] = node; } return pxm2node[pxm]; } @@ -140,7 +143,7 @@ acpi_numa_memory_affinity_init(struct acpi_table_memory_affinity *ma) return; } nd = &nodes[node]; - if (!test_and_set_bit(node, &nodes_parsed)) { + if (!node_test_and_set(node, nodes_parsed)) { nd->start = start; nd->end = end; } else { @@ -163,7 +166,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) int i; if (acpi_numa <= 0) return -1; - memnode_shift = compute_hash_shift(nodes); + memnode_shift = compute_hash_shift(nodes, nodes_weight(nodes_parsed)); if (memnode_shift < 0) { printk(KERN_ERR "SRAT: No NUMA node hash function found. Contact maintainer\n"); @@ -171,7 +174,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) return -1; } for (i = 0; i < MAX_NUMNODES; i++) { - if (!test_bit(i, &nodes_parsed)) + if (!node_isset(i, nodes_parsed)) continue; cutoff_node(i, start, end); if (nodes[i].start == nodes[i].end) diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c index 8075fdff2364..a53d9a9e0de4 100644 --- a/fs/binfmt_aout.c +++ b/fs/binfmt_aout.c @@ -50,7 +50,10 @@ static int set_brk(unsigned long start, unsigned long end) start = PAGE_ALIGN(start); end = PAGE_ALIGN(end); if (end > start) { - unsigned long addr = do_brk(start, end - start); + unsigned long addr; + down_write(¤t->mm->mmap_sem); + addr = do_brk(start, end - start); + up_write(¤t->mm->mmap_sem); if (BAD_ADDR(addr)) return addr; } @@ -323,10 +326,14 @@ static int load_aout_binary(struct linux_binprm * bprm, struct pt_regs * regs) loff_t pos = fd_offset; /* Fuck me plenty... */ /* <AOL></AOL> */ + down_write(¤t->mm->mmap_sem); error = do_brk(N_TXTADDR(ex), ex.a_text); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char *) N_TXTADDR(ex), ex.a_text, &pos); + down_write(¤t->mm->mmap_sem); error = do_brk(N_DATADDR(ex), ex.a_data); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char *) N_DATADDR(ex), ex.a_data, &pos); goto beyond_if; @@ -346,8 +353,9 @@ static int load_aout_binary(struct linux_binprm * bprm, struct pt_regs * regs) pos = 32; map_size = ex.a_text+ex.a_data; #endif - + down_write(¤t->mm->mmap_sem); error = do_brk(text_addr & PAGE_MASK, map_size); + up_write(¤t->mm->mmap_sem); if (error != (text_addr & PAGE_MASK)) { send_sig(SIGKILL, current, 0); return error; @@ -382,7 +390,9 @@ static int load_aout_binary(struct linux_binprm * bprm, struct pt_regs * regs) if (!bprm->file->f_op->mmap||((fd_offset & ~PAGE_MASK) != 0)) { loff_t pos = fd_offset; + down_write(¤t->mm->mmap_sem); do_brk(N_TXTADDR(ex), ex.a_text+ex.a_data); + up_write(¤t->mm->mmap_sem); bprm->file->f_op->read(bprm->file, (char __user *)N_TXTADDR(ex), ex.a_text+ex.a_data, &pos); @@ -487,8 +497,9 @@ static int load_aout_library(struct file *file) file->f_dentry->d_name.name); error_time = jiffies; } - + down_write(¤t->mm->mmap_sem); do_brk(start_addr, ex.a_text + ex.a_data + ex.a_bss); + up_write(¤t->mm->mmap_sem); file->f_op->read(file, (char __user *)start_addr, ex.a_text + ex.a_data, &pos); diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index a66bc8df6d6f..6eb6971bdee2 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -88,7 +88,10 @@ static int set_brk(unsigned long start, unsigned long end) start = ELF_PAGEALIGN(start); end = ELF_PAGEALIGN(end); if (end > start) { - unsigned long addr = do_brk(start, end - start); + unsigned long addr; + down_write(¤t->mm->mmap_sem); + addr = do_brk(start, end - start); + up_write(¤t->mm->mmap_sem); if (BAD_ADDR(addr)) return addr; } @@ -409,7 +412,9 @@ static unsigned long load_elf_interp(struct elfhdr * interp_elf_ex, /* Map the last of the bss segment */ if (last_bss > elf_bss) { + down_write(¤t->mm->mmap_sem); error = do_brk(elf_bss, last_bss - elf_bss); + up_write(¤t->mm->mmap_sem); if (BAD_ADDR(error)) goto out_close; } @@ -449,7 +454,9 @@ static unsigned long load_aout_interp(struct exec * interp_ex, goto out; } + down_write(¤t->mm->mmap_sem); do_brk(0, text_data); + up_write(¤t->mm->mmap_sem); if (!interpreter->f_op || !interpreter->f_op->read) goto out; if (interpreter->f_op->read(interpreter, addr, text_data, &offset) < 0) @@ -457,8 +464,11 @@ static unsigned long load_aout_interp(struct exec * interp_ex, flush_icache_range((unsigned long)addr, (unsigned long)addr + text_data); + + down_write(¤t->mm->mmap_sem); do_brk(ELF_PAGESTART(text_data + ELF_MIN_ALIGN - 1), interp_ex->a_bss); + up_write(¤t->mm->mmap_sem); elf_entry = interp_ex->a_entry; out: diff --git a/include/asm-x86_64/numa.h b/include/asm-x86_64/numa.h index 21843b35ef18..3574f604542b 100644 --- a/include/asm-x86_64/numa.h +++ b/include/asm-x86_64/numa.h @@ -8,7 +8,7 @@ struct node { u64 start,end; }; -extern int compute_hash_shift(struct node *nodes); +extern int compute_hash_shift(struct node *nodes, int numnodes); #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT)) diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 4de843d94147..16475a23efa7 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -38,6 +38,8 @@ * * int first_node(mask) Number lowest set bit, or MAX_NUMNODES * int next_node(node, mask) Next node past 'node', or MAX_NUMNODES + * int first_unset_node(mask) First node not set in mask, or + * MAX_NUMNODES. * * nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set * NODE_MASK_ALL Initializer - all bits set @@ -235,6 +237,13 @@ static inline int __next_node(int n, const nodemask_t *srcp, int nbits) m; \ }) +#define first_unset_node(mask) __first_unset_node(&(mask)) +static inline int __first_unset_node(const nodemask_t *maskp) +{ + return min_t(int,MAX_NUMNODES, + find_first_zero_bit(maskp->bits, MAX_NUMNODES)); +} + #define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES) #if MAX_NUMNODES <= BITS_PER_LONG diff --git a/mm/mmap.c b/mm/mmap.c index f0d1f464ee00..21329bf3dea3 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1475,7 +1475,6 @@ static int acct_stack_growth(struct vm_area_struct * vma, unsigned long size, un int expand_stack(struct vm_area_struct * vma, unsigned long address) { int error; - unsigned long size, grow; if (!(vma->vm_flags & VM_GROWSUP)) return -EFAULT; @@ -1495,12 +1494,19 @@ int expand_stack(struct vm_area_struct * vma, unsigned long address) */ address += 4 + PAGE_SIZE - 1; address &= PAGE_MASK; - size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + error = 0; - error = acct_stack_growth(vma, size, grow); - if (!error) - vma->vm_end = address; + /* Somebody else might have raced and expanded it already */ + if (address > vma->vm_end) { + unsigned long size, grow; + + size = address - vma->vm_start; + grow = (address - vma->vm_end) >> PAGE_SHIFT; + + error = acct_stack_growth(vma, size, grow); + if (!error) + vma->vm_end = address; + } anon_vma_unlock(vma); return error; } @@ -1528,7 +1534,6 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) int expand_stack(struct vm_area_struct *vma, unsigned long address) { int error; - unsigned long size, grow; /* * We must make sure the anon_vma is allocated @@ -1544,13 +1549,20 @@ int expand_stack(struct vm_area_struct *vma, unsigned long address) * anon_vma lock to serialize against concurrent expand_stacks. */ address &= PAGE_MASK; - size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + error = 0; + + /* Somebody else might have raced and expanded it already */ + if (address < vma->vm_start) { + unsigned long size, grow; - error = acct_stack_growth(vma, size, grow); - if (!error) { - vma->vm_start = address; - vma->vm_pgoff -= grow; + size = vma->vm_end - address; + grow = (vma->vm_start - address) >> PAGE_SHIFT; + + error = acct_stack_growth(vma, size, grow); + if (!error) { + vma->vm_start = address; + vma->vm_pgoff -= grow; + } } anon_vma_unlock(vma); return error; @@ -1858,6 +1870,16 @@ asmlinkage long sys_munmap(unsigned long addr, size_t len) return ret; } +static inline void verify_mm_writelocked(struct mm_struct *mm) +{ +#ifdef CONFIG_DEBUG_KERNEL + if (unlikely(down_read_trylock(&mm->mmap_sem))) { + WARN_ON(1); + up_read(&mm->mmap_sem); + } +#endif +} + /* * this is really a simplified "do_mmap". it only handles * anonymous maps. eventually we may be able to do some @@ -1891,6 +1913,12 @@ unsigned long do_brk(unsigned long addr, unsigned long len) } /* + * mm->mmap_sem is required to protect against another thread + * changing the mappings in case we sleep. + */ + verify_mm_writelocked(mm); + + /* * Clear old maps. this also does some error checking for us */ munmap_back: |
