From 001430c1910df65ab805116c563aebd8a40ff23a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:29 +0200 Subject: arm64: add CPU field to struct thread_info The CPU field will be moved back into thread_info even when THREAD_INFO_IN_TASK is enabled, so add it back to arm64's definition of struct thread_info. Note that arm64 always has CONFIG_SMP=y so there is no point in guarding the CPU field with an #ifdef. Signed-off-by: Ard Biesheuvel Acked-by: Catalin Marinas Acked-by: Mark Rutland --- arch/arm64/include/asm/thread_info.h | 1 + arch/arm64/kernel/asm-offsets.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 6623c99f0984..c02bc8c183c3 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -42,6 +42,7 @@ struct thread_info { void *scs_base; void *scs_sp; #endif + u32 cpu; }; #define thread_saved_pc(tsk) \ diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 551427ae8cc5..cee9f3e9f906 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -29,6 +29,7 @@ int main(void) DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); DEFINE(TSK_CPU, offsetof(struct task_struct, cpu)); BLANK(); + DEFINE(TSK_TI_CPU, offsetof(struct task_struct, thread_info.cpu)); DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); #ifdef CONFIG_ARM64_SW_TTBR0_PAN -- cgit v1.2.3 From 5443f98fb9e06e765e24f9d894bf028accad8f71 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:30 +0200 Subject: x86: add CPU field to struct thread_info The CPU field will be moved back into thread_info even when THREAD_INFO_IN_TASK is enabled, so add it back to x86's definition of struct thread_info. Signed-off-by: Ard Biesheuvel Acked-by: Borislav Petkov Acked-by: Mark Rutland --- arch/x86/include/asm/thread_info.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index cf132663c219..ebec69c35e95 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -57,6 +57,9 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ u32 status; /* thread synchronous flags */ +#ifdef CONFIG_SMP + u32 cpu; /* current CPU */ +#endif }; #define INIT_THREAD_INFO(tsk) \ -- cgit v1.2.3 From bd2e2632556a464bd07cc8e21f60738611a934af Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:31 +0200 Subject: s390: add CPU field to struct thread_info The CPU field will be moved back into thread_info even when THREAD_INFO_IN_TASK is enabled, so add it back to s390's definition of struct thread_info. Note that s390 always has CONFIG_SMP=y so there is no point in guarding the CPU field with an #ifdef. Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Acked-by: Heiko Carstens --- arch/s390/include/asm/thread_info.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index e6674796aa6f..b2ffcb4fe000 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -37,6 +37,7 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ + unsigned int cpu; /* current CPU */ }; /* -- cgit v1.2.3 From 227d735d889e0403f1659df6e2dece7633f380bc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:32 +0200 Subject: powerpc: add CPU field to struct thread_info The CPU field will be moved back into thread_info even when THREAD_INFO_IN_TASK is enabled, so add it back to powerpc's definition of struct thread_info. Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Acked-by: Michael Ellerman --- arch/powerpc/include/asm/thread_info.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index b4ec6c7dd72e..5725029aaa29 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -47,6 +47,9 @@ struct thread_info { int preempt_count; /* 0 => preemptable, <0 => BUG */ +#ifdef CONFIG_SMP + unsigned int cpu; +#endif unsigned long local_flags; /* private flags for thread */ #ifdef CONFIG_LIVEPATCH unsigned long *livepatch_sp; -- cgit v1.2.3 From bcf9033e5449bdcaa9bed46467a7141a8049dadb Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:33 +0200 Subject: sched: move CPU field back into thread_info if THREAD_INFO_IN_TASK=y THREAD_INFO_IN_TASK moved the CPU field out of thread_info, but this causes some issues on architectures that define raw_smp_processor_id() in terms of this field, due to the fact that #include'ing linux/sched.h to get at struct task_struct is problematic in terms of circular dependencies. Given that thread_info and task_struct are the same data structure anyway when THREAD_INFO_IN_TASK=y, let's move it back so that having access to the type definition of struct thread_info is sufficient to reference the CPU number of the current task. Note that this requires THREAD_INFO_IN_TASK's definition of the task_thread_info() helper to be updated, as task_cpu() takes a pointer-to-const, whereas task_thread_info() (which is used to generate lvalues as well), needs a non-const pointer. So make it a macro instead. Signed-off-by: Ard Biesheuvel Acked-by: Catalin Marinas Acked-by: Mark Rutland Acked-by: Michael Ellerman --- arch/arm64/kernel/asm-offsets.c | 1 - arch/arm64/kernel/head.S | 2 +- arch/powerpc/kernel/asm-offsets.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- include/linux/sched.h | 13 +------------ kernel/sched/sched.h | 4 ---- 6 files changed, 4 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index cee9f3e9f906..0bfc048221af 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -27,7 +27,6 @@ int main(void) { DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); - DEFINE(TSK_CPU, offsetof(struct task_struct, cpu)); BLANK(); DEFINE(TSK_TI_CPU, offsetof(struct task_struct, thread_info.cpu)); DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 17962452e31d..6a98f1a38c29 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -412,7 +412,7 @@ SYM_FUNC_END(__create_page_tables) scs_load \tsk adr_l \tmp1, __per_cpu_offset - ldr w\tmp2, [\tsk, #TSK_CPU] + ldr w\tmp2, [\tsk, #TSK_TI_CPU] ldr \tmp1, [\tmp1, \tmp2, lsl #3] set_this_cpu_offset \tmp1 .endm diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index e563d3222d69..e37e4546034e 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -93,7 +93,7 @@ int main(void) #endif /* CONFIG_PPC64 */ OFFSET(TASK_STACK, task_struct, stack); #ifdef CONFIG_SMP - OFFSET(TASK_CPU, task_struct, cpu); + OFFSET(TASK_CPU, task_struct, thread_info.cpu); #endif #ifdef CONFIG_LIVEPATCH diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 9cc7d3dbf439..512d875b45e0 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1223,7 +1223,7 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) + THREAD_SIZE - STACK_FRAME_OVERHEAD; #endif - idle->cpu = cpu; + task_thread_info(idle)->cpu = cpu; secondary_current = current_set[cpu] = idle; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 39039ce8ac4c..8699594e3f99 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -750,10 +750,6 @@ struct task_struct { #ifdef CONFIG_SMP int on_cpu; struct __call_single_node wake_entry; -#ifdef CONFIG_THREAD_INFO_IN_TASK - /* Current CPU: */ - unsigned int cpu; -#endif unsigned int wakee_flips; unsigned long wakee_flip_decay_ts; struct task_struct *last_wakee; @@ -1886,10 +1882,7 @@ extern struct thread_info init_thread_info; extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)]; #ifdef CONFIG_THREAD_INFO_IN_TASK -static inline struct thread_info *task_thread_info(struct task_struct *task) -{ - return &task->thread_info; -} +# define task_thread_info(task) (&(task)->thread_info) #elif !defined(__HAVE_THREAD_FUNCTIONS) # define task_thread_info(task) ((struct thread_info *)(task)->stack) #endif @@ -2114,11 +2107,7 @@ static __always_inline bool need_resched(void) static inline unsigned int task_cpu(const struct task_struct *p) { -#ifdef CONFIG_THREAD_INFO_IN_TASK - return READ_ONCE(p->cpu); -#else return READ_ONCE(task_thread_info(p)->cpu); -#endif } extern void set_task_cpu(struct task_struct *p, unsigned int cpu); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3d3e5793e117..79fcbad11450 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1926,11 +1926,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) * per-task data have been completed by this moment. */ smp_wmb(); -#ifdef CONFIG_THREAD_INFO_IN_TASK - WRITE_ONCE(p->cpu, cpu); -#else WRITE_ONCE(task_thread_info(p)->cpu, cpu); -#endif p->wake_cpu = cpu; #endif } -- cgit v1.2.3 From 336868afbaae2d153fc20268a21747c31e5071b8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:34 +0200 Subject: powerpc: smp: remove hack to obtain offset of task_struct::cpu Instead of relying on awful hacks to obtain the offset of the cpu field in struct task_struct, move it back into struct thread_info, which does not create the same level of circular dependency hell when trying to include the header file that defines it. Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Acked-by: Michael Ellerman --- arch/powerpc/Makefile | 11 ----------- arch/powerpc/include/asm/smp.h | 17 +---------------- arch/powerpc/kernel/asm-offsets.c | 2 -- 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index aa6808e70647..54cad1faa5d0 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -446,17 +446,6 @@ else endif endif -ifdef CONFIG_SMP -ifdef CONFIG_PPC32 -prepare: task_cpu_prepare - -PHONY += task_cpu_prepare -task_cpu_prepare: prepare0 - $(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h)) - -endif # CONFIG_PPC32 -endif # CONFIG_SMP - PHONY += checkbin # Check toolchain versions: # - gcc-4.6 is the minimum kernel-wide version so nothing required. diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index 7ef1cd8168a0..007332a4a732 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -87,22 +87,7 @@ int is_cpu_dead(unsigned int cpu); /* 32-bit */ extern int smp_hw_index[]; -/* - * This is particularly ugly: it appears we can't actually get the definition - * of task_struct here, but we need access to the CPU this task is running on. - * Instead of using task_struct we're using _TASK_CPU which is extracted from - * asm-offsets.h by kbuild to get the current processor ID. - * - * This also needs to be safeguarded when building asm-offsets.s because at - * that time _TASK_CPU is not defined yet. It could have been guarded by - * _TASK_CPU itself, but we want the build to fail if _TASK_CPU is missing - * when building something else than asm-offsets.s - */ -#ifdef GENERATING_ASM_OFFSETS -#define raw_smp_processor_id() (0) -#else -#define raw_smp_processor_id() (*(unsigned int *)((void *)current + _TASK_CPU)) -#endif +#define raw_smp_processor_id() (current_thread_info()->cpu) #define hard_smp_processor_id() (smp_hw_index[smp_processor_id()]) static inline int get_hard_smp_processor_id(int cpu) diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index e37e4546034e..cc05522f50bf 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -9,8 +9,6 @@ * #defines from the assembly-language output. */ -#define GENERATING_ASM_OFFSETS /* asm/smp.h */ - #include #include #include -- cgit v1.2.3 From 8aa0fb0fbb82a4d2395be7eaeb994653b2d869fc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 14 Sep 2021 14:10:35 +0200 Subject: riscv: rely on core code to keep thread_info::cpu updated Now that the core code switched back to using thread_info::cpu to keep a task's CPU number, we no longer need to keep it in sync explicitly. So just drop the code that does this. Signed-off-by: Ard Biesheuvel Acked-by: Palmer Dabbelt Acked-by: Mark Rutland --- arch/riscv/kernel/asm-offsets.c | 1 - arch/riscv/kernel/entry.S | 5 ----- arch/riscv/kernel/head.S | 1 - 3 files changed, 7 deletions(-) diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 90f8ce64fa6f..478d9f02dab5 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -33,7 +33,6 @@ void asm_offsets(void) OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count); OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp); OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp); - OFFSET(TASK_TI_CPU, task_struct, thread_info.cpu); OFFSET(TASK_THREAD_F0, task_struct, thread.fstate.f[0]); OFFSET(TASK_THREAD_F1, task_struct, thread.fstate.f[1]); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 98f502654edd..459eb1714353 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -544,11 +544,6 @@ ENTRY(__switch_to) REG_L s9, TASK_THREAD_S9_RA(a4) REG_L s10, TASK_THREAD_S10_RA(a4) REG_L s11, TASK_THREAD_S11_RA(a4) - /* Swap the CPU entry around. */ - lw a3, TASK_TI_CPU(a0) - lw a4, TASK_TI_CPU(a1) - sw a3, TASK_TI_CPU(a1) - sw a4, TASK_TI_CPU(a0) /* The offset of thread_info in task_struct is zero. */ move tp, a1 ret diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index fce5184b22c3..d5ec30ef6f5d 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -317,7 +317,6 @@ clear_bss_done: call setup_trap_vector /* Restore C environment */ la tp, init_task - sw zero, TASK_TI_CPU(tp) la sp, init_thread_union + THREAD_SIZE #ifdef CONFIG_KASAN -- cgit v1.2.3