From fa5e5dc39669b4427830c546ede8709323b8276c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 6 May 2021 21:33:58 +0200 Subject: jump_label, x86: Introduce jump_entry_size() This allows architectures to have variable sized jumps. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210506194157.786777050@infradead.org --- include/linux/jump_label.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 05f5554d860f..8c45f58292ac 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -176,6 +176,15 @@ static inline void jump_entry_set_init(struct jump_entry *entry) entry->key |= 2; } +static inline int jump_entry_size(struct jump_entry *entry) +{ +#ifdef JUMP_LABEL_NOP_SIZE + return JUMP_LABEL_NOP_SIZE; +#else + return arch_jump_entry_size(entry); +#endif +} + #endif #endif -- cgit v1.2.3 From 5af0ea293d78c8b8f0b87ae2b13f7ac584057bc3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 6 May 2021 21:34:00 +0200 Subject: jump_label: Free jump_entry::key bit1 for build use Have jump_label_init() set jump_entry::key bit1 to either 0 ot 1 unconditionally. This makes it available for build-time games. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210506194157.906893264@infradead.org --- include/linux/jump_label.h | 7 +++++-- kernel/jump_label.c | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 8c45f58292ac..48b9b2a82767 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -171,9 +171,12 @@ static inline bool jump_entry_is_init(const struct jump_entry *entry) return (unsigned long)entry->key & 2UL; } -static inline void jump_entry_set_init(struct jump_entry *entry) +static inline void jump_entry_set_init(struct jump_entry *entry, bool set) { - entry->key |= 2; + if (set) + entry->key |= 2; + else + entry->key &= ~2; } static inline int jump_entry_size(struct jump_entry *entry) diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 521cafcfcb69..bdb0681bece8 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -483,13 +483,14 @@ void __init jump_label_init(void) for (iter = iter_start; iter < iter_stop; iter++) { struct static_key *iterk; + bool in_init; /* rewrite NOPs */ if (jump_label_type(iter) == JUMP_LABEL_NOP) arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); - if (init_section_contains((void *)jump_entry_code(iter), 1)) - jump_entry_set_init(iter); + in_init = init_section_contains((void *)jump_entry_code(iter), 1); + jump_entry_set_init(iter, in_init); iterk = jump_entry_key(iter); if (iterk == key) @@ -634,9 +635,10 @@ static int jump_label_add_module(struct module *mod) for (iter = iter_start; iter < iter_stop; iter++) { struct static_key *iterk; + bool in_init; - if (within_module_init(jump_entry_code(iter), mod)) - jump_entry_set_init(iter); + in_init = within_module_init(jump_entry_code(iter), mod); + jump_entry_set_init(iter, in_init); iterk = jump_entry_key(iter); if (iterk == key) -- cgit v1.2.3 From f1069a8756b9e9f6c055e709740d2d66650f0fb0 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 19 May 2021 15:03:08 +0200 Subject: compiler.h: Avoid using inline asm operand modifiers The expansion of annotate_reachable/annotate_unreachable on s390 will result in a compiler error if the __COUNTER__ value is high enough. For example with "i" (154) the "%c0" operand of annotate_reachable will be expanded to -102: -102: .pushsection .discard.reachable .long -102b - . .popsection This is a quirk of the gcc backend for s390, it interprets the %c0 as a signed byte value. Avoid using operand modifiers in this case by simply converting __COUNTER__ to string, with the same result, but in an arch assembler independent way. Signed-off-by: Vasily Gorbik Signed-off-by: Josh Poimboeuf Link: https://lore.kernel.org/r/patch-1.thread-1a26be.git-930d1b44844a.your-ad-here.call-01621428935-ext-2104@work.hours Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Miroslav Benes Cc: Borislav Petkov Cc: linux-kernel@vger.kernel.org --- include/linux/compiler.h | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler.h b/include/linux/compiler.h index df5b405e6305..77047904cf70 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -115,18 +115,24 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, * The __COUNTER__ based labels are a hack to make each instance of the macros * unique, to convince GCC not to merge duplicate inline asm statements. */ -#define annotate_reachable() ({ \ - asm volatile("%c0:\n\t" \ +#define __stringify_label(n) #n + +#define __annotate_reachable(c) ({ \ + asm volatile(__stringify_label(c) ":\n\t" \ ".pushsection .discard.reachable\n\t" \ - ".long %c0b - .\n\t" \ - ".popsection\n\t" : : "i" (__COUNTER__)); \ + ".long " __stringify_label(c) "b - .\n\t" \ + ".popsection\n\t"); \ }) -#define annotate_unreachable() ({ \ - asm volatile("%c0:\n\t" \ +#define annotate_reachable() __annotate_reachable(__COUNTER__) + +#define __annotate_unreachable(c) ({ \ + asm volatile(__stringify_label(c) ":\n\t" \ ".pushsection .discard.unreachable\n\t" \ - ".long %c0b - .\n\t" \ - ".popsection\n\t" : : "i" (__COUNTER__)); \ + ".long " __stringify_label(c) "b - .\n\t" \ + ".popsection\n\t"); \ }) +#define annotate_unreachable() __annotate_unreachable(__COUNTER__) + #define ASM_UNREACHABLE \ "999:\n\t" \ ".pushsection .discard.unreachable\n\t" \ -- cgit v1.2.3 From c199f64ff93c48a45add92eee4456ffcabfc838e Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 19 May 2021 15:03:13 +0200 Subject: instrumentation.h: Avoid using inline asm operand modifiers The expansion of instrumentation_begin/instrumentation_end on s390 will result in a compiler error if the __COUNTER__ value is high enough. For example with "i" (154) the "%c0" operand of annotate_reachable will be expanded to -102: -102: .pushsection .discard.instr_begin .long -102b - . .popsection This is a quirk of the gcc backend for s390, it interprets the %c0 as a signed byte value. Avoid using operand modifiers in this case by simply converting __COUNTER__ to string, with the same result, but in an arch assembler independent way. Signed-off-by: Vasily Gorbik Signed-off-by: Josh Poimboeuf Link: https://lore.kernel.org/r/patch-2.thread-1a26be.git-1a26be80cb18.your-ad-here.call-01621428935-ext-2104@work.hours Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Miroslav Benes Cc: Borislav Petkov Cc: linux-kernel@vger.kernel.org --- include/linux/instrumentation.h | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/instrumentation.h b/include/linux/instrumentation.h index 93e2ad67fc10..fa2cd8c63dcc 100644 --- a/include/linux/instrumentation.h +++ b/include/linux/instrumentation.h @@ -4,13 +4,16 @@ #if defined(CONFIG_DEBUG_ENTRY) && defined(CONFIG_STACK_VALIDATION) +#include + /* Begin/end of an instrumentation safe region */ -#define instrumentation_begin() ({ \ - asm volatile("%c0: nop\n\t" \ +#define __instrumentation_begin(c) ({ \ + asm volatile(__stringify(c) ": nop\n\t" \ ".pushsection .discard.instr_begin\n\t" \ - ".long %c0b - .\n\t" \ - ".popsection\n\t" : : "i" (__COUNTER__)); \ + ".long " __stringify(c) "b - .\n\t" \ + ".popsection\n\t"); \ }) +#define instrumentation_begin() __instrumentation_begin(__COUNTER__) /* * Because instrumentation_{begin,end}() can nest, objtool validation considers @@ -43,12 +46,13 @@ * To avoid this, have _end() be a NOP instruction, this ensures it will be * part of the condition block and does not escape. */ -#define instrumentation_end() ({ \ - asm volatile("%c0: nop\n\t" \ +#define __instrumentation_end(c) ({ \ + asm volatile(__stringify(c) ": nop\n\t" \ ".pushsection .discard.instr_end\n\t" \ - ".long %c0b - .\n\t" \ - ".popsection\n\t" : : "i" (__COUNTER__)); \ + ".long " __stringify(c) "b - .\n\t" \ + ".popsection\n\t"); \ }) +#define instrumentation_end() __instrumentation_end(__COUNTER__) #else # define instrumentation_begin() do { } while(0) # define instrumentation_end() do { } while(0) -- cgit v1.2.3