From 74523c06ae20b83c5508a98af62393ac34913362 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 6 Nov 2023 20:57:23 -0800 Subject: bpf: Add __bpf_dynptr_data* for in kernel use Different types of bpf dynptr have different internal data storage. Specifically, SKB and XDP type of dynptr may have non-continuous data. Therefore, it is not always safe to directly access dynptr->data. Add __bpf_dynptr_data and __bpf_dynptr_data_rw to replace direct access to dynptr->data. Update bpf_verify_pkcs7_signature to use __bpf_dynptr_data instead of dynptr->data. Signed-off-by: Song Liu Signed-off-by: Andrii Nakryiko Acked-by: Vadim Fedorenko Link: https://lore.kernel.org/bpf/20231107045725.2278852-2-song@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b4825d3cdb29..eb84caf133df 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1222,6 +1222,8 @@ enum bpf_dynptr_type { int bpf_dynptr_check_size(u32 size); u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr); +const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len); +void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len); #ifdef CONFIG_BPF_JIT int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); -- cgit v1.2.3 From 790ce3cfefb1b768dccd4eee324ddef0f0ce3db4 Mon Sep 17 00:00:00 2001 From: Dave Marchevsky Date: Tue, 7 Nov 2023 00:56:37 -0800 Subject: bpf: Move GRAPH_{ROOT,NODE}_MASK macros into btf_field_type enum This refactoring patch removes the unused BPF_GRAPH_NODE_OR_ROOT btf_field_type and moves BPF_GRAPH_{NODE,ROOT} macros into the btf_field_type enum. Further patches in the series will use BPF_GRAPH_NODE, so let's move this useful definition out of btf.c. Signed-off-by: Dave Marchevsky Link: https://lore.kernel.org/r/20231107085639.3016113-5-davemarchevsky@fb.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 4 ++-- kernel/bpf/btf.c | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index eb84caf133df..4001d11be151 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -186,8 +186,8 @@ enum btf_field_type { BPF_LIST_NODE = (1 << 6), BPF_RB_ROOT = (1 << 7), BPF_RB_NODE = (1 << 8), - BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD | - BPF_RB_NODE | BPF_RB_ROOT, + BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE, + BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD, BPF_REFCOUNT = (1 << 9), }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 15d71d2986d3..63cf4128fc05 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3840,9 +3840,6 @@ end: return ERR_PTR(ret); } -#define GRAPH_ROOT_MASK (BPF_LIST_HEAD | BPF_RB_ROOT) -#define GRAPH_NODE_MASK (BPF_LIST_NODE | BPF_RB_NODE) - int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) { int i; @@ -3855,13 +3852,13 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) * Hence we only need to ensure that bpf_{list_head,rb_root} ownership * does not form cycles. */ - if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & GRAPH_ROOT_MASK)) + if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & BPF_GRAPH_ROOT)) return 0; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *meta; u32 btf_id; - if (!(rec->fields[i].type & GRAPH_ROOT_MASK)) + if (!(rec->fields[i].type & BPF_GRAPH_ROOT)) continue; btf_id = rec->fields[i].graph_root.value_btf_id; meta = btf_find_struct_meta(btf, btf_id); @@ -3873,7 +3870,7 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) * to check ownership cycle for a type unless it's also a * node type. */ - if (!(rec->field_mask & GRAPH_NODE_MASK)) + if (!(rec->field_mask & BPF_GRAPH_NODE)) continue; /* We need to ensure ownership acyclicity among all types. The @@ -3909,7 +3906,7 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) * - A is both an root and node. * - B is only an node. */ - if (meta->record->field_mask & GRAPH_ROOT_MASK) + if (meta->record->field_mask & BPF_GRAPH_ROOT) return -ELOOP; } return 0; -- cgit v1.2.3 From 689b097a06bafb461ec162fc3b3ecc9765cea67b Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Mon, 6 Nov 2023 03:18:02 +0000 Subject: compiler-gcc: Suppress -Wmissing-prototypes warning for all supported GCC The kernel supports a minimum GCC version of 5.1.0 for building. However, the "__diag_ignore_all" directive only suppresses the "-Wmissing-prototypes" warning for GCC versions >= 8.0.0. As a result, when building the kernel with older GCC versions, warnings may be triggered. The example below illustrates the warnings reported by the kernel test robot using GCC 7.5.0: compiler: gcc-7 (Ubuntu 7.5.0-6ubuntu2) 7.5.0 All warnings (new ones prefixed by >>): kernel/bpf/helpers.c:1893:19: warning: no previous prototype for 'bpf_obj_new_impl' [-Wmissing-prototypes] __bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign) ^~~~~~~~~~~~~~~~ kernel/bpf/helpers.c:1907:19: warning: no previous prototype for 'bpf_percpu_obj_new_impl' [-Wmissing-prototypes] __bpf_kfunc void *bpf_percpu_obj_new_impl(u64 local_type_id__k, void *meta__ign) [...] To address this, we should also suppress the "-Wmissing-prototypes" warning for older GCC versions. "#pragma GCC diagnostic push" is supported as of GCC 4.6, and both "-Wmissing-prototypes" and "-Wmissing-declarations" are supported for all the GCC versions that we currently support. Therefore, it is reasonable to suppress these warnings for all supported GCC versions. With this adjustment, it's important to note that after implementing "__diag_ignore_all", it will effectively suppress warnings for all the supported GCC versions. In the future, if you wish to suppress warnings that are only supported on higher GCC versions, it is advisable to explicitly use "__diag_ignore" to specify the GCC version you are targeting. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202311031651.A7crZEur-lkp@intel.com/ Suggested-by: Arnd Bergmann Signed-off-by: Yafang Shao Cc: Kumar Kartikeya Dwivedi Cc: Arnd Bergmann Acked-by: Arnd Bergmann Link: https://lore.kernel.org/r/20231106031802.4188-1-laoar.shao@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/compiler-gcc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 2ceba3fe4ec1..aebb65bf95a7 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -136,7 +136,7 @@ #endif #define __diag_ignore_all(option, comment) \ - __diag_GCC(8, ignore, option) + __diag(__diag_GCC_ignore option) /* * Prior to 9.1, -Wno-alloc-size-larger-than (and therefore the "alloc_size" -- cgit v1.2.3 From 4aea6a6d61cd6e3df9ed98345638abad1b1e5276 Mon Sep 17 00:00:00 2001 From: Rahul Rameshbabu Date: Thu, 30 Mar 2023 15:05:38 -0700 Subject: net/mlx5: Query maximum frequency adjustment of the PTP hardware clock Some mlx5 devices do not support the default advertised maximum frequency adjustment value for the PTP hardware clock that is set by the driver. These devices need to be queried when initializing the clock functionality in order to get the maximum supported frequency adjustment value. This value can be greater than the minimum supported frequency adjustment across mlx5 devices (50 million ppb). Signed-off-by: Rahul Rameshbabu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/lib/clock.c | 22 ++++++++++++++++++++++ include/linux/mlx5/mlx5_ifc.h | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index 1daa4b019513..cac60a841e1d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -1000,6 +1000,25 @@ static void mlx5_init_clock_info(struct mlx5_core_dev *mdev) info->frac = timer->tc.frac; } +static void mlx5_init_timer_max_freq_adjustment(struct mlx5_core_dev *mdev) +{ + struct mlx5_clock *clock = &mdev->clock; + u32 out[MLX5_ST_SZ_DW(mtutc_reg)] = {}; + u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; + u8 log_max_freq_adjustment = 0; + int err; + + err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out), + MLX5_REG_MTUTC, 0, 0); + if (!err) + log_max_freq_adjustment = + MLX5_GET(mtutc_reg, out, log_max_freq_adjustment); + + if (log_max_freq_adjustment) + clock->ptp_info.max_adj = + min(S32_MAX, 1 << log_max_freq_adjustment); +} + static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) { struct mlx5_clock *clock = &mdev->clock; @@ -1007,6 +1026,9 @@ static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) /* Configure the PHC */ clock->ptp_info = mlx5_ptp_clock_info; + if (MLX5_CAP_MCAM_REG(mdev, mtutc)) + mlx5_init_timer_max_freq_adjustment(mdev); + mlx5_timecounter_init(mdev); mlx5_init_clock_info(mdev); mlx5_init_overflow_period(clock); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 6f3631425f38..ce2e71cd6d2a 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -10103,7 +10103,10 @@ enum { struct mlx5_ifc_mtutc_reg_bits { u8 reserved_at_0[0x5]; u8 freq_adj_units[0x3]; - u8 reserved_at_8[0x14]; + u8 reserved_at_8[0x3]; + u8 log_max_freq_adjustment[0x5]; + + u8 reserved_at_10[0xc]; u8 operation[0x4]; u8 freq_adjustment[0x20]; -- cgit v1.2.3 From 67420501e8681ae18f9f0ea0a69cd2f432100e70 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Sat, 11 Nov 2023 17:05:57 -0800 Subject: bpf: generalize reg_set_min_max() to handle non-const register comparisons Generalize bounds adjustment logic of reg_set_min_max() to handle not just register vs constant case, but in general any register vs any register cases. For most of the operations it's trivial extension based on range vs range comparison logic, we just need to properly pick min/max of a range to compare against min/max of the other range. For BPF_JSET we keep the original capabilities, just make sure JSET is integrated in the common framework. This is manifested in the internal-only BPF_JSET + BPF_X "opcode" to allow for simpler and more uniform rev_opcode() handling. See the code for details. This allows to reuse the same code exactly both for TRUE and FALSE branches without explicitly handling both conditions with custom code. Note also that now we don't need a special handling of BPF_JEQ/BPF_JNE case none of the registers are constants. This is now just a normal generic case handled by reg_set_min_max(). To make tnum handling cleaner, tnum_with_subreg() helper is added, as that's a common operator when dealing with 32-bit subregister bounds. This keeps the overall logic much less noisy when it comes to tnums. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Acked-by: Shung-Hsi Yu Link: https://lore.kernel.org/r/20231112010609.848406-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/tnum.h | 4 + kernel/bpf/tnum.c | 7 +- kernel/bpf/verifier.c | 314 ++++++++++++++++++++++---------------------------- 3 files changed, 146 insertions(+), 179 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tnum.h b/include/linux/tnum.h index 1c3948a1d6ad..3c13240077b8 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -106,6 +106,10 @@ int tnum_sbin(char *str, size_t size, struct tnum a); struct tnum tnum_subreg(struct tnum a); /* Returns the tnum with the lower 32-bit subreg cleared */ struct tnum tnum_clear_subreg(struct tnum a); +/* Returns the tnum with the lower 32-bit subreg in *reg* set to the lower + * 32-bit subreg in *subreg* + */ +struct tnum tnum_with_subreg(struct tnum reg, struct tnum subreg); /* Returns the tnum with the lower 32-bit subreg set to value */ struct tnum tnum_const_subreg(struct tnum a, u32 value); /* Returns true if 32-bit subreg @a is a known constant*/ diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index 3d7127f439a1..f4c91c9b27d7 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -208,7 +208,12 @@ struct tnum tnum_clear_subreg(struct tnum a) return tnum_lshift(tnum_rshift(a, 32), 32); } +struct tnum tnum_with_subreg(struct tnum reg, struct tnum subreg) +{ + return tnum_or(tnum_clear_subreg(reg), tnum_subreg(subreg)); +} + struct tnum tnum_const_subreg(struct tnum a, u32 value) { - return tnum_or(tnum_clear_subreg(a), tnum_const(value)); + return tnum_with_subreg(a, tnum_const(value)); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9ae6eae13471..39ce141c55d3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14453,218 +14453,186 @@ static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32); } -/* Adjusts the register min/max values in the case that the dst_reg and - * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K - * check, in which case we havea fake SCALAR_VALUE representing insn->imm). - * Technically we can do similar adjustments for pointers to the same object, - * but we don't support that right now. +/* Opcode that corresponds to a *false* branch condition. + * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */ -static void reg_set_min_max(struct bpf_reg_state *true_reg1, - struct bpf_reg_state *true_reg2, - struct bpf_reg_state *false_reg1, - struct bpf_reg_state *false_reg2, - u8 opcode, bool is_jmp32) +static u8 rev_opcode(u8 opcode) { - struct tnum false_32off, false_64off; - struct tnum true_32off, true_64off; - u64 uval; - u32 uval32; - s64 sval; - s32 sval32; - - /* If either register is a pointer, we can't learn anything about its - * variable offset from the compare (unless they were a pointer into - * the same object, but we don't bother with that). + switch (opcode) { + case BPF_JEQ: return BPF_JNE; + case BPF_JNE: return BPF_JEQ; + /* JSET doesn't have it's reverse opcode in BPF, so add + * BPF_X flag to denote the reverse of that operation */ - if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE) - return; - - /* we expect right-hand registers (src ones) to be constants, for now */ - if (!is_reg_const(false_reg2, is_jmp32)) { - opcode = flip_opcode(opcode); - swap(true_reg1, true_reg2); - swap(false_reg1, false_reg2); + case BPF_JSET: return BPF_JSET | BPF_X; + case BPF_JSET | BPF_X: return BPF_JSET; + case BPF_JGE: return BPF_JLT; + case BPF_JGT: return BPF_JLE; + case BPF_JLE: return BPF_JGT; + case BPF_JLT: return BPF_JGE; + case BPF_JSGE: return BPF_JSLT; + case BPF_JSGT: return BPF_JSLE; + case BPF_JSLE: return BPF_JSGT; + case BPF_JSLT: return BPF_JSGE; + default: return 0; } - if (!is_reg_const(false_reg2, is_jmp32)) - return; +} - false_32off = tnum_subreg(false_reg1->var_off); - false_64off = false_reg1->var_off; - true_32off = tnum_subreg(true_reg1->var_off); - true_64off = true_reg1->var_off; - uval = false_reg2->var_off.value; - uval32 = (u32)tnum_subreg(false_reg2->var_off).value; - sval = (s64)uval; - sval32 = (s32)uval32; +/* Refine range knowledge for 2 conditional operation. */ +static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, + u8 opcode, bool is_jmp32) +{ + struct tnum t; + u64 val; +again: switch (opcode) { - /* JEQ/JNE comparison doesn't change the register equivalence. - * - * r1 = r2; - * if (r1 == 42) goto label; - * ... - * label: // here both r1 and r2 are known to be 42. - * - * Hence when marking register as known preserve it's ID. - */ case BPF_JEQ: if (is_jmp32) { - __mark_reg32_known(true_reg1, uval32); - true_32off = tnum_subreg(true_reg1->var_off); + reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); + reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); + reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); + reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); + reg2->u32_min_value = reg1->u32_min_value; + reg2->u32_max_value = reg1->u32_max_value; + reg2->s32_min_value = reg1->s32_min_value; + reg2->s32_max_value = reg1->s32_max_value; + + t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); + reg1->var_off = tnum_with_subreg(reg1->var_off, t); + reg2->var_off = tnum_with_subreg(reg2->var_off, t); } else { - ___mark_reg_known(true_reg1, uval); - true_64off = true_reg1->var_off; + reg1->umin_value = max(reg1->umin_value, reg2->umin_value); + reg1->umax_value = min(reg1->umax_value, reg2->umax_value); + reg1->smin_value = max(reg1->smin_value, reg2->smin_value); + reg1->smax_value = min(reg1->smax_value, reg2->smax_value); + reg2->umin_value = reg1->umin_value; + reg2->umax_value = reg1->umax_value; + reg2->smin_value = reg1->smin_value; + reg2->smax_value = reg1->smax_value; + + reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); + reg2->var_off = reg1->var_off; } break; case BPF_JNE: - if (is_jmp32) { - __mark_reg32_known(false_reg1, uval32); - false_32off = tnum_subreg(false_reg1->var_off); - } else { - ___mark_reg_known(false_reg1, uval); - false_64off = false_reg1->var_off; - } + /* we don't derive any new information for inequality yet */ break; case BPF_JSET: + if (!is_reg_const(reg2, is_jmp32)) + swap(reg1, reg2); + if (!is_reg_const(reg2, is_jmp32)) + break; + val = reg_const_value(reg2, is_jmp32); + /* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X) + * requires single bit to learn something useful. E.g., if we + * know that `r1 & 0x3` is true, then which bits (0, 1, or both) + * are actually set? We can learn something definite only if + * it's a single-bit value to begin with. + * + * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have + * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor + * bit 1 is set, which we can readily use in adjustments. + */ + if (!is_power_of_2(val)) + break; if (is_jmp32) { - false_32off = tnum_and(false_32off, tnum_const(~uval32)); - if (is_power_of_2(uval32)) - true_32off = tnum_or(true_32off, - tnum_const(uval32)); + t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); + reg1->var_off = tnum_with_subreg(reg1->var_off, t); } else { - false_64off = tnum_and(false_64off, tnum_const(~uval)); - if (is_power_of_2(uval)) - true_64off = tnum_or(true_64off, - tnum_const(uval)); + reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; - case BPF_JGE: - case BPF_JGT: - { + case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ + if (!is_reg_const(reg2, is_jmp32)) + swap(reg1, reg2); + if (!is_reg_const(reg2, is_jmp32)) + break; + val = reg_const_value(reg2, is_jmp32); if (is_jmp32) { - u32 false_umax = opcode == BPF_JGT ? uval32 : uval32 - 1; - u32 true_umin = opcode == BPF_JGT ? uval32 + 1 : uval32; - - false_reg1->u32_max_value = min(false_reg1->u32_max_value, - false_umax); - true_reg1->u32_min_value = max(true_reg1->u32_min_value, - true_umin); + t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); + reg1->var_off = tnum_with_subreg(reg1->var_off, t); } else { - u64 false_umax = opcode == BPF_JGT ? uval : uval - 1; - u64 true_umin = opcode == BPF_JGT ? uval + 1 : uval; - - false_reg1->umax_value = min(false_reg1->umax_value, false_umax); - true_reg1->umin_value = max(true_reg1->umin_value, true_umin); + reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val)); } break; - } - case BPF_JSGE: - case BPF_JSGT: - { + case BPF_JLE: if (is_jmp32) { - s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; - s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; - - false_reg1->s32_max_value = min(false_reg1->s32_max_value, false_smax); - true_reg1->s32_min_value = max(true_reg1->s32_min_value, true_smin); + reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); + reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); } else { - s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; - s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; - - false_reg1->smax_value = min(false_reg1->smax_value, false_smax); - true_reg1->smin_value = max(true_reg1->smin_value, true_smin); + reg1->umax_value = min(reg1->umax_value, reg2->umax_value); + reg2->umin_value = max(reg1->umin_value, reg2->umin_value); } break; - } - case BPF_JLE: case BPF_JLT: - { if (is_jmp32) { - u32 false_umin = opcode == BPF_JLT ? uval32 : uval32 + 1; - u32 true_umax = opcode == BPF_JLT ? uval32 - 1 : uval32; - - false_reg1->u32_min_value = max(false_reg1->u32_min_value, - false_umin); - true_reg1->u32_max_value = min(true_reg1->u32_max_value, - true_umax); + reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1); + reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value); } else { - u64 false_umin = opcode == BPF_JLT ? uval : uval + 1; - u64 true_umax = opcode == BPF_JLT ? uval - 1 : uval; - - false_reg1->umin_value = max(false_reg1->umin_value, false_umin); - true_reg1->umax_value = min(true_reg1->umax_value, true_umax); + reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1); + reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value); } break; - } case BPF_JSLE: + if (is_jmp32) { + reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); + reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); + } else { + reg1->smax_value = min(reg1->smax_value, reg2->smax_value); + reg2->smin_value = max(reg1->smin_value, reg2->smin_value); + } + break; case BPF_JSLT: - { if (is_jmp32) { - s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; - s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; - - false_reg1->s32_min_value = max(false_reg1->s32_min_value, false_smin); - true_reg1->s32_max_value = min(true_reg1->s32_max_value, true_smax); + reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1); + reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value); } else { - s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; - s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; - - false_reg1->smin_value = max(false_reg1->smin_value, false_smin); - true_reg1->smax_value = min(true_reg1->smax_value, true_smax); + reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1); + reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value); } break; - } + case BPF_JGE: + case BPF_JGT: + case BPF_JSGE: + case BPF_JSGT: + /* just reuse LE/LT logic above */ + opcode = flip_opcode(opcode); + swap(reg1, reg2); + goto again; default: return; } - - if (is_jmp32) { - false_reg1->var_off = tnum_or(tnum_clear_subreg(false_64off), - tnum_subreg(false_32off)); - true_reg1->var_off = tnum_or(tnum_clear_subreg(true_64off), - tnum_subreg(true_32off)); - reg_bounds_sync(false_reg1); - reg_bounds_sync(true_reg1); - } else { - false_reg1->var_off = false_64off; - true_reg1->var_off = true_64off; - reg_bounds_sync(false_reg1); - reg_bounds_sync(true_reg1); - } -} - -/* Regs are known to be equal, so intersect their min/max/var_off */ -static void __reg_combine_min_max(struct bpf_reg_state *src_reg, - struct bpf_reg_state *dst_reg) -{ - src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, - dst_reg->umin_value); - src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, - dst_reg->umax_value); - src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, - dst_reg->smin_value); - src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, - dst_reg->smax_value); - src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, - dst_reg->var_off); - reg_bounds_sync(src_reg); - reg_bounds_sync(dst_reg); } -static void reg_combine_min_max(struct bpf_reg_state *true_src, - struct bpf_reg_state *true_dst, - struct bpf_reg_state *false_src, - struct bpf_reg_state *false_dst, - u8 opcode) +/* Adjusts the register min/max values in the case that the dst_reg and + * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K + * check, in which case we havea fake SCALAR_VALUE representing insn->imm). + * Technically we can do similar adjustments for pointers to the same object, + * but we don't support that right now. + */ +static void reg_set_min_max(struct bpf_reg_state *true_reg1, + struct bpf_reg_state *true_reg2, + struct bpf_reg_state *false_reg1, + struct bpf_reg_state *false_reg2, + u8 opcode, bool is_jmp32) { - switch (opcode) { - case BPF_JEQ: - __reg_combine_min_max(true_src, true_dst); - break; - case BPF_JNE: - __reg_combine_min_max(false_src, false_dst); - break; - } + /* If either register is a pointer, we can't learn anything about its + * variable offset from the compare (unless they were a pointer into + * the same object, but we don't bother with that). + */ + if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE) + return; + + /* fallthrough (FALSE) branch */ + regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32); + reg_bounds_sync(false_reg1); + reg_bounds_sync(false_reg2); + + /* jump (TRUE) branch */ + regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32); + reg_bounds_sync(true_reg1); + reg_bounds_sync(true_reg2); } static void mark_ptr_or_null_reg(struct bpf_func_state *state, @@ -14961,22 +14929,12 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, reg_set_min_max(&other_branch_regs[insn->dst_reg], &other_branch_regs[insn->src_reg], dst_reg, src_reg, opcode, is_jmp32); - - if (dst_reg->type == SCALAR_VALUE && - src_reg->type == SCALAR_VALUE && - !is_jmp32 && (opcode == BPF_JEQ || opcode == BPF_JNE)) { - /* Comparing for equality, we can combine knowledge */ - reg_combine_min_max(&other_branch_regs[insn->src_reg], - &other_branch_regs[insn->dst_reg], - src_reg, dst_reg, opcode); - } } else /* BPF_SRC(insn->code) == BPF_K */ { reg_set_min_max(&other_branch_regs[insn->dst_reg], src_reg /* fake one */, dst_reg, src_reg /* same fake one */, opcode, is_jmp32); } - if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id && !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { -- cgit v1.2.3 From 5f99f312bd3bedb3b266b0d26376a8c500cdc97f Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Sat, 11 Nov 2023 17:06:00 -0800 Subject: bpf: add register bounds sanity checks and sanitization Add simple sanity checks that validate well-formed ranges (min <= max) across u64, s64, u32, and s32 ranges. Also for cases when the value is constant (either 64-bit or 32-bit), we validate that ranges and tnums are in agreement. These bounds checks are performed at the end of BPF_ALU/BPF_ALU64 operations, on conditional jumps, and for LDX instructions (where subreg zero/sign extension is probably the most important to check). This covers most of the interesting cases. Also, we validate the sanity of the return register when manually adjusting it for some special helpers. By default, sanity violation will trigger a warning in verifier log and resetting register bounds to "unbounded" ones. But to aid development and debugging, BPF_F_TEST_SANITY_STRICT flag is added, which will trigger hard failure of verification with -EFAULT on register bounds violations. This allows selftests to catch such issues. veristat will also gain a CLI option to enable this behavior. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Acked-by: Shung-Hsi Yu Link: https://lore.kernel.org/r/20231112010609.848406-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 1 + include/uapi/linux/bpf.h | 3 ++ kernel/bpf/syscall.c | 3 +- kernel/bpf/verifier.c | 117 ++++++++++++++++++++++++++++++++--------- tools/include/uapi/linux/bpf.h | 3 ++ 5 files changed, 101 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 24213a99cc79..402b6bc44a1b 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -602,6 +602,7 @@ struct bpf_verifier_env { int stack_size; /* number of states to be processed */ bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ + bool test_sanity_strict; /* fail verification on sanity violations */ struct bpf_verifier_state *cur_state; /* current verifier state */ struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ struct bpf_verifier_state_list *free_list; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 7cf8bcf9f6a2..8a5855fcee69 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1200,6 +1200,9 @@ enum bpf_perf_event_type { */ #define BPF_F_XDP_DEV_BOUND_ONLY (1U << 6) +/* The verifier internal test flag. Behavior is undefined */ +#define BPF_F_TEST_SANITY_STRICT (1U << 7) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0ed286b8a0f0..f266e03ba342 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2573,7 +2573,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_SLEEPABLE | BPF_F_TEST_RND_HI32 | BPF_F_XDP_HAS_FRAGS | - BPF_F_XDP_DEV_BOUND_ONLY)) + BPF_F_XDP_DEV_BOUND_ONLY | + BPF_F_TEST_SANITY_STRICT)) return -EINVAL; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 65570eedfe88..e7edacf86e0f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2615,6 +2615,56 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) __update_reg_bounds(reg); } +static int reg_bounds_sanity_check(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, const char *ctx) +{ + const char *msg; + + if (reg->umin_value > reg->umax_value || + reg->smin_value > reg->smax_value || + reg->u32_min_value > reg->u32_max_value || + reg->s32_min_value > reg->s32_max_value) { + msg = "range bounds violation"; + goto out; + } + + if (tnum_is_const(reg->var_off)) { + u64 uval = reg->var_off.value; + s64 sval = (s64)uval; + + if (reg->umin_value != uval || reg->umax_value != uval || + reg->smin_value != sval || reg->smax_value != sval) { + msg = "const tnum out of sync with range bounds"; + goto out; + } + } + + if (tnum_subreg_is_const(reg->var_off)) { + u32 uval32 = tnum_subreg(reg->var_off).value; + s32 sval32 = (s32)uval32; + + if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 || + reg->s32_min_value != sval32 || reg->s32_max_value != sval32) { + msg = "const subreg tnum out of sync with range bounds"; + goto out; + } + } + + return 0; +out: + verbose(env, "REG SANITY VIOLATION (%s): %s u64=[%#llx, %#llx] " + "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n", + ctx, msg, reg->umin_value, reg->umax_value, + reg->smin_value, reg->smax_value, + reg->u32_min_value, reg->u32_max_value, + reg->s32_min_value, reg->s32_max_value, + reg->var_off.value, reg->var_off.mask); + if (env->test_sanity_strict) + return -EFAULT; + __mark_reg_unbounded(reg); + return 0; +} + static bool __reg32_bound_s64(s32 a) { return a >= 0 && a <= S32_MAX; @@ -9982,14 +10032,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return 0; } -static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, - int func_id, - struct bpf_call_arg_meta *meta) +static int do_refine_retval_range(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, int ret_type, + int func_id, + struct bpf_call_arg_meta *meta) { struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; if (ret_type != RET_INTEGER) - return; + return 0; switch (func_id) { case BPF_FUNC_get_stack: @@ -10015,6 +10066,8 @@ static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, reg_bounds_sync(ret_reg); break; } + + return reg_bounds_sanity_check(env, ret_reg, "retval"); } static int @@ -10666,7 +10719,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].ref_obj_id = id; } - do_refine_retval_range(regs, fn->ret_type, func_id, &meta); + err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); + if (err) + return err; err = check_map_func_compatibility(env, meta.map_ptr, func_id); if (err) @@ -14166,13 +14221,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) /* check dest operand */ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); + err = err ?: adjust_reg_min_max_vals(env, insn); if (err) return err; - - return adjust_reg_min_max_vals(env, insn); } - return 0; + return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); } static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, @@ -14653,18 +14707,21 @@ again: * Technically we can do similar adjustments for pointers to the same object, * but we don't support that right now. */ -static void reg_set_min_max(struct bpf_reg_state *true_reg1, - struct bpf_reg_state *true_reg2, - struct bpf_reg_state *false_reg1, - struct bpf_reg_state *false_reg2, - u8 opcode, bool is_jmp32) +static int reg_set_min_max(struct bpf_verifier_env *env, + struct bpf_reg_state *true_reg1, + struct bpf_reg_state *true_reg2, + struct bpf_reg_state *false_reg1, + struct bpf_reg_state *false_reg2, + u8 opcode, bool is_jmp32) { + int err; + /* If either register is a pointer, we can't learn anything about its * variable offset from the compare (unless they were a pointer into * the same object, but we don't bother with that). */ if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE) - return; + return 0; /* fallthrough (FALSE) branch */ regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32); @@ -14675,6 +14732,12 @@ static void reg_set_min_max(struct bpf_reg_state *true_reg1, regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32); reg_bounds_sync(true_reg1); reg_bounds_sync(true_reg2); + + err = reg_bounds_sanity_check(env, true_reg1, "true_reg1"); + err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2"); + err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1"); + err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2"); + return err; } static void mark_ptr_or_null_reg(struct bpf_func_state *state, @@ -14968,15 +15031,20 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, other_branch_regs = other_branch->frame[other_branch->curframe]->regs; if (BPF_SRC(insn->code) == BPF_X) { - reg_set_min_max(&other_branch_regs[insn->dst_reg], - &other_branch_regs[insn->src_reg], - dst_reg, src_reg, opcode, is_jmp32); + err = reg_set_min_max(env, + &other_branch_regs[insn->dst_reg], + &other_branch_regs[insn->src_reg], + dst_reg, src_reg, opcode, is_jmp32); } else /* BPF_SRC(insn->code) == BPF_K */ { - reg_set_min_max(&other_branch_regs[insn->dst_reg], - src_reg /* fake one */, - dst_reg, src_reg /* same fake one */, - opcode, is_jmp32); + err = reg_set_min_max(env, + &other_branch_regs[insn->dst_reg], + src_reg /* fake one */, + dst_reg, src_reg /* same fake one */, + opcode, is_jmp32); } + if (err) + return err; + if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id && !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { @@ -17479,10 +17547,8 @@ static int do_check(struct bpf_verifier_env *env) insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, false, BPF_MODE(insn->code) == BPF_MEMSX); - if (err) - return err; - - err = save_aux_ptr_type(env, src_reg_type, true); + err = err ?: save_aux_ptr_type(env, src_reg_type, true); + err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], "ldx"); if (err) return err; } else if (class == BPF_STX) { @@ -20769,6 +20835,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; + env->test_sanity_strict = attr->prog_flags & BPF_F_TEST_SANITY_STRICT; env->explored_states = kvcalloc(state_htab_size(env), sizeof(struct bpf_verifier_state_list *), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 7cf8bcf9f6a2..8a5855fcee69 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1200,6 +1200,9 @@ enum bpf_perf_event_type { */ #define BPF_F_XDP_DEV_BOUND_ONLY (1U << 6) +/* The verifier internal test flag. Behavior is undefined */ +#define BPF_F_TEST_SANITY_STRICT (1U << 7) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ -- cgit v1.2.3 From 3185d57cfcd34fadbe28f4ed57a6cb5122277ece Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Tue, 14 Nov 2023 11:42:02 +0100 Subject: indirect_call_wrapper: Fix typo in INDIRECT_CALL_$NR kerneldoc Fix a small typo in the kerneldoc comment of the INDIRECT_CALL_$NR macro. Signed-off-by: Tobias Klauser Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20231114104202.4680-1-tklauser@distanz.ch Signed-off-by: Paolo Abeni --- include/linux/indirect_call_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index c1c76a70a6ce..adb83a42a6b9 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -11,7 +11,7 @@ * @__VA_ARGS__: arguments for @f * * Avoid retpoline overhead for known builtin, checking @f vs each of them and - * eventually invoking directly the builtin function. The functions are check + * eventually invoking directly the builtin function. The functions are checked * in the given order. Fallback to the indirect call. */ #define INDIRECT_CALL_1(f, f1, ...) \ -- cgit v1.2.3 From 96fa96e198f9707285003075fbbce7db6a485112 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 15 Nov 2023 11:39:18 +0000 Subject: net: linkmode: add linkmode_fill() helper Add a linkmode_fill() helper, which will allow us to convert phylink's open coded bitmap_fill() operations. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/linkmode.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/linkmode.h b/include/linux/linkmode.h index 7303b4bc2ce0..287f590ed56b 100644 --- a/include/linux/linkmode.h +++ b/include/linux/linkmode.h @@ -10,6 +10,11 @@ static inline void linkmode_zero(unsigned long *dst) bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS); } +static inline void linkmode_fill(unsigned long *dst) +{ + bitmap_fill(dst, __ETHTOOL_LINK_MODE_MASK_NBITS); +} + static inline void linkmode_copy(unsigned long *dst, const unsigned long *src) { bitmap_copy(dst, src, __ETHTOOL_LINK_MODE_MASK_NBITS); -- cgit v1.2.3 From ff8867af01daa7ea770bebf5f91199b7434b74e5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 17 Nov 2023 09:14:04 -0800 Subject: bpf: rename BPF_F_TEST_SANITY_STRICT to BPF_F_TEST_REG_INVARIANTS Rename verifier internal flag BPF_F_TEST_SANITY_STRICT to more neutral BPF_F_TEST_REG_INVARIANTS. This is a follow up to [0]. A few selftests and veristat need to be adjusted in the same patch as well. [0] https://patchwork.kernel.org/project/netdevbpf/patch/20231112010609.848406-5-andrii@kernel.org/ Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231117171404.225508-1-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- include/uapi/linux/bpf.h | 2 +- kernel/bpf/syscall.c | 2 +- kernel/bpf/verifier.c | 6 +++--- tools/include/uapi/linux/bpf.h | 2 +- tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c | 2 +- tools/testing/selftests/bpf/prog_tests/reg_bounds.c | 2 +- tools/testing/selftests/bpf/progs/verifier_bounds.c | 4 ++-- tools/testing/selftests/bpf/test_loader.c | 6 +++--- tools/testing/selftests/bpf/test_sock_addr.c | 3 +-- tools/testing/selftests/bpf/test_verifier.c | 2 +- tools/testing/selftests/bpf/testing_helpers.c | 4 ++-- tools/testing/selftests/bpf/veristat.c | 12 ++++++------ 13 files changed, 24 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 402b6bc44a1b..52a4012b8255 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -602,7 +602,7 @@ struct bpf_verifier_env { int stack_size; /* number of states to be processed */ bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ - bool test_sanity_strict; /* fail verification on sanity violations */ + bool test_reg_invariants; /* fail verification on register invariants violations */ struct bpf_verifier_state *cur_state; /* current verifier state */ struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ struct bpf_verifier_state_list *free_list; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 8a5855fcee69..7a5498242eaa 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1201,7 +1201,7 @@ enum bpf_perf_event_type { #define BPF_F_XDP_DEV_BOUND_ONLY (1U << 6) /* The verifier internal test flag. Behavior is undefined */ -#define BPF_F_TEST_SANITY_STRICT (1U << 7) +#define BPF_F_TEST_REG_INVARIANTS (1U << 7) /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f266e03ba342..5e43ddd1b83f 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2574,7 +2574,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_TEST_RND_HI32 | BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | - BPF_F_TEST_SANITY_STRICT)) + BPF_F_TEST_REG_INVARIANTS)) return -EINVAL; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 59505881e7a7..7c3461b89513 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2608,14 +2608,14 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, return 0; out: - verbose(env, "REG SANITY VIOLATION (%s): %s u64=[%#llx, %#llx] " + verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n", ctx, msg, reg->umin_value, reg->umax_value, reg->smin_value, reg->smax_value, reg->u32_min_value, reg->u32_max_value, reg->s32_min_value, reg->s32_max_value, reg->var_off.value, reg->var_off.mask); - if (env->test_sanity_strict) + if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); return 0; @@ -20791,7 +20791,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; - env->test_sanity_strict = attr->prog_flags & BPF_F_TEST_SANITY_STRICT; + env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; env->explored_states = kvcalloc(state_htab_size(env), sizeof(struct bpf_verifier_state_list *), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 8a5855fcee69..7a5498242eaa 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1201,7 +1201,7 @@ enum bpf_perf_event_type { #define BPF_F_XDP_DEV_BOUND_ONLY (1U << 6) /* The verifier internal test flag. Behavior is undefined */ -#define BPF_F_TEST_SANITY_STRICT (1U << 7) +#define BPF_F_TEST_REG_INVARIANTS (1U << 7) /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 3f2d70831873..e770912fc1d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32 | BPF_F_TEST_SANITY_STRICT); + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index fe0cb906644b..7a8b0bf0a7f8 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -838,7 +838,7 @@ static int load_range_cmp_prog(struct range x, struct range y, enum op op, .log_level = 2, .log_buf = log_buf, .log_size = log_sz, - .prog_flags = BPF_F_TEST_SANITY_STRICT, + .prog_flags = BPF_F_TEST_REG_INVARIANTS, ); /* ; skip exit block below diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 0c1460936373..ec430b71730b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -965,7 +965,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("bound check with JMP_JSLT for crossing 64-bit signed boundary") __success __retval(0) -__flag(!BPF_F_TEST_SANITY_STRICT) /* known sanity violation */ +__flag(!BPF_F_TEST_REG_INVARIANTS) /* known invariants violation */ __naked void crossing_64_bit_signed_boundary_2(void) { asm volatile (" \ @@ -1047,7 +1047,7 @@ l0_%=: r0 = 0; \ SEC("xdp") __description("bound check with JMP32_JSLT for crossing 32-bit signed boundary") __success __retval(0) -__flag(!BPF_F_TEST_SANITY_STRICT) /* known sanity violation */ +__flag(!BPF_F_TEST_REG_INVARIANTS) /* known invariants violation */ __naked void crossing_32_bit_signed_boundary_2(void) { asm volatile (" \ diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 57e27b1a73a6..a350ecdfba4a 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -179,7 +179,7 @@ static int parse_test_spec(struct test_loader *tester, memset(spec, 0, sizeof(*spec)); spec->prog_name = bpf_program__name(prog); - spec->prog_flags = BPF_F_TEST_SANITY_STRICT; /* by default be strict */ + spec->prog_flags = BPF_F_TEST_REG_INVARIANTS; /* by default be strict */ btf = bpf_object__btf(obj); if (!btf) { @@ -280,8 +280,8 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear); } else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) { update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); - } else if (strcmp(val, "BPF_F_TEST_SANITY_STRICT") == 0) { - update_flags(&spec->prog_flags, BPF_F_TEST_SANITY_STRICT, clear); + } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { + update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c index 878c077e0fa7..b0068a9d2cfe 100644 --- a/tools/testing/selftests/bpf/test_sock_addr.c +++ b/tools/testing/selftests/bpf/test_sock_addr.c @@ -679,8 +679,7 @@ static int load_path(const struct sock_addr_test *test, const char *path) bpf_program__set_type(prog, BPF_PROG_TYPE_CGROUP_SOCK_ADDR); bpf_program__set_expected_attach_type(prog, test->expected_attach_type); - bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); - bpf_program__set_flags(prog, BPF_F_TEST_SANITY_STRICT); + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS); err = bpf_object__load(obj); if (err) { diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 4992022f3137..f36e41435be7 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1588,7 +1588,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, if (fixup_skips != skips) return; - pflags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_SANITY_STRICT; + pflags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS; if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT) pflags |= BPF_F_STRICT_ALIGNMENT; if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 9786a94a666c..d2458c1b1671 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -276,7 +276,7 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type, if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type) bpf_program__set_type(prog, type); - flags = bpf_program__flags(prog) | BPF_F_TEST_RND_HI32 | BPF_F_TEST_SANITY_STRICT; + flags = bpf_program__flags(prog) | BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS; bpf_program__set_flags(prog, flags); err = bpf_object__load(obj); @@ -299,7 +299,7 @@ int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, { LIBBPF_OPTS(bpf_prog_load_opts, opts, .kern_version = kern_version, - .prog_flags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_SANITY_STRICT, + .prog_flags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS, .log_level = extra_prog_load_log_flags, .log_buf = log_buf, .log_size = log_buf_sz, diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index 609fd9753af0..1d418d66e375 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -145,7 +145,7 @@ static struct env { bool debug; bool quiet; bool force_checkpoints; - bool strict_range_sanity; + bool force_reg_invariants; enum resfmt out_fmt; bool show_version; bool comparison_mode; @@ -225,8 +225,8 @@ static const struct argp_option opts[] = { { "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." }, { "test-states", 't', NULL, 0, "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" }, - { "test-sanity", 'r', NULL, 0, - "Force strict BPF verifier register sanity behavior (BPF_F_TEST_SANITY_STRICT program flag)" }, + { "test-reg-invariants", 'r', NULL, 0, + "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" }, {}, }; @@ -299,7 +299,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) env.force_checkpoints = true; break; case 'r': - env.strict_range_sanity = true; + env.force_reg_invariants = true; break; case 'n': errno = 0; @@ -1028,8 +1028,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf if (env.force_checkpoints) bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); - if (env.strict_range_sanity) - bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_SANITY_STRICT); + if (env.force_reg_invariants) + bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); err = bpf_object__load(obj); env.progs_processed++; -- cgit v1.2.3 From 446e2305827b76e8081057ce56bbd2703b4da8a9 Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Tue, 14 Nov 2023 12:28:29 +0100 Subject: net: Convert PHYs hwtstamp callback to use kernel_hwtstamp_config The PHYs hwtstamp callback are still getting the timestamp config from ifreq and using copy_from/to_user. Get rid of these functions by using timestamp configuration in parameter. This also allow to move on to kernel_hwtstamp_config and be similar to net devices using the new ndo_hwstamp_get/set. This adds the possibility to manipulate the timestamp configuration from the kernel which was not possible with the copy_from/to_user. Signed-off-by: Kory Maincent Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/phy/bcm-phy-ptp.c | 15 ++++++--------- drivers/net/phy/dp83640.c | 24 +++++++++++------------- drivers/net/phy/micrel.c | 38 +++++++++++++++++--------------------- drivers/net/phy/mscc/mscc_ptp.c | 18 ++++++++---------- drivers/net/phy/nxp-c45-tja11xx.c | 17 +++++++---------- drivers/net/phy/phy.c | 21 +++++++++++++++++++-- drivers/ptp/ptp_ines.c | 16 +++++++--------- include/linux/mii_timestamper.h | 4 +++- include/linux/phy.h | 6 ++++-- 9 files changed, 82 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/bcm-phy-ptp.c b/drivers/net/phy/bcm-phy-ptp.c index cb4b91af5e17..617d384d4551 100644 --- a/drivers/net/phy/bcm-phy-ptp.c +++ b/drivers/net/phy/bcm-phy-ptp.c @@ -782,16 +782,13 @@ out: } static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts, - struct ifreq *ifr) + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct bcm_ptp_private *priv = mii2priv(mii_ts); - struct hwtstamp_config cfg; u16 mode, ctrl; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: priv->hwts_rx = false; break; @@ -804,14 +801,14 @@ static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts, case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; priv->hwts_rx = true; break; default: return -ERANGE; } - priv->tx_type = cfg.tx_type; + priv->tx_type = cfg->tx_type; ctrl = priv->hwts_rx ? SLICE_RX_EN : 0; ctrl |= priv->tx_type != HWTSTAMP_TX_OFF ? SLICE_TX_EN : 0; @@ -840,7 +837,7 @@ static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts, /* purge existing data */ skb_queue_purge(&priv->tx_queue); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static int bcm_ptp_ts_info(struct mii_timestamper *mii_ts, diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c index 2657be7cc049..5c42c47dc564 100644 --- a/drivers/net/phy/dp83640.c +++ b/drivers/net/phy/dp83640.c @@ -1207,22 +1207,20 @@ static irqreturn_t dp83640_handle_interrupt(struct phy_device *phydev) return IRQ_HANDLED; } -static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) +static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct dp83640_private *dp83640 = container_of(mii_ts, struct dp83640_private, mii_ts); - struct hwtstamp_config cfg; u16 txcfg0, rxcfg0; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - if (cfg.tx_type < 0 || cfg.tx_type > HWTSTAMP_TX_ONESTEP_SYNC) + if (cfg->tx_type < 0 || cfg->tx_type > HWTSTAMP_TX_ONESTEP_SYNC) return -ERANGE; - dp83640->hwts_tx_en = cfg.tx_type; + dp83640->hwts_tx_en = cfg->tx_type; - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: dp83640->hwts_rx_en = 0; dp83640->layer = 0; @@ -1234,7 +1232,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) dp83640->hwts_rx_en = 1; dp83640->layer = PTP_CLASS_L4; dp83640->version = PTP_CLASS_V1; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; break; case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: @@ -1242,7 +1240,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) dp83640->hwts_rx_en = 1; dp83640->layer = PTP_CLASS_L4; dp83640->version = PTP_CLASS_V2; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; break; case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: @@ -1250,7 +1248,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) dp83640->hwts_rx_en = 1; dp83640->layer = PTP_CLASS_L2; dp83640->version = PTP_CLASS_V2; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_SYNC: @@ -1258,7 +1256,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) dp83640->hwts_rx_en = 1; dp83640->layer = PTP_CLASS_L4 | PTP_CLASS_L2; dp83640->version = PTP_CLASS_V2; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; break; default: return -ERANGE; @@ -1292,7 +1290,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) mutex_unlock(&dp83640->clock->extreg_lock); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static void rx_timestamp_work(struct work_struct *work) diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 08e3915001c3..99af1e500c6c 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -2395,24 +2395,22 @@ static void lan8814_flush_fifo(struct phy_device *phydev, bool egress) lanphy_read_page_reg(phydev, 5, PTP_TSU_INT_STS); } -static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) +static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) { struct kszphy_ptp_priv *ptp_priv = container_of(mii_ts, struct kszphy_ptp_priv, mii_ts); struct phy_device *phydev = ptp_priv->phydev; struct lan8814_shared_priv *shared = phydev->shared->priv; struct lan8814_ptp_rx_ts *rx_ts, *tmp; - struct hwtstamp_config config; int txcfg = 0, rxcfg = 0; int pkt_ts_enable; - if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) - return -EFAULT; + ptp_priv->hwts_tx_type = config->tx_type; + ptp_priv->rx_filter = config->rx_filter; - ptp_priv->hwts_tx_type = config.tx_type; - ptp_priv->rx_filter = config.rx_filter; - - switch (config.rx_filter) { + switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: ptp_priv->layer = 0; ptp_priv->version = 0; @@ -2458,13 +2456,13 @@ static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) lanphy_write_page_reg(ptp_priv->phydev, 5, PTP_TX_MOD, PTP_TX_MOD_TX_PTP_SYNC_TS_INSERT_); - if (config.rx_filter != HWTSTAMP_FILTER_NONE) + if (config->rx_filter != HWTSTAMP_FILTER_NONE) lan8814_config_ts_intr(ptp_priv->phydev, true); else lan8814_config_ts_intr(ptp_priv->phydev, false); mutex_lock(&shared->shared_lock); - if (config.rx_filter != HWTSTAMP_FILTER_NONE) + if (config->rx_filter != HWTSTAMP_FILTER_NONE) shared->ref++; else shared->ref--; @@ -2488,7 +2486,7 @@ static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) lan8814_flush_fifo(ptp_priv->phydev, false); lan8814_flush_fifo(ptp_priv->phydev, true); - return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; + return 0; } static void lan8814_txtstamp(struct mii_timestamper *mii_ts, @@ -3703,21 +3701,19 @@ static void lan8841_ptp_enable_processing(struct kszphy_ptp_priv *ptp_priv, #define LAN8841_PTP_TX_TIMESTAMP_EN 443 #define LAN8841_PTP_TX_MOD 445 -static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) +static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) { struct kszphy_ptp_priv *ptp_priv = container_of(mii_ts, struct kszphy_ptp_priv, mii_ts); struct phy_device *phydev = ptp_priv->phydev; - struct hwtstamp_config config; int txcfg = 0, rxcfg = 0; int pkt_ts_enable; - if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) - return -EFAULT; + ptp_priv->hwts_tx_type = config->tx_type; + ptp_priv->rx_filter = config->rx_filter; - ptp_priv->hwts_tx_type = config.tx_type; - ptp_priv->rx_filter = config.rx_filter; - - switch (config.rx_filter) { + switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: ptp_priv->layer = 0; ptp_priv->version = 0; @@ -3771,13 +3767,13 @@ static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) /* Now enable/disable the timestamping */ lan8841_ptp_enable_processing(ptp_priv, - config.rx_filter != HWTSTAMP_FILTER_NONE); + config->rx_filter != HWTSTAMP_FILTER_NONE); skb_queue_purge(&ptp_priv->tx_queue); lan8841_ptp_flush_fifo(ptp_priv); - return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; + return 0; } static bool lan8841_rxtstamp(struct mii_timestamper *mii_ts, diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c index cf728bfd83e2..eb0b032cb613 100644 --- a/drivers/net/phy/mscc/mscc_ptp.c +++ b/drivers/net/phy/mscc/mscc_ptp.c @@ -1045,19 +1045,17 @@ static void vsc85xx_ts_reset_fifo(struct phy_device *phydev) val); } -static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) +static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct vsc8531_private *vsc8531 = container_of(mii_ts, struct vsc8531_private, mii_ts); struct phy_device *phydev = vsc8531->ptp->phydev; - struct hwtstamp_config cfg; bool one_step = false; u32 val; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - switch (cfg.tx_type) { + switch (cfg->tx_type) { case HWTSTAMP_TX_ONESTEP_SYNC: one_step = true; break; @@ -1069,9 +1067,9 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) return -ERANGE; } - vsc8531->ptp->tx_type = cfg.tx_type; + vsc8531->ptp->tx_type = cfg->tx_type; - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: break; case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: @@ -1084,7 +1082,7 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) return -ERANGE; } - vsc8531->ptp->rx_filter = cfg.rx_filter; + vsc8531->ptp->rx_filter = cfg->rx_filter; mutex_lock(&vsc8531->ts_lock); @@ -1132,7 +1130,7 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) vsc8531->ptp->configured = 1; mutex_unlock(&vsc8531->ts_lock); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static int vsc85xx_ts_info(struct mii_timestamper *mii_ts, diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c index 7ab080ff02df..780ad353cf55 100644 --- a/drivers/net/phy/nxp-c45-tja11xx.c +++ b/drivers/net/phy/nxp-c45-tja11xx.c @@ -1022,24 +1022,21 @@ static bool nxp_c45_rxtstamp(struct mii_timestamper *mii_ts, } static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, - struct ifreq *ifreq) + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct nxp_c45_phy *priv = container_of(mii_ts, struct nxp_c45_phy, mii_ts); struct phy_device *phydev = priv->phydev; const struct nxp_c45_phy_data *data; - struct hwtstamp_config cfg; - if (copy_from_user(&cfg, ifreq->ifr_data, sizeof(cfg))) - return -EFAULT; - - if (cfg.tx_type < 0 || cfg.tx_type > HWTSTAMP_TX_ON) + if (cfg->tx_type < 0 || cfg->tx_type > HWTSTAMP_TX_ON) return -ERANGE; data = nxp_c45_get_data(phydev); - priv->hwts_tx = cfg.tx_type; + priv->hwts_tx = cfg->tx_type; - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: priv->hwts_rx = 0; break; @@ -1047,7 +1044,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: priv->hwts_rx = 1; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; break; default: return -ERANGE; @@ -1074,7 +1071,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts, nxp_c45_clear_reg_field(phydev, &data->regmap->irq_egr_ts_en); nxp_c45_no_ptp_irq: - return copy_to_user(ifreq->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static int nxp_c45_ts_info(struct mii_timestamper *mii_ts, diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index a5fa077650e8..d058316666ba 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -325,9 +325,13 @@ EXPORT_SYMBOL(phy_ethtool_ksettings_get); int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd) { struct mii_ioctl_data *mii_data = if_mii(ifr); + struct kernel_hwtstamp_config kernel_cfg; + struct netlink_ext_ack extack = {}; u16 val = mii_data->val_in; bool change_autoneg = false; + struct hwtstamp_config cfg; int prtad, devad; + int ret; switch (cmd) { case SIOCGMIIPHY: @@ -411,8 +415,21 @@ int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd) return 0; case SIOCSHWTSTAMP: - if (phydev->mii_ts && phydev->mii_ts->hwtstamp) - return phydev->mii_ts->hwtstamp(phydev->mii_ts, ifr); + if (phydev->mii_ts && phydev->mii_ts->hwtstamp) { + if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) + return -EFAULT; + + hwtstamp_config_to_kernel(&kernel_cfg, &cfg); + ret = phydev->mii_ts->hwtstamp(phydev->mii_ts, &kernel_cfg, &extack); + if (ret) + return ret; + + hwtstamp_config_from_kernel(&cfg, &kernel_cfg); + if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg))) + return -EFAULT; + + return 0; + } fallthrough; default: diff --git a/drivers/ptp/ptp_ines.c b/drivers/ptp/ptp_ines.c index ed215b458183..1d2940a78455 100644 --- a/drivers/ptp/ptp_ines.c +++ b/drivers/ptp/ptp_ines.c @@ -328,17 +328,15 @@ static u64 ines_find_txts(struct ines_port *port, struct sk_buff *skb) return ns; } -static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) +static int ines_hwtstamp(struct mii_timestamper *mii_ts, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { struct ines_port *port = container_of(mii_ts, struct ines_port, mii_ts); u32 cm_one_step = 0, port_conf, ts_stat_rx, ts_stat_tx; - struct hwtstamp_config cfg; unsigned long flags; - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - switch (cfg.tx_type) { + switch (cfg->tx_type) { case HWTSTAMP_TX_OFF: ts_stat_tx = 0; break; @@ -353,7 +351,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) return -ERANGE; } - switch (cfg.rx_filter) { + switch (cfg->rx_filter) { case HWTSTAMP_FILTER_NONE: ts_stat_rx = 0; break; @@ -372,7 +370,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: ts_stat_rx = TS_ENABLE; - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; break; default: return -ERANGE; @@ -393,7 +391,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr) spin_unlock_irqrestore(&port->lock, flags); - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; + return 0; } static void ines_link_state(struct mii_timestamper *mii_ts, diff --git a/include/linux/mii_timestamper.h b/include/linux/mii_timestamper.h index fa940bbaf8ae..26b04f73f214 100644 --- a/include/linux/mii_timestamper.h +++ b/include/linux/mii_timestamper.h @@ -9,6 +9,7 @@ #include #include #include +#include struct phy_device; @@ -51,7 +52,8 @@ struct mii_timestamper { struct sk_buff *skb, int type); int (*hwtstamp)(struct mii_timestamper *mii_ts, - struct ifreq *ifreq); + struct kernel_hwtstamp_config *kernel_config, + struct netlink_ext_ack *extack); void (*link_state)(struct mii_timestamper *mii_ts, struct phy_device *phydev); diff --git a/include/linux/phy.h b/include/linux/phy.h index 3cc52826f18e..e5f1f41e399c 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1560,9 +1560,11 @@ static inline bool phy_has_txtstamp(struct phy_device *phydev) return phydev && phydev->mii_ts && phydev->mii_ts->txtstamp; } -static inline int phy_hwtstamp(struct phy_device *phydev, struct ifreq *ifr) +static inline int phy_hwtstamp(struct phy_device *phydev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { - return phydev->mii_ts->hwtstamp(phydev->mii_ts, ifr); + return phydev->mii_ts->hwtstamp(phydev->mii_ts, cfg, extack); } static inline bool phy_rxtstamp(struct phy_device *phydev, struct sk_buff *skb, -- cgit v1.2.3 From b8768dc4077712915f045ba1b198f521493c7914 Mon Sep 17 00:00:00 2001 From: Richard Cochran Date: Tue, 14 Nov 2023 12:28:31 +0100 Subject: net: ethtool: Refactor identical get_ts_info implementations. The vlan, macvlan and the bonding drivers call their "real" device driver in order to report the time stamping capabilities. Provide a core ethtool helper function to avoid copy/paste in the stack. Signed-off-by: Richard Cochran Signed-off-by: Kory Maincent Reviewed-by: Florian Fainelli Reviewed-by: Jay Vosburgh Signed-off-by: David S. Miller --- drivers/net/bonding/bond_main.c | 29 ++--------------------------- drivers/net/macvlan.c | 14 +------------- include/linux/ethtool.h | 8 ++++++++ net/8021q/vlan_dev.c | 15 +-------------- net/ethtool/common.c | 6 ++++++ 5 files changed, 18 insertions(+), 54 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 8e6cc0e133b7..4e0600c7b050 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5755,10 +5755,8 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev, { struct bonding *bond = netdev_priv(bond_dev); struct ethtool_ts_info ts_info; - const struct ethtool_ops *ops; struct net_device *real_dev; bool sw_tx_support = false; - struct phy_device *phydev; struct list_head *iter; struct slave *slave; int ret = 0; @@ -5769,29 +5767,12 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev, rcu_read_unlock(); if (real_dev) { - ops = real_dev->ethtool_ops; - phydev = real_dev->phydev; - - if (phy_has_tsinfo(phydev)) { - ret = phy_ts_info(phydev, info); - goto out; - } else if (ops->get_ts_info) { - ret = ops->get_ts_info(real_dev, info); - goto out; - } + ret = ethtool_get_ts_info_by_layer(real_dev, info); } else { /* Check if all slaves support software tx timestamping */ rcu_read_lock(); bond_for_each_slave_rcu(bond, slave, iter) { - ret = -1; - ops = slave->dev->ethtool_ops; - phydev = slave->dev->phydev; - - if (phy_has_tsinfo(phydev)) - ret = phy_ts_info(phydev, &ts_info); - else if (ops->get_ts_info) - ret = ops->get_ts_info(slave->dev, &ts_info); - + ret = ethtool_get_ts_info_by_layer(slave->dev, &ts_info); if (!ret && (ts_info.so_timestamping & SOF_TIMESTAMPING_TX_SOFTWARE)) { sw_tx_support = true; continue; @@ -5803,15 +5784,9 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev, rcu_read_unlock(); } - ret = 0; - info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE; if (sw_tx_support) info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE; - info->phc_index = -1; - -out: dev_put(real_dev); return ret; } diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c8da94af4161..88dd8a2245b0 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1086,20 +1086,8 @@ static int macvlan_ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { struct net_device *real_dev = macvlan_dev_real_dev(dev); - const struct ethtool_ops *ops = real_dev->ethtool_ops; - struct phy_device *phydev = real_dev->phydev; - if (phy_has_tsinfo(phydev)) { - return phy_ts_info(phydev, info); - } else if (ops->get_ts_info) { - return ops->get_ts_info(real_dev, info); - } else { - info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE; - info->phc_index = -1; - } - - return 0; + return ethtool_get_ts_info_by_layer(real_dev, info); } static netdev_features_t macvlan_fix_features(struct net_device *dev, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 689028257fcc..c2bb74143eda 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -1043,6 +1043,14 @@ static inline int ethtool_mm_frag_size_min_to_add(u32 val_min, u32 *val_add, return -EINVAL; } +/** + * ethtool_get_ts_info_by_layer - Obtains time stamping capabilities from the MAC or PHY layer. + * @dev: pointer to net_device structure + * @info: buffer to hold the result + * Returns zero on success, non-zero otherwise. + */ +int ethtool_get_ts_info_by_layer(struct net_device *dev, struct ethtool_ts_info *info); + /** * ethtool_sprintf - Write formatted string to ethtool string data * @data: Pointer to a pointer to the start of string to update diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 2a7f1b15714a..407b2335f091 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -702,20 +702,7 @@ static int vlan_ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { const struct vlan_dev_priv *vlan = vlan_dev_priv(dev); - const struct ethtool_ops *ops = vlan->real_dev->ethtool_ops; - struct phy_device *phydev = vlan->real_dev->phydev; - - if (phy_has_tsinfo(phydev)) { - return phy_ts_info(phydev, info); - } else if (ops->get_ts_info) { - return ops->get_ts_info(vlan->real_dev, info); - } else { - info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE; - info->phc_index = -1; - } - - return 0; + return ethtool_get_ts_info_by_layer(vlan->real_dev, info); } static void vlan_dev_get_stats64(struct net_device *dev, diff --git a/net/ethtool/common.c b/net/ethtool/common.c index b4419fb6df6a..11d8797f63f6 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -661,6 +661,12 @@ int ethtool_get_phc_vclocks(struct net_device *dev, int **vclock_index) } EXPORT_SYMBOL(ethtool_get_phc_vclocks); +int ethtool_get_ts_info_by_layer(struct net_device *dev, struct ethtool_ts_info *info) +{ + return __ethtool_get_ts_info(dev, info); +} +EXPORT_SYMBOL(ethtool_get_ts_info_by_layer); + const struct ethtool_phy_ops *ethtool_phy_ops; void ethtool_set_ethtool_phy_ops(const struct ethtool_phy_ops *ops) -- cgit v1.2.3 From 011dd3b3f83f9c89605c640424e05845b84f2dad Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Tue, 14 Nov 2023 12:28:33 +0100 Subject: net: Make dev_set_hwtstamp_phylib accessible Make the dev_set_hwtstamp_phylib function accessible in prevision to use it from ethtool to reset the tstamp current configuration. Reviewed-by: Florian Fainelli Signed-off-by: Kory Maincent Signed-off-by: David S. Miller --- include/linux/netdevice.h | 3 +++ net/core/dev_ioctl.c | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a16c9cc063fe..2d840d7056f2 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3942,6 +3942,9 @@ int generic_hwtstamp_get_lower(struct net_device *dev, int generic_hwtstamp_set_lower(struct net_device *dev, struct kernel_hwtstamp_config *kernel_cfg, struct netlink_ext_ack *extack); +int dev_set_hwtstamp_phylib(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack); int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata); unsigned int dev_get_flags(const struct net_device *); int __dev_change_flags(struct net_device *dev, unsigned int flags, diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index feeddf95f450..9a66cf5015f2 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -322,9 +322,9 @@ static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in * dev->priv_flags. */ -static int dev_set_hwtstamp_phylib(struct net_device *dev, - struct kernel_hwtstamp_config *cfg, - struct netlink_ext_ack *extack) +int dev_set_hwtstamp_phylib(struct net_device *dev, + struct kernel_hwtstamp_config *cfg, + struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; bool phy_ts = phy_has_hwtstamp(dev->phydev); @@ -363,6 +363,7 @@ static int dev_set_hwtstamp_phylib(struct net_device *dev, return 0; } +EXPORT_SYMBOL_GPL(dev_set_hwtstamp_phylib); static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) { -- cgit v1.2.3 From 51bdf3165f012827644c474a6d905baa3de3f1ea Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Tue, 14 Nov 2023 12:28:40 +0100 Subject: net: Replace hwtstamp_source by timestamping layer Replace hwtstamp_source which is only used by the kernel_hwtstamp_config structure by the more widely use timestamp_layer structure. This is done to prepare the support of selectable timestamping source. Signed-off-by: Kory Maincent Signed-off-by: David S. Miller --- drivers/net/ethernet/microchip/lan966x/lan966x_main.c | 6 +++--- include/linux/net_tstamp.h | 11 +++-------- net/core/dev_ioctl.c | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index 2635ef8958c8..fbe56b1bb386 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -470,15 +470,15 @@ static int lan966x_port_hwtstamp_set(struct net_device *dev, struct lan966x_port *port = netdev_priv(dev); int err; - if (cfg->source != HWTSTAMP_SOURCE_NETDEV && - cfg->source != HWTSTAMP_SOURCE_PHYLIB) + if (cfg->source != MAC_TIMESTAMPING && + cfg->source != PHY_TIMESTAMPING) return -EOPNOTSUPP; err = lan966x_ptp_setup_traps(port, cfg); if (err) return err; - if (cfg->source == HWTSTAMP_SOURCE_NETDEV) { + if (cfg->source == MAC_TIMESTAMPING) { if (!port->lan966x->ptp) return -EOPNOTSUPP; diff --git a/include/linux/net_tstamp.h b/include/linux/net_tstamp.h index eb01c37e71e0..bb289c2ad376 100644 --- a/include/linux/net_tstamp.h +++ b/include/linux/net_tstamp.h @@ -5,11 +5,6 @@ #include -enum hwtstamp_source { - HWTSTAMP_SOURCE_NETDEV, - HWTSTAMP_SOURCE_PHYLIB, -}; - /** * struct kernel_hwtstamp_config - Kernel copy of struct hwtstamp_config * @@ -20,8 +15,8 @@ enum hwtstamp_source { * a legacy implementation of a lower driver * @copied_to_user: request was passed to a legacy implementation which already * copied the ioctl request back to user space - * @source: indication whether timestamps should come from the netdev or from - * an attached phylib PHY + * @source: indication whether timestamps should come from software, the netdev + * or from an attached phylib PHY * * Prefer using this structure for in-kernel processing of hardware * timestamping configuration, over the inextensible struct hwtstamp_config @@ -33,7 +28,7 @@ struct kernel_hwtstamp_config { int rx_filter; struct ifreq *ifr; bool copied_to_user; - enum hwtstamp_source source; + enum timestamping_layer source; }; static inline void hwtstamp_config_to_kernel(struct kernel_hwtstamp_config *kernel_cfg, diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 9a66cf5015f2..267cd00269d0 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -332,7 +332,7 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, bool changed = false; int err; - cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; + cfg->source = phy_ts ? PHY_TIMESTAMPING : MAC_TIMESTAMPING; if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { err = ops->ndo_hwtstamp_get(dev, &old_cfg); -- cgit v1.2.3 From 0f7f463d4821a4f52fa5c0a961389e651d50c384 Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Tue, 14 Nov 2023 12:28:41 +0100 Subject: net: Change the API of PHY default timestamp to MAC Change the API to select MAC default time stamping instead of the PHY. Indeed the PHY is closer to the wire therefore theoretically it has less delay than the MAC timestamping but the reality is different. Due to lower time stamping clock frequency, latency in the MDIO bus and no PHC hardware synchronization between different PHY, the PHY PTP is often less precise than the MAC. The exception is for PHY designed specially for PTP case but these devices are not very widespread. For not breaking the compatibility I introduce a default_timestamp flag in phy_device that is set by the phy driver to know we are using the old API behavior. The phy_set_timestamp function is called at each call of phy_attach_direct. In case of MAC driver using phylink this function is called when the interface is turned up. Then if the interface goes down and up again the last choice of timestamp will be overwritten by the default choice. A solution could be to cache the timestamp status but it can bring other issues. In case of SFP, if we change the module, it doesn't make sense to blindly re-set the timestamp back to PHY, if the new module has a PHY with mediocre timestamping capabilities. Signed-off-by: Kory Maincent Signed-off-by: David S. Miller --- drivers/net/phy/bcm-phy-ptp.c | 3 +++ drivers/net/phy/dp83640.c | 3 +++ drivers/net/phy/micrel.c | 6 ++++++ drivers/net/phy/mscc/mscc_ptp.c | 2 ++ drivers/net/phy/nxp-c45-tja11xx.c | 3 +++ drivers/net/phy/phy_device.c | 37 +++++++++++++++++++++++++++++++++++++ include/linux/netdevice.h | 5 +++++ include/linux/phy.h | 4 ++++ net/core/dev.c | 3 +++ net/core/dev_ioctl.c | 36 ++++++++++++++++++++++-------------- net/core/timestamping.c | 10 ++++++++++ net/ethtool/common.c | 19 +++++++++++++++++-- 12 files changed, 115 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/bcm-phy-ptp.c b/drivers/net/phy/bcm-phy-ptp.c index 617d384d4551..d3e825c951ee 100644 --- a/drivers/net/phy/bcm-phy-ptp.c +++ b/drivers/net/phy/bcm-phy-ptp.c @@ -931,6 +931,9 @@ struct bcm_ptp_private *bcm_ptp_probe(struct phy_device *phydev) return ERR_CAST(clock); priv->ptp_clock = clock; + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; + priv->phydev = phydev; bcm_ptp_init(priv); diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c index 5c42c47dc564..64fd1a109c0f 100644 --- a/drivers/net/phy/dp83640.c +++ b/drivers/net/phy/dp83640.c @@ -1450,6 +1450,9 @@ static int dp83640_probe(struct phy_device *phydev) phydev->mii_ts = &dp83640->mii_ts; phydev->priv = dp83640; + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; + spin_lock_init(&dp83640->rx_lock); skb_queue_head_init(&dp83640->rx_queue); skb_queue_head_init(&dp83640->tx_queue); diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index bd4cd082662f..2b8dd0131926 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -3158,6 +3158,9 @@ static void lan8814_ptp_init(struct phy_device *phydev) ptp_priv->mii_ts.ts_info = lan8814_ts_info; phydev->mii_ts = &ptp_priv->mii_ts; + + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; } static int lan8814_ptp_probe_once(struct phy_device *phydev) @@ -4586,6 +4589,9 @@ static int lan8841_probe(struct phy_device *phydev) phydev->mii_ts = &ptp_priv->mii_ts; + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; + return 0; } diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c index eb0b032cb613..fd174eb06d4a 100644 --- a/drivers/net/phy/mscc/mscc_ptp.c +++ b/drivers/net/phy/mscc/mscc_ptp.c @@ -1570,6 +1570,8 @@ int vsc8584_ptp_probe(struct phy_device *phydev) return PTR_ERR(vsc8531->load_save); } + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; vsc8531->ptp->phydev = phydev; return 0; diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c index 780ad353cf55..0515c7b979db 100644 --- a/drivers/net/phy/nxp-c45-tja11xx.c +++ b/drivers/net/phy/nxp-c45-tja11xx.c @@ -1658,6 +1658,9 @@ static int nxp_c45_probe(struct phy_device *phydev) priv->mii_ts.ts_info = nxp_c45_ts_info; phydev->mii_ts = &priv->mii_ts; ret = nxp_c45_init_ptp_clock(priv); + + /* Timestamp selected by default to keep legacy API */ + phydev->default_timestamp = true; } else { phydev_dbg(phydev, "PTP support not enabled even if the phy supports it"); } diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 2ce74593d6e4..8c4794631daa 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1411,6 +1411,26 @@ int phy_sfp_probe(struct phy_device *phydev, } EXPORT_SYMBOL(phy_sfp_probe); +/** + * phy_set_timestamp - set the default selected timestamping device + * @dev: Pointer to net_device + * @phydev: Pointer to phy_device + * + * This is used to set default timestamping device taking into account + * the new API choice, which is selecting the timestamping from MAC by + * default if the phydev does not have default_timestamp flag enabled. + */ +static void phy_set_timestamp(struct net_device *dev, struct phy_device *phydev) +{ + const struct ethtool_ops *ops = dev->ethtool_ops; + + if (!phy_has_tsinfo(phydev)) + return; + + if (!ops->get_ts_info || phydev->default_timestamp) + dev->ts_layer = PHY_TIMESTAMPING; +} + /** * phy_attach_direct - attach a network device to a given PHY device pointer * @dev: network device to attach @@ -1484,6 +1504,7 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, phydev->phy_link_change = phy_link_change; if (dev) { + phy_set_timestamp(dev, phydev); phydev->attached_dev = dev; dev->phydev = phydev; @@ -1812,6 +1833,22 @@ void phy_detach(struct phy_device *phydev) phy_suspend(phydev); if (dev) { + const struct ethtool_ops *ops = dev->ethtool_ops; + struct ethtool_ts_info ts_info = {0}; + + if (ops->get_ts_info) { + ops->get_ts_info(dev, &ts_info); + if ((ts_info.so_timestamping & + SOF_TIMESTAMPING_HARDWARE_MASK) == + SOF_TIMESTAMPING_HARDWARE_MASK) + dev->ts_layer = MAC_TIMESTAMPING; + else if ((ts_info.so_timestamping & + SOF_TIMESTAMPING_SOFTWARE_MASK) == + SOF_TIMESTAMPING_SOFTWARE_MASK) + dev->ts_layer = SOFTWARE_TIMESTAMPING; + } else { + dev->ts_layer = NO_TIMESTAMPING; + } phydev->attached_dev->phydev = NULL; phydev->attached_dev = NULL; } diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 2d840d7056f2..f020d2790c12 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -2074,6 +2075,8 @@ enum netdev_ml_priv_type { * * @dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem, * where the clock is recovered. + * @ts_layer: Tracks which network device + * performs packet time stamping. * * FIXME: cleanup struct net_device such that network protocol info * moves out. @@ -2435,6 +2438,8 @@ struct net_device { #if IS_ENABLED(CONFIG_DPLL) struct dpll_pin *dpll_pin; #endif + + enum timestamping_layer ts_layer; }; #define to_net_dev(d) container_of(d, struct net_device, dev) diff --git a/include/linux/phy.h b/include/linux/phy.h index e5f1f41e399c..317def2a7843 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -604,6 +604,8 @@ struct macsec_ops; * handling shall be postponed until PHY has resumed * @irq_rerun: Flag indicating interrupts occurred while PHY was suspended, * requiring a rerun of the interrupt handler after resume + * @default_timestamp: Flag indicating whether we are using the phy + * timestamp as the default one * @interface: enum phy_interface_t value * @skb: Netlink message for cable diagnostics * @nest: Netlink nest used for cable diagnostics @@ -667,6 +669,8 @@ struct phy_device { unsigned irq_suspended:1; unsigned irq_rerun:1; + unsigned default_timestamp:1; + int rate_matching; enum phy_state state; diff --git a/net/core/dev.c b/net/core/dev.c index af53f6d838ce..05ce00632892 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10212,6 +10212,9 @@ int register_netdevice(struct net_device *dev) dev->rtnl_link_state == RTNL_LINK_INITIALIZED) rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); + if (dev->ethtool_ops->get_ts_info) + dev->ts_layer = MAC_TIMESTAMPING; + out: return ret; diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index 267cd00269d0..bc8be9749376 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -259,9 +259,7 @@ static int dev_eth_ioctl(struct net_device *dev, * @dev: Network device * @cfg: Timestamping configuration structure * - * Helper for enforcing a common policy that phylib timestamping, if available, - * should take precedence in front of hardware timestamping provided by the - * netdev. + * Helper for calling the selected hardware provider timestamping. * * Note: phy_mii_ioctl() only handles SIOCSHWTSTAMP (not SIOCGHWTSTAMP), and * there only exists a phydev->mii_ts->hwtstamp() method. So this will return @@ -271,10 +269,14 @@ static int dev_eth_ioctl(struct net_device *dev, static int dev_get_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg) { - if (phy_has_hwtstamp(dev->phydev)) + enum timestamping_layer ts_layer = dev->ts_layer; + + if (ts_layer == PHY_TIMESTAMPING) return phy_hwtstamp_get(dev->phydev, cfg); + else if (ts_layer == MAC_TIMESTAMPING) + return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); - return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); + return -EOPNOTSUPP; } static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) @@ -315,9 +317,8 @@ static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) * @cfg: Timestamping configuration structure * @extack: Netlink extended ack message structure, for error reporting * - * Helper for enforcing a common policy that phylib timestamping, if available, - * should take precedence in front of hardware timestamping provided by the - * netdev. If the netdev driver needs to perform specific actions even for PHY + * Helper for calling the selected hardware provider timestamping. + * If the netdev driver needs to perform specific actions even for PHY * timestamping to work properly (a switch port must trap the timestamped * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in * dev->priv_flags. @@ -327,20 +328,26 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; - bool phy_ts = phy_has_hwtstamp(dev->phydev); + enum timestamping_layer ts_layer = dev->ts_layer; struct kernel_hwtstamp_config old_cfg = {}; bool changed = false; int err; - cfg->source = phy_ts ? PHY_TIMESTAMPING : MAC_TIMESTAMPING; + cfg->source = ts_layer; + + if (ts_layer != PHY_TIMESTAMPING && + ts_layer != MAC_TIMESTAMPING) + return -EOPNOTSUPP; - if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { + if (ts_layer == PHY_TIMESTAMPING && + dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) { err = ops->ndo_hwtstamp_get(dev, &old_cfg); if (err) return err; } - if (!phy_ts || (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { + if (ts_layer == MAC_TIMESTAMPING || + dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) { err = ops->ndo_hwtstamp_set(dev, cfg, extack); if (err) { if (extack->_msg) @@ -349,10 +356,11 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, } } - if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) + if (ts_layer == PHY_TIMESTAMPING && + dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) changed = kernel_hwtstamp_config_changed(&old_cfg, cfg); - if (phy_ts) { + if (ts_layer == PHY_TIMESTAMPING) { err = phy_hwtstamp_set(dev->phydev, cfg, extack); if (err) { if (changed) diff --git a/net/core/timestamping.c b/net/core/timestamping.c index 04840697fe79..5cf51a523fb3 100644 --- a/net/core/timestamping.c +++ b/net/core/timestamping.c @@ -21,6 +21,7 @@ static unsigned int classify(const struct sk_buff *skb) void skb_clone_tx_timestamp(struct sk_buff *skb) { + enum timestamping_layer ts_layer; struct mii_timestamper *mii_ts; struct sk_buff *clone; unsigned int type; @@ -28,6 +29,10 @@ void skb_clone_tx_timestamp(struct sk_buff *skb) if (!skb->sk) return; + ts_layer = skb->dev->ts_layer; + if (ts_layer != PHY_TIMESTAMPING) + return; + type = classify(skb); if (type == PTP_CLASS_NONE) return; @@ -44,12 +49,17 @@ EXPORT_SYMBOL_GPL(skb_clone_tx_timestamp); bool skb_defer_rx_timestamp(struct sk_buff *skb) { + enum timestamping_layer ts_layer; struct mii_timestamper *mii_ts; unsigned int type; if (!skb->dev || !skb->dev->phydev || !skb->dev->phydev->mii_ts) return false; + ts_layer = skb->dev->ts_layer; + if (ts_layer != PHY_TIMESTAMPING) + return false; + if (skb_headroom(skb) < ETH_HLEN) return false; diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 11d8797f63f6..9f6e3b2c74e2 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -633,13 +633,28 @@ int __ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { const struct ethtool_ops *ops = dev->ethtool_ops; struct phy_device *phydev = dev->phydev; + enum timestamping_layer ts_layer; + int ret; memset(info, 0, sizeof(*info)); info->cmd = ETHTOOL_GET_TS_INFO; - if (phy_has_tsinfo(phydev)) + ts_layer = dev->ts_layer; + if (ts_layer == SOFTWARE_TIMESTAMPING) { + ret = ops->get_ts_info(dev, info); + if (ret) + return ret; + info->so_timestamping &= ~SOF_TIMESTAMPING_HARDWARE_MASK; + info->phc_index = -1; + info->rx_filters = 0; + info->tx_types = 0; + return 0; + } + + if (ts_layer == PHY_TIMESTAMPING) return phy_ts_info(phydev, info); - if (ops->get_ts_info) + + if (ts_layer == MAC_TIMESTAMPING) return ops->get_ts_info(dev, info); info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | -- cgit v1.2.3 From db840d389bad60ce6f3aadc1079da13e7e993a16 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 17 Nov 2023 19:46:16 -0800 Subject: bpf: move verbose_linfo() into kernel/bpf/log.c verifier.c is huge. Let's try to move out parts that are logging-related into log.c, as we previously did with bpf_log() and other related stuff. This patch moves line info verbose output routines: it's pretty self-contained and isolated code, so there is no problem with this. Acked-by: Eduard Zingerman Acked-by: Stanislav Fomichev Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231118034623.3320920-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 4 +++ kernel/bpf/log.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 57 ------------------------------------------ 3 files changed, 63 insertions(+), 57 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 52a4012b8255..d896f3db6a22 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -680,6 +680,10 @@ int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level, void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos); int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); +__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, + u32 insn_off, + const char *prefix_fmt, ...); + static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env) { struct bpf_verifier_state *cur = env->cur_state; diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 850494423530..f20e1449c882 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -10,6 +10,8 @@ #include #include +#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) + static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log) { /* ubuf and len_total should both be specified (or not) together */ @@ -325,3 +327,60 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, va_end(args); } EXPORT_SYMBOL_GPL(bpf_log); + +static const struct bpf_line_info * +find_linfo(const struct bpf_verifier_env *env, u32 insn_off) +{ + const struct bpf_line_info *linfo; + const struct bpf_prog *prog; + u32 i, nr_linfo; + + prog = env->prog; + nr_linfo = prog->aux->nr_linfo; + + if (!nr_linfo || insn_off >= prog->len) + return NULL; + + linfo = prog->aux->linfo; + for (i = 1; i < nr_linfo; i++) + if (insn_off < linfo[i].insn_off) + break; + + return &linfo[i - 1]; +} + +static const char *ltrim(const char *s) +{ + while (isspace(*s)) + s++; + + return s; +} + +__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, + u32 insn_off, + const char *prefix_fmt, ...) +{ + const struct bpf_line_info *linfo; + + if (!bpf_verifier_log_needed(&env->log)) + return; + + linfo = find_linfo(env, insn_off); + if (!linfo || linfo == env->prev_linfo) + return; + + if (prefix_fmt) { + va_list args; + + va_start(args, prefix_fmt); + bpf_verifier_vlog(&env->log, prefix_fmt, args); + va_end(args); + } + + verbose(env, "%s\n", + ltrim(btf_name_by_offset(env->prog->aux->btf, + linfo->line_off))); + + env->prev_linfo = linfo; +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7c3461b89513..683fdda25c13 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -337,27 +337,6 @@ struct btf *btf_vmlinux; static DEFINE_MUTEX(bpf_verifier_lock); -static const struct bpf_line_info * -find_linfo(const struct bpf_verifier_env *env, u32 insn_off) -{ - const struct bpf_line_info *linfo; - const struct bpf_prog *prog; - u32 i, nr_linfo; - - prog = env->prog; - nr_linfo = prog->aux->nr_linfo; - - if (!nr_linfo || insn_off >= prog->len) - return NULL; - - linfo = prog->aux->linfo; - for (i = 1; i < nr_linfo; i++) - if (insn_off < linfo[i].insn_off) - break; - - return &linfo[i - 1]; -} - __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) { struct bpf_verifier_env *env = private_data; @@ -371,42 +350,6 @@ __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) va_end(args); } -static const char *ltrim(const char *s) -{ - while (isspace(*s)) - s++; - - return s; -} - -__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, - u32 insn_off, - const char *prefix_fmt, ...) -{ - const struct bpf_line_info *linfo; - - if (!bpf_verifier_log_needed(&env->log)) - return; - - linfo = find_linfo(env, insn_off); - if (!linfo || linfo == env->prev_linfo) - return; - - if (prefix_fmt) { - va_list args; - - va_start(args, prefix_fmt); - bpf_verifier_vlog(&env->log, prefix_fmt, args); - va_end(args); - } - - verbose(env, "%s\n", - ltrim(btf_name_by_offset(env->prog->aux->btf, - linfo->line_off))); - - env->prev_linfo = linfo; -} - static void verbose_invalid_scalar(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct tnum *range, const char *ctx, -- cgit v1.2.3 From 42feb6620accded89cad5f455665e21281813d79 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Fri, 17 Nov 2023 19:46:17 -0800 Subject: bpf: move verifier state printing code to kernel/bpf/log.c Move a good chunk of code from verifier.c to log.c: verifier state verbose printing logic. This is an important and very much logging/debugging oriented code. It fits the overlall log.c's focus on verifier logging, and moving it allows to keep growing it without unnecessarily adding to verifier.c code that otherwise contains a core verification logic. There are not many shared dependencies between this code and the rest of verifier.c code, except a few single-line helpers for various register type checks and a bit of state "scratching" helpers. We move all such trivial helpers into include/bpf/bpf_verifier.h as static inlines. No functional changes in this patch. Acked-by: Eduard Zingerman Acked-by: Stanislav Fomichev Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231118034623.3320920-3-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 72 ++++++++ kernel/bpf/log.c | 342 ++++++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 403 ------------------------------------------- 3 files changed, 414 insertions(+), 403 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d896f3db6a22..39edc76f436e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -783,4 +783,76 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type) return type_flag(type) & ~BPF_REG_TRUSTED_MODIFIERS; } +static inline bool type_is_ptr_alloc_obj(u32 type) +{ + return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; +} + +static inline bool type_is_non_owning_ref(u32 type) +{ + return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; +} + +static inline bool type_is_pkt_pointer(enum bpf_reg_type type) +{ + type = base_type(type); + return type == PTR_TO_PACKET || + type == PTR_TO_PACKET_META; +} + +static inline bool type_is_sk_pointer(enum bpf_reg_type type) +{ + return type == PTR_TO_SOCKET || + type == PTR_TO_SOCK_COMMON || + type == PTR_TO_TCP_SOCK || + type == PTR_TO_XDP_SOCK; +} + +static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) +{ + env->scratched_regs |= 1U << regno; +} + +static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) +{ + env->scratched_stack_slots |= 1ULL << spi; +} + +static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) +{ + return (env->scratched_regs >> regno) & 1; +} + +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +{ + return (env->scratched_stack_slots >> regno) & 1; +} + +static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) +{ + return env->scratched_regs || env->scratched_stack_slots; +} + +static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) +{ + env->scratched_regs = 0U; + env->scratched_stack_slots = 0ULL; +} + +/* Used for printing the entire verifier state. */ +static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) +{ + env->scratched_regs = ~0U; + env->scratched_stack_slots = ~0ULL; +} + +const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type); +const char *dynptr_type_str(enum bpf_dynptr_type type); +const char *iter_type_str(const struct btf *btf, u32 btf_id); +const char *iter_state_str(enum bpf_iter_state state); + +void print_verifier_state(struct bpf_verifier_env *env, + const struct bpf_func_state *state, bool print_all); +void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index f20e1449c882..c1b257eac21b 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -384,3 +384,345 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, env->prev_linfo = linfo; } + +static const char *btf_type_name(const struct btf *btf, u32 id) +{ + return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); +} + +/* string representation of 'enum bpf_reg_type' + * + * Note that reg_type_str() can not appear more than once in a single verbose() + * statement. + */ +const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type) +{ + char postfix[16] = {0}, prefix[64] = {0}; + static const char * const str[] = { + [NOT_INIT] = "?", + [SCALAR_VALUE] = "scalar", + [PTR_TO_CTX] = "ctx", + [CONST_PTR_TO_MAP] = "map_ptr", + [PTR_TO_MAP_VALUE] = "map_value", + [PTR_TO_STACK] = "fp", + [PTR_TO_PACKET] = "pkt", + [PTR_TO_PACKET_META] = "pkt_meta", + [PTR_TO_PACKET_END] = "pkt_end", + [PTR_TO_FLOW_KEYS] = "flow_keys", + [PTR_TO_SOCKET] = "sock", + [PTR_TO_SOCK_COMMON] = "sock_common", + [PTR_TO_TCP_SOCK] = "tcp_sock", + [PTR_TO_TP_BUFFER] = "tp_buffer", + [PTR_TO_XDP_SOCK] = "xdp_sock", + [PTR_TO_BTF_ID] = "ptr_", + [PTR_TO_MEM] = "mem", + [PTR_TO_BUF] = "buf", + [PTR_TO_FUNC] = "func", + [PTR_TO_MAP_KEY] = "map_key", + [CONST_PTR_TO_DYNPTR] = "dynptr_ptr", + }; + + if (type & PTR_MAYBE_NULL) { + if (base_type(type) == PTR_TO_BTF_ID) + strncpy(postfix, "or_null_", 16); + else + strncpy(postfix, "_or_null", 16); + } + + snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", + type & MEM_RDONLY ? "rdonly_" : "", + type & MEM_RINGBUF ? "ringbuf_" : "", + type & MEM_USER ? "user_" : "", + type & MEM_PERCPU ? "percpu_" : "", + type & MEM_RCU ? "rcu_" : "", + type & PTR_UNTRUSTED ? "untrusted_" : "", + type & PTR_TRUSTED ? "trusted_" : "" + ); + + snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s", + prefix, str[base_type(type)], postfix); + return env->tmp_str_buf; +} + +const char *dynptr_type_str(enum bpf_dynptr_type type) +{ + switch (type) { + case BPF_DYNPTR_TYPE_LOCAL: + return "local"; + case BPF_DYNPTR_TYPE_RINGBUF: + return "ringbuf"; + case BPF_DYNPTR_TYPE_SKB: + return "skb"; + case BPF_DYNPTR_TYPE_XDP: + return "xdp"; + case BPF_DYNPTR_TYPE_INVALID: + return ""; + default: + WARN_ONCE(1, "unknown dynptr type %d\n", type); + return ""; + } +} + +const char *iter_type_str(const struct btf *btf, u32 btf_id) +{ + if (!btf || btf_id == 0) + return ""; + + /* we already validated that type is valid and has conforming name */ + return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1; +} + +const char *iter_state_str(enum bpf_iter_state state) +{ + switch (state) { + case BPF_ITER_STATE_ACTIVE: + return "active"; + case BPF_ITER_STATE_DRAINED: + return "drained"; + case BPF_ITER_STATE_INVALID: + return ""; + default: + WARN_ONCE(1, "unknown iter state %d\n", state); + return ""; + } +} + +static char slot_type_char[] = { + [STACK_INVALID] = '?', + [STACK_SPILL] = 'r', + [STACK_MISC] = 'm', + [STACK_ZERO] = '0', + [STACK_DYNPTR] = 'd', + [STACK_ITER] = 'i', +}; + +static void print_liveness(struct bpf_verifier_env *env, + enum bpf_reg_liveness live) +{ + if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) + verbose(env, "_"); + if (live & REG_LIVE_READ) + verbose(env, "r"); + if (live & REG_LIVE_WRITTEN) + verbose(env, "w"); + if (live & REG_LIVE_DONE) + verbose(env, "D"); +} + +static void print_scalar_ranges(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + const char **sep) +{ + struct { + const char *name; + u64 val; + bool omit; + } minmaxs[] = { + {"smin", reg->smin_value, reg->smin_value == S64_MIN}, + {"smax", reg->smax_value, reg->smax_value == S64_MAX}, + {"umin", reg->umin_value, reg->umin_value == 0}, + {"umax", reg->umax_value, reg->umax_value == U64_MAX}, + {"smin32", (s64)reg->s32_min_value, reg->s32_min_value == S32_MIN}, + {"smax32", (s64)reg->s32_max_value, reg->s32_max_value == S32_MAX}, + {"umin32", reg->u32_min_value, reg->u32_min_value == 0}, + {"umax32", reg->u32_max_value, reg->u32_max_value == U32_MAX}, + }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)]; + bool neg1, neg2; + + for (m1 = &minmaxs[0]; m1 < mend; m1++) { + if (m1->omit) + continue; + + neg1 = m1->name[0] == 's' && (s64)m1->val < 0; + + verbose(env, "%s%s=", *sep, m1->name); + *sep = ","; + + for (m2 = m1 + 2; m2 < mend; m2 += 2) { + if (m2->omit || m2->val != m1->val) + continue; + /* don't mix negatives with positives */ + neg2 = m2->name[0] == 's' && (s64)m2->val < 0; + if (neg2 != neg1) + continue; + m2->omit = true; + verbose(env, "%s=", m2->name); + } + + verbose(env, m1->name[0] == 's' ? "%lld" : "%llu", m1->val); + } +} + +void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state, + bool print_all) +{ + const struct bpf_reg_state *reg; + enum bpf_reg_type t; + int i; + + if (state->frameno) + verbose(env, " frame%d:", state->frameno); + for (i = 0; i < MAX_BPF_REG; i++) { + reg = &state->regs[i]; + t = reg->type; + if (t == NOT_INIT) + continue; + if (!print_all && !reg_scratched(env, i)) + continue; + verbose(env, " R%d", i); + print_liveness(env, reg->live); + verbose(env, "="); + if (t == SCALAR_VALUE && reg->precise) + verbose(env, "P"); + if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && + tnum_is_const(reg->var_off)) { + /* reg->off should be 0 for SCALAR_VALUE */ + verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); + verbose(env, "%lld", reg->var_off.value + reg->off); + } else { + const char *sep = ""; + + verbose(env, "%s", reg_type_str(env, t)); + if (base_type(t) == PTR_TO_BTF_ID) + verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); + verbose(env, "("); +/* + * _a stands for append, was shortened to avoid multiline statements below. + * This macro is used to output a comma separated list of attributes. + */ +#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) + + if (reg->id) + verbose_a("id=%d", reg->id); + if (reg->ref_obj_id) + verbose_a("ref_obj_id=%d", reg->ref_obj_id); + if (type_is_non_owning_ref(reg->type)) + verbose_a("%s", "non_own_ref"); + if (t != SCALAR_VALUE) + verbose_a("off=%d", reg->off); + if (type_is_pkt_pointer(t)) + verbose_a("r=%d", reg->range); + else if (base_type(t) == CONST_PTR_TO_MAP || + base_type(t) == PTR_TO_MAP_KEY || + base_type(t) == PTR_TO_MAP_VALUE) + verbose_a("ks=%d,vs=%d", + reg->map_ptr->key_size, + reg->map_ptr->value_size); + if (tnum_is_const(reg->var_off)) { + /* Typically an immediate SCALAR_VALUE, but + * could be a pointer whose offset is too big + * for reg->off + */ + verbose_a("imm=%llx", reg->var_off.value); + } else { + print_scalar_ranges(env, reg, &sep); + if (!tnum_is_unknown(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose_a("var_off=%s", tn_buf); + } + } +#undef verbose_a + + verbose(env, ")"); + } + } + for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { + char types_buf[BPF_REG_SIZE + 1]; + bool valid = false; + int j; + + for (j = 0; j < BPF_REG_SIZE; j++) { + if (state->stack[i].slot_type[j] != STACK_INVALID) + valid = true; + types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; + } + types_buf[BPF_REG_SIZE] = 0; + if (!valid) + continue; + if (!print_all && !stack_slot_scratched(env, i)) + continue; + switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { + case STACK_SPILL: + reg = &state->stack[i].spilled_ptr; + t = reg->type; + + verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); + print_liveness(env, reg->live); + verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); + if (t == SCALAR_VALUE && reg->precise) + verbose(env, "P"); + if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) + verbose(env, "%lld", reg->var_off.value + reg->off); + break; + case STACK_DYNPTR: + i += BPF_DYNPTR_NR_SLOTS - 1; + reg = &state->stack[i].spilled_ptr; + + verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); + print_liveness(env, reg->live); + verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type)); + if (reg->ref_obj_id) + verbose(env, "(ref_id=%d)", reg->ref_obj_id); + break; + case STACK_ITER: + /* only main slot has ref_obj_id set; skip others */ + reg = &state->stack[i].spilled_ptr; + if (!reg->ref_obj_id) + continue; + + verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); + print_liveness(env, reg->live); + verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)", + iter_type_str(reg->iter.btf, reg->iter.btf_id), + reg->ref_obj_id, iter_state_str(reg->iter.state), + reg->iter.depth); + break; + case STACK_MISC: + case STACK_ZERO: + default: + reg = &state->stack[i].spilled_ptr; + + for (j = 0; j < BPF_REG_SIZE; j++) + types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; + types_buf[BPF_REG_SIZE] = 0; + + verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); + print_liveness(env, reg->live); + verbose(env, "=%s", types_buf); + break; + } + } + if (state->acquired_refs && state->refs[0].id) { + verbose(env, " refs=%d", state->refs[0].id); + for (i = 1; i < state->acquired_refs; i++) + if (state->refs[i].id) + verbose(env, ",%d", state->refs[i].id); + } + if (state->in_callback_fn) + verbose(env, " cb"); + if (state->in_async_callback_fn) + verbose(env, " async_cb"); + verbose(env, "\n"); + if (!print_all) + mark_verifier_state_clean(env); +} + +static inline u32 vlog_alignment(u32 pos) +{ + return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), + BPF_LOG_MIN_ALIGNMENT) - pos - 1; +} + +void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state) +{ + if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { + /* remove new line character */ + bpf_vlog_reset(&env->log, env->prev_log_pos - 1); + verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' '); + } else { + verbose(env, "%d:", env->insn_idx); + } + print_verifier_state(env, state, false); +} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 683fdda25c13..8c2d31aa3d31 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -368,21 +368,6 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env, verbose(env, " should have been in %s\n", tn_buf); } -static bool type_is_pkt_pointer(enum bpf_reg_type type) -{ - type = base_type(type); - return type == PTR_TO_PACKET || - type == PTR_TO_PACKET_META; -} - -static bool type_is_sk_pointer(enum bpf_reg_type type) -{ - return type == PTR_TO_SOCKET || - type == PTR_TO_SOCK_COMMON || - type == PTR_TO_TCP_SOCK || - type == PTR_TO_XDP_SOCK; -} - static bool type_may_be_null(u32 type) { return type & PTR_MAYBE_NULL; @@ -406,16 +391,6 @@ static bool reg_not_null(const struct bpf_reg_state *reg) type == PTR_TO_MEM; } -static bool type_is_ptr_alloc_obj(u32 type) -{ - return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; -} - -static bool type_is_non_owning_ref(u32 type) -{ - return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; -} - static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) { struct btf_record *rec = NULL; @@ -532,83 +507,6 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) insn->imm == BPF_CMPXCHG; } -/* string representation of 'enum bpf_reg_type' - * - * Note that reg_type_str() can not appear more than once in a single verbose() - * statement. - */ -static const char *reg_type_str(struct bpf_verifier_env *env, - enum bpf_reg_type type) -{ - char postfix[16] = {0}, prefix[64] = {0}; - static const char * const str[] = { - [NOT_INIT] = "?", - [SCALAR_VALUE] = "scalar", - [PTR_TO_CTX] = "ctx", - [CONST_PTR_TO_MAP] = "map_ptr", - [PTR_TO_MAP_VALUE] = "map_value", - [PTR_TO_STACK] = "fp", - [PTR_TO_PACKET] = "pkt", - [PTR_TO_PACKET_META] = "pkt_meta", - [PTR_TO_PACKET_END] = "pkt_end", - [PTR_TO_FLOW_KEYS] = "flow_keys", - [PTR_TO_SOCKET] = "sock", - [PTR_TO_SOCK_COMMON] = "sock_common", - [PTR_TO_TCP_SOCK] = "tcp_sock", - [PTR_TO_TP_BUFFER] = "tp_buffer", - [PTR_TO_XDP_SOCK] = "xdp_sock", - [PTR_TO_BTF_ID] = "ptr_", - [PTR_TO_MEM] = "mem", - [PTR_TO_BUF] = "buf", - [PTR_TO_FUNC] = "func", - [PTR_TO_MAP_KEY] = "map_key", - [CONST_PTR_TO_DYNPTR] = "dynptr_ptr", - }; - - if (type & PTR_MAYBE_NULL) { - if (base_type(type) == PTR_TO_BTF_ID) - strncpy(postfix, "or_null_", 16); - else - strncpy(postfix, "_or_null", 16); - } - - snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", - type & MEM_RDONLY ? "rdonly_" : "", - type & MEM_RINGBUF ? "ringbuf_" : "", - type & MEM_USER ? "user_" : "", - type & MEM_PERCPU ? "percpu_" : "", - type & MEM_RCU ? "rcu_" : "", - type & PTR_UNTRUSTED ? "untrusted_" : "", - type & PTR_TRUSTED ? "trusted_" : "" - ); - - snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s", - prefix, str[base_type(type)], postfix); - return env->tmp_str_buf; -} - -static char slot_type_char[] = { - [STACK_INVALID] = '?', - [STACK_SPILL] = 'r', - [STACK_MISC] = 'm', - [STACK_ZERO] = '0', - [STACK_DYNPTR] = 'd', - [STACK_ITER] = 'i', -}; - -static void print_liveness(struct bpf_verifier_env *env, - enum bpf_reg_liveness live) -{ - if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) - verbose(env, "_"); - if (live & REG_LIVE_READ) - verbose(env, "r"); - if (live & REG_LIVE_WRITTEN) - verbose(env, "w"); - if (live & REG_LIVE_DONE) - verbose(env, "D"); -} - static int __get_spi(s32 off) { return (-off - 1) / BPF_REG_SIZE; @@ -678,87 +576,6 @@ static const char *btf_type_name(const struct btf *btf, u32 id) return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); } -static const char *dynptr_type_str(enum bpf_dynptr_type type) -{ - switch (type) { - case BPF_DYNPTR_TYPE_LOCAL: - return "local"; - case BPF_DYNPTR_TYPE_RINGBUF: - return "ringbuf"; - case BPF_DYNPTR_TYPE_SKB: - return "skb"; - case BPF_DYNPTR_TYPE_XDP: - return "xdp"; - case BPF_DYNPTR_TYPE_INVALID: - return ""; - default: - WARN_ONCE(1, "unknown dynptr type %d\n", type); - return ""; - } -} - -static const char *iter_type_str(const struct btf *btf, u32 btf_id) -{ - if (!btf || btf_id == 0) - return ""; - - /* we already validated that type is valid and has conforming name */ - return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1; -} - -static const char *iter_state_str(enum bpf_iter_state state) -{ - switch (state) { - case BPF_ITER_STATE_ACTIVE: - return "active"; - case BPF_ITER_STATE_DRAINED: - return "drained"; - case BPF_ITER_STATE_INVALID: - return ""; - default: - WARN_ONCE(1, "unknown iter state %d\n", state); - return ""; - } -} - -static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) -{ - env->scratched_regs |= 1U << regno; -} - -static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) -{ - env->scratched_stack_slots |= 1ULL << spi; -} - -static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) -{ - return (env->scratched_regs >> regno) & 1; -} - -static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) -{ - return (env->scratched_stack_slots >> regno) & 1; -} - -static bool verifier_state_scratched(const struct bpf_verifier_env *env) -{ - return env->scratched_regs || env->scratched_stack_slots; -} - -static void mark_verifier_state_clean(struct bpf_verifier_env *env) -{ - env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; -} - -/* Used for printing the entire verifier state. */ -static void mark_verifier_state_scratched(struct bpf_verifier_env *env) -{ - env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; -} - static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) { switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { @@ -1298,226 +1115,6 @@ static void scrub_spilled_slot(u8 *stype) *stype = STACK_MISC; } -static void print_scalar_ranges(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, - const char **sep) -{ - struct { - const char *name; - u64 val; - bool omit; - } minmaxs[] = { - {"smin", reg->smin_value, reg->smin_value == S64_MIN}, - {"smax", reg->smax_value, reg->smax_value == S64_MAX}, - {"umin", reg->umin_value, reg->umin_value == 0}, - {"umax", reg->umax_value, reg->umax_value == U64_MAX}, - {"smin32", (s64)reg->s32_min_value, reg->s32_min_value == S32_MIN}, - {"smax32", (s64)reg->s32_max_value, reg->s32_max_value == S32_MAX}, - {"umin32", reg->u32_min_value, reg->u32_min_value == 0}, - {"umax32", reg->u32_max_value, reg->u32_max_value == U32_MAX}, - }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)]; - bool neg1, neg2; - - for (m1 = &minmaxs[0]; m1 < mend; m1++) { - if (m1->omit) - continue; - - neg1 = m1->name[0] == 's' && (s64)m1->val < 0; - - verbose(env, "%s%s=", *sep, m1->name); - *sep = ","; - - for (m2 = m1 + 2; m2 < mend; m2 += 2) { - if (m2->omit || m2->val != m1->val) - continue; - /* don't mix negatives with positives */ - neg2 = m2->name[0] == 's' && (s64)m2->val < 0; - if (neg2 != neg1) - continue; - m2->omit = true; - verbose(env, "%s=", m2->name); - } - - verbose(env, m1->name[0] == 's' ? "%lld" : "%llu", m1->val); - } -} - -static void print_verifier_state(struct bpf_verifier_env *env, - const struct bpf_func_state *state, - bool print_all) -{ - const struct bpf_reg_state *reg; - enum bpf_reg_type t; - int i; - - if (state->frameno) - verbose(env, " frame%d:", state->frameno); - for (i = 0; i < MAX_BPF_REG; i++) { - reg = &state->regs[i]; - t = reg->type; - if (t == NOT_INIT) - continue; - if (!print_all && !reg_scratched(env, i)) - continue; - verbose(env, " R%d", i); - print_liveness(env, reg->live); - verbose(env, "="); - if (t == SCALAR_VALUE && reg->precise) - verbose(env, "P"); - if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && - tnum_is_const(reg->var_off)) { - /* reg->off should be 0 for SCALAR_VALUE */ - verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); - verbose(env, "%lld", reg->var_off.value + reg->off); - } else { - const char *sep = ""; - - verbose(env, "%s", reg_type_str(env, t)); - if (base_type(t) == PTR_TO_BTF_ID) - verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); - verbose(env, "("); -/* - * _a stands for append, was shortened to avoid multiline statements below. - * This macro is used to output a comma separated list of attributes. - */ -#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) - - if (reg->id) - verbose_a("id=%d", reg->id); - if (reg->ref_obj_id) - verbose_a("ref_obj_id=%d", reg->ref_obj_id); - if (type_is_non_owning_ref(reg->type)) - verbose_a("%s", "non_own_ref"); - if (t != SCALAR_VALUE) - verbose_a("off=%d", reg->off); - if (type_is_pkt_pointer(t)) - verbose_a("r=%d", reg->range); - else if (base_type(t) == CONST_PTR_TO_MAP || - base_type(t) == PTR_TO_MAP_KEY || - base_type(t) == PTR_TO_MAP_VALUE) - verbose_a("ks=%d,vs=%d", - reg->map_ptr->key_size, - reg->map_ptr->value_size); - if (tnum_is_const(reg->var_off)) { - /* Typically an immediate SCALAR_VALUE, but - * could be a pointer whose offset is too big - * for reg->off - */ - verbose_a("imm=%llx", reg->var_off.value); - } else { - print_scalar_ranges(env, reg, &sep); - if (!tnum_is_unknown(reg->var_off)) { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose_a("var_off=%s", tn_buf); - } - } -#undef verbose_a - - verbose(env, ")"); - } - } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - char types_buf[BPF_REG_SIZE + 1]; - bool valid = false; - int j; - - for (j = 0; j < BPF_REG_SIZE; j++) { - if (state->stack[i].slot_type[j] != STACK_INVALID) - valid = true; - types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; - } - types_buf[BPF_REG_SIZE] = 0; - if (!valid) - continue; - if (!print_all && !stack_slot_scratched(env, i)) - continue; - switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { - case STACK_SPILL: - reg = &state->stack[i].spilled_ptr; - t = reg->type; - - verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); - print_liveness(env, reg->live); - verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); - if (t == SCALAR_VALUE && reg->precise) - verbose(env, "P"); - if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) - verbose(env, "%lld", reg->var_off.value + reg->off); - break; - case STACK_DYNPTR: - i += BPF_DYNPTR_NR_SLOTS - 1; - reg = &state->stack[i].spilled_ptr; - - verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); - print_liveness(env, reg->live); - verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type)); - if (reg->ref_obj_id) - verbose(env, "(ref_id=%d)", reg->ref_obj_id); - break; - case STACK_ITER: - /* only main slot has ref_obj_id set; skip others */ - reg = &state->stack[i].spilled_ptr; - if (!reg->ref_obj_id) - continue; - - verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); - print_liveness(env, reg->live); - verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)", - iter_type_str(reg->iter.btf, reg->iter.btf_id), - reg->ref_obj_id, iter_state_str(reg->iter.state), - reg->iter.depth); - break; - case STACK_MISC: - case STACK_ZERO: - default: - reg = &state->stack[i].spilled_ptr; - - for (j = 0; j < BPF_REG_SIZE; j++) - types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; - types_buf[BPF_REG_SIZE] = 0; - - verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); - print_liveness(env, reg->live); - verbose(env, "=%s", types_buf); - break; - } - } - if (state->acquired_refs && state->refs[0].id) { - verbose(env, " refs=%d", state->refs[0].id); - for (i = 1; i < state->acquired_refs; i++) - if (state->refs[i].id) - verbose(env, ",%d", state->refs[i].id); - } - if (state->in_callback_fn) - verbose(env, " cb"); - if (state->in_async_callback_fn) - verbose(env, " async_cb"); - verbose(env, "\n"); - if (!print_all) - mark_verifier_state_clean(env); -} - -static inline u32 vlog_alignment(u32 pos) -{ - return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), - BPF_LOG_MIN_ALIGNMENT) - pos - 1; -} - -static void print_insn_state(struct bpf_verifier_env *env, - const struct bpf_func_state *state) -{ - if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { - /* remove new line character */ - bpf_vlog_reset(&env->log, env->prev_log_pos - 1); - verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' '); - } else { - verbose(env, "%d:", env->insn_idx); - } - print_verifier_state(env, state, false); -} - /* copy array src of length n * size bytes to dst. dst is reallocated if it's too * small to hold src. This is different from krealloc since we don't want to preserve * the contents of dst. -- cgit v1.2.3 From 289354f21b2c3fac93e956efd45f256a88a4d997 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Sat, 18 Nov 2023 18:38:05 -0800 Subject: net: partial revert of the "Make timestamping selectable: series Revert following commits: commit acec05fb78ab ("net_tstamp: Add TIMESTAMPING SOFTWARE and HARDWARE mask") commit 11d55be06df0 ("net: ethtool: Add a command to expose current time stamping layer") commit bb8645b00ced ("netlink: specs: Introduce new netlink command to get current timestamp") commit d905f9c75329 ("net: ethtool: Add a command to list available time stamping layers") commit aed5004ee7a0 ("netlink: specs: Introduce new netlink command to list available time stamping layers") commit 51bdf3165f01 ("net: Replace hwtstamp_source by timestamping layer") commit 0f7f463d4821 ("net: Change the API of PHY default timestamp to MAC") commit 091fab122869 ("net: ethtool: ts: Update GET_TS to reply the current selected timestamp") commit 152c75e1d002 ("net: ethtool: ts: Let the active time stamping layer be selectable") commit ee60ea6be0d3 ("netlink: specs: Introduce time stamping set command") They need more time for reviews. Link: https://lore.kernel.org/all/20231118183529.6e67100c@kernel.org/ Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/ethtool.yaml | 57 ----- Documentation/networking/ethtool-netlink.rst | 63 ------ .../net/ethernet/microchip/lan966x/lan966x_main.c | 6 +- drivers/net/phy/bcm-phy-ptp.c | 3 - drivers/net/phy/dp83640.c | 3 - drivers/net/phy/micrel.c | 6 - drivers/net/phy/mscc/mscc_ptp.c | 2 - drivers/net/phy/nxp-c45-tja11xx.c | 3 - drivers/net/phy/phy_device.c | 37 ---- include/linux/net_tstamp.h | 11 +- include/linux/netdevice.h | 5 - include/linux/phy.h | 4 - include/uapi/linux/ethtool_netlink.h | 29 --- include/uapi/linux/net_tstamp.h | 18 -- net/core/dev.c | 3 - net/core/dev_ioctl.c | 36 ++- net/core/timestamping.c | 10 - net/ethtool/Makefile | 2 +- net/ethtool/common.c | 19 +- net/ethtool/common.h | 1 - net/ethtool/netlink.c | 28 --- net/ethtool/netlink.h | 4 - net/ethtool/ts.c | 244 --------------------- 23 files changed, 28 insertions(+), 566 deletions(-) delete mode 100644 net/ethtool/ts.c (limited to 'include/linux') diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 06d9120543d3..5c7a65b009b4 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -939,26 +939,6 @@ attribute-sets: - name: burst-tmr type: u32 - - - name: ts - attributes: - - - name: header - type: nest - nested-attributes: header - - - name: ts-layer - type: u32 - - - name: ts-list - attributes: - - - name: header - type: nest - nested-attributes: header - - - name: ts-list-layer - type: binary operations: enum-model: directional @@ -1709,40 +1689,3 @@ operations: name: mm-ntf doc: Notification for change in MAC Merge configuration. notify: mm-get - - - name: ts-get - doc: Get current timestamp - - attribute-set: ts - - do: - request: - attributes: - - header - reply: - attributes: &ts - - header - - ts-layer - - - name: ts-list-get - doc: Get list of timestamp devices available on an interface - - attribute-set: ts-list - - do: - request: - attributes: - - header - reply: - attributes: - - header - - ts-list-layer - - - name: ts-set - doc: Set the timestamp device - - attribute-set: ts - - do: - request: - attributes: *ts diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 530c1775e5f4..2540c70952ff 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -225,9 +225,6 @@ Userspace to kernel: ``ETHTOOL_MSG_RSS_GET`` get RSS settings ``ETHTOOL_MSG_MM_GET`` get MAC merge layer state ``ETHTOOL_MSG_MM_SET`` set MAC merge layer parameters - ``ETHTOOL_MSG_TS_GET`` get current timestamping - ``ETHTOOL_MSG_TS_LIST_GET`` list available timestampings - ``ETHTOOL_MSG_TS_SET`` set current timestamping ===================================== ================================= Kernel to userspace: @@ -271,8 +268,6 @@ Kernel to userspace: ``ETHTOOL_MSG_PSE_GET_REPLY`` PSE parameters ``ETHTOOL_MSG_RSS_GET_REPLY`` RSS settings ``ETHTOOL_MSG_MM_GET_REPLY`` MAC merge layer status - ``ETHTOOL_MSG_TS_GET_REPLY`` current timestamping - ``ETHTOOL_MSG_TS_LIST_GET_REPLY`` available timestampings ======================================== ================================= ``GET`` requests are sent by userspace applications to retrieve device @@ -1999,61 +1994,6 @@ The attributes are propagated to the driver through the following structure: .. kernel-doc:: include/linux/ethtool.h :identifiers: ethtool_mm_cfg -TS_GET -====== - -Gets current timestamping. - -Request contents: - - ================================= ====== ==================== - ``ETHTOOL_A_TS_HEADER`` nested request header - ================================= ====== ==================== - -Kernel response contents: - - ======================= ====== ============================== - ``ETHTOOL_A_TS_HEADER`` nested reply header - ``ETHTOOL_A_TS_LAYER`` u32 current timestamping - ======================= ====== ============================== - -This command get the current timestamp layer. - -TS_LIST_GET -=========== - -Get the list of available timestampings. - -Request contents: - - ================================= ====== ==================== - ``ETHTOOL_A_TS_HEADER`` nested request header - ================================= ====== ==================== - -Kernel response contents: - - =========================== ====== ============================== - ``ETHTOOL_A_TS_HEADER`` nested reply header - ``ETHTOOL_A_TS_LIST_LAYER`` binary available timestampings - =========================== ====== ============================== - -This command lists all the possible timestamp layer available. - -TS_SET -====== - -Modify the selected timestamping. - -Request contents: - - ======================= ====== =================== - ``ETHTOOL_A_TS_HEADER`` nested reply header - ``ETHTOOL_A_TS_LAYER`` u32 timestamping - ======================= ====== =================== - -This command set the timestamping with one that should be listed by the -TSLIST_GET command. - Request translation =================== @@ -2160,7 +2100,4 @@ are netlink only. n/a ``ETHTOOL_MSG_PLCA_GET_STATUS`` n/a ``ETHTOOL_MSG_MM_GET`` n/a ``ETHTOOL_MSG_MM_SET`` - n/a ``ETHTOOL_MSG_TS_GET`` - n/a ``ETHTOOL_MSG_TS_LIST_GET`` - n/a ``ETHTOOL_MSG_TS_SET`` =================================== ===================================== diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index fbe56b1bb386..2635ef8958c8 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -470,15 +470,15 @@ static int lan966x_port_hwtstamp_set(struct net_device *dev, struct lan966x_port *port = netdev_priv(dev); int err; - if (cfg->source != MAC_TIMESTAMPING && - cfg->source != PHY_TIMESTAMPING) + if (cfg->source != HWTSTAMP_SOURCE_NETDEV && + cfg->source != HWTSTAMP_SOURCE_PHYLIB) return -EOPNOTSUPP; err = lan966x_ptp_setup_traps(port, cfg); if (err) return err; - if (cfg->source == MAC_TIMESTAMPING) { + if (cfg->source == HWTSTAMP_SOURCE_NETDEV) { if (!port->lan966x->ptp) return -EOPNOTSUPP; diff --git a/drivers/net/phy/bcm-phy-ptp.c b/drivers/net/phy/bcm-phy-ptp.c index d3e825c951ee..617d384d4551 100644 --- a/drivers/net/phy/bcm-phy-ptp.c +++ b/drivers/net/phy/bcm-phy-ptp.c @@ -931,9 +931,6 @@ struct bcm_ptp_private *bcm_ptp_probe(struct phy_device *phydev) return ERR_CAST(clock); priv->ptp_clock = clock; - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; - priv->phydev = phydev; bcm_ptp_init(priv); diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c index 64fd1a109c0f..5c42c47dc564 100644 --- a/drivers/net/phy/dp83640.c +++ b/drivers/net/phy/dp83640.c @@ -1450,9 +1450,6 @@ static int dp83640_probe(struct phy_device *phydev) phydev->mii_ts = &dp83640->mii_ts; phydev->priv = dp83640; - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; - spin_lock_init(&dp83640->rx_lock); skb_queue_head_init(&dp83640->rx_queue); skb_queue_head_init(&dp83640->tx_queue); diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 2b8dd0131926..bd4cd082662f 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -3158,9 +3158,6 @@ static void lan8814_ptp_init(struct phy_device *phydev) ptp_priv->mii_ts.ts_info = lan8814_ts_info; phydev->mii_ts = &ptp_priv->mii_ts; - - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; } static int lan8814_ptp_probe_once(struct phy_device *phydev) @@ -4589,9 +4586,6 @@ static int lan8841_probe(struct phy_device *phydev) phydev->mii_ts = &ptp_priv->mii_ts; - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; - return 0; } diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c index fd174eb06d4a..eb0b032cb613 100644 --- a/drivers/net/phy/mscc/mscc_ptp.c +++ b/drivers/net/phy/mscc/mscc_ptp.c @@ -1570,8 +1570,6 @@ int vsc8584_ptp_probe(struct phy_device *phydev) return PTR_ERR(vsc8531->load_save); } - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; vsc8531->ptp->phydev = phydev; return 0; diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c index 0515c7b979db..780ad353cf55 100644 --- a/drivers/net/phy/nxp-c45-tja11xx.c +++ b/drivers/net/phy/nxp-c45-tja11xx.c @@ -1658,9 +1658,6 @@ static int nxp_c45_probe(struct phy_device *phydev) priv->mii_ts.ts_info = nxp_c45_ts_info; phydev->mii_ts = &priv->mii_ts; ret = nxp_c45_init_ptp_clock(priv); - - /* Timestamp selected by default to keep legacy API */ - phydev->default_timestamp = true; } else { phydev_dbg(phydev, "PTP support not enabled even if the phy supports it"); } diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 8c4794631daa..2ce74593d6e4 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1411,26 +1411,6 @@ int phy_sfp_probe(struct phy_device *phydev, } EXPORT_SYMBOL(phy_sfp_probe); -/** - * phy_set_timestamp - set the default selected timestamping device - * @dev: Pointer to net_device - * @phydev: Pointer to phy_device - * - * This is used to set default timestamping device taking into account - * the new API choice, which is selecting the timestamping from MAC by - * default if the phydev does not have default_timestamp flag enabled. - */ -static void phy_set_timestamp(struct net_device *dev, struct phy_device *phydev) -{ - const struct ethtool_ops *ops = dev->ethtool_ops; - - if (!phy_has_tsinfo(phydev)) - return; - - if (!ops->get_ts_info || phydev->default_timestamp) - dev->ts_layer = PHY_TIMESTAMPING; -} - /** * phy_attach_direct - attach a network device to a given PHY device pointer * @dev: network device to attach @@ -1504,7 +1484,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, phydev->phy_link_change = phy_link_change; if (dev) { - phy_set_timestamp(dev, phydev); phydev->attached_dev = dev; dev->phydev = phydev; @@ -1833,22 +1812,6 @@ void phy_detach(struct phy_device *phydev) phy_suspend(phydev); if (dev) { - const struct ethtool_ops *ops = dev->ethtool_ops; - struct ethtool_ts_info ts_info = {0}; - - if (ops->get_ts_info) { - ops->get_ts_info(dev, &ts_info); - if ((ts_info.so_timestamping & - SOF_TIMESTAMPING_HARDWARE_MASK) == - SOF_TIMESTAMPING_HARDWARE_MASK) - dev->ts_layer = MAC_TIMESTAMPING; - else if ((ts_info.so_timestamping & - SOF_TIMESTAMPING_SOFTWARE_MASK) == - SOF_TIMESTAMPING_SOFTWARE_MASK) - dev->ts_layer = SOFTWARE_TIMESTAMPING; - } else { - dev->ts_layer = NO_TIMESTAMPING; - } phydev->attached_dev->phydev = NULL; phydev->attached_dev = NULL; } diff --git a/include/linux/net_tstamp.h b/include/linux/net_tstamp.h index bb289c2ad376..eb01c37e71e0 100644 --- a/include/linux/net_tstamp.h +++ b/include/linux/net_tstamp.h @@ -5,6 +5,11 @@ #include +enum hwtstamp_source { + HWTSTAMP_SOURCE_NETDEV, + HWTSTAMP_SOURCE_PHYLIB, +}; + /** * struct kernel_hwtstamp_config - Kernel copy of struct hwtstamp_config * @@ -15,8 +20,8 @@ * a legacy implementation of a lower driver * @copied_to_user: request was passed to a legacy implementation which already * copied the ioctl request back to user space - * @source: indication whether timestamps should come from software, the netdev - * or from an attached phylib PHY + * @source: indication whether timestamps should come from the netdev or from + * an attached phylib PHY * * Prefer using this structure for in-kernel processing of hardware * timestamping configuration, over the inextensible struct hwtstamp_config @@ -28,7 +33,7 @@ struct kernel_hwtstamp_config { int rx_filter; struct ifreq *ifr; bool copied_to_user; - enum timestamping_layer source; + enum hwtstamp_source source; }; static inline void hwtstamp_config_to_kernel(struct kernel_hwtstamp_config *kernel_cfg, diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index f020d2790c12..2d840d7056f2 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -2075,8 +2074,6 @@ enum netdev_ml_priv_type { * * @dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem, * where the clock is recovered. - * @ts_layer: Tracks which network device - * performs packet time stamping. * * FIXME: cleanup struct net_device such that network protocol info * moves out. @@ -2438,8 +2435,6 @@ struct net_device { #if IS_ENABLED(CONFIG_DPLL) struct dpll_pin *dpll_pin; #endif - - enum timestamping_layer ts_layer; }; #define to_net_dev(d) container_of(d, struct net_device, dev) diff --git a/include/linux/phy.h b/include/linux/phy.h index 317def2a7843..e5f1f41e399c 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -604,8 +604,6 @@ struct macsec_ops; * handling shall be postponed until PHY has resumed * @irq_rerun: Flag indicating interrupts occurred while PHY was suspended, * requiring a rerun of the interrupt handler after resume - * @default_timestamp: Flag indicating whether we are using the phy - * timestamp as the default one * @interface: enum phy_interface_t value * @skb: Netlink message for cable diagnostics * @nest: Netlink nest used for cable diagnostics @@ -669,8 +667,6 @@ struct phy_device { unsigned irq_suspended:1; unsigned irq_rerun:1; - unsigned default_timestamp:1; - int rate_matching; enum phy_state state; diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index df6c4fcc62c1..73e2c10dc2cc 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -57,9 +57,6 @@ enum { ETHTOOL_MSG_PLCA_GET_STATUS, ETHTOOL_MSG_MM_GET, ETHTOOL_MSG_MM_SET, - ETHTOOL_MSG_TS_GET, - ETHTOOL_MSG_TS_LIST_GET, - ETHTOOL_MSG_TS_SET, /* add new constants above here */ __ETHTOOL_MSG_USER_CNT, @@ -112,8 +109,6 @@ enum { ETHTOOL_MSG_PLCA_NTF, ETHTOOL_MSG_MM_GET_REPLY, ETHTOOL_MSG_MM_NTF, - ETHTOOL_MSG_TS_GET_REPLY, - ETHTOOL_MSG_TS_LIST_GET_REPLY, /* add new constants above here */ __ETHTOOL_MSG_KERNEL_CNT, @@ -980,30 +975,6 @@ enum { ETHTOOL_A_MM_MAX = (__ETHTOOL_A_MM_CNT - 1) }; -/* TS LAYER */ - -enum { - ETHTOOL_A_TS_UNSPEC, - ETHTOOL_A_TS_HEADER, /* nest - _A_HEADER_* */ - ETHTOOL_A_TS_LAYER, /* u32 */ - - /* add new constants above here */ - __ETHTOOL_A_TS_CNT, - ETHTOOL_A_TS_MAX = (__ETHTOOL_A_TS_CNT - 1) -}; - -/* TS LIST LAYER */ - -enum { - ETHTOOL_A_TS_LIST_UNSPEC, - ETHTOOL_A_TS_LIST_HEADER, /* nest - _A_HEADER_* */ - ETHTOOL_A_TS_LIST_LAYER, /* array, u32 */ - - /* add new constants above here */ - __ETHTOOL_A_TS_LIST_CNT, - ETHTOOL_A_TS_LIST_MAX = (__ETHTOOL_A_TS_LIST_CNT - 1) -}; - /* generic netlink info */ #define ETHTOOL_GENL_NAME "ethtool" #define ETHTOOL_GENL_VERSION 1 diff --git a/include/uapi/linux/net_tstamp.h b/include/uapi/linux/net_tstamp.h index 4551fb3d7720..a2c66b3d7f0f 100644 --- a/include/uapi/linux/net_tstamp.h +++ b/include/uapi/linux/net_tstamp.h @@ -13,16 +13,6 @@ #include #include /* for SO_TIMESTAMPING */ -/* Layer of the TIMESTAMPING provider */ -enum timestamping_layer { - NO_TIMESTAMPING, - SOFTWARE_TIMESTAMPING, - MAC_TIMESTAMPING, - PHY_TIMESTAMPING, - - __TIMESTAMPING_COUNT, -}; - /* SO_TIMESTAMPING flags */ enum { SOF_TIMESTAMPING_TX_HARDWARE = (1<<0), @@ -58,14 +48,6 @@ enum { SOF_TIMESTAMPING_TX_SCHED | \ SOF_TIMESTAMPING_TX_ACK) -#define SOF_TIMESTAMPING_SOFTWARE_MASK (SOF_TIMESTAMPING_RX_SOFTWARE | \ - SOF_TIMESTAMPING_TX_SOFTWARE | \ - SOF_TIMESTAMPING_SOFTWARE) - -#define SOF_TIMESTAMPING_HARDWARE_MASK (SOF_TIMESTAMPING_RX_HARDWARE | \ - SOF_TIMESTAMPING_TX_HARDWARE | \ - SOF_TIMESTAMPING_RAW_HARDWARE) - /** * struct so_timestamping - SO_TIMESTAMPING parameter * diff --git a/net/core/dev.c b/net/core/dev.c index 05ce00632892..af53f6d838ce 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10212,9 +10212,6 @@ int register_netdevice(struct net_device *dev) dev->rtnl_link_state == RTNL_LINK_INITIALIZED) rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); - if (dev->ethtool_ops->get_ts_info) - dev->ts_layer = MAC_TIMESTAMPING; - out: return ret; diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index bc8be9749376..9a66cf5015f2 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -259,7 +259,9 @@ static int dev_eth_ioctl(struct net_device *dev, * @dev: Network device * @cfg: Timestamping configuration structure * - * Helper for calling the selected hardware provider timestamping. + * Helper for enforcing a common policy that phylib timestamping, if available, + * should take precedence in front of hardware timestamping provided by the + * netdev. * * Note: phy_mii_ioctl() only handles SIOCSHWTSTAMP (not SIOCGHWTSTAMP), and * there only exists a phydev->mii_ts->hwtstamp() method. So this will return @@ -269,14 +271,10 @@ static int dev_eth_ioctl(struct net_device *dev, static int dev_get_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg) { - enum timestamping_layer ts_layer = dev->ts_layer; - - if (ts_layer == PHY_TIMESTAMPING) + if (phy_has_hwtstamp(dev->phydev)) return phy_hwtstamp_get(dev->phydev, cfg); - else if (ts_layer == MAC_TIMESTAMPING) - return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); - return -EOPNOTSUPP; + return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); } static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) @@ -317,8 +315,9 @@ static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) * @cfg: Timestamping configuration structure * @extack: Netlink extended ack message structure, for error reporting * - * Helper for calling the selected hardware provider timestamping. - * If the netdev driver needs to perform specific actions even for PHY + * Helper for enforcing a common policy that phylib timestamping, if available, + * should take precedence in front of hardware timestamping provided by the + * netdev. If the netdev driver needs to perform specific actions even for PHY * timestamping to work properly (a switch port must trap the timestamped * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in * dev->priv_flags. @@ -328,26 +327,20 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; - enum timestamping_layer ts_layer = dev->ts_layer; + bool phy_ts = phy_has_hwtstamp(dev->phydev); struct kernel_hwtstamp_config old_cfg = {}; bool changed = false; int err; - cfg->source = ts_layer; - - if (ts_layer != PHY_TIMESTAMPING && - ts_layer != MAC_TIMESTAMPING) - return -EOPNOTSUPP; + cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; - if (ts_layer == PHY_TIMESTAMPING && - dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) { + if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { err = ops->ndo_hwtstamp_get(dev, &old_cfg); if (err) return err; } - if (ts_layer == MAC_TIMESTAMPING || - dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) { + if (!phy_ts || (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) { err = ops->ndo_hwtstamp_set(dev, cfg, extack); if (err) { if (extack->_msg) @@ -356,11 +349,10 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, } } - if (ts_layer == PHY_TIMESTAMPING && - dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS) + if (phy_ts && (dev->priv_flags & IFF_SEE_ALL_HWTSTAMP_REQUESTS)) changed = kernel_hwtstamp_config_changed(&old_cfg, cfg); - if (ts_layer == PHY_TIMESTAMPING) { + if (phy_ts) { err = phy_hwtstamp_set(dev->phydev, cfg, extack); if (err) { if (changed) diff --git a/net/core/timestamping.c b/net/core/timestamping.c index 5cf51a523fb3..04840697fe79 100644 --- a/net/core/timestamping.c +++ b/net/core/timestamping.c @@ -21,7 +21,6 @@ static unsigned int classify(const struct sk_buff *skb) void skb_clone_tx_timestamp(struct sk_buff *skb) { - enum timestamping_layer ts_layer; struct mii_timestamper *mii_ts; struct sk_buff *clone; unsigned int type; @@ -29,10 +28,6 @@ void skb_clone_tx_timestamp(struct sk_buff *skb) if (!skb->sk) return; - ts_layer = skb->dev->ts_layer; - if (ts_layer != PHY_TIMESTAMPING) - return; - type = classify(skb); if (type == PTP_CLASS_NONE) return; @@ -49,17 +44,12 @@ EXPORT_SYMBOL_GPL(skb_clone_tx_timestamp); bool skb_defer_rx_timestamp(struct sk_buff *skb) { - enum timestamping_layer ts_layer; struct mii_timestamper *mii_ts; unsigned int type; if (!skb->dev || !skb->dev->phydev || !skb->dev->phydev->mii_ts) return false; - ts_layer = skb->dev->ts_layer; - if (ts_layer != PHY_TIMESTAMPING) - return false; - if (skb_headroom(skb) < ETH_HLEN) return false; diff --git a/net/ethtool/Makefile b/net/ethtool/Makefile index 4ea64c080639..504f954a1b28 100644 --- a/net/ethtool/Makefile +++ b/net/ethtool/Makefile @@ -8,4 +8,4 @@ ethtool_nl-y := netlink.o bitset.o strset.o linkinfo.o linkmodes.o rss.o \ linkstate.o debug.o wol.o features.o privflags.o rings.o \ channels.o coalesce.o pause.o eee.o tsinfo.o cabletest.o \ tunnels.o fec.o eeprom.o stats.o phc_vclocks.o mm.o \ - module.o pse-pd.o plca.o mm.o ts.o + module.o pse-pd.o plca.o mm.o diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 9f6e3b2c74e2..11d8797f63f6 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -633,28 +633,13 @@ int __ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { const struct ethtool_ops *ops = dev->ethtool_ops; struct phy_device *phydev = dev->phydev; - enum timestamping_layer ts_layer; - int ret; memset(info, 0, sizeof(*info)); info->cmd = ETHTOOL_GET_TS_INFO; - ts_layer = dev->ts_layer; - if (ts_layer == SOFTWARE_TIMESTAMPING) { - ret = ops->get_ts_info(dev, info); - if (ret) - return ret; - info->so_timestamping &= ~SOF_TIMESTAMPING_HARDWARE_MASK; - info->phc_index = -1; - info->rx_filters = 0; - info->tx_types = 0; - return 0; - } - - if (ts_layer == PHY_TIMESTAMPING) + if (phy_has_tsinfo(phydev)) return phy_ts_info(phydev, info); - - if (ts_layer == MAC_TIMESTAMPING) + if (ops->get_ts_info) return ops->get_ts_info(dev, info); info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | diff --git a/net/ethtool/common.h b/net/ethtool/common.h index a264b635f7d3..28b8aaaf9bcb 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -35,7 +35,6 @@ extern const char wol_mode_names[][ETH_GSTRING_LEN]; extern const char sof_timestamping_names[][ETH_GSTRING_LEN]; extern const char ts_tx_type_names[][ETH_GSTRING_LEN]; extern const char ts_rx_filter_names[][ETH_GSTRING_LEN]; -extern const char ts_layer_names[][ETH_GSTRING_LEN]; extern const char udp_tunnel_type_names[][ETH_GSTRING_LEN]; int __ethtool_get_link(struct net_device *dev); diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 8322bf71f80d..3bbd5afb7b31 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -306,9 +306,6 @@ ethnl_default_requests[__ETHTOOL_MSG_USER_CNT] = { [ETHTOOL_MSG_PLCA_GET_STATUS] = ðnl_plca_status_request_ops, [ETHTOOL_MSG_MM_GET] = ðnl_mm_request_ops, [ETHTOOL_MSG_MM_SET] = ðnl_mm_request_ops, - [ETHTOOL_MSG_TS_GET] = ðnl_ts_request_ops, - [ETHTOOL_MSG_TS_LIST_GET] = ðnl_ts_list_request_ops, - [ETHTOOL_MSG_TS_SET] = ðnl_ts_request_ops, }; static struct ethnl_dump_ctx *ethnl_dump_context(struct netlink_callback *cb) @@ -1131,31 +1128,6 @@ static const struct genl_ops ethtool_genl_ops[] = { .policy = ethnl_mm_set_policy, .maxattr = ARRAY_SIZE(ethnl_mm_set_policy) - 1, }, - { - .cmd = ETHTOOL_MSG_TS_GET, - .doit = ethnl_default_doit, - .start = ethnl_default_start, - .dumpit = ethnl_default_dumpit, - .done = ethnl_default_done, - .policy = ethnl_ts_get_policy, - .maxattr = ARRAY_SIZE(ethnl_ts_get_policy) - 1, - }, - { - .cmd = ETHTOOL_MSG_TS_LIST_GET, - .doit = ethnl_default_doit, - .start = ethnl_default_start, - .dumpit = ethnl_default_dumpit, - .done = ethnl_default_done, - .policy = ethnl_ts_get_policy, - .maxattr = ARRAY_SIZE(ethnl_ts_get_policy) - 1, - }, - { - .cmd = ETHTOOL_MSG_TS_SET, - .flags = GENL_UNS_ADMIN_PERM, - .doit = ethnl_default_set_doit, - .policy = ethnl_ts_set_policy, - .maxattr = ARRAY_SIZE(ethnl_ts_set_policy) - 1, - }, }; static const struct genl_multicast_group ethtool_nl_mcgrps[] = { diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 8fedf234b824..9a333a8d04c1 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -395,8 +395,6 @@ extern const struct ethnl_request_ops ethnl_rss_request_ops; extern const struct ethnl_request_ops ethnl_plca_cfg_request_ops; extern const struct ethnl_request_ops ethnl_plca_status_request_ops; extern const struct ethnl_request_ops ethnl_mm_request_ops; -extern const struct ethnl_request_ops ethnl_ts_request_ops; -extern const struct ethnl_request_ops ethnl_ts_list_request_ops; extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_FLAGS + 1]; extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_FLAGS + 1]; @@ -443,8 +441,6 @@ extern const struct nla_policy ethnl_plca_set_cfg_policy[ETHTOOL_A_PLCA_MAX + 1] extern const struct nla_policy ethnl_plca_get_status_policy[ETHTOOL_A_PLCA_HEADER + 1]; extern const struct nla_policy ethnl_mm_get_policy[ETHTOOL_A_MM_HEADER + 1]; extern const struct nla_policy ethnl_mm_set_policy[ETHTOOL_A_MM_MAX + 1]; -extern const struct nla_policy ethnl_ts_get_policy[ETHTOOL_A_TS_HEADER + 1]; -extern const struct nla_policy ethnl_ts_set_policy[ETHTOOL_A_TS_MAX + 1]; int ethnl_set_features(struct sk_buff *skb, struct genl_info *info); int ethnl_act_cable_test(struct sk_buff *skb, struct genl_info *info); diff --git a/net/ethtool/ts.c b/net/ethtool/ts.c deleted file mode 100644 index 357265e74e08..000000000000 --- a/net/ethtool/ts.c +++ /dev/null @@ -1,244 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -#include -#include - -#include "netlink.h" -#include "common.h" -#include "bitset.h" - -struct ts_req_info { - struct ethnl_req_info base; -}; - -struct ts_reply_data { - struct ethnl_reply_data base; - enum timestamping_layer ts_layer; -}; - -#define TS_REPDATA(__reply_base) \ - container_of(__reply_base, struct ts_reply_data, base) - -/* TS_GET */ -const struct nla_policy ethnl_ts_get_policy[] = { - [ETHTOOL_A_TS_HEADER] = - NLA_POLICY_NESTED(ethnl_header_policy), -}; - -static int ts_prepare_data(const struct ethnl_req_info *req_base, - struct ethnl_reply_data *reply_base, - const struct genl_info *info) -{ - struct ts_reply_data *data = TS_REPDATA(reply_base); - struct net_device *dev = reply_base->dev; - int ret; - - ret = ethnl_ops_begin(dev); - if (ret < 0) - return ret; - - data->ts_layer = dev->ts_layer; - - ethnl_ops_complete(dev); - - return ret; -} - -static int ts_reply_size(const struct ethnl_req_info *req_base, - const struct ethnl_reply_data *reply_base) -{ - return nla_total_size(sizeof(u32)); -} - -static int ts_fill_reply(struct sk_buff *skb, - const struct ethnl_req_info *req_base, - const struct ethnl_reply_data *reply_base) -{ - struct ts_reply_data *data = TS_REPDATA(reply_base); - - return nla_put_u32(skb, ETHTOOL_A_TS_LAYER, data->ts_layer); -} - -/* TS_SET */ -const struct nla_policy ethnl_ts_set_policy[] = { - [ETHTOOL_A_TS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), - [ETHTOOL_A_TS_LAYER] = NLA_POLICY_RANGE(NLA_U32, 0, - __TIMESTAMPING_COUNT - 1) -}; - -static int ethnl_set_ts_validate(struct ethnl_req_info *req_info, - struct genl_info *info) -{ - struct nlattr **tb = info->attrs; - const struct net_device_ops *ops = req_info->dev->netdev_ops; - - if (!ops->ndo_hwtstamp_set) - return -EOPNOTSUPP; - - if (!tb[ETHTOOL_A_TS_LAYER]) - return 0; - - return 1; -} - -static int ethnl_set_ts(struct ethnl_req_info *req_info, struct genl_info *info) -{ - struct net_device *dev = req_info->dev; - const struct ethtool_ops *ops = dev->ethtool_ops; - struct kernel_hwtstamp_config config = {0}; - struct nlattr **tb = info->attrs; - enum timestamping_layer ts_layer; - bool mod = false; - int ret; - - ts_layer = dev->ts_layer; - ethnl_update_u32(&ts_layer, tb[ETHTOOL_A_TS_LAYER], &mod); - - if (!mod) - return 0; - - if (ts_layer == SOFTWARE_TIMESTAMPING) { - struct ethtool_ts_info ts_info = {0}; - - if (!ops->get_ts_info) { - NL_SET_ERR_MSG_ATTR(info->extack, - tb[ETHTOOL_A_TS_LAYER], - "this net device cannot support timestamping"); - return -EINVAL; - } - - ops->get_ts_info(dev, &ts_info); - if ((ts_info.so_timestamping & - SOF_TIMESTAMPING_SOFTWARE_MASK) != - SOF_TIMESTAMPING_SOFTWARE_MASK) { - NL_SET_ERR_MSG_ATTR(info->extack, - tb[ETHTOOL_A_TS_LAYER], - "this net device cannot support software timestamping"); - return -EINVAL; - } - } else if (ts_layer == MAC_TIMESTAMPING) { - struct ethtool_ts_info ts_info = {0}; - - if (!ops->get_ts_info) { - NL_SET_ERR_MSG_ATTR(info->extack, - tb[ETHTOOL_A_TS_LAYER], - "this net device cannot support timestamping"); - return -EINVAL; - } - - ops->get_ts_info(dev, &ts_info); - if ((ts_info.so_timestamping & - SOF_TIMESTAMPING_HARDWARE_MASK) != - SOF_TIMESTAMPING_HARDWARE_MASK) { - NL_SET_ERR_MSG_ATTR(info->extack, - tb[ETHTOOL_A_TS_LAYER], - "this net device cannot support hardware timestamping"); - return -EINVAL; - } - } else if (ts_layer == PHY_TIMESTAMPING && !phy_has_tsinfo(dev->phydev)) { - NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_TS_LAYER], - "this phy device cannot support timestamping"); - return -EINVAL; - } - - /* Disable time stamping in the current layer. */ - if (netif_device_present(dev) && - (dev->ts_layer == PHY_TIMESTAMPING || - dev->ts_layer == MAC_TIMESTAMPING)) { - ret = dev_set_hwtstamp_phylib(dev, &config, info->extack); - if (ret < 0) - return ret; - } - - dev->ts_layer = ts_layer; - - return 1; -} - -const struct ethnl_request_ops ethnl_ts_request_ops = { - .request_cmd = ETHTOOL_MSG_TS_GET, - .reply_cmd = ETHTOOL_MSG_TS_GET_REPLY, - .hdr_attr = ETHTOOL_A_TS_HEADER, - .req_info_size = sizeof(struct ts_req_info), - .reply_data_size = sizeof(struct ts_reply_data), - - .prepare_data = ts_prepare_data, - .reply_size = ts_reply_size, - .fill_reply = ts_fill_reply, - - .set_validate = ethnl_set_ts_validate, - .set = ethnl_set_ts, -}; - -/* TS_LIST_GET */ -struct ts_list_reply_data { - struct ethnl_reply_data base; - enum timestamping_layer ts_layer[__TIMESTAMPING_COUNT]; - u8 num_ts; -}; - -#define TS_LIST_REPDATA(__reply_base) \ - container_of(__reply_base, struct ts_list_reply_data, base) - -static int ts_list_prepare_data(const struct ethnl_req_info *req_base, - struct ethnl_reply_data *reply_base, - const struct genl_info *info) -{ - struct ts_list_reply_data *data = TS_LIST_REPDATA(reply_base); - struct net_device *dev = reply_base->dev; - const struct ethtool_ops *ops = dev->ethtool_ops; - int ret, i = 0; - - ret = ethnl_ops_begin(dev); - if (ret < 0) - return ret; - - if (phy_has_tsinfo(dev->phydev)) - data->ts_layer[i++] = PHY_TIMESTAMPING; - if (ops->get_ts_info) { - struct ethtool_ts_info ts_info = {0}; - - ops->get_ts_info(dev, &ts_info); - if (ts_info.so_timestamping & - SOF_TIMESTAMPING_HARDWARE_MASK) - data->ts_layer[i++] = MAC_TIMESTAMPING; - - if (ts_info.so_timestamping & - SOF_TIMESTAMPING_SOFTWARE_MASK) - data->ts_layer[i++] = SOFTWARE_TIMESTAMPING; - } - - data->num_ts = i; - ethnl_ops_complete(dev); - - return ret; -} - -static int ts_list_reply_size(const struct ethnl_req_info *req_base, - const struct ethnl_reply_data *reply_base) -{ - struct ts_list_reply_data *data = TS_LIST_REPDATA(reply_base); - - return nla_total_size(sizeof(u32)) * data->num_ts; -} - -static int ts_list_fill_reply(struct sk_buff *skb, - const struct ethnl_req_info *req_base, - const struct ethnl_reply_data *reply_base) -{ - struct ts_list_reply_data *data = TS_LIST_REPDATA(reply_base); - - return nla_put(skb, ETHTOOL_A_TS_LIST_LAYER, sizeof(u32) * data->num_ts, data->ts_layer); -} - -const struct ethnl_request_ops ethnl_ts_list_request_ops = { - .request_cmd = ETHTOOL_MSG_TS_LIST_GET, - .reply_cmd = ETHTOOL_MSG_TS_LIST_GET_REPLY, - .hdr_attr = ETHTOOL_A_TS_HEADER, - .req_info_size = sizeof(struct ts_req_info), - .reply_data_size = sizeof(struct ts_list_reply_data), - - .prepare_data = ts_list_prepare_data, - .reply_size = ts_list_reply_size, - .fill_reply = ts_list_fill_reply, -}; -- cgit v1.2.3 From ac40916a3f7243efbe6e129ebf495b5c33a3adfe Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 15 Nov 2023 20:01:08 +0800 Subject: rtnetlink: introduce nlmsg_new_large and use it in rtnl_getlink if a PF has 256 or more VFs, ip link command will allocate an order 3 memory or more, and maybe trigger OOM due to memory fragment, the VFs needed memory size is computed in rtnl_vfinfo_size. so introduce nlmsg_new_large which calls netlink_alloc_large_skb in which vmalloc is used for large memory, to avoid the failure of allocating memory ip invoked oom-killer: gfp_mask=0xc2cc0(GFP_KERNEL|__GFP_NOWARN|\ __GFP_COMP|__GFP_NOMEMALLOC), order=3, oom_score_adj=0 CPU: 74 PID: 204414 Comm: ip Kdump: loaded Tainted: P OE Call Trace: dump_stack+0x57/0x6a dump_header+0x4a/0x210 oom_kill_process+0xe4/0x140 out_of_memory+0x3e8/0x790 __alloc_pages_slowpath.constprop.116+0x953/0xc50 __alloc_pages_nodemask+0x2af/0x310 kmalloc_large_node+0x38/0xf0 __kmalloc_node_track_caller+0x417/0x4d0 __kmalloc_reserve.isra.61+0x2e/0x80 __alloc_skb+0x82/0x1c0 rtnl_getlink+0x24f/0x370 rtnetlink_rcv_msg+0x12c/0x350 netlink_rcv_skb+0x50/0x100 netlink_unicast+0x1b2/0x280 netlink_sendmsg+0x355/0x4a0 sock_sendmsg+0x5b/0x60 ____sys_sendmsg+0x1ea/0x250 ___sys_sendmsg+0x88/0xd0 __sys_sendmsg+0x5e/0xa0 do_syscall_64+0x33/0x40 entry_SYSCALL_64_after_hwframe+0x44/0xa9 RIP: 0033:0x7f95a65a5b70 Cc: Yunsheng Lin Signed-off-by: Li RongQing Link: https://lore.kernel.org/r/20231115120108.3711-1-lirongqing@baidu.com Signed-off-by: Jakub Kicinski --- include/linux/netlink.h | 1 + include/net/netlink.h | 14 ++++++++++++++ net/core/rtnetlink.c | 2 +- net/netlink/af_netlink.c | 3 +-- 4 files changed, 17 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netlink.h b/include/linux/netlink.h index 75d7de34c908..abe91ed6b9aa 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -351,5 +351,6 @@ bool netlink_ns_capable(const struct sk_buff *skb, struct user_namespace *ns, int cap); bool netlink_capable(const struct sk_buff *skb, int cap); bool netlink_net_capable(const struct sk_buff *skb, int cap); +struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast); #endif /* __LINUX_NETLINK_H */ diff --git a/include/net/netlink.h b/include/net/netlink.h index 83bdf787aeee..167b91348e57 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -1010,6 +1010,20 @@ static inline struct sk_buff *nlmsg_new(size_t payload, gfp_t flags) return alloc_skb(nlmsg_total_size(payload), flags); } +/** + * nlmsg_new_large - Allocate a new netlink message with non-contiguous + * physical memory + * @payload: size of the message payload + * + * The allocated skb is unable to have frag page for shinfo->frags*, + * as the NULL setting for skb->head in netlink_skb_destructor() will + * bypass most of the handling in skb_release_data() + */ +static inline struct sk_buff *nlmsg_new_large(size_t payload) +{ + return netlink_alloc_large_skb(nlmsg_total_size(payload), 0); +} + /** * nlmsg_end - Finalize a netlink message * @skb: socket buffer the message is stored in diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index e8431c6c8490..592164c2a540 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -3849,7 +3849,7 @@ static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh, goto out; err = -ENOBUFS; - nskb = nlmsg_new(if_nlmsg_size(dev, ext_filter_mask), GFP_KERNEL); + nskb = nlmsg_new_large(if_nlmsg_size(dev, ext_filter_mask)); if (nskb == NULL) goto out; diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index eb086b06d60d..177126fb0484 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -1204,8 +1204,7 @@ struct sock *netlink_getsockbyfilp(struct file *filp) return sock; } -static struct sk_buff *netlink_alloc_large_skb(unsigned int size, - int broadcast) +struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast) { struct sk_buff *skb; void *data; -- cgit v1.2.3 From 2afae08c9dcb8ac648414277cec70c2fe6a34d9e Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 23 Nov 2023 19:59:36 -0800 Subject: bpf: Validate global subprogs lazily Slightly change BPF verifier logic around eagerness and order of global subprog validation. Instead of going over every global subprog eagerly and validating it before main (entry) BPF program is verified, turn it around. Validate main program first, mark subprogs that were called from main program for later verification, but otherwise assume it is valid. Afterwards, go over marked global subprogs and validate those, potentially marking some more global functions as being called. Continue this process until all (transitively) callable global subprogs are validated. It's a BFS traversal at its heart and will always converge. This is an important change because it allows to feature-gate some subprograms that might not be verifiable on some older kernel, depending on supported set of features. E.g., at some point, global functions were allowed to accept a pointer to memory, which size is identified by user-provided type. Unfortunately, older kernels don't support this feature. With BPF CO-RE approach, the natural way would be to still compile BPF object file once and guard calls to this global subprog with some CO-RE check or using .rodata variables. That's what people do to guard usage of new helpers or kfuncs, and any other new BPF-side feature that might be missing on old kernels. That's currently impossible to do with global subprogs, unfortunately, because they are eagerly and unconditionally validated. This patch set aims to change this, so that in the future when global funcs gain new features, those can be guarded using BPF CO-RE techniques in the same fashion as any other new kernel feature. Two selftests had to be adjusted in sync with these changes. test_global_func12 relied on eager global subprog validation failing before main program failure is detected (unknown return value). Fix by making sure that main program is always valid. verifier_subprog_precision's parent_stack_slot_precise subtest relied on verifier checkpointing heuristic to do a checkpoint at instruction #5, but that's no longer true because we don't have enough jumps validated before reaching insn #5 due to global subprogs being validated later. Other than that, no changes, as one would expect. Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Acked-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20231124035937.403208-3-andrii@kernel.org --- include/linux/bpf.h | 2 + kernel/bpf/verifier.c | 48 +++++++++++++++++++--- .../selftests/bpf/progs/test_global_func12.c | 4 +- .../bpf/progs/verifier_subprog_precision.c | 4 +- 4 files changed, 48 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 258ba232e302..eb447b0a9423 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1347,6 +1347,8 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) struct bpf_func_info_aux { u16 linkage; bool unreliable; + bool called : 1; + bool verified : 1; }; enum bpf_jit_poke_reason { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a2939ebf2638..8e7b6072e3f4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -434,6 +434,11 @@ static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) return btf_type_name(env->prog->aux->btf, info->type_id); } +static struct bpf_func_info_aux *subprog_aux(const struct bpf_verifier_env *env, int subprog) +{ + return &env->prog->aux->func_info_aux[subprog]; +} + static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) { return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); @@ -9290,6 +9295,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + /* mark global subprog for verifying after main prog */ + subprog_aux(env, subprog)->called = true; clear_caller_saved_regs(env, caller->regs); /* All global functions return a 64-bit SCALAR_VALUE */ @@ -19873,8 +19880,11 @@ out: return ret; } -/* Verify all global functions in a BPF program one by one based on their BTF. - * All global functions must pass verification. Otherwise the whole program is rejected. +/* Lazily verify all global functions based on their BTF, if they are called + * from main BPF program or any of subprograms transitively. + * BPF global subprogs called from dead code are not validated. + * All callable global functions must pass verification. + * Otherwise the whole program is rejected. * Consider: * int bar(int); * int foo(int f) @@ -19893,14 +19903,26 @@ out: static int do_check_subprogs(struct bpf_verifier_env *env) { struct bpf_prog_aux *aux = env->prog->aux; - int i, ret; + struct bpf_func_info_aux *sub_aux; + int i, ret, new_cnt; if (!aux->func_info) return 0; + /* exception callback is presumed to be always called */ + if (env->exception_callback_subprog) + subprog_aux(env, env->exception_callback_subprog)->called = true; + +again: + new_cnt = 0; for (i = 1; i < env->subprog_cnt; i++) { - if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) + if (!subprog_is_global(env, i)) + continue; + + sub_aux = subprog_aux(env, i); + if (!sub_aux->called || sub_aux->verified) continue; + env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i, env->exception_callback_subprog == i); @@ -19910,7 +19932,21 @@ static int do_check_subprogs(struct bpf_verifier_env *env) verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", i, subprog_name(env, i)); } + + /* We verified new global subprog, it might have called some + * more global subprogs that we haven't verified yet, so we + * need to do another pass over subprogs to verify those. + */ + sub_aux->verified = true; + new_cnt++; } + + /* We can't loop forever as we verify at least one global subprog on + * each pass. + */ + if (new_cnt) + goto again; + return 0; } @@ -20556,8 +20592,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (ret < 0) goto skip_full_check; - ret = do_check_subprogs(env); - ret = ret ?: do_check_main(env); + ret = do_check_main(env); + ret = ret ?: do_check_subprogs(env); if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); diff --git a/tools/testing/selftests/bpf/progs/test_global_func12.c b/tools/testing/selftests/bpf/progs/test_global_func12.c index 7f159d83c6f6..6e03d42519a6 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func12.c +++ b/tools/testing/selftests/bpf/progs/test_global_func12.c @@ -19,5 +19,7 @@ int global_func12(struct __sk_buff *skb) { const struct S s = {.x = skb->len }; - return foo(&s); + foo(&s); + + return 1; } diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index f61d623b1ce8..b5efcaeaa1ae 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -370,12 +370,10 @@ __naked int parent_stack_slot_precise(void) SEC("?raw_tp") __success __log_level(2) __msg("9: (0f) r1 += r6") -__msg("mark_precise: frame0: last_idx 9 first_idx 6") +__msg("mark_precise: frame0: last_idx 9 first_idx 0") __msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") __msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") __msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") -__msg("mark_precise: frame0: parent state regs= stack=-8:") -__msg("mark_precise: frame0: last_idx 5 first_idx 0") __msg("mark_precise: frame0: regs= stack=-8 before 5: (85) call pc+6") __msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") __msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") -- cgit v1.2.3 From d3ca4ab4f16eb81dc3e7721251adcba49b229d54 Mon Sep 17 00:00:00 2001 From: Liam Kearney Date: Wed, 25 Oct 2023 11:27:55 +1100 Subject: wifi: ieee80211: fix PV1 frame control field name Update PV1 frame control field TODS to FROMDS to match 802.11 standard Signed-off-by: Liam Kearney Reviewed-by: Jeff Johnson Link: https://lore.kernel.org/r/20231025002755.1752983-1-liam.kearney@morsemicro.com Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 958771bac9c0..5e5ea216f341 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -172,11 +172,11 @@ #define IEEE80211_SN_MODULO (IEEE80211_MAX_SN + 1) -/* PV1 Layout 11ah 9.8.3.1 */ +/* PV1 Layout IEEE 802.11-2020 9.8.3.1 */ #define IEEE80211_PV1_FCTL_VERS 0x0003 #define IEEE80211_PV1_FCTL_FTYPE 0x001c #define IEEE80211_PV1_FCTL_STYPE 0x00e0 -#define IEEE80211_PV1_FCTL_TODS 0x0100 +#define IEEE80211_PV1_FCTL_FROMDS 0x0100 #define IEEE80211_PV1_FCTL_MOREFRAGS 0x0200 #define IEEE80211_PV1_FCTL_PM 0x0400 #define IEEE80211_PV1_FCTL_MOREDATA 0x0800 -- cgit v1.2.3 From a066f906ba396ab00d4af19fc5fad42b2605582a Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Wed, 22 Nov 2023 14:52:43 +0100 Subject: firmware_loader: Expand Firmware upload error codes with firmware invalid error No error code are available to signal an invalid firmware content. Drivers that can check the firmware content validity can not return this specific failure to the user-space Expand the firmware error code with an additional code: - "firmware invalid" code which can be used when the provided firmware is invalid Sync lib/test_firmware.c file accordingly. Acked-by: Luis Chamberlain Acked-by: Greg Kroah-Hartman Signed-off-by: Kory Maincent Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20231122-feature_firmware_error_code-v3-1-04ec753afb71@bootlin.com Signed-off-by: Jakub Kicinski --- drivers/base/firmware_loader/sysfs_upload.c | 1 + include/linux/firmware.h | 2 ++ lib/test_firmware.c | 1 + 3 files changed, 4 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c index a0af8f5f13d8..829270067d16 100644 --- a/drivers/base/firmware_loader/sysfs_upload.c +++ b/drivers/base/firmware_loader/sysfs_upload.c @@ -27,6 +27,7 @@ static const char * const fw_upload_err_str[] = { [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", + [FW_UPLOAD_ERR_FW_INVALID] = "firmware-invalid", }; static const char *fw_upload_progress(struct device *dev, diff --git a/include/linux/firmware.h b/include/linux/firmware.h index de7fea3bca51..0311858b46ce 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -27,6 +27,7 @@ struct firmware { * @FW_UPLOAD_ERR_INVALID_SIZE: invalid firmware image size * @FW_UPLOAD_ERR_RW_ERROR: read or write to HW failed, see kernel log * @FW_UPLOAD_ERR_WEAROUT: FLASH device is approaching wear-out, wait & retry + * @FW_UPLOAD_ERR_FW_INVALID: invalid firmware file * @FW_UPLOAD_ERR_MAX: Maximum error code marker */ enum fw_upload_err { @@ -38,6 +39,7 @@ enum fw_upload_err { FW_UPLOAD_ERR_INVALID_SIZE, FW_UPLOAD_ERR_RW_ERROR, FW_UPLOAD_ERR_WEAROUT, + FW_UPLOAD_ERR_FW_INVALID, FW_UPLOAD_ERR_MAX }; diff --git a/lib/test_firmware.c b/lib/test_firmware.c index add4699fc6cd..9cfdcd6d21db 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -1132,6 +1132,7 @@ static const char * const fw_upload_err_str[] = { [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", + [FW_UPLOAD_ERR_FW_INVALID] = "firmware-invalid", }; static void upload_err_inject_error(struct test_firmware_upload *tst, -- cgit v1.2.3 From 243ad8df7a1bd24c2e01bd99d9f0bb88844dae91 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 24 Nov 2023 12:27:52 +0000 Subject: net: phy: add possible interfaces Add a possible_interfaces member to struct phy_device to indicate which interfaces a clause 45 PHY may switch between depending on the media. This must be populated by the PHY driver by the time the .config_init() method completes according to the PHYs host-side configuration. For example, the Marvell 88x3310 PHY can switch between 10GBASE-R, 5GBASE-R, 2500BASE-X, and SGMII on the host side depending on the media side speed, so all these interface modes are set in the possible_interfaces member. This allows phylib users (such as phylink) to know in advance which interface modes to expect, which allows them to appropriately restrict the advertised link modes according to the capabilities of other parts of the link. Tested-by: Luo Jie Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/E1r6VHk-00DDLN-I7@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy_device.c | 2 ++ include/linux/phy.h | 3 +++ 2 files changed, 5 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 478126f6b5bc..400fb09d9cd6 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1246,6 +1246,8 @@ int phy_init_hw(struct phy_device *phydev) if (ret < 0) return ret; + phy_interface_zero(phydev->possible_interfaces); + if (phydev->drv->config_init) { ret = phydev->drv->config_init(phydev); if (ret < 0) diff --git a/include/linux/phy.h b/include/linux/phy.h index e5f1f41e399c..6e7ebcc50b85 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -605,6 +605,8 @@ struct macsec_ops; * @irq_rerun: Flag indicating interrupts occurred while PHY was suspended, * requiring a rerun of the interrupt handler after resume * @interface: enum phy_interface_t value + * @possible_interfaces: bitmap if interface modes that the attached PHY + * will switch between depending on media speed. * @skb: Netlink message for cable diagnostics * @nest: Netlink nest used for cable diagnostics * @ehdr: nNtlink header for cable diagnostics @@ -674,6 +676,7 @@ struct phy_device { u32 dev_flags; phy_interface_t interface; + DECLARE_PHY_INTERFACE_MASK(possible_interfaces); /* * forced speed & duplex (no autoneg) -- cgit v1.2.3 From 083772c9f972dcc248913b52a0dec1025baa1e16 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Sun, 26 Nov 2023 15:07:30 -0800 Subject: net: page_pool: record pools per netdev Link the page pools with netdevs. This needs to be netns compatible so we have two options. Either we record the pools per netns and have to worry about moving them as the netdev gets moved. Or we record them directly on the netdev so they move with the netdev without any extra work. Implement the latter option. Since pools may outlast netdev we need a place to store orphans. In time honored tradition use loopback for this purpose. Reviewed-by: Mina Almasry Reviewed-by: Eric Dumazet Acked-by: Jesper Dangaard Brouer Signed-off-by: Jakub Kicinski Signed-off-by: Paolo Abeni --- include/linux/list.h | 20 +++++++++++ include/linux/netdevice.h | 4 +++ include/linux/poison.h | 2 ++ include/net/page_pool/types.h | 4 +++ net/core/page_pool_user.c | 84 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+) (limited to 'include/linux') diff --git a/include/linux/list.h b/include/linux/list.h index 1837caedf723..059aa1fff41e 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -1119,6 +1119,26 @@ static inline void hlist_move_list(struct hlist_head *old, old->first = NULL; } +/** + * hlist_splice_init() - move all entries from one list to another + * @from: hlist_head from which entries will be moved + * @last: last entry on the @from list + * @to: hlist_head to which entries will be moved + * + * @to can be empty, @from must contain at least @last. + */ +static inline void hlist_splice_init(struct hlist_head *from, + struct hlist_node *last, + struct hlist_head *to) +{ + if (to->first) + to->first->pprev = &last->next; + last->next = to->first; + to->first = from->first; + from->first->pprev = &to->first; + from->first = NULL; +} + #define hlist_entry(ptr, type, member) container_of(ptr,type,member) #define hlist_for_each(pos, head) \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e87caa81f70c..998c7aaa98b8 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2447,6 +2447,10 @@ struct net_device { #if IS_ENABLED(CONFIG_DPLL) struct dpll_pin *dpll_pin; #endif +#if IS_ENABLED(CONFIG_PAGE_POOL) + /** @page_pools: page pools created for this netdevice */ + struct hlist_head page_pools; +#endif }; #define to_net_dev(d) container_of(d, struct net_device, dev) diff --git a/include/linux/poison.h b/include/linux/poison.h index 851a855d3868..27a7dad17eef 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -83,6 +83,8 @@ /********** net/core/skbuff.c **********/ #define SKB_LIST_POISON_NEXT ((void *)(0x800 + POISON_POINTER_DELTA)) +/********** net/ **********/ +#define NET_PTR_POISON ((void *)(0x801 + POISON_POINTER_DELTA)) /********** kernel/bpf/ **********/ #define BPF_PTR_POISON ((void *)(0xeB9FUL + POISON_POINTER_DELTA)) diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index c19f0df3bf0b..b258a571201e 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -5,6 +5,7 @@ #include #include +#include #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA * map/unmap @@ -48,6 +49,7 @@ struct pp_alloc_cache { * @pool_size: size of the ptr_ring * @nid: NUMA node id to allocate from pages from * @dev: device, for DMA pre-mapping purposes + * @netdev: netdev this pool will serve (leave as NULL if none or multiple) * @napi: NAPI which is the sole consumer of pages, otherwise NULL * @dma_dir: DMA mapping direction * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV @@ -66,6 +68,7 @@ struct page_pool_params { unsigned int offset; ); struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; /* private: used by test code only */ void (*init_callback)(struct page *page, void *arg); void *init_arg; @@ -189,6 +192,7 @@ struct page_pool { struct page_pool_params_slow slow; /* User-facing fields, protected by page_pools_lock */ struct { + struct hlist_node list; u32 id; } user; }; diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c index 630d1eeecf2a..e5c7f078fbd4 100644 --- a/net/core/page_pool_user.c +++ b/net/core/page_pool_user.c @@ -1,14 +1,31 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include +#include #include #include "page_pool_priv.h" static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); +/* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user. + * Ordering: inside rtnl_lock + */ static DEFINE_MUTEX(page_pools_lock); +/* Page pools are only reachable from user space (via netlink) if they are + * linked to a netdev at creation time. Following page pool "visibility" + * states are possible: + * - normal + * - user.list: linked to real netdev, netdev: real netdev + * - orphaned - real netdev has disappeared + * - user.list: linked to lo, netdev: lo + * - invisible - either (a) created without netdev linking, (b) unlisted due + * to error, or (c) the entire namespace which owned this pool disappeared + * - user.list: unhashed, netdev: unknown + */ + int page_pool_list(struct page_pool *pool) { static u32 id_alloc_next; @@ -20,6 +37,10 @@ int page_pool_list(struct page_pool *pool) if (err < 0) goto err_unlock; + if (pool->slow.netdev) + hlist_add_head(&pool->user.list, + &pool->slow.netdev->page_pools); + mutex_unlock(&page_pools_lock); return 0; @@ -32,5 +53,68 @@ void page_pool_unlist(struct page_pool *pool) { mutex_lock(&page_pools_lock); xa_erase(&page_pools, pool->user.id); + hlist_del(&pool->user.list); + mutex_unlock(&page_pools_lock); +} + +static void page_pool_unreg_netdev_wipe(struct net_device *netdev) +{ + struct page_pool *pool; + struct hlist_node *n; + + mutex_lock(&page_pools_lock); + hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { + hlist_del_init(&pool->user.list); + pool->slow.netdev = NET_PTR_POISON; + } + mutex_unlock(&page_pools_lock); +} + +static void page_pool_unreg_netdev(struct net_device *netdev) +{ + struct page_pool *pool, *last; + struct net_device *lo; + + lo = dev_net(netdev)->loopback_dev; + + mutex_lock(&page_pools_lock); + last = NULL; + hlist_for_each_entry(pool, &netdev->page_pools, user.list) { + pool->slow.netdev = lo; + last = pool; + } + if (last) + hlist_splice_init(&netdev->page_pools, &last->user.list, + &lo->page_pools); mutex_unlock(&page_pools_lock); } + +static int +page_pool_netdevice_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + + if (event != NETDEV_UNREGISTER) + return NOTIFY_DONE; + + if (hlist_empty(&netdev->page_pools)) + return NOTIFY_OK; + + if (netdev->ifindex != LOOPBACK_IFINDEX) + page_pool_unreg_netdev(netdev); + else + page_pool_unreg_netdev_wipe(netdev); + return NOTIFY_OK; +} + +static struct notifier_block page_pool_netdevice_nb = { + .notifier_call = page_pool_netdevice_event, +}; + +static int __init page_pool_user_init(void) +{ + return register_netdevice_notifier(&page_pool_netdevice_nb); +} + +subsys_initcall(page_pool_user_init); -- cgit v1.2.3 From 48eb03dd26304c24f03bdbb9382e89c8564e71df Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Mon, 27 Nov 2023 11:03:08 -0800 Subject: xsk: Add TX timestamp and TX checksum offload support This change actually defines the (initial) metadata layout that should be used by AF_XDP userspace (xsk_tx_metadata). The first field is flags which requests appropriate offloads, followed by the offload-specific fields. The supported per-device offloads are exported via netlink (new xsk-flags). The offloads themselves are still implemented in a bit of a framework-y fashion that's left from my initial kfunc attempt. I'm introducing new xsk_tx_metadata_ops which drivers are supposed to implement. The drivers are also supposed to call xsk_tx_metadata_request/xsk_tx_metadata_complete in the right places. Since xsk_tx_metadata_{request,_complete} are static inline, we don't incur any extra overhead doing indirect calls. The benefit of this scheme is as follows: - keeps all metadata layout parsing away from driver code - makes it easy to grep and see which drivers implement what - don't need any extra flags to maintain to keep track of what offloads are implemented; if the callback is implemented - the offload is supported (used by netlink reporting code) Two offloads are defined right now: 1. XDP_TXMD_FLAGS_CHECKSUM: skb-style csum_start+csum_offset 2. XDP_TXMD_FLAGS_TIMESTAMP: writes TX timestamp back into metadata area upon completion (tx_timestamp field) XDP_TXMD_FLAGS_TIMESTAMP is also implemented for XDP_COPY mode: it writes SW timestamp from the skb destructor (note I'm reusing hwtstamps to pass metadata pointer). The struct is forward-compatible and can be extended in the future by appending more fields. Reviewed-by: Song Yoong Siang Signed-off-by: Stanislav Fomichev Acked-by: Jakub Kicinski Link: https://lore.kernel.org/r/20231127190319.1190813-3-sdf@google.com Signed-off-by: Alexei Starovoitov --- Documentation/netlink/specs/netdev.yaml | 19 +++++- include/linux/netdevice.h | 2 + include/linux/skbuff.h | 14 +++- include/net/xdp_sock.h | 110 ++++++++++++++++++++++++++++++++ include/net/xdp_sock_drv.h | 13 ++++ include/net/xsk_buff_pool.h | 6 ++ include/uapi/linux/if_xdp.h | 38 +++++++++++ include/uapi/linux/netdev.h | 16 +++++ net/core/netdev-genl.c | 13 +++- net/xdp/xsk.c | 34 ++++++++++ net/xdp/xsk_queue.h | 2 +- tools/include/uapi/linux/if_xdp.h | 52 +++++++++++++-- tools/include/uapi/linux/netdev.h | 16 +++++ tools/net/ynl/generated/netdev-user.c | 19 ++++++ tools/net/ynl/generated/netdev-user.h | 3 + 15 files changed, 348 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index 14511b13f305..00439bcbd2e3 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -45,7 +45,6 @@ definitions: - type: flags name: xdp-rx-metadata - render-max: true entries: - name: timestamp @@ -55,6 +54,18 @@ definitions: name: hash doc: Device is capable of exposing receive packet hash via bpf_xdp_metadata_rx_hash(). + - + type: flags + name: xsk-flags + entries: + - + name: tx-timestamp + doc: + HW timestamping egress packets is supported by the driver. + - + name: tx-checksum + doc: + L3 checksum HW offload is supported by the driver. attribute-sets: - @@ -86,6 +97,11 @@ attribute-sets: See Documentation/networking/xdp-rx-metadata.rst for more details. type: u64 enum: xdp-rx-metadata + - + name: xsk-features + doc: Bitmask of enabled AF_XDP features. + type: u64 + enum: xsk-flags operations: list: @@ -103,6 +119,7 @@ operations: - xdp-features - xdp-zc-max-segs - xdp-rx-metadata-features + - xsk-features dump: reply: *dev-all - diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e87caa81f70c..08da8b28c816 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1865,6 +1865,7 @@ enum netdev_stat_type { * @netdev_ops: Includes several pointers to callbacks, * if one wants to override the ndo_*() functions * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks. + * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks. * @ethtool_ops: Management operations * @l3mdev_ops: Layer 3 master device operations * @ndisc_ops: Includes callbacks for different IPv6 neighbour @@ -2128,6 +2129,7 @@ struct net_device { unsigned long long priv_flags; const struct net_device_ops *netdev_ops; const struct xdp_metadata_ops *xdp_metadata_ops; + const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops; int ifindex; unsigned short gflags; unsigned short hard_header_len; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 27998f73183e..b370eb8d70f7 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -566,6 +566,15 @@ struct ubuf_info_msgzc { int mm_account_pinned_pages(struct mmpin *mmp, size_t size); void mm_unaccount_pinned_pages(struct mmpin *mmp); +/* Preserve some data across TX submission and completion. + * + * Note, this state is stored in the driver. Extending the layout + * might need some special care. + */ +struct xsk_tx_metadata_compl { + __u64 *tx_timestamp; +}; + /* This data is invariant across clones and lives at * the end of the header data, ie. at skb->end. */ @@ -578,7 +587,10 @@ struct skb_shared_info { /* Warning: this field is not always filled in (UFO)! */ unsigned short gso_segs; struct sk_buff *frag_list; - struct skb_shared_hwtstamps hwtstamps; + union { + struct skb_shared_hwtstamps hwtstamps; + struct xsk_tx_metadata_compl xsk_meta; + }; unsigned int gso_type; u32 tskey; diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index bcf765124f72..3cb4dc9bd70e 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -93,12 +93,105 @@ struct xdp_sock { struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ }; +/* + * AF_XDP TX metadata hooks for network devices. + * The following hooks can be defined; unless noted otherwise, they are + * optional and can be filled with a null pointer. + * + * void (*tmo_request_timestamp)(void *priv) + * Called when AF_XDP frame requested egress timestamp. + * + * u64 (*tmo_fill_timestamp)(void *priv) + * Called when AF_XDP frame, that had requested egress timestamp, + * received a completion. The hook needs to return the actual HW timestamp. + * + * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv) + * Called when AF_XDP frame requested HW checksum offload. csum_start + * indicates position where checksumming should start. + * csum_offset indicates position where checksum should be stored. + * + */ +struct xsk_tx_metadata_ops { + void (*tmo_request_timestamp)(void *priv); + u64 (*tmo_fill_timestamp)(void *priv); + void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv); +}; + #ifdef CONFIG_XDP_SOCKETS int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); void __xsk_map_flush(void); +/** + * xsk_tx_metadata_to_compl - Save enough relevant metadata information + * to perform tx completion in the future. + * @meta: pointer to AF_XDP metadata area + * @compl: pointer to output struct xsk_tx_metadata_to_compl + * + * This function should be called by the networking device when + * it prepares AF_XDP egress packet. The value of @compl should be stored + * and passed to xsk_tx_metadata_complete upon TX completion. + */ +static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, + struct xsk_tx_metadata_compl *compl) +{ + if (!meta) + return; + + if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) + compl->tx_timestamp = &meta->completion.tx_timestamp; + else + compl->tx_timestamp = NULL; +} + +/** + * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission + * and call appropriate xsk_tx_metadata_ops operation. + * @meta: pointer to AF_XDP metadata area + * @ops: pointer to struct xsk_tx_metadata_ops + * @priv: pointer to driver-private aread + * + * This function should be called by the networking device when + * it prepares AF_XDP egress packet. + */ +static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, + const struct xsk_tx_metadata_ops *ops, + void *priv) +{ + if (!meta) + return; + + if (ops->tmo_request_timestamp) + if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) + ops->tmo_request_timestamp(priv); + + if (ops->tmo_request_checksum) + if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) + ops->tmo_request_checksum(meta->request.csum_start, + meta->request.csum_offset, priv); +} + +/** + * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion + * and call appropriate xsk_tx_metadata_ops operation. + * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl + * @ops: pointer to struct xsk_tx_metadata_ops + * @priv: pointer to driver-private aread + * + * This function should be called by the networking device upon + * AF_XDP egress completion. + */ +static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, + const struct xsk_tx_metadata_ops *ops, + void *priv) +{ + if (!compl) + return; + + *compl->tx_timestamp = ops->tmo_fill_timestamp(priv); +} + #else static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) @@ -115,6 +208,23 @@ static inline void __xsk_map_flush(void) { } +static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, + struct xsk_tx_metadata_compl *compl) +{ +} + +static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, + const struct xsk_tx_metadata_ops *ops, + void *priv) +{ +} + +static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, + const struct xsk_tx_metadata_ops *ops, + void *priv) +{ +} + #endif /* CONFIG_XDP_SOCKETS */ #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 1f6fc8c7a84c..e2558ac3e195 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -165,6 +165,14 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) return xp_raw_get_data(pool, addr); } +static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +{ + if (!pool->tx_metadata_len) + return NULL; + + return xp_raw_get_data(pool, addr) - pool->tx_metadata_len; +} + static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); @@ -324,6 +332,11 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) return NULL; } +static inline struct xsk_tx_metadata *xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +{ + return NULL; +} + static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool) { } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 1985ffaf9b0c..97f5cc10d79e 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -33,6 +33,7 @@ struct xdp_buff_xsk { }; #define XSK_CHECK_PRIV_TYPE(t) BUILD_BUG_ON(sizeof(t) > offsetofend(struct xdp_buff_xsk, cb)) +#define XSK_TX_COMPL_FITS(t) BUILD_BUG_ON(sizeof(struct xsk_tx_metadata_compl) > sizeof(t)) struct xsk_dma_map { dma_addr_t *dma_pages; @@ -234,4 +235,9 @@ static inline u64 xp_get_handle(struct xdp_buff_xsk *xskb) return xskb->orig_addr + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT); } +static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool) +{ + return pool->tx_metadata_len > 0; +} + #endif /* XSK_BUFF_POOL_H_ */ diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h index 2ecf79282c26..95de66d5a26c 100644 --- a/include/uapi/linux/if_xdp.h +++ b/include/uapi/linux/if_xdp.h @@ -106,6 +106,41 @@ struct xdp_options { #define XSK_UNALIGNED_BUF_ADDR_MASK \ ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) +/* Request transmit timestamp. Upon completion, put it into tx_timestamp + * field of struct xsk_tx_metadata. + */ +#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) + +/* Request transmit checksum offload. Checksum start position and offset + * are communicated via csum_start and csum_offset fields of struct + * xsk_tx_metadata. + */ +#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1) + +/* AF_XDP offloads request. 'request' union member is consumed by the driver + * when the packet is being transmitted. 'completion' union member is + * filled by the driver when the transmit completion arrives. + */ +struct xsk_tx_metadata { + __u64 flags; + + union { + struct { + /* XDP_TXMD_FLAGS_CHECKSUM */ + + /* Offset from desc->addr where checksumming should start. */ + __u16 csum_start; + /* Offset from csum_start where checksum should be stored. */ + __u16 csum_offset; + } request; + + struct { + /* XDP_TXMD_FLAGS_TIMESTAMP */ + __u64 tx_timestamp; + } completion; + }; +}; + /* Rx/Tx descriptor */ struct xdp_desc { __u64 addr; @@ -122,4 +157,7 @@ struct xdp_desc { */ #define XDP_PKT_CONTD (1 << 0) +/* TX packet carries valid metadata. */ +#define XDP_TX_METADATA (1 << 1) + #endif /* _LINUX_IF_XDP_H */ diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h index 2943a151d4f1..48d5477a668c 100644 --- a/include/uapi/linux/netdev.h +++ b/include/uapi/linux/netdev.h @@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { NETDEV_XDP_RX_METADATA_MASK = 3, }; +/** + * enum netdev_xsk_flags + * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported + * by the driver. + * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the + * driver. + */ +enum netdev_xsk_flags { + NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1, + NETDEV_XSK_FLAGS_TX_CHECKSUM = 2, + + /* private: */ + NETDEV_XSK_FLAGS_MASK = 3, +}; + enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, NETDEV_A_DEV_XDP_FEATURES, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, + NETDEV_A_DEV_XSK_FEATURES, __NETDEV_A_DEV_MAX, NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index fe61f85bcf33..10f2124e9e23 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "netdev-genl-gen.h" @@ -13,6 +14,7 @@ static int netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, const struct genl_info *info) { + u64 xsk_features = 0; u64 xdp_rx_meta = 0; void *hdr; @@ -26,11 +28,20 @@ netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, XDP_METADATA_KFUNC_xxx #undef XDP_METADATA_KFUNC + if (netdev->xsk_tx_metadata_ops) { + if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) + xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; + if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) + xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; + } + if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, netdev->xdp_features, NETDEV_A_DEV_PAD) || nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, - xdp_rx_meta, NETDEV_A_DEV_PAD)) { + xdp_rx_meta, NETDEV_A_DEV_PAD) || + nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, + xsk_features, NETDEV_A_DEV_PAD)) { genlmsg_cancel(rsp, hdr); return -EINVAL; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index c904356e2800..e83ade32f1fd 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -571,6 +571,13 @@ static u32 xsk_get_num_desc(struct sk_buff *skb) static void xsk_destruct_skb(struct sk_buff *skb) { + struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta; + + if (compl->tx_timestamp) { + /* sw completion timestamp, not a real one */ + *compl->tx_timestamp = ktime_get_tai_fast_ns(); + } + xsk_cq_submit_locked(xdp_sk(skb->sk), xsk_get_num_desc(skb)); sock_wfree(skb); } @@ -655,8 +662,10 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, struct xdp_desc *desc) { + struct xsk_tx_metadata *meta = NULL; struct net_device *dev = xs->dev; struct sk_buff *skb = xs->skb; + bool first_frag = false; int err; if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) { @@ -687,6 +696,8 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, kfree_skb(skb); goto free_err; } + + first_frag = true; } else { int nr_frags = skb_shinfo(skb)->nr_frags; struct page *page; @@ -709,12 +720,35 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb_add_rx_frag(skb, nr_frags, page, 0, len, 0); } + + if (first_frag && desc->options & XDP_TX_METADATA) { + if (unlikely(xs->pool->tx_metadata_len == 0)) { + err = -EINVAL; + goto free_err; + } + + meta = buffer - xs->pool->tx_metadata_len; + + if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { + if (unlikely(meta->request.csum_start + + meta->request.csum_offset + + sizeof(__sum16) > len)) { + err = -EINVAL; + goto free_err; + } + + skb->csum_start = hr + meta->request.csum_start; + skb->csum_offset = meta->request.csum_offset; + skb->ip_summed = CHECKSUM_PARTIAL; + } + } } skb->dev = dev; skb->priority = READ_ONCE(xs->sk.sk_priority); skb->mark = READ_ONCE(xs->sk.sk_mark); skb->destructor = xsk_destruct_skb; + xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); xsk_set_destructor_arg(skb); return skb; diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index c74a1372bcb9..6f2d1621c992 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -137,7 +137,7 @@ static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr) static inline bool xp_unused_options_set(u32 options) { - return options & ~XDP_PKT_CONTD; + return options & ~(XDP_PKT_CONTD | XDP_TX_METADATA); } static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool, diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h index 34411a2e5b6c..d0882edc1642 100644 --- a/tools/include/uapi/linux/if_xdp.h +++ b/tools/include/uapi/linux/if_xdp.h @@ -26,11 +26,11 @@ */ #define XDP_USE_NEED_WAKEUP (1 << 3) /* By setting this option, userspace application indicates that it can - * handle multiple descriptors per packet thus enabling xsk core to split + * handle multiple descriptors per packet thus enabling AF_XDP to split * multi-buffer XDP frames into multiple Rx descriptors. Without this set - * such frames will be dropped by xsk. + * such frames will be dropped. */ -#define XDP_USE_SG (1 << 4) +#define XDP_USE_SG (1 << 4) /* Flags for xsk_umem_config flags */ #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0) @@ -106,6 +106,41 @@ struct xdp_options { #define XSK_UNALIGNED_BUF_ADDR_MASK \ ((1ULL << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1) +/* Request transmit timestamp. Upon completion, put it into tx_timestamp + * field of union xsk_tx_metadata. + */ +#define XDP_TXMD_FLAGS_TIMESTAMP (1 << 0) + +/* Request transmit checksum offload. Checksum start position and offset + * are communicated via csum_start and csum_offset fields of union + * xsk_tx_metadata. + */ +#define XDP_TXMD_FLAGS_CHECKSUM (1 << 1) + +/* AF_XDP offloads request. 'request' union member is consumed by the driver + * when the packet is being transmitted. 'completion' union member is + * filled by the driver when the transmit completion arrives. + */ +struct xsk_tx_metadata { + __u64 flags; + + union { + struct { + /* XDP_TXMD_FLAGS_CHECKSUM */ + + /* Offset from desc->addr where checksumming should start. */ + __u16 csum_start; + /* Offset from csum_start where checksum should be stored. */ + __u16 csum_offset; + } request; + + struct { + /* XDP_TXMD_FLAGS_TIMESTAMP */ + __u64 tx_timestamp; + } completion; + }; +}; + /* Rx/Tx descriptor */ struct xdp_desc { __u64 addr; @@ -113,9 +148,16 @@ struct xdp_desc { __u32 options; }; -/* Flag indicating packet constitutes of multiple buffers*/ +/* UMEM descriptor is __u64 */ + +/* Flag indicating that the packet continues with the buffer pointed out by the + * next frame in the ring. The end of the packet is signalled by setting this + * bit to zero. For single buffer packets, every descriptor has 'options' set + * to 0 and this maintains backward compatibility. + */ #define XDP_PKT_CONTD (1 << 0) -/* UMEM descriptor is __u64 */ +/* TX packet carries valid metadata. */ +#define XDP_TX_METADATA (1 << 1) #endif /* _LINUX_IF_XDP_H */ diff --git a/tools/include/uapi/linux/netdev.h b/tools/include/uapi/linux/netdev.h index 2943a151d4f1..48d5477a668c 100644 --- a/tools/include/uapi/linux/netdev.h +++ b/tools/include/uapi/linux/netdev.h @@ -53,12 +53,28 @@ enum netdev_xdp_rx_metadata { NETDEV_XDP_RX_METADATA_MASK = 3, }; +/** + * enum netdev_xsk_flags + * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported + * by the driver. + * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the + * driver. + */ +enum netdev_xsk_flags { + NETDEV_XSK_FLAGS_TX_TIMESTAMP = 1, + NETDEV_XSK_FLAGS_TX_CHECKSUM = 2, + + /* private: */ + NETDEV_XSK_FLAGS_MASK = 3, +}; + enum { NETDEV_A_DEV_IFINDEX = 1, NETDEV_A_DEV_PAD, NETDEV_A_DEV_XDP_FEATURES, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, + NETDEV_A_DEV_XSK_FEATURES, __NETDEV_A_DEV_MAX, NETDEV_A_DEV_MAX = (__NETDEV_A_DEV_MAX - 1) diff --git a/tools/net/ynl/generated/netdev-user.c b/tools/net/ynl/generated/netdev-user.c index b5ffe8cd1144..6283d87dad37 100644 --- a/tools/net/ynl/generated/netdev-user.c +++ b/tools/net/ynl/generated/netdev-user.c @@ -58,6 +58,19 @@ const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value) return netdev_xdp_rx_metadata_strmap[value]; } +static const char * const netdev_xsk_flags_strmap[] = { + [0] = "tx-timestamp", + [1] = "tx-checksum", +}; + +const char *netdev_xsk_flags_str(enum netdev_xsk_flags value) +{ + value = ffs(value) - 1; + if (value < 0 || value >= (int)MNL_ARRAY_SIZE(netdev_xsk_flags_strmap)) + return NULL; + return netdev_xsk_flags_strmap[value]; +} + /* Policies */ struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { [NETDEV_A_DEV_IFINDEX] = { .name = "ifindex", .type = YNL_PT_U32, }, @@ -65,6 +78,7 @@ struct ynl_policy_attr netdev_dev_policy[NETDEV_A_DEV_MAX + 1] = { [NETDEV_A_DEV_XDP_FEATURES] = { .name = "xdp-features", .type = YNL_PT_U64, }, [NETDEV_A_DEV_XDP_ZC_MAX_SEGS] = { .name = "xdp-zc-max-segs", .type = YNL_PT_U32, }, [NETDEV_A_DEV_XDP_RX_METADATA_FEATURES] = { .name = "xdp-rx-metadata-features", .type = YNL_PT_U64, }, + [NETDEV_A_DEV_XSK_FEATURES] = { .name = "xsk-features", .type = YNL_PT_U64, }, }; struct ynl_policy_nest netdev_dev_nest = { @@ -116,6 +130,11 @@ int netdev_dev_get_rsp_parse(const struct nlmsghdr *nlh, void *data) return MNL_CB_ERROR; dst->_present.xdp_rx_metadata_features = 1; dst->xdp_rx_metadata_features = mnl_attr_get_u64(attr); + } else if (type == NETDEV_A_DEV_XSK_FEATURES) { + if (ynl_attr_validate(yarg, attr)) + return MNL_CB_ERROR; + dst->_present.xsk_features = 1; + dst->xsk_features = mnl_attr_get_u64(attr); } } diff --git a/tools/net/ynl/generated/netdev-user.h b/tools/net/ynl/generated/netdev-user.h index 4fafac879df3..39af1908444b 100644 --- a/tools/net/ynl/generated/netdev-user.h +++ b/tools/net/ynl/generated/netdev-user.h @@ -19,6 +19,7 @@ extern const struct ynl_family ynl_netdev_family; const char *netdev_op_str(int op); const char *netdev_xdp_act_str(enum netdev_xdp_act value); const char *netdev_xdp_rx_metadata_str(enum netdev_xdp_rx_metadata value); +const char *netdev_xsk_flags_str(enum netdev_xsk_flags value); /* Common nested types */ /* ============== NETDEV_CMD_DEV_GET ============== */ @@ -50,12 +51,14 @@ struct netdev_dev_get_rsp { __u32 xdp_features:1; __u32 xdp_zc_max_segs:1; __u32 xdp_rx_metadata_features:1; + __u32 xsk_features:1; } _present; __u32 ifindex; __u64 xdp_features; __u32 xdp_zc_max_segs; __u64 xdp_rx_metadata_features; + __u64 xsk_features; }; void netdev_dev_get_rsp_free(struct netdev_dev_get_rsp *rsp); -- cgit v1.2.3 From 7577bc8249c3fc86096ef1b1c9a8f4b6232231e7 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Tue, 28 Nov 2023 18:29:20 -0800 Subject: tcp: Don't pass cookie to __cookie_v[46]_check(). tcp_hdr(skb) and SYN Cookie are passed to __cookie_v[46]_check(), but none of the callers passes cookie other than ntohl(th->ack_seq) - 1. Let's fetch it in __cookie_v[46]_check() instead of passing the cookie over and over. Signed-off-by: Kuniyuki Iwashima Reviewed-by: Simon Horman Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/20231129022924.96156-5-kuniyu@amazon.com Signed-off-by: Jakub Kicinski --- include/linux/netfilter_ipv6.h | 8 ++++---- include/net/tcp.h | 6 ++---- net/core/filter.c | 15 ++++----------- net/ipv4/syncookies.c | 15 ++++++++------- net/ipv6/syncookies.c | 15 ++++++++------- net/netfilter/nf_synproxy_core.c | 4 ++-- 6 files changed, 28 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter_ipv6.h b/include/linux/netfilter_ipv6.h index 7834c0be2831..61aa48f46dd7 100644 --- a/include/linux/netfilter_ipv6.h +++ b/include/linux/netfilter_ipv6.h @@ -51,7 +51,7 @@ struct nf_ipv6_ops { u32 (*cookie_init_sequence)(const struct ipv6hdr *iph, const struct tcphdr *th, u16 *mssp); int (*cookie_v6_check)(const struct ipv6hdr *iph, - const struct tcphdr *th, __u32 cookie); + const struct tcphdr *th); #endif void (*route_input)(struct sk_buff *skb); int (*fragment)(struct net *net, struct sock *sk, struct sk_buff *skb, @@ -179,16 +179,16 @@ static inline u32 nf_ipv6_cookie_init_sequence(const struct ipv6hdr *iph, } static inline int nf_cookie_v6_check(const struct ipv6hdr *iph, - const struct tcphdr *th, __u32 cookie) + const struct tcphdr *th) { #if IS_ENABLED(CONFIG_SYN_COOKIES) #if IS_MODULE(CONFIG_IPV6) const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); if (v6_ops) - return v6_ops->cookie_v6_check(iph, th, cookie); + return v6_ops->cookie_v6_check(iph, th); #elif IS_BUILTIN(CONFIG_IPV6) - return __cookie_v6_check(iph, th, cookie); + return __cookie_v6_check(iph, th); #endif #endif return 0; diff --git a/include/net/tcp.h b/include/net/tcp.h index d2f0736b76b8..2b2c79c7bbcd 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -491,8 +491,7 @@ void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, u32 tsoff); -int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, - u32 cookie); +int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th); struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, const struct tcp_request_sock_ops *af_ops, @@ -586,8 +585,7 @@ bool cookie_ecn_ok(const struct tcp_options_received *opt, const struct net *net, const struct dst_entry *dst); /* From net/ipv6/syncookies.c */ -int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, - u32 cookie); +int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th); struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, diff --git a/net/core/filter.c b/net/core/filter.c index 7e4d7c3bcc84..0adaa4afa35f 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7238,7 +7238,6 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len struct tcphdr *, th, u32, th_len) { #ifdef CONFIG_SYN_COOKIES - u32 cookie; int ret; if (unlikely(!sk || th_len < sizeof(*th))) @@ -7260,8 +7259,6 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len if (tcp_synq_no_recent_overflow(sk)) return -ENOENT; - cookie = ntohl(th->ack_seq) - 1; - /* Both struct iphdr and struct ipv6hdr have the version field at the * same offset so we can cast to the shorter header (struct iphdr). */ @@ -7270,7 +7267,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk)) return -EINVAL; - ret = __cookie_v4_check((struct iphdr *)iph, th, cookie); + ret = __cookie_v4_check((struct iphdr *)iph, th); break; #if IS_BUILTIN(CONFIG_IPV6) @@ -7281,7 +7278,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len if (sk->sk_family != AF_INET6) return -EINVAL; - ret = __cookie_v6_check((struct ipv6hdr *)iph, th, cookie); + ret = __cookie_v6_check((struct ipv6hdr *)iph, th); break; #endif /* CONFIG_IPV6 */ @@ -7734,9 +7731,7 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph, struct tcphdr *, th) { - u32 cookie = ntohl(th->ack_seq) - 1; - - if (__cookie_v4_check(iph, th, cookie) > 0) + if (__cookie_v4_check(iph, th) > 0) return 0; return -EACCES; @@ -7757,9 +7752,7 @@ BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv6, struct ipv6hdr *, iph, struct tcphdr *, th) { #if IS_BUILTIN(CONFIG_IPV6) - u32 cookie = ntohl(th->ack_seq) - 1; - - if (__cookie_v6_check(iph, th, cookie) > 0) + if (__cookie_v6_check(iph, th) > 0) return 0; return -EACCES; diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 8b7d7d7788af..c08428d63d11 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -189,12 +189,14 @@ __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mssp) * Check if a ack sequence number is a valid syncookie. * Return the decoded mss if it is, or 0 if not. */ -int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th, - u32 cookie) +int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th) { + __u32 cookie = ntohl(th->ack_seq) - 1; __u32 seq = ntohl(th->seq) - 1; - __u32 mssind = check_tcp_syn_cookie(cookie, iph->saddr, iph->daddr, - th->source, th->dest, seq); + __u32 mssind; + + mssind = check_tcp_syn_cookie(cookie, iph->saddr, iph->daddr, + th->source, th->dest, seq); return mssind < ARRAY_SIZE(msstab) ? msstab[mssind] : 0; } @@ -332,7 +334,6 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb) { struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; const struct tcphdr *th = tcp_hdr(skb); - __u32 cookie = ntohl(th->ack_seq) - 1; struct tcp_options_received tcp_opt; struct tcp_sock *tp = tcp_sk(sk); struct inet_request_sock *ireq; @@ -354,7 +355,7 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb) if (tcp_synq_no_recent_overflow(sk)) goto out; - mss = __cookie_v4_check(ip_hdr(skb), th, cookie); + mss = __cookie_v4_check(ip_hdr(skb), th); if (mss == 0) { __NET_INC_STATS(net, LINUX_MIB_SYNCOOKIESFAILED); goto out; @@ -384,7 +385,7 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb) ireq = inet_rsk(req); treq = tcp_rsk(req); treq->rcv_isn = ntohl(th->seq) - 1; - treq->snt_isn = cookie; + treq->snt_isn = ntohl(th->ack_seq) - 1; treq->ts_off = 0; treq->txhash = net_tx_rndhash(); req->mss = mss; diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 106376cbc9de..4cd26c481168 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -114,12 +114,14 @@ __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mssp) return __cookie_v6_init_sequence(iph, th, mssp); } -int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th, - __u32 cookie) +int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th) { + __u32 cookie = ntohl(th->ack_seq) - 1; __u32 seq = ntohl(th->seq) - 1; - __u32 mssind = check_tcp_syn_cookie(cookie, &iph->saddr, &iph->daddr, - th->source, th->dest, seq); + __u32 mssind; + + mssind = check_tcp_syn_cookie(cookie, &iph->saddr, &iph->daddr, + th->source, th->dest, seq); return mssind < ARRAY_SIZE(msstab) ? msstab[mssind] : 0; } @@ -128,7 +130,6 @@ EXPORT_SYMBOL_GPL(__cookie_v6_check); struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) { const struct tcphdr *th = tcp_hdr(skb); - __u32 cookie = ntohl(th->ack_seq) - 1; struct ipv6_pinfo *np = inet6_sk(sk); struct tcp_options_received tcp_opt; struct tcp_sock *tp = tcp_sk(sk); @@ -150,7 +151,7 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) if (tcp_synq_no_recent_overflow(sk)) goto out; - mss = __cookie_v6_check(ipv6_hdr(skb), th, cookie); + mss = __cookie_v6_check(ipv6_hdr(skb), th); if (mss == 0) { __NET_INC_STATS(net, LINUX_MIB_SYNCOOKIESFAILED); goto out; @@ -213,7 +214,7 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) req->ts_recent = tcp_opt.saw_tstamp ? tcp_opt.rcv_tsval : 0; treq->snt_synack = 0; treq->rcv_isn = ntohl(th->seq) - 1; - treq->snt_isn = cookie; + treq->snt_isn = ntohl(th->ack_seq) - 1; treq->ts_off = 0; treq->txhash = net_tx_rndhash(); diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c index 467671f2d42f..fbbc4fd37349 100644 --- a/net/netfilter/nf_synproxy_core.c +++ b/net/netfilter/nf_synproxy_core.c @@ -617,7 +617,7 @@ synproxy_recv_client_ack(struct net *net, struct synproxy_net *snet = synproxy_pernet(net); int mss; - mss = __cookie_v4_check(ip_hdr(skb), th, ntohl(th->ack_seq) - 1); + mss = __cookie_v4_check(ip_hdr(skb), th); if (mss == 0) { this_cpu_inc(snet->stats->cookie_invalid); return false; @@ -1034,7 +1034,7 @@ synproxy_recv_client_ack_ipv6(struct net *net, struct synproxy_net *snet = synproxy_pernet(net); int mss; - mss = nf_cookie_v6_check(ipv6_hdr(skb), th, ntohl(th->ack_seq) - 1); + mss = nf_cookie_v6_check(ipv6_hdr(skb), th); if (mss == 0) { this_cpu_inc(snet->stats->cookie_invalid); return false; -- cgit v1.2.3 From df16c1c51d8166958f533c0c886766f7ee9dd50f Mon Sep 17 00:00:00 2001 From: Andrew Halaney Date: Mon, 27 Nov 2023 15:41:10 -0600 Subject: net: phy: mdio_device: Reset device only when necessary Currently the phy reset sequence is as shown below for a devicetree described mdio phy on boot: 1. Assert the phy_device's reset as part of registering 2. Deassert the phy_device's reset as part of registering 3. Deassert the phy_device's reset as part of phy_probe 4. Deassert the phy_device's reset as part of phy_hw_init The extra two deasserts include waiting the deassert delay afterwards, which is adding unnecessary delay. This applies to both possible types of resets (reset controller reference and a reset gpio) that can be used. Here's some snipped tracing output using the following command line params "trace_event=gpio:* trace_options=stacktrace" illustrating the reset handling and where its coming from: /* Assert */ systemd-udevd-283 [002] ..... 6.780434: gpio_value: 544 set 0 systemd-udevd-283 [002] ..... 6.783849: => gpiod_set_raw_value_commit => gpiod_set_value_nocheck => gpiod_set_value_cansleep => mdio_device_reset => mdiobus_register_device => phy_device_register => fwnode_mdiobus_phy_device_register => fwnode_mdiobus_register_phy => __of_mdiobus_register => stmmac_mdio_register => stmmac_dvr_probe => stmmac_pltfr_probe => devm_stmmac_pltfr_probe => qcom_ethqos_probe => platform_probe /* Deassert */ systemd-udevd-283 [002] ..... 6.802480: gpio_value: 544 set 1 systemd-udevd-283 [002] ..... 6.805886: => gpiod_set_raw_value_commit => gpiod_set_value_nocheck => gpiod_set_value_cansleep => mdio_device_reset => phy_device_register => fwnode_mdiobus_phy_device_register => fwnode_mdiobus_register_phy => __of_mdiobus_register => stmmac_mdio_register => stmmac_dvr_probe => stmmac_pltfr_probe => devm_stmmac_pltfr_probe => qcom_ethqos_probe => platform_probe /* Deassert */ systemd-udevd-283 [002] ..... 6.882601: gpio_value: 544 set 1 systemd-udevd-283 [002] ..... 6.886014: => gpiod_set_raw_value_commit => gpiod_set_value_nocheck => gpiod_set_value_cansleep => mdio_device_reset => phy_probe => really_probe => __driver_probe_device => driver_probe_device => __device_attach_driver => bus_for_each_drv => __device_attach => device_initial_probe => bus_probe_device => device_add => phy_device_register => fwnode_mdiobus_phy_device_register => fwnode_mdiobus_register_phy => __of_mdiobus_register => stmmac_mdio_register => stmmac_dvr_probe => stmmac_pltfr_probe => devm_stmmac_pltfr_probe => qcom_ethqos_probe => platform_probe /* Deassert */ NetworkManager-477 [000] ..... 7.023144: gpio_value: 544 set 1 NetworkManager-477 [000] ..... 7.026596: => gpiod_set_raw_value_commit => gpiod_set_value_nocheck => gpiod_set_value_cansleep => mdio_device_reset => phy_init_hw => phy_attach_direct => phylink_fwnode_phy_connect => __stmmac_open => stmmac_open There's a lot of paths where the device is getting its reset asserted and deasserted. Let's track the state and only actually do the assert/deassert when it changes. Reported-by: Sagar Cheluvegowda Signed-off-by: Andrew Halaney Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20231127-net-phy-reset-once-v2-1-448e8658779e@redhat.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/mdio_device.c | 6 ++++++ drivers/net/phy/phy_device.c | 1 + include/linux/mdio.h | 1 + 3 files changed, 8 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/mdio_device.c b/drivers/net/phy/mdio_device.c index 044828d081d2..73f6539b9e50 100644 --- a/drivers/net/phy/mdio_device.c +++ b/drivers/net/phy/mdio_device.c @@ -62,6 +62,7 @@ struct mdio_device *mdio_device_create(struct mii_bus *bus, int addr) mdiodev->device_remove = mdio_device_remove; mdiodev->bus = bus; mdiodev->addr = addr; + mdiodev->reset_state = -1; dev_set_name(&mdiodev->dev, PHY_ID_FMT, bus->id, addr); @@ -122,6 +123,9 @@ void mdio_device_reset(struct mdio_device *mdiodev, int value) if (!mdiodev->reset_gpio && !mdiodev->reset_ctrl) return; + if (mdiodev->reset_state == value) + return; + if (mdiodev->reset_gpio) gpiod_set_value_cansleep(mdiodev->reset_gpio, value); @@ -135,6 +139,8 @@ void mdio_device_reset(struct mdio_device *mdiodev, int value) d = value ? mdiodev->reset_assert_delay : mdiodev->reset_deassert_delay; if (d) fsleep(d); + + mdiodev->reset_state = value; } EXPORT_SYMBOL(mdio_device_reset); diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 400fb09d9cd6..d8e9335d415c 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -654,6 +654,7 @@ struct phy_device *phy_device_create(struct mii_bus *bus, int addr, u32 phy_id, mdiodev->flags = MDIO_DEVICE_FLAG_PHY; mdiodev->device_free = phy_mdio_device_free; mdiodev->device_remove = phy_mdio_device_remove; + mdiodev->reset_state = -1; dev->speed = SPEED_UNKNOWN; dev->duplex = DUPLEX_UNKNOWN; diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 007fd9c3e4b6..79ceee3c8673 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -38,6 +38,7 @@ struct mdio_device { /* Bus address of the MDIO device (0-31) */ int addr; int flags; + int reset_state; struct gpio_desc *reset_gpio; struct reset_control *reset_ctrl; unsigned int reset_assert_delay; -- cgit v1.2.3 From 45b5623f2d721c25d1a2fdc8c4600fb4b7b61c75 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Sat, 2 Dec 2023 09:56:55 -0800 Subject: bpf: rearrange bpf_func_state fields to save a bit of memory It's a trivial rearrangement saving 8 bytes. We have 4 bytes of padding at the end which can be filled with another field without increasing struct bpf_func_state. copy_func_state() logic remains correct without any further changes. BEFORE ====== struct bpf_func_state { struct bpf_reg_state regs[11]; /* 0 1320 */ /* --- cacheline 20 boundary (1280 bytes) was 40 bytes ago --- */ int callsite; /* 1320 4 */ u32 frameno; /* 1324 4 */ u32 subprogno; /* 1328 4 */ u32 async_entry_cnt; /* 1332 4 */ bool in_callback_fn; /* 1336 1 */ /* XXX 7 bytes hole, try to pack */ /* --- cacheline 21 boundary (1344 bytes) --- */ struct tnum callback_ret_range; /* 1344 16 */ bool in_async_callback_fn; /* 1360 1 */ bool in_exception_callback_fn; /* 1361 1 */ /* XXX 2 bytes hole, try to pack */ int acquired_refs; /* 1364 4 */ struct bpf_reference_state * refs; /* 1368 8 */ int allocated_stack; /* 1376 4 */ /* XXX 4 bytes hole, try to pack */ struct bpf_stack_state * stack; /* 1384 8 */ /* size: 1392, cachelines: 22, members: 13 */ /* sum members: 1379, holes: 3, sum holes: 13 */ /* last cacheline: 48 bytes */ }; AFTER ===== struct bpf_func_state { struct bpf_reg_state regs[11]; /* 0 1320 */ /* --- cacheline 20 boundary (1280 bytes) was 40 bytes ago --- */ int callsite; /* 1320 4 */ u32 frameno; /* 1324 4 */ u32 subprogno; /* 1328 4 */ u32 async_entry_cnt; /* 1332 4 */ struct tnum callback_ret_range; /* 1336 16 */ /* --- cacheline 21 boundary (1344 bytes) was 8 bytes ago --- */ bool in_callback_fn; /* 1352 1 */ bool in_async_callback_fn; /* 1353 1 */ bool in_exception_callback_fn; /* 1354 1 */ /* XXX 1 byte hole, try to pack */ int acquired_refs; /* 1356 4 */ struct bpf_reference_state * refs; /* 1360 8 */ struct bpf_stack_state * stack; /* 1368 8 */ int allocated_stack; /* 1376 4 */ /* size: 1384, cachelines: 22, members: 13 */ /* sum members: 1379, holes: 1, sum holes: 1 */ /* padding: 4 */ /* last cacheline: 40 bytes */ }; Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231202175705.885270-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d99a636d36a7..0c0e1bccad45 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -297,8 +297,8 @@ struct bpf_func_state { * void foo(void) { bpf_timer_set_callback(,foo); } */ u32 async_entry_cnt; - bool in_callback_fn; struct tnum callback_ret_range; + bool in_callback_fn; bool in_async_callback_fn; bool in_exception_callback_fn; /* For callback calling functions that limit number of possible @@ -316,8 +316,8 @@ struct bpf_func_state { /* The following fields should be last. See copy_func_state() */ int acquired_refs; struct bpf_reference_state *refs; - int allocated_stack; struct bpf_stack_state *stack; + int allocated_stack; }; struct bpf_idx_pair { -- cgit v1.2.3 From 8fa4ecd49b81ccd9d1d87f1c8b2260e218644878 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Sat, 2 Dec 2023 09:56:58 -0800 Subject: bpf: enforce exact retval range on subprog/callback exit Instead of relying on potentially imprecise tnum representation of expected return value range for callbacks and subprogs, validate that smin/smax range satisfy exact expected range of return values. E.g., if callback would need to return [0, 2] range, tnum can't represent this precisely and instead will allow [0, 3] range. By checking smin/smax range, we can make sure that subprog/callback indeed returns only valid [0, 2] range. Acked-by: Eduard Zingerman Acked-by: Shung-Hsi Yu Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231202175705.885270-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 7 ++++++- kernel/bpf/verifier.c | 33 ++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 0c0e1bccad45..3378cc753061 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -275,6 +275,11 @@ struct bpf_reference_state { int callback_ref; }; +struct bpf_retval_range { + s32 minval; + s32 maxval; +}; + /* state of the program: * type of all registers and stack info */ @@ -297,7 +302,7 @@ struct bpf_func_state { * void foo(void) { bpf_timer_set_callback(,foo); } */ u32 async_entry_cnt; - struct tnum callback_ret_range; + struct bpf_retval_range callback_ret_range; bool in_callback_fn; bool in_async_callback_fn; bool in_exception_callback_fn; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 849fbf47b5f3..f3d9d7de68da 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2305,6 +2305,11 @@ static void init_reg_state(struct bpf_verifier_env *env, regs[BPF_REG_FP].frameno = state->frameno; } +static struct bpf_retval_range retval_range(s32 minval, s32 maxval) +{ + return (struct bpf_retval_range){ minval, maxval }; +} + #define BPF_MAIN_FUNC (-1) static void init_func_state(struct bpf_verifier_env *env, struct bpf_func_state *state, @@ -2313,7 +2318,7 @@ static void init_func_state(struct bpf_verifier_env *env, state->callsite = callsite; state->frameno = frameno; state->subprogno = subprogno; - state->callback_ret_range = tnum_range(0, 0); + state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); mark_verifier_state_scratched(env); } @@ -9396,7 +9401,7 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, return err; callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9418,7 +9423,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9448,7 +9453,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_async_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9476,7 +9481,7 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9499,7 +9504,7 @@ static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9531,7 +9536,7 @@ static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; - callee->callback_ret_range = tnum_range(0, 1); + callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -9560,6 +9565,11 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) return is_rbtree_lock_required_kfunc(kfunc_btf_id); } +static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) +{ + return range.minval <= reg->smin_value && reg->smax_value <= range.maxval; +} + static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) { struct bpf_verifier_state *state = env->cur_state, *prev_st; @@ -9583,9 +9593,6 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) caller = state->frame[state->curframe - 1]; if (callee->in_callback_fn) { - /* enforce R0 return value range [0, 1]. */ - struct tnum range = callee->callback_ret_range; - if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); return -EACCES; @@ -9597,7 +9604,11 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) if (err) return err; - if (!tnum_in(range, r0->var_off)) { + /* enforce R0 return value range */ + if (!retval_range_within(callee->callback_ret_range, r0)) { + struct tnum range = tnum_range(callee->callback_ret_range.minval, + callee->callback_ret_range.maxval); + verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); return -EINVAL; } -- cgit v1.2.3 From aeb9ce058d7c6193dc41e06b3a5b29d22c446b14 Mon Sep 17 00:00:00 2001 From: Coco Li Date: Wed, 29 Nov 2023 07:27:53 +0000 Subject: cache: enforce cache groups Set up build time warnings to safeguard against future header changes of organized structs. Warning includes: 1) whether all variables are still in the same cache group 2) whether all the cache groups have the sum of the members size (in the maximum condition, including all members defined in configs) The __cache_group* variables are ignored in kernel-doc check in the various header files they appear in to enforce the cache groups. Suggested-by: Daniel Borkmann Acked-by: Daniel Borkmann Signed-off-by: Coco Li Reviewed-by: Eric Dumazet Reviewed-by: Shakeel Butt Signed-off-by: David S. Miller --- include/linux/cache.h | 25 +++++++++++++++++++++++++ scripts/kernel-doc | 5 +++++ 2 files changed, 30 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cache.h b/include/linux/cache.h index 9900d20b76c2..0ecb17bb6883 100644 --- a/include/linux/cache.h +++ b/include/linux/cache.h @@ -85,6 +85,31 @@ #define cache_line_size() L1_CACHE_BYTES #endif +#ifndef __cacheline_group_begin +#define __cacheline_group_begin(GROUP) \ + __u8 __cacheline_group_begin__##GROUP[0] +#endif + +#ifndef __cacheline_group_end +#define __cacheline_group_end(GROUP) \ + __u8 __cacheline_group_end__##GROUP[0] +#endif + +#ifndef CACHELINE_ASSERT_GROUP_MEMBER +#define CACHELINE_ASSERT_GROUP_MEMBER(TYPE, GROUP, MEMBER) \ + BUILD_BUG_ON(!(offsetof(TYPE, MEMBER) >= \ + offsetofend(TYPE, __cacheline_group_begin__##GROUP) && \ + offsetofend(TYPE, MEMBER) <= \ + offsetof(TYPE, __cacheline_group_end__##GROUP))) +#endif + +#ifndef CACHELINE_ASSERT_GROUP_SIZE +#define CACHELINE_ASSERT_GROUP_SIZE(TYPE, GROUP, SIZE) \ + BUILD_BUG_ON(offsetof(TYPE, __cacheline_group_end__##GROUP) - \ + offsetofend(TYPE, __cacheline_group_begin__##GROUP) > \ + SIZE) +#endif + /* * Helper to add padding within a struct to ensure data fall into separate * cachelines. diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 08a3e603db19..0a890fe4d22b 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1592,6 +1592,11 @@ sub push_parameter($$$$$) { $parameterdescs{$param} = "anonymous\n"; $anon_struct_union = 1; } + elsif ($param =~ "__cacheline_group" ) + # handle cache group enforcing variables: they do not need be described in header files + { + return; # ignore __cacheline_group_begin and __cacheline_group_end + } # warn if parameter has no description # (but ignore ones starting with # as these are not parameters -- cgit v1.2.3 From 20c20bd11a0702ce4dc9300c3da58acf551d9725 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Mon, 4 Dec 2023 22:04:20 +0800 Subject: bpf: Add map and need_defer parameters to .map_fd_put_ptr() map is the pointer of outer map, and need_defer needs some explanation. need_defer tells the implementation to defer the reference release of the passed element and ensure that the element is still alive before the bpf program, which may manipulate it, exits. The following three cases will invoke map_fd_put_ptr() and different need_defer values will be passed to these callers: 1) release the reference of the old element in the map during map update or map deletion. The release must be deferred, otherwise the bpf program may incur use-after-free problem, so need_defer needs to be true. 2) release the reference of the to-be-added element in the error path of map update. The to-be-added element is not visible to any bpf program, so it is OK to pass false for need_defer parameter. 3) release the references of all elements in the map during map release. Any bpf program which has access to the map must have been exited and released, so need_defer=false will be OK. These two parameters will be used by the following patches to fix the potential use-after-free problem for map-in-map. Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20231204140425.1480317-3-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 6 +++++- kernel/bpf/arraymap.c | 12 +++++++----- kernel/bpf/hashtab.c | 6 +++--- kernel/bpf/map_in_map.c | 2 +- kernel/bpf/map_in_map.h | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index eb447b0a9423..d273348cfb2f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -106,7 +106,11 @@ struct bpf_map_ops { /* funcs called by prog_array and perf_event_array map */ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, int fd); - void (*map_fd_put_ptr)(void *ptr); + /* If need_defer is true, the implementation should guarantee that + * the to-be-put element is still alive before the bpf program, which + * may manipulate it, exists. + */ + void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer); int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); u32 (*map_fd_sys_lookup_elem)(void *ptr); void (*map_seq_show_elem)(struct bpf_map *map, void *key, diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 2058e89b5ddd..f9aed5909d6e 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -867,7 +867,7 @@ int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, } if (old_ptr) - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, true); return 0; } @@ -890,7 +890,7 @@ static long fd_array_map_delete_elem(struct bpf_map *map, void *key) } if (old_ptr) { - map->ops->map_fd_put_ptr(old_ptr); + map->ops->map_fd_put_ptr(map, old_ptr, true); return 0; } else { return -ENOENT; @@ -913,8 +913,9 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, return prog; } -static void prog_fd_array_put_ptr(void *ptr) +static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_prog is freed after one RCU or tasks trace grace period */ bpf_prog_put(ptr); } @@ -1239,8 +1240,9 @@ err_out: return ee; } -static void perf_event_fd_array_put_ptr(void *ptr) +static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { + /* bpf_perf_event is freed after one RCU grace period */ bpf_event_entry_free_rcu(ptr); } @@ -1294,7 +1296,7 @@ static void *cgroup_fd_array_get_ptr(struct bpf_map *map, return cgroup_get_from_fd(fd); } -static void cgroup_fd_array_put_ptr(void *ptr) +static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { /* cgroup_put free cgrp after a rcu grace period */ cgroup_put(ptr); diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index fd8d4b0addfc..5b9146fa825f 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -897,7 +897,7 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) if (map->ops->map_fd_put_ptr) { ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, true); } } @@ -2484,7 +2484,7 @@ static void fd_htab_map_free(struct bpf_map *map) hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { void *ptr = fd_htab_map_get_ptr(map, l); - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); } } @@ -2525,7 +2525,7 @@ int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, ret = htab_map_update_elem(map, key, &ptr, map_flags); if (ret) - map->ops->map_fd_put_ptr(ptr); + map->ops->map_fd_put_ptr(map, ptr, false); return ret; } diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index cd5eafaba97e..2dfeb5835e16 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -127,7 +127,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return inner_map; } -void bpf_map_fd_put_ptr(void *ptr) +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { /* ptr->ops->map_free() has to go through one * rcu grace period by itself. diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h index bcb7534afb3c..7d61602354de 100644 --- a/kernel/bpf/map_in_map.h +++ b/kernel/bpf/map_in_map.h @@ -13,7 +13,7 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd); void bpf_map_meta_free(struct bpf_map *map_meta); void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file, int ufd); -void bpf_map_fd_put_ptr(void *ptr); +void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer); u32 bpf_map_fd_sys_lookup_elem(void *ptr); #endif -- cgit v1.2.3 From 876673364161da50eed6b472d746ef88242b2368 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Mon, 4 Dec 2023 22:04:22 +0800 Subject: bpf: Defer the free of inner map when necessary When updating or deleting an inner map in map array or map htab, the map may still be accessed by non-sleepable program or sleepable program. However bpf_map_fd_put_ptr() decreases the ref-counter of the inner map directly through bpf_map_put(), if the ref-counter is the last one (which is true for most cases), the inner map will be freed by ops->map_free() in a kworker. But for now, most .map_free() callbacks don't use synchronize_rcu() or its variants to wait for the elapse of a RCU grace period, so after the invocation of ops->map_free completes, the bpf program which is accessing the inner map may incur use-after-free problem. Fix the free of inner map by invoking bpf_map_free_deferred() after both one RCU grace period and one tasks trace RCU grace period if the inner map has been removed from the outer map before. The deferment is accomplished by using call_rcu() or call_rcu_tasks_trace() when releasing the last ref-counter of bpf map. The newly-added rcu_head field in bpf_map shares the same storage space with work field to reduce the size of bpf_map. Fixes: bba1dc0b55ac ("bpf: Remove redundant synchronize_rcu.") Fixes: 638e4b825d52 ("bpf: Allows per-cpu maps and map-in-map in sleepable programs") Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20231204140425.1480317-5-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 7 ++++++- kernel/bpf/map_in_map.c | 11 ++++++++--- kernel/bpf/syscall.c | 32 +++++++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d273348cfb2f..de3bd03cbeea 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -276,7 +276,11 @@ struct bpf_map { */ atomic64_t refcnt ____cacheline_aligned; atomic64_t usercnt; - struct work_struct work; + /* rcu is used before freeing and work is only used during freeing */ + union { + struct work_struct work; + struct rcu_head rcu; + }; struct mutex freeze_mutex; atomic64_t writecnt; /* 'Ownership' of program-containing map is claimed by the first program @@ -292,6 +296,7 @@ struct bpf_map { } owner; bool bypass_spec_v1; bool frozen; /* write-once; write-protected by freeze_mutex */ + bool free_after_mult_rcu_gp; s64 __percpu *elem_count; }; diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 2dfeb5835e16..3248ff5d8161 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -129,10 +129,15 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { - /* ptr->ops->map_free() has to go through one - * rcu grace period by itself. + struct bpf_map *inner_map = ptr; + + /* The inner map may still be used by both non-sleepable and sleepable + * bpf program, so free it after one RCU grace period and one tasks + * trace RCU grace period. */ - bpf_map_put(ptr); + if (need_defer) + WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true); + bpf_map_put(inner_map); } u32 bpf_map_fd_sys_lookup_elem(void *ptr) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 5e43ddd1b83f..dd515f6b9741 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -719,6 +719,28 @@ static void bpf_map_put_uref(struct bpf_map *map) } } +static void bpf_map_free_in_work(struct bpf_map *map) +{ + INIT_WORK(&map->work, bpf_map_free_deferred); + /* Avoid spawning kworkers, since they all might contend + * for the same mutex like slab_mutex. + */ + queue_work(system_unbound_wq, &map->work); +} + +static void bpf_map_free_rcu_gp(struct rcu_head *rcu) +{ + bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu)); +} + +static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu) +{ + if (rcu_trace_implies_rcu_gp()) + bpf_map_free_rcu_gp(rcu); + else + call_rcu(rcu, bpf_map_free_rcu_gp); +} + /* decrement map refcnt and schedule it for freeing via workqueue * (underlying map implementation ops->map_free() might sleep) */ @@ -728,11 +750,11 @@ void bpf_map_put(struct bpf_map *map) /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); btf_put(map->btf); - INIT_WORK(&map->work, bpf_map_free_deferred); - /* Avoid spawning kworkers, since they all might contend - * for the same mutex like slab_mutex. - */ - queue_work(system_unbound_wq, &map->work); + + if (READ_ONCE(map->free_after_mult_rcu_gp)) + call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp); + else + bpf_map_free_in_work(map); } } EXPORT_SYMBOL_GPL(bpf_map_put); -- cgit v1.2.3 From af66bfd3c8538ed21cf72af18426fc4a408665cf Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Mon, 4 Dec 2023 22:04:23 +0800 Subject: bpf: Optimize the free of inner map When removing the inner map from the outer map, the inner map will be freed after one RCU grace period and one RCU tasks trace grace period, so it is certain that the bpf program, which may access the inner map, has exited before the inner map is freed. However there is no need to wait for one RCU tasks trace grace period if the outer map is only accessed by non-sleepable program. So adding sleepable_refcnt in bpf_map and increasing sleepable_refcnt when adding the outer map into env->used_maps for sleepable program. Although the max number of bpf program is INT_MAX - 1, the number of bpf programs which are being loaded may be greater than INT_MAX, so using atomic64_t instead of atomic_t for sleepable_refcnt. When removing the inner map from the outer map, using sleepable_refcnt to decide whether or not a RCU tasks trace grace period is needed before freeing the inner map. Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20231204140425.1480317-6-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 ++ kernel/bpf/core.c | 4 ++++ kernel/bpf/map_in_map.c | 14 +++++++++----- kernel/bpf/syscall.c | 8 ++++++++ kernel/bpf/verifier.c | 4 +++- 5 files changed, 26 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index de3bd03cbeea..10e5e4d8a00f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -297,6 +297,8 @@ struct bpf_map { bool bypass_spec_v1; bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; + bool free_after_rcu_gp; + atomic64_t sleepable_refcnt; s64 __percpu *elem_count; }; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index cd3afe57ece3..4b813da8d6c0 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2664,12 +2664,16 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, struct bpf_map **used_maps, u32 len) { struct bpf_map *map; + bool sleepable; u32 i; + sleepable = aux->sleepable; for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) map->ops->map_poke_untrack(map, aux); + if (sleepable) + atomic64_dec(&map->sleepable_refcnt); bpf_map_put(map); } } diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index 3248ff5d8161..8ef269e66ba5 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -131,12 +131,16 @@ void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) { struct bpf_map *inner_map = ptr; - /* The inner map may still be used by both non-sleepable and sleepable - * bpf program, so free it after one RCU grace period and one tasks - * trace RCU grace period. + /* Defer the freeing of inner map according to the sleepable attribute + * of bpf program which owns the outer map, so unnecessary waiting for + * RCU tasks trace grace period can be avoided. */ - if (need_defer) - WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true); + if (need_defer) { + if (atomic64_read(&map->sleepable_refcnt)) + WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true); + else + WRITE_ONCE(inner_map->free_after_rcu_gp, true); + } bpf_map_put(inner_map); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index dd515f6b9741..ebaccf77d56e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -751,8 +751,11 @@ void bpf_map_put(struct bpf_map *map) bpf_map_free_id(map); btf_put(map->btf); + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); if (READ_ONCE(map->free_after_mult_rcu_gp)) call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp); + else if (READ_ONCE(map->free_after_rcu_gp)) + call_rcu(&map->rcu, bpf_map_free_rcu_gp); else bpf_map_free_in_work(map); } @@ -5345,6 +5348,11 @@ static int bpf_prog_bind_map(union bpf_attr *attr) goto out_unlock; } + /* The bpf program will not access the bpf map, but for the sake of + * simplicity, increase sleepable_refcnt for sleepable program as well. + */ + if (prog->aux->sleepable) + atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); used_maps_new[prog->aux->used_map_cnt] = map; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index cdb4f5f0ba79..1ed39665f802 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17889,10 +17889,12 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) return -E2BIG; } + if (env->prog->aux->sleepable) + atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, * the map will be released by release_maps() or it * will be used by the valid program until it's unloaded - * and all maps are released in free_used_maps() + * and all maps are released in bpf_free_used_maps() */ bpf_map_inc(map); -- cgit v1.2.3 From 2a502ff0c4e42a739b5aa550c901bf3852795532 Mon Sep 17 00:00:00 2001 From: Amritha Nambiar Date: Fri, 1 Dec 2023 15:28:34 -0800 Subject: net: Add queue and napi association Add the napi pointer in netdev queue for tracking the napi instance for each queue. This achieves the queue<->napi mapping. Signed-off-by: Amritha Nambiar Reviewed-by: Sridhar Samudrala Link: https://lore.kernel.org/r/170147331483.5260.15723438819994285695.stgit@anambiarhost.jf.intel.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 8 ++++++++ include/net/netdev_rx_queue.h | 4 ++++ net/core/dev.c | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c2d74bc112dd..5ddff11cbe26 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -665,6 +665,10 @@ struct netdev_queue { #ifdef CONFIG_XDP_SOCKETS struct xsk_buff_pool *pool; #endif + /* NAPI instance for the queue + * Readers and writers must hold RTNL + */ + struct napi_struct *napi; /* * write-mostly part */ @@ -2657,6 +2661,10 @@ static inline void *netdev_priv(const struct net_device *dev) */ #define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype)) +void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, + enum netdev_queue_type type, + struct napi_struct *napi); + /* Default NAPI poll() weight * Device drivers are strongly advised to not use bigger value */ diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h index cdcafb30d437..aa1716fb0e53 100644 --- a/include/net/netdev_rx_queue.h +++ b/include/net/netdev_rx_queue.h @@ -21,6 +21,10 @@ struct netdev_rx_queue { #ifdef CONFIG_XDP_SOCKETS struct xsk_buff_pool *pool; #endif + /* NAPI instance for the queue + * Readers and writers must hold RTNL + */ + struct napi_struct *napi; } ____cacheline_aligned_in_smp; /* diff --git a/net/core/dev.c b/net/core/dev.c index 3950ced396b5..0a2eecbeef38 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6400,6 +6400,43 @@ int dev_set_threaded(struct net_device *dev, bool threaded) } EXPORT_SYMBOL(dev_set_threaded); +/** + * netif_queue_set_napi - Associate queue with the napi + * @dev: device to which NAPI and queue belong + * @queue_index: Index of queue + * @type: queue type as RX or TX + * @napi: NAPI context, pass NULL to clear previously set NAPI + * + * Set queue with its corresponding napi context. This should be done after + * registering the NAPI handler for the queue-vector and the queues have been + * mapped to the corresponding interrupt vector. + */ +void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, + enum netdev_queue_type type, struct napi_struct *napi) +{ + struct netdev_rx_queue *rxq; + struct netdev_queue *txq; + + if (WARN_ON_ONCE(napi && !napi->dev)) + return; + if (dev->reg_state >= NETREG_REGISTERED) + ASSERT_RTNL(); + + switch (type) { + case NETDEV_QUEUE_TYPE_RX: + rxq = __netif_get_rx_queue(dev, queue_index); + rxq->napi = napi; + return; + case NETDEV_QUEUE_TYPE_TX: + txq = netdev_get_tx_queue(dev, queue_index); + txq->napi = napi; + return; + default: + return; + } +} +EXPORT_SYMBOL(netif_queue_set_napi); + void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi, int (*poll)(struct napi_struct *, int), int weight) { -- cgit v1.2.3 From 26793bfb5d6072326d1465343e7cbf6156abca4f Mon Sep 17 00:00:00 2001 From: Amritha Nambiar Date: Fri, 1 Dec 2023 15:29:07 -0800 Subject: net: Add NAPI IRQ support Add support to associate the interrupt vector number for a NAPI instance. Signed-off-by: Amritha Nambiar Reviewed-by: Sridhar Samudrala Link: https://lore.kernel.org/r/170147334728.5260.13221803396905901904.stgit@anambiarhost.jf.intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/ice/ice_lib.c | 2 ++ include/linux/netdevice.h | 6 ++++++ net/core/dev.c | 1 + net/core/netdev-genl.c | 4 ++++ 4 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 83f6977fad22..626577c7d5b2 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2978,6 +2978,8 @@ void ice_q_vector_set_napi_queues(struct ice_q_vector *q_vector, bool locked) ice_queue_set_napi(q_vector->vsi->netdev, tx_ring->q_index, NETDEV_QUEUE_TYPE_TX, &q_vector->napi, locked); + /* Also set the interrupt number for the NAPI */ + netif_napi_set_irq(&q_vector->napi, q_vector->irq.virq); } /** diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5ddff11cbe26..5551177e024e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -382,6 +382,7 @@ struct napi_struct { /* control-path-only fields follow */ struct list_head dev_list; struct hlist_node napi_hash_node; + int irq; }; enum { @@ -2665,6 +2666,11 @@ void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, enum netdev_queue_type type, struct napi_struct *napi); +static inline void netif_napi_set_irq(struct napi_struct *napi, int irq) +{ + napi->irq = irq; +} + /* Default NAPI poll() weight * Device drivers are strongly advised to not use bigger value */ diff --git a/net/core/dev.c b/net/core/dev.c index 13c9d1580e62..1d720fc59161 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6471,6 +6471,7 @@ void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi, */ if (dev->threaded && napi_kthread_create(napi)) dev->threaded = 0; + netif_napi_set_irq(napi, -1); } EXPORT_SYMBOL(netif_napi_add_weight); diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index d55c92f9f26c..9753c19e36de 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -180,7 +180,11 @@ netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) goto nla_put_failure; + if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) + goto nla_put_failure; + genlmsg_end(rsp, hdr); + return 0; nla_put_failure: -- cgit v1.2.3 From 41f6f64e6999a837048b1bd13a2f8742964eca6b Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 5 Dec 2023 10:42:39 -0800 Subject: bpf: support non-r10 register spill/fill to/from stack in precision tracking Use instruction (jump) history to record instructions that performed register spill/fill to/from stack, regardless if this was done through read-only r10 register, or any other register after copying r10 into it *and* potentially adjusting offset. To make this work reliably, we push extra per-instruction flags into instruction history, encoding stack slot index (spi) and stack frame number in extra 10 bit flags we take away from prev_idx in instruction history. We don't touch idx field for maximum performance, as it's checked most frequently during backtracking. This change removes basically the last remaining practical limitation of precision backtracking logic in BPF verifier. It fixes known deficiencies, but also opens up new opportunities to reduce number of verified states, explored in the subsequent patches. There are only three differences in selftests' BPF object files according to veristat, all in the positive direction (less states). File Program Insns (A) Insns (B) Insns (DIFF) States (A) States (B) States (DIFF) -------------------------------------- ------------- --------- --------- ------------- ---------- ---------- ------------- test_cls_redirect_dynptr.bpf.linked3.o cls_redirect 2987 2864 -123 (-4.12%) 240 231 -9 (-3.75%) xdp_synproxy_kern.bpf.linked3.o syncookie_tc 82848 82661 -187 (-0.23%) 5107 5073 -34 (-0.67%) xdp_synproxy_kern.bpf.linked3.o syncookie_xdp 85116 84964 -152 (-0.18%) 5162 5130 -32 (-0.62%) Note, I avoided renaming jmp_history to more generic insn_hist to minimize number of lines changed and potential merge conflicts between bpf and bpf-next trees. Notice also cur_hist_entry pointer reset to NULL at the beginning of instruction verification loop. This pointer avoids the problem of relying on last jump history entry's insn_idx to determine whether we already have entry for current instruction or not. It can happen that we added jump history entry because current instruction is_jmp_point(), but also we need to add instruction flags for stack access. In this case, we don't want to entries, so we need to reuse last added entry, if it is present. Relying on insn_idx comparison has the same ambiguity problem as the one that was fixed recently in [0], so we avoid that. [0] https://patchwork.kernel.org/project/netdevbpf/patch/20231110002638.4168352-3-andrii@kernel.org/ Acked-by: Eduard Zingerman Reported-by: Tao Lyu Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231205184248.1502704-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 31 +++- kernel/bpf/verifier.c | 175 ++++++++++++--------- .../bpf/progs/verifier_subprog_precision.c | 23 ++- tools/testing/selftests/bpf/verifier/precise.c | 38 +++-- 4 files changed, 169 insertions(+), 98 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3378cc753061..bada59812e00 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -325,12 +325,34 @@ struct bpf_func_state { int allocated_stack; }; -struct bpf_idx_pair { - u32 prev_idx; +#define MAX_CALL_FRAMES 8 + +/* instruction history flags, used in bpf_jmp_history_entry.flags field */ +enum { + /* instruction references stack slot through PTR_TO_STACK register; + * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8) + * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512, + * 8 bytes per slot, so slot index (spi) is [0, 63]) + */ + INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */ + + INSN_F_SPI_MASK = 0x3f, /* 6 bits */ + INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */ + + INSN_F_STACK_ACCESS = BIT(9), /* we need 10 bits total */ +}; + +static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES); +static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8); + +struct bpf_jmp_history_entry { u32 idx; + /* insn idx can't be bigger than 1 million */ + u32 prev_idx : 22; + /* special flags, e.g., whether insn is doing register stack spill/load */ + u32 flags : 10; }; -#define MAX_CALL_FRAMES 8 /* Maximum number of register states that can exist at once */ #define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES) struct bpf_verifier_state { @@ -413,7 +435,7 @@ struct bpf_verifier_state { * For most states jmp_history_cnt is [0-3]. * For loops can go up to ~40. */ - struct bpf_idx_pair *jmp_history; + struct bpf_jmp_history_entry *jmp_history; u32 jmp_history_cnt; u32 dfs_depth; u32 callback_unroll_depth; @@ -656,6 +678,7 @@ struct bpf_verifier_env { int cur_stack; } cfg; struct backtrack_state bt; + struct bpf_jmp_history_entry *cur_hist_ent; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1ed39665f802..9bc16dc66465 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1355,8 +1355,8 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, int i, err; dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, - src->jmp_history_cnt, sizeof(struct bpf_idx_pair), - GFP_USER); + src->jmp_history_cnt, sizeof(*dst_state->jmp_history), + GFP_USER); if (!dst_state->jmp_history) return -ENOMEM; dst_state->jmp_history_cnt = src->jmp_history_cnt; @@ -3221,6 +3221,21 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, return __check_reg_arg(env, state->regs, regno, t); } +static int insn_stack_access_flags(int frameno, int spi) +{ + return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; +} + +static int insn_stack_access_spi(int insn_flags) +{ + return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; +} + +static int insn_stack_access_frameno(int insn_flags) +{ + return insn_flags & INSN_F_FRAMENO_MASK; +} + static void mark_jmp_point(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].jmp_point = true; @@ -3232,28 +3247,51 @@ static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) } /* for any branch, call, exit record the history of jmps in the given state */ -static int push_jmp_history(struct bpf_verifier_env *env, - struct bpf_verifier_state *cur) +static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, + int insn_flags) { u32 cnt = cur->jmp_history_cnt; - struct bpf_idx_pair *p; + struct bpf_jmp_history_entry *p; size_t alloc_size; - if (!is_jmp_point(env, env->insn_idx)) + /* combine instruction flags if we already recorded this instruction */ + if (env->cur_hist_ent) { + /* atomic instructions push insn_flags twice, for READ and + * WRITE sides, but they should agree on stack slot + */ + WARN_ONCE((env->cur_hist_ent->flags & insn_flags) && + (env->cur_hist_ent->flags & insn_flags) != insn_flags, + "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n", + env->insn_idx, env->cur_hist_ent->flags, insn_flags); + env->cur_hist_ent->flags |= insn_flags; return 0; + } cnt++; alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); p = krealloc(cur->jmp_history, alloc_size, GFP_USER); if (!p) return -ENOMEM; - p[cnt - 1].idx = env->insn_idx; - p[cnt - 1].prev_idx = env->prev_insn_idx; cur->jmp_history = p; + + p = &cur->jmp_history[cnt - 1]; + p->idx = env->insn_idx; + p->prev_idx = env->prev_insn_idx; + p->flags = insn_flags; cur->jmp_history_cnt = cnt; + env->cur_hist_ent = p; + return 0; } +static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, + u32 hist_end, int insn_idx) +{ + if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) + return &st->jmp_history[hist_end - 1]; + return NULL; +} + /* Backtrack one insn at a time. If idx is not at the top of recorded * history then previous instruction came from straight line execution. * Return -ENOENT if we exhausted all instructions within given state. @@ -3415,9 +3453,14 @@ static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) return bt->reg_masks[bt->frame] & (1 << reg); } +static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) +{ + return bt->stack_masks[frame] & (1ull << slot); +} + static inline bool bt_is_slot_set(struct backtrack_state *bt, u32 slot) { - return bt->stack_masks[bt->frame] & (1ull << slot); + return bt_is_frame_slot_set(bt, bt->frame, slot); } /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ @@ -3471,7 +3514,7 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); * - *was* processed previously during backtracking. */ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, - struct backtrack_state *bt) + struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) { const struct bpf_insn_cbs cbs = { .cb_call = disasm_kfunc_name, @@ -3484,7 +3527,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, u8 mode = BPF_MODE(insn->code); u32 dreg = insn->dst_reg; u32 sreg = insn->src_reg; - u32 spi, i; + u32 spi, i, fr; if (insn->code == 0) return 0; @@ -3545,20 +3588,15 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * by 'precise' mark in corresponding register of this state. * No further tracking necessary. */ - if (insn->src_reg != BPF_REG_FP) + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; - /* dreg = *(u64 *)[fp - off] was a fill from the stack. * that [fp - off] slot contains scalar that needs to be * tracked with precision */ - spi = (-insn->off - 1) / BPF_REG_SIZE; - if (spi >= 64) { - verbose(env, "BUG spi %d\n", spi); - WARN_ONCE(1, "verifier backtracking bug"); - return -EFAULT; - } - bt_set_slot(bt, spi); + spi = insn_stack_access_spi(hist->flags); + fr = insn_stack_access_frameno(hist->flags); + bt_set_frame_slot(bt, fr, spi); } else if (class == BPF_STX || class == BPF_ST) { if (bt_is_reg_set(bt, dreg)) /* stx & st shouldn't be using _scalar_ dst_reg @@ -3567,17 +3605,13 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ return -ENOTSUPP; /* scalars can only be spilled into stack */ - if (insn->dst_reg != BPF_REG_FP) + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; - spi = (-insn->off - 1) / BPF_REG_SIZE; - if (spi >= 64) { - verbose(env, "BUG spi %d\n", spi); - WARN_ONCE(1, "verifier backtracking bug"); - return -EFAULT; - } - if (!bt_is_slot_set(bt, spi)) + spi = insn_stack_access_spi(hist->flags); + fr = insn_stack_access_frameno(hist->flags); + if (!bt_is_frame_slot_set(bt, fr, spi)) return 0; - bt_clear_slot(bt, spi); + bt_clear_frame_slot(bt, fr, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { @@ -3621,10 +3655,14 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, WARN_ONCE(1, "verifier backtracking bug"); return -EFAULT; } - /* we don't track register spills perfectly, - * so fallback to force-precise instead of failing */ - if (bt_stack_mask(bt) != 0) - return -ENOTSUPP; + /* we are now tracking register spills correctly, + * so any instance of leftover slots is a bug + */ + if (bt_stack_mask(bt) != 0) { + verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); + WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)"); + return -EFAULT; + } /* propagate r1-r5 to the caller */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (bt_is_reg_set(bt, i)) { @@ -3649,8 +3687,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, WARN_ONCE(1, "verifier backtracking bug"); return -EFAULT; } - if (bt_stack_mask(bt) != 0) - return -ENOTSUPP; + if (bt_stack_mask(bt) != 0) { + verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); + WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)"); + return -EFAULT; + } /* clear r1-r5 in callback subprog's mask */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); @@ -4087,6 +4128,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) for (;;) { DECLARE_BITMAP(mask, 64); u32 history = st->jmp_history_cnt; + struct bpf_jmp_history_entry *hist; if (env->log.level & BPF_LOG_LEVEL2) { verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", @@ -4150,7 +4192,8 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) err = 0; skip_first = false; } else { - err = backtrack_insn(env, i, subseq_idx, bt); + hist = get_jmp_hist_entry(st, history, i); + err = backtrack_insn(env, i, subseq_idx, hist, bt); } if (err == -ENOTSUPP) { mark_all_scalars_precise(env, env->cur_state); @@ -4203,22 +4246,10 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); for_each_set_bit(i, mask, 64) { if (i >= func->allocated_stack / BPF_REG_SIZE) { - /* the sequence of instructions: - * 2: (bf) r3 = r10 - * 3: (7b) *(u64 *)(r3 -8) = r0 - * 4: (79) r4 = *(u64 *)(r10 -8) - * doesn't contain jmps. It's backtracked - * as a single block. - * During backtracking insn 3 is not recognized as - * stack access, so at the end of backtracking - * stack slot fp-8 is still marked in stack_mask. - * However the parent state may not have accessed - * fp-8 and it's "unallocated" stack space. - * In such case fallback to conservative. - */ - mark_all_scalars_precise(env, env->cur_state); - bt_reset(bt); - return 0; + verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n", + i, func->allocated_stack / BPF_REG_SIZE); + WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)"); + return -EFAULT; } if (!is_spilled_scalar_reg(&func->stack[i])) { @@ -4391,7 +4422,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_reg_state *reg = NULL; - u32 dst_reg = insn->dst_reg; + int insn_flags = insn_stack_access_flags(state->frameno, spi); err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); if (err) @@ -4432,17 +4463,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && !register_is_null(reg) && env->bpf_capable) { - if (dst_reg != BPF_REG_FP) { - /* The backtracking logic can only recognize explicit - * stack slot address like [fp - 8]. Other spill of - * scalar via different register has to be conservative. - * Backtrack from here and mark all registers as precise - * that contributed into 'reg' being a constant. - */ - err = mark_chain_precision(env, value_regno); - if (err) - return err; - } save_register_state(state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (fls64(reg->umax_value) > BITS_PER_BYTE * size) @@ -4454,6 +4474,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, __mark_reg_known(&fake_reg, insn->imm); fake_reg.type = SCALAR_VALUE; save_register_state(state, spi, &fake_reg, size); + insn_flags = 0; /* not a register spill */ } else if (reg && is_spillable_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { @@ -4499,9 +4520,12 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = - type; + state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + insn_flags = 0; /* not a register spill */ } + + if (insn_flags) + return push_jmp_history(env, env->cur_state, insn_flags); return 0; } @@ -4694,6 +4718,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int insn_flags = insn_stack_access_flags(reg_state->frameno, spi); stype = reg_state->stack[spi].slot_type; reg = ®_state->stack[spi].spilled_ptr; @@ -4739,12 +4764,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } mark_reg_unknown(env, state->regs, dst_regno); + insn_flags = 0; /* not restoring original register state */ } state->regs[dst_regno].live |= REG_LIVE_WRITTEN; - return 0; - } - - if (dst_regno >= 0) { + } else if (dst_regno >= 0) { /* restore register state from stack */ copy_register_state(&state->regs[dst_regno], reg); /* mark reg as written since spilled pointer state likely @@ -4780,7 +4803,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); if (dst_regno >= 0) mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + insn_flags = 0; /* we are not restoring spilled register */ } + if (insn_flags) + return push_jmp_history(env, env->cur_state, insn_flags); return 0; } @@ -6940,7 +6966,6 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); if (err) return err; - return 0; } @@ -16910,7 +16935,8 @@ hit: * the precision needs to be propagated back in * the current state. */ - err = err ? : push_jmp_history(env, cur); + if (is_jmp_point(env, env->insn_idx)) + err = err ? : push_jmp_history(env, cur, 0); err = err ? : propagate_precision(env, &sl->state); if (err) return err; @@ -17135,6 +17161,9 @@ static int do_check(struct bpf_verifier_env *env) u8 class; int err; + /* reset current history entry on each new instruction */ + env->cur_hist_ent = NULL; + env->prev_insn_idx = prev_insn_idx; if (env->insn_idx >= insn_cnt) { verbose(env, "invalid insn idx %d insn_cnt %d\n", @@ -17174,7 +17203,7 @@ static int do_check(struct bpf_verifier_env *env) } if (is_jmp_point(env, env->insn_idx)) { - err = push_jmp_history(env, state); + err = push_jmp_history(env, state, 0); if (err) return err; } diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index 0dfe3f8b69ac..eba98fab2f54 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -589,11 +589,24 @@ static __u64 subprog_spill_reg_precise(void) SEC("?raw_tp") __success __log_level(2) -/* precision backtracking can't currently handle stack access not through r10, - * so we won't be able to mark stack slot fp-8 as precise, and so will - * fallback to forcing all as precise - */ -__msg("mark_precise: frame0: falling back to forcing all scalars precise") +__msg("10: (0f) r1 += r7") +__msg("mark_precise: frame0: last_idx 10 first_idx 7 subseq_idx -1") +__msg("mark_precise: frame0: regs=r7 stack= before 9: (bf) r1 = r8") +__msg("mark_precise: frame0: regs=r7 stack= before 8: (27) r7 *= 4") +__msg("mark_precise: frame0: regs=r7 stack= before 7: (79) r7 = *(u64 *)(r10 -8)") +__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=2 R6_w=1 R8_rw=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8_rw=P1") +__msg("mark_precise: frame0: last_idx 18 first_idx 0 subseq_idx 7") +__msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") +__msg("mark_precise: frame1: regs= stack= before 17: (0f) r0 += r2") +__msg("mark_precise: frame1: regs= stack= before 16: (79) r2 = *(u64 *)(r1 +0)") +__msg("mark_precise: frame1: regs= stack= before 15: (79) r0 = *(u64 *)(r10 -16)") +__msg("mark_precise: frame1: regs= stack= before 14: (7b) *(u64 *)(r10 -16) = r2") +__msg("mark_precise: frame1: regs= stack= before 13: (7b) *(u64 *)(r1 +0) = r2") +__msg("mark_precise: frame1: regs=r2 stack= before 6: (85) call pc+6") +__msg("mark_precise: frame0: regs=r2 stack= before 5: (bf) r2 = r6") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") __naked int subprog_spill_into_parent_stack_slot_precise(void) { asm volatile ( diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 0d84dd1f38b6..8a2ff81d8350 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -140,10 +140,11 @@ .result = REJECT, }, { - "precise: ST insn causing spi > allocated_stack", + "precise: ST zero to stack insn is supported", .insns = { BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 123, 0), + /* not a register spill, so we stop precision propagation for R4 here */ BPF_ST_MEM(BPF_DW, BPF_REG_3, -8, 0), BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8), BPF_MOV64_IMM(BPF_REG_0, -1), @@ -157,11 +158,11 @@ mark_precise: frame0: last_idx 4 first_idx 2\ mark_precise: frame0: regs=r4 stack= before 4\ mark_precise: frame0: regs=r4 stack= before 3\ - mark_precise: frame0: regs= stack=-8 before 2\ - mark_precise: frame0: falling back to forcing all scalars precise\ - force_precise: frame0: forcing r0 to be precise\ mark_precise: frame0: last_idx 5 first_idx 5\ - mark_precise: frame0: parent state regs= stack=:", + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 4 first_idx 2\ + mark_precise: frame0: regs=r0 stack= before 4\ + 5: R0=-1 R4=0", .result = VERBOSE_ACCEPT, .retval = -1, }, @@ -169,6 +170,8 @@ "precise: STX insn causing spi > allocated_stack", .insns = { BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32), + /* make later reg spill more interesting by having somewhat known scalar */ + BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xff), BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 123, 0), BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, -8), @@ -179,18 +182,21 @@ }, .prog_type = BPF_PROG_TYPE_XDP, .flags = BPF_F_TEST_STATE_FREQ, - .errstr = "mark_precise: frame0: last_idx 6 first_idx 6\ + .errstr = "mark_precise: frame0: last_idx 7 first_idx 7\ mark_precise: frame0: parent state regs=r4 stack=:\ - mark_precise: frame0: last_idx 5 first_idx 3\ - mark_precise: frame0: regs=r4 stack= before 5\ - mark_precise: frame0: regs=r4 stack= before 4\ - mark_precise: frame0: regs= stack=-8 before 3\ - mark_precise: frame0: falling back to forcing all scalars precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - force_precise: frame0: forcing r0 to be precise\ - mark_precise: frame0: last_idx 6 first_idx 6\ + mark_precise: frame0: last_idx 6 first_idx 4\ + mark_precise: frame0: regs=r4 stack= before 6: (b7) r0 = -1\ + mark_precise: frame0: regs=r4 stack= before 5: (79) r4 = *(u64 *)(r10 -8)\ + mark_precise: frame0: regs= stack=-8 before 4: (7b) *(u64 *)(r3 -8) = r0\ + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 3 first_idx 3\ + mark_precise: frame0: regs=r0 stack= before 3: (55) if r3 != 0x7b goto pc+0\ + mark_precise: frame0: regs=r0 stack= before 2: (bf) r3 = r10\ + mark_precise: frame0: regs=r0 stack= before 1: (57) r0 &= 255\ + mark_precise: frame0: parent state regs=r0 stack=:\ + mark_precise: frame0: last_idx 0 first_idx 0\ + mark_precise: frame0: regs=r0 stack= before 0: (85) call bpf_get_prandom_u32#7\ + mark_precise: frame0: last_idx 7 first_idx 7\ mark_precise: frame0: parent state regs= stack=:", .result = VERBOSE_ACCEPT, .retval = -1, -- cgit v1.2.3 From 43a71cd66b9c0a4af3d15d8644359fde35bdbed0 Mon Sep 17 00:00:00 2001 From: Coco Li Date: Mon, 4 Dec 2023 20:12:30 +0000 Subject: net-device: reorganize net_device fast path variables Reorganize fast path variables on tx-txrx-rx order Fastpath variables end after npinfo. Below data generated with pahole on x86 architecture. Fast path variables span cache lines before change: 12 Fast path variables span cache lines after change: 4 Suggested-by: Eric Dumazet Signed-off-by: Coco Li Reviewed-by: Eric Dumazet Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20231204201232.520025-2-lixiaoyan@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 117 +++++++++++++++++++++++++--------------------- net/core/dev.c | 56 ++++++++++++++++++++++ 2 files changed, 120 insertions(+), 53 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5551177e024e..cb96aad6a6ee 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2097,6 +2097,70 @@ enum netdev_stat_type { */ struct net_device { + /* Cacheline organization can be found documented in + * Documentation/networking/net_cachelines/net_device.rst. + * Please update the document when adding new fields. + */ + + /* TX read-mostly hotpath */ + __cacheline_group_begin(net_device_read_tx); + unsigned long long priv_flags; + const struct net_device_ops *netdev_ops; + const struct header_ops *header_ops; + struct netdev_queue *_tx; + unsigned int real_num_tx_queues; + unsigned int gso_max_size; + unsigned int gso_ipv4_max_size; + u16 gso_max_segs; + s16 num_tc; + /* Note : dev->mtu is often read without holding a lock. + * Writers usually hold RTNL. + * It is recommended to use READ_ONCE() to annotate the reads, + * and to use WRITE_ONCE() to annotate the writes. + */ + unsigned int mtu; + unsigned short needed_headroom; + struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE]; +#ifdef CONFIG_XPS + struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX]; +#endif +#ifdef CONFIG_NETFILTER_EGRESS + struct nf_hook_entries __rcu *nf_hooks_egress; +#endif +#ifdef CONFIG_NET_XGRESS + struct bpf_mprog_entry __rcu *tcx_egress; +#endif + __cacheline_group_end(net_device_read_tx); + + /* TXRX read-mostly hotpath */ + __cacheline_group_begin(net_device_read_txrx); + unsigned int flags; + unsigned short hard_header_len; + netdev_features_t features; + struct inet6_dev __rcu *ip6_ptr; + __cacheline_group_end(net_device_read_txrx); + + /* RX read-mostly hotpath */ + __cacheline_group_begin(net_device_read_rx); + struct list_head ptype_specific; + int ifindex; + unsigned int real_num_rx_queues; + struct netdev_rx_queue *_rx; + unsigned long gro_flush_timeout; + int napi_defer_hard_irqs; + unsigned int gro_max_size; + unsigned int gro_ipv4_max_size; + rx_handler_func_t __rcu *rx_handler; + void __rcu *rx_handler_data; + possible_net_t nd_net; +#ifdef CONFIG_NETPOLL + struct netpoll_info __rcu *npinfo; +#endif +#ifdef CONFIG_NET_XGRESS + struct bpf_mprog_entry __rcu *tcx_ingress; +#endif + __cacheline_group_end(net_device_read_rx); + char name[IFNAMSIZ]; struct netdev_name_node *name_node; struct dev_ifalias __rcu *ifalias; @@ -2121,7 +2185,6 @@ struct net_device { struct list_head unreg_list; struct list_head close_list; struct list_head ptype_all; - struct list_head ptype_specific; struct { struct list_head upper; @@ -2129,26 +2192,13 @@ struct net_device { } adj_list; /* Read-mostly cache-line for fast-path access */ - unsigned int flags; xdp_features_t xdp_features; - unsigned long long priv_flags; - const struct net_device_ops *netdev_ops; const struct xdp_metadata_ops *xdp_metadata_ops; const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops; - int ifindex; unsigned short gflags; - unsigned short hard_header_len; - /* Note : dev->mtu is often read without holding a lock. - * Writers usually hold RTNL. - * It is recommended to use READ_ONCE() to annotate the reads, - * and to use WRITE_ONCE() to annotate the writes. - */ - unsigned int mtu; - unsigned short needed_headroom; unsigned short needed_tailroom; - netdev_features_t features; netdev_features_t hw_features; netdev_features_t wanted_features; netdev_features_t vlan_features; @@ -2192,8 +2242,6 @@ struct net_device { const struct tlsdev_ops *tlsdev_ops; #endif - const struct header_ops *header_ops; - unsigned char operstate; unsigned char link_mode; @@ -2234,9 +2282,7 @@ struct net_device { /* Protocol-specific pointers */ - struct in_device __rcu *ip_ptr; - struct inet6_dev __rcu *ip6_ptr; #if IS_ENABLED(CONFIG_VLAN_8021Q) struct vlan_info __rcu *vlan_info; #endif @@ -2271,26 +2317,14 @@ struct net_device { /* Interface address info used in eth_type_trans() */ const unsigned char *dev_addr; - struct netdev_rx_queue *_rx; unsigned int num_rx_queues; - unsigned int real_num_rx_queues; - struct bpf_prog __rcu *xdp_prog; - unsigned long gro_flush_timeout; - int napi_defer_hard_irqs; #define GRO_LEGACY_MAX_SIZE 65536u /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE), * and shinfo->gso_segs is a 16bit field. */ #define GRO_MAX_SIZE (8 * 65535u) - unsigned int gro_max_size; - unsigned int gro_ipv4_max_size; unsigned int xdp_zc_max_segs; - rx_handler_func_t __rcu *rx_handler; - void __rcu *rx_handler_data; -#ifdef CONFIG_NET_XGRESS - struct bpf_mprog_entry __rcu *tcx_ingress; -#endif struct netdev_queue __rcu *ingress_queue; #ifdef CONFIG_NETFILTER_INGRESS struct nf_hook_entries __rcu *nf_hooks_ingress; @@ -2305,25 +2339,13 @@ struct net_device { /* * Cache lines mostly used on transmit path */ - struct netdev_queue *_tx ____cacheline_aligned_in_smp; unsigned int num_tx_queues; - unsigned int real_num_tx_queues; struct Qdisc __rcu *qdisc; unsigned int tx_queue_len; spinlock_t tx_global_lock; struct xdp_dev_bulk_queue __percpu *xdp_bulkq; -#ifdef CONFIG_XPS - struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX]; -#endif -#ifdef CONFIG_NET_XGRESS - struct bpf_mprog_entry __rcu *tcx_egress; -#endif -#ifdef CONFIG_NETFILTER_EGRESS - struct nf_hook_entries __rcu *nf_hooks_egress; -#endif - #ifdef CONFIG_NET_SCHED DECLARE_HASHTABLE (qdisc_hash, 4); #endif @@ -2362,12 +2384,6 @@ struct net_device { bool needs_free_netdev; void (*priv_destructor)(struct net_device *dev); -#ifdef CONFIG_NETPOLL - struct netpoll_info __rcu *npinfo; -#endif - - possible_net_t nd_net; - /* mid-layer private */ void *ml_priv; enum netdev_ml_priv_type ml_priv_type; @@ -2402,20 +2418,15 @@ struct net_device { */ #define GSO_MAX_SIZE (8 * GSO_MAX_SEGS) - unsigned int gso_max_size; #define TSO_LEGACY_MAX_SIZE 65536 #define TSO_MAX_SIZE UINT_MAX unsigned int tso_max_size; - u16 gso_max_segs; #define TSO_MAX_SEGS U16_MAX u16 tso_max_segs; - unsigned int gso_ipv4_max_size; #ifdef CONFIG_DCB const struct dcbnl_rtnl_ops *dcbnl_ops; #endif - s16 num_tc; - struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE]; u8 prio_tc_map[TC_BITMASK + 1]; #if IS_ENABLED(CONFIG_FCOE) diff --git a/net/core/dev.c b/net/core/dev.c index 1d720fc59161..c5679dfbaa70 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -11609,6 +11609,60 @@ static struct pernet_operations __net_initdata default_device_ops = { .exit_batch = default_device_exit_batch, }; +static void __init net_dev_struct_check(void) +{ + /* TX read-mostly hotpath */ + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq); +#ifdef CONFIG_XPS + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps); +#endif +#ifdef CONFIG_NETFILTER_EGRESS + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress); +#endif +#ifdef CONFIG_NET_XGRESS + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress); +#endif + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 152); + + /* TXRX read-mostly hotpath */ + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr); + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 30); + + /* RX read-mostly hotpath */ + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_flush_timeout); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, napi_defer_hard_irqs); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net); +#ifdef CONFIG_NETPOLL + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo); +#endif +#ifdef CONFIG_NET_XGRESS + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress); +#endif + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 96); +} + /* * Initialize the DEV module. At boot time this walks the device list and * unhooks any devices that fail to initialise (normally hardware not @@ -11626,6 +11680,8 @@ static int __init net_dev_init(void) BUG_ON(!dev_boot_phase); + net_dev_struct_check(); + if (dev_proc_init()) goto out; -- cgit v1.2.3 From d5fed5addb2b6bc13035de4338b7ea2052a2e006 Mon Sep 17 00:00:00 2001 From: Coco Li Date: Mon, 4 Dec 2023 20:12:31 +0000 Subject: tcp: reorganize tcp_sock fast path variables The variables are organized according in the following way: - TX read-mostly hotpath cache lines - TXRX read-mostly hotpath cache lines - RX read-mostly hotpath cache lines - TX read-write hotpath cache line - TXRX read-write hotpath cache line - RX read-write hotpath cache line Fastpath cachelines end after rcvq_space. Cache line boundaries are enforced only between read-mostly and read-write. That is, if read-mostly tx cachelines bleed into read-mostly txrx cachelines, we do not care. We care about the boundaries between read and write cachelines because we want to prevent false sharing. Fast path variables span cache lines before change: 12 Fast path variables span cache lines after change: 8 Suggested-by: Eric Dumazet Reviewed-by: Wei Wang Signed-off-by: Coco Li Reviewed-by: Eric Dumazet Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20231204201232.520025-3-lixiaoyan@google.com Signed-off-by: Jakub Kicinski --- include/linux/tcp.h | 248 ++++++++++++++++++++++++++++------------------------ net/ipv4/tcp.c | 93 ++++++++++++++++++++ 2 files changed, 227 insertions(+), 114 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 68f3d315d2e1..f55ec155f5b7 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -194,23 +194,121 @@ static inline bool tcp_rsk_used_ao(const struct request_sock *req) #define TCP_RMEM_TO_WIN_SCALE 8 struct tcp_sock { + /* Cacheline organization can be found documented in + * Documentation/networking/net_cachelines/tcp_sock.rst. + * Please update the document when adding new fields. + */ + /* inet_connection_sock has to be the first member of tcp_sock */ struct inet_connection_sock inet_conn; - u16 tcp_header_len; /* Bytes of tcp header to send */ + + /* TX read-mostly hotpath cache lines */ + __cacheline_group_begin(tcp_sock_read_tx); + /* timestamp of last sent data packet (for restart window) */ + u32 max_window; /* Maximal window ever seen from peer */ + u32 rcv_ssthresh; /* Current window clamp */ + u32 reordering; /* Packet reordering metric. */ + u32 notsent_lowat; /* TCP_NOTSENT_LOWAT */ u16 gso_segs; /* Max number of segs per GSO packet */ + /* from STCP, retrans queue hinting */ + struct sk_buff *lost_skb_hint; + struct sk_buff *retransmit_skb_hint; + __cacheline_group_end(tcp_sock_read_tx); + + /* TXRX read-mostly hotpath cache lines */ + __cacheline_group_begin(tcp_sock_read_txrx); + u32 tsoffset; /* timestamp offset */ + u32 snd_wnd; /* The window we expect to receive */ + u32 mss_cache; /* Cached effective mss, not including SACKS */ + u32 snd_cwnd; /* Sending congestion window */ + u32 prr_out; /* Total number of pkts sent during Recovery. */ + u32 lost_out; /* Lost packets */ + u32 sacked_out; /* SACK'd packets */ + u16 tcp_header_len; /* Bytes of tcp header to send */ + u8 chrono_type : 2, /* current chronograph type */ + repair : 1, + is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ + is_cwnd_limited:1;/* forward progress limited by snd_cwnd? */ + __cacheline_group_end(tcp_sock_read_txrx); + + /* RX read-mostly hotpath cache lines */ + __cacheline_group_begin(tcp_sock_read_rx); + u32 copied_seq; /* Head of yet unread data */ + u32 rcv_tstamp; /* timestamp of last received ACK (for keepalives) */ + u32 snd_wl1; /* Sequence for window update */ + u32 tlp_high_seq; /* snd_nxt at the time of TLP */ + u32 rttvar_us; /* smoothed mdev_max */ + u32 retrans_out; /* Retransmitted packets out */ + u16 advmss; /* Advertised MSS */ + u16 urg_data; /* Saved octet of OOB data and control flags */ + u32 lost; /* Total data packets lost incl. rexmits */ + struct minmax rtt_min; + /* OOO segments go in this rbtree. Socket lock must be held. */ + struct rb_root out_of_order_queue; + u32 snd_ssthresh; /* Slow start size threshold */ + __cacheline_group_end(tcp_sock_read_rx); + /* TX read-write hotpath cache lines */ + __cacheline_group_begin(tcp_sock_write_tx) ____cacheline_aligned; + u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut + * The total number of segments sent. + */ + u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut + * total number of data segments sent. + */ + u64 bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut + * total number of data bytes sent. + */ + u32 snd_sml; /* Last byte of the most recently transmitted small packet */ + u32 chrono_start; /* Start time in jiffies of a TCP chrono */ + u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ + u32 write_seq; /* Tail(+1) of data held in tcp send buffer */ + u32 pushed_seq; /* Last pushed seq, required to talk to windows */ + u32 lsndtime; + u32 mdev_us; /* medium deviation */ + u64 tcp_wstamp_ns; /* departure time for next sent data packet */ + u64 tcp_clock_cache; /* cache last tcp_clock_ns() (see tcp_mstamp_refresh()) */ + u64 tcp_mstamp; /* most recent packet received/sent */ + u32 rtt_seq; /* sequence number to update rttvar */ + struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */ + struct sk_buff *highest_sack; /* skb just after the highest + * skb with SACKed bit set + * (validity guaranteed only if + * sacked_out > 0) + */ + u8 ecn_flags; /* ECN status bits. */ + __cacheline_group_end(tcp_sock_write_tx); + + /* TXRX read-write hotpath cache lines */ + __cacheline_group_begin(tcp_sock_write_txrx); /* * Header prediction flags * 0x5?10 << 16 + snd_wnd in net byte order */ __be32 pred_flags; - + u32 rcv_nxt; /* What we want to receive next */ + u32 snd_nxt; /* Next sequence we send */ + u32 snd_una; /* First byte we want an ack for */ + u32 window_clamp; /* Maximal window to advertise */ + u32 srtt_us; /* smoothed round trip time << 3 in usecs */ + u32 packets_out; /* Packets which are "in flight" */ + u32 snd_up; /* Urgent pointer */ + u32 delivered; /* Total data packets delivered incl. rexmits */ + u32 delivered_ce; /* Like the above but only ECE marked packets */ + u32 app_limited; /* limited until "delivered" reaches this val */ + u32 rcv_wnd; /* Current receiver window */ /* - * RFC793 variables by their proper names. This means you can - * read the code and the spec side by side (and laugh ...) - * See RFC793 and RFC1122. The RFC writes these in capitals. + * Options received (usually on last packet, some only on SYN packets). */ - u64 bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived + struct tcp_options_received rx_opt; + u8 nonagle : 4,/* Disable Nagle algorithm? */ + rate_app_limited:1; /* rate_{delivered,interval_us} limited? */ + __cacheline_group_end(tcp_sock_write_txrx); + + /* RX read-write hotpath cache lines */ + __cacheline_group_begin(tcp_sock_write_rx); + u64 bytes_received; + /* RFC4898 tcpEStatsAppHCThruOctetsReceived * sum(delta(rcv_nxt)), or how many bytes * were acked. */ @@ -220,45 +318,44 @@ struct tcp_sock { u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn * total number of data segments in. */ - u32 rcv_nxt; /* What we want to receive next */ - u32 copied_seq; /* Head of yet unread data */ u32 rcv_wup; /* rcv_nxt on last window update sent */ - u32 snd_nxt; /* Next sequence we send */ - u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut - * The total number of segments sent. - */ - u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut - * total number of data segments sent. - */ - u64 bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut - * total number of data bytes sent. - */ + u32 max_packets_out; /* max packets_out in last window */ + u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */ + u32 rate_delivered; /* saved rate sample: packets delivered */ + u32 rate_interval_us; /* saved rate sample: time elapsed */ + u32 rcv_rtt_last_tsecr; + u64 first_tx_mstamp; /* start of window send phase */ + u64 delivered_mstamp; /* time we reached "delivered" */ u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked * sum(delta(snd_una)), or how many bytes * were acked. */ + struct { + u32 rtt_us; + u32 seq; + u64 time; + } rcv_rtt_est; +/* Receiver queue space */ + struct { + u32 space; + u32 seq; + u64 time; + } rcvq_space; + __cacheline_group_end(tcp_sock_write_rx); + /* End of Hot Path */ + +/* + * RFC793 variables by their proper names. This means you can + * read the code and the spec side by side (and laugh ...) + * See RFC793 and RFC1122. The RFC writes these in capitals. + */ u32 dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups * total number of DSACK blocks received */ - u32 snd_una; /* First byte we want an ack for */ - u32 snd_sml; /* Last byte of the most recently transmitted small packet */ - u32 rcv_tstamp; /* timestamp of last received ACK (for keepalives) */ - u32 lsndtime; /* timestamp of last sent data packet (for restart window) */ u32 last_oow_ack_time; /* timestamp of last out-of-window ACK */ u32 compressed_ack_rcv_nxt; - - u32 tsoffset; /* timestamp offset */ - struct list_head tsq_node; /* anchor in tsq_tasklet.head list */ - struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */ - - u32 snd_wl1; /* Sequence for window update */ - u32 snd_wnd; /* The window we expect to receive */ - u32 max_window; /* Maximal window ever seen from peer */ - u32 mss_cache; /* Cached effective mss, not including SACKS */ - u32 window_clamp; /* Maximal window to advertise */ - u32 rcv_ssthresh; /* Current window clamp */ u8 scaling_ratio; /* see tcp_win_from_space() */ /* Information of the most recently (s)acked skb */ struct tcp_rack { @@ -272,24 +369,16 @@ struct tcp_sock { dsack_seen:1, /* Whether DSACK seen after last adj */ advanced:1; /* mstamp advanced since last lost marking */ } rack; - u16 advmss; /* Advertised MSS */ u8 compressed_ack; u8 dup_ack_counter:2, tlp_retrans:1, /* TLP is a retransmission */ tcp_usec_ts:1, /* TSval values in usec */ unused:4; - u32 chrono_start; /* Start time in jiffies of a TCP chrono */ - u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ - u8 chrono_type:2, /* current chronograph type */ - rate_app_limited:1, /* rate_{delivered,interval_us} limited? */ + u8 thin_lto : 1,/* Use linear timeouts for thin streams */ + recvmsg_inq : 1,/* Indicate # of bytes in queue upon recvmsg */ fastopen_connect:1, /* FASTOPEN_CONNECT sockopt */ fastopen_no_cookie:1, /* Allow send/recv SYN+data without a cookie */ - is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ - fastopen_client_fail:2; /* reason why fastopen failed */ - u8 nonagle : 4,/* Disable Nagle algorithm? */ - thin_lto : 1,/* Use linear timeouts for thin streams */ - recvmsg_inq : 1,/* Indicate # of bytes in queue upon recvmsg */ - repair : 1, + fastopen_client_fail:2, /* reason why fastopen failed */ frto : 1;/* F-RTO (RFC5682) activated in CA_Loss */ u8 repair_queue; u8 save_syn:2, /* Save headers of SYN packet */ @@ -297,45 +386,19 @@ struct tcp_sock { syn_fastopen:1, /* SYN includes Fast Open option */ syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */ syn_fastopen_ch:1, /* Active TFO re-enabling probe */ - syn_data_acked:1,/* data in SYN is acked by SYN-ACK */ - is_cwnd_limited:1;/* forward progress limited by snd_cwnd? */ - u32 tlp_high_seq; /* snd_nxt at the time of TLP */ + syn_data_acked:1;/* data in SYN is acked by SYN-ACK */ u32 tcp_tx_delay; /* delay (in usec) added to TX packets */ - u64 tcp_wstamp_ns; /* departure time for next sent data packet */ - u64 tcp_clock_cache; /* cache last tcp_clock_ns() (see tcp_mstamp_refresh()) */ /* RTT measurement */ - u64 tcp_mstamp; /* most recent packet received/sent */ - u32 srtt_us; /* smoothed round trip time << 3 in usecs */ - u32 mdev_us; /* medium deviation */ u32 mdev_max_us; /* maximal mdev for the last rtt period */ - u32 rttvar_us; /* smoothed mdev_max */ - u32 rtt_seq; /* sequence number to update rttvar */ - struct minmax rtt_min; - u32 packets_out; /* Packets which are "in flight" */ - u32 retrans_out; /* Retransmitted packets out */ - u32 max_packets_out; /* max packets_out in last window */ - u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */ - - u16 urg_data; /* Saved octet of OOB data and control flags */ - u8 ecn_flags; /* ECN status bits. */ u8 keepalive_probes; /* num of allowed keep alive probes */ - u32 reordering; /* Packet reordering metric. */ u32 reord_seen; /* number of data packet reordering events */ - u32 snd_up; /* Urgent pointer */ - -/* - * Options received (usually on last packet, some only on SYN packets). - */ - struct tcp_options_received rx_opt; /* * Slow start and congestion control (see also Nagle, and Karn & Partridge) */ - u32 snd_ssthresh; /* Slow start size threshold */ - u32 snd_cwnd; /* Sending congestion window */ u32 snd_cwnd_cnt; /* Linear increase counter */ u32 snd_cwnd_clamp; /* Do not allow snd_cwnd to grow above this */ u32 snd_cwnd_used; @@ -343,32 +406,10 @@ struct tcp_sock { u32 prior_cwnd; /* cwnd right before starting loss recovery */ u32 prr_delivered; /* Number of newly delivered packets to * receiver in Recovery. */ - u32 prr_out; /* Total number of pkts sent during Recovery. */ - u32 delivered; /* Total data packets delivered incl. rexmits */ - u32 delivered_ce; /* Like the above but only ECE marked packets */ - u32 lost; /* Total data packets lost incl. rexmits */ - u32 app_limited; /* limited until "delivered" reaches this val */ - u64 first_tx_mstamp; /* start of window send phase */ - u64 delivered_mstamp; /* time we reached "delivered" */ - u32 rate_delivered; /* saved rate sample: packets delivered */ - u32 rate_interval_us; /* saved rate sample: time elapsed */ - - u32 rcv_wnd; /* Current receiver window */ - u32 write_seq; /* Tail(+1) of data held in tcp send buffer */ - u32 notsent_lowat; /* TCP_NOTSENT_LOWAT */ - u32 pushed_seq; /* Last pushed seq, required to talk to windows */ - u32 lost_out; /* Lost packets */ - u32 sacked_out; /* SACK'd packets */ struct hrtimer pacing_timer; struct hrtimer compressed_ack_timer; - /* from STCP, retrans queue hinting */ - struct sk_buff* lost_skb_hint; - struct sk_buff *retransmit_skb_hint; - - /* OOO segments go in this rbtree. Socket lock must be held. */ - struct rb_root out_of_order_queue; struct sk_buff *ooo_last_skb; /* cache rb_last(out_of_order_queue) */ /* SACKs data, these 2 need to be together (see tcp_options_write) */ @@ -377,12 +418,6 @@ struct tcp_sock { struct tcp_sack_block recv_sack_cache[4]; - struct sk_buff *highest_sack; /* skb just after the highest - * skb with SACKed bit set - * (validity guaranteed only if - * sacked_out > 0) - */ - int lost_cnt_hint; u32 prior_ssthresh; /* ssthresh saved at recovery start */ @@ -433,21 +468,6 @@ struct tcp_sock { u32 rcv_ooopack; /* Received out-of-order packets, for tcpinfo */ -/* Receiver side RTT estimation */ - u32 rcv_rtt_last_tsecr; - struct { - u32 rtt_us; - u32 seq; - u64 time; - } rcv_rtt_est; - -/* Receiver queue space */ - struct { - u32 space; - u32 seq; - u64 time; - } rcvq_space; - /* TCP-specific MTU probe information. */ struct { u32 probe_seq_start; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index a100df07d34a..70a1bafbefba 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4564,6 +4564,97 @@ static void __init tcp_init_mem(void) sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */ } +static void __init tcp_struct_check(void) +{ + /* TX read-mostly hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, lost_skb_hint); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_tx, 40); + + /* TXRX read-mostly hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_txrx, 31); + + /* RX read-mostly hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rcv_tstamp); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 69); + + /* TX read-write hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_clock_cache); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_mstamp); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 113); + + /* TXRX read-write hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 76); + + /* RX read-write hotpath cache lines */ + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est); + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space); + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_rx, 99); +} + void __init tcp_init(void) { int max_rshare, max_wshare, cnt; @@ -4574,6 +4665,8 @@ void __init tcp_init(void) BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof_field(struct sk_buff, cb)); + tcp_struct_check(); + percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL); timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE); -- cgit v1.2.3 From facd15dfd69122042502d99ab8c9f888b48ee994 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 4 Dec 2023 21:47:07 +0100 Subject: net: core: synchronize link-watch when carrier is queried There are multiple ways to query for the carrier state: through rtnetlink, sysfs, and (possibly) ethtool. Synchronize linkwatch work before these operations so that we don't have a situation where userspace queries the carrier state between the driver's carrier off->on transition and linkwatch running and expects it to work, when really (at least) TX cannot work until linkwatch has run. I previously posted a longer explanation of how this applies to wireless [1] but with this wireless can simply query the state before sending data, to ensure the kernel is ready for it. [1] https://lore.kernel.org/all/346b21d87c69f817ea3c37caceb34f1f56255884.camel@sipsolutions.net/ Signed-off-by: Johannes Berg Reviewed-by: Jiri Pirko Link: https://lore.kernel.org/r/20231204214706.303c62768415.I1caedccae72ee5a45c9085c5eb49c145ce1c0dd5@changeid Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 9 +++++++++ net/core/dev.c | 2 +- net/core/dev.h | 1 - net/core/link_watch.c | 2 +- net/core/net-sysfs.c | 8 +++++++- net/core/rtnetlink.c | 8 ++++++++ net/ethtool/ioctl.c | 3 +++ 7 files changed, 29 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index cb96aad6a6ee..1b935ee341b4 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4229,6 +4229,15 @@ static inline void netdev_ref_replace(struct net_device *odev, */ void linkwatch_fire_event(struct net_device *dev); +/** + * linkwatch_sync_dev - sync linkwatch for the given device + * @dev: network device to sync linkwatch for + * + * Sync linkwatch for the given device, removing it from the + * pending work list (if queued). + */ +void linkwatch_sync_dev(struct net_device *dev); + /** * netif_carrier_ok - test if carrier present * @dev: network device diff --git a/net/core/dev.c b/net/core/dev.c index c5679dfbaa70..0432b04cf9b0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10548,7 +10548,7 @@ void netdev_run_todo(void) write_lock(&dev_base_lock); dev->reg_state = NETREG_UNREGISTERED; write_unlock(&dev_base_lock); - linkwatch_forget_dev(dev); + linkwatch_sync_dev(dev); } while (!list_empty(&list)) { diff --git a/net/core/dev.h b/net/core/dev.h index 7795b8ad841d..cf93e188785b 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -30,7 +30,6 @@ int __init dev_proc_init(void); #endif void linkwatch_init_dev(struct net_device *dev); -void linkwatch_forget_dev(struct net_device *dev); void linkwatch_run_queue(void); void dev_addr_flush(struct net_device *dev); diff --git a/net/core/link_watch.c b/net/core/link_watch.c index c469d1c4db5d..a19f21403339 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -245,7 +245,7 @@ static void __linkwatch_run_queue(int urgent_only) spin_unlock_irq(&lweventlist_lock); } -void linkwatch_forget_dev(struct net_device *dev) +void linkwatch_sync_dev(struct net_device *dev) { unsigned long flags; int clean = 0; diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index fccaa5bac0ed..d9b33e923b18 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -194,8 +194,14 @@ static ssize_t carrier_show(struct device *dev, { struct net_device *netdev = to_net_dev(dev); - if (netif_running(netdev)) + if (netif_running(netdev)) { + /* Synchronize carrier state with link watch, + * see also rtnl_getlink(). + */ + linkwatch_sync_dev(netdev); + return sysfs_emit(buf, fmt_dec, !!netif_carrier_ok(netdev)); + } return -EINVAL; } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 592164c2a540..5e0ab4c08f72 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -3853,6 +3853,14 @@ static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh, if (nskb == NULL) goto out; + /* Synchronize the carrier state so we don't report a state + * that we're not actually going to honour immediately; if + * the driver just did a carrier off->on transition, we can + * only TX if link watch work has run, but without this we'd + * already report carrier on, even if it doesn't work yet. + */ + linkwatch_sync_dev(dev); + err = rtnl_fill_ifinfo(nskb, dev, net, RTM_NEWLINK, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 0, 0, ext_filter_mask, diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 0b0ce4f81c01..a977f8903467 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -58,6 +58,9 @@ static struct devlink *netdev_to_devlink_get(struct net_device *dev) u32 ethtool_op_get_link(struct net_device *dev) { + /* Synchronize carrier state with link watch, see also rtnl_getlink() */ + linkwatch_sync_dev(dev); + return netif_carrier_ok(dev) ? 1 : 0; } EXPORT_SYMBOL(ethtool_op_get_link); -- cgit v1.2.3 From 40bba140c60fbb3ee8df6203c82fbd3de9f19d95 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:14 -0800 Subject: bpf: add BPF token delegation mount options to BPF FS Add few new mount options to BPF FS that allow to specify that a given BPF FS instance allows creation of BPF token (added in the next patch), and what sort of operations are allowed under BPF token. As such, we get 4 new mount options, each is a bit mask - `delegate_cmds` allow to specify which bpf() syscall commands are allowed with BPF token derived from this BPF FS instance; - if BPF_MAP_CREATE command is allowed, `delegate_maps` specifies a set of allowable BPF map types that could be created with BPF token; - if BPF_PROG_LOAD command is allowed, `delegate_progs` specifies a set of allowable BPF program types that could be loaded with BPF token; - if BPF_PROG_LOAD command is allowed, `delegate_attachs` specifies a set of allowable BPF program attach types that could be loaded with BPF token; delegate_progs and delegate_attachs are meant to be used together, as full BPF program type is, in general, determined through both program type and program attach type. Currently, these mount options accept the following forms of values: - a special value "any", that enables all possible values of a given bit set; - numeric value (decimal or hexadecimal, determined by kernel automatically) that specifies a bit mask value directly; - all the values for a given mount option are combined, if specified multiple times. E.g., `mount -t bpf nodev /path/to/mount -o delegate_maps=0x1 -o delegate_maps=0x2` will result in a combined 0x3 mask. Ideally, more convenient (for humans) symbolic form derived from corresponding UAPI enums would be accepted (e.g., `-o delegate_progs=kprobe|tracepoint`) and I intend to implement this, but it requires a bunch of UAPI header churn, so I postponed it until this feature lands upstream or at least there is a definite consensus that this feature is acceptable and is going to make it, just to minimize amount of wasted effort and not increase amount of non-essential code to be reviewed. Attentive reader will notice that BPF FS is now marked as FS_USERNS_MOUNT, which theoretically makes it mountable inside non-init user namespace as long as the process has sufficient *namespaced* capabilities within that user namespace. But in reality we still restrict BPF FS to be mountable only by processes with CAP_SYS_ADMIN *in init userns* (extra check in bpf_fill_super()). FS_USERNS_MOUNT is added to allow creating BPF FS context object (i.e., fsopen("bpf")) from inside unprivileged process inside non-init userns, to capture that userns as the owning userns. It will still be required to pass this context object back to privileged process to instantiate and mount it. This manipulation is important, because capturing non-init userns as the owning userns of BPF FS instance (super block) allows to use that userns to constraint BPF token to that userns later on (see next patch). So creating BPF FS with delegation inside unprivileged userns will restrict derived BPF token objects to only "work" inside that intended userns, making it scoped to a intended "container". Also, setting these delegation options requires capable(CAP_SYS_ADMIN), so unprivileged process cannot set this up without involvement of a privileged process. There is a set of selftests at the end of the patch set that simulates this sequence of steps and validates that everything works as intended. But careful review is requested to make sure there are no missed gaps in the implementation and testing. This somewhat subtle set of aspects is the result of previous discussions ([0]) about various user namespace implications and interactions with BPF token functionality and is necessary to contain BPF token inside intended user namespace. [0] https://lore.kernel.org/bpf/20230704-hochverdient-lehne-eeb9eeef785e@brauner/ Acked-by: Christian Brauner Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-3-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 10 ++++++ kernel/bpf/inode.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 10e5e4d8a00f..d3c9acc593ea 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1581,6 +1581,16 @@ struct bpf_link_primer { u32 id; }; +struct bpf_mount_opts { + umode_t mode; + + /* BPF token-related delegation options */ + u64 delegate_cmds; + u64 delegate_maps; + u64 delegate_progs; + u64 delegate_attachs; +}; + struct bpf_struct_ops_value; struct btf_member; diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 1aafb2ff2e95..220fe0f99095 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "preload/bpf_preload.h" enum bpf_type { @@ -599,10 +600,31 @@ EXPORT_SYMBOL(bpf_prog_get_type_path); */ static int bpf_show_options(struct seq_file *m, struct dentry *root) { + struct bpf_mount_opts *opts = root->d_sb->s_fs_info; umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); + + if (opts->delegate_cmds == ~0ULL) + seq_printf(m, ",delegate_cmds=any"); + else if (opts->delegate_cmds) + seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); + + if (opts->delegate_maps == ~0ULL) + seq_printf(m, ",delegate_maps=any"); + else if (opts->delegate_maps) + seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); + + if (opts->delegate_progs == ~0ULL) + seq_printf(m, ",delegate_progs=any"); + else if (opts->delegate_progs) + seq_printf(m, ",delegate_progs=0x%llx", opts->delegate_progs); + + if (opts->delegate_attachs == ~0ULL) + seq_printf(m, ",delegate_attachs=any"); + else if (opts->delegate_attachs) + seq_printf(m, ",delegate_attachs=0x%llx", opts->delegate_attachs); return 0; } @@ -626,22 +648,27 @@ static const struct super_operations bpf_super_ops = { enum { OPT_MODE, + OPT_DELEGATE_CMDS, + OPT_DELEGATE_MAPS, + OPT_DELEGATE_PROGS, + OPT_DELEGATE_ATTACHS, }; static const struct fs_parameter_spec bpf_fs_parameters[] = { fsparam_u32oct ("mode", OPT_MODE), + fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), + fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), + fsparam_string ("delegate_progs", OPT_DELEGATE_PROGS), + fsparam_string ("delegate_attachs", OPT_DELEGATE_ATTACHS), {} }; -struct bpf_mount_opts { - umode_t mode; -}; - static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = fc->s_fs_info; struct fs_parse_result result; - int opt; + int opt, err; + u64 msk; opt = fs_parse(fc, bpf_fs_parameters, param, &result); if (opt < 0) { @@ -665,6 +692,28 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; + case OPT_DELEGATE_CMDS: + case OPT_DELEGATE_MAPS: + case OPT_DELEGATE_PROGS: + case OPT_DELEGATE_ATTACHS: + if (strcmp(param->string, "any") == 0) { + msk = ~0ULL; + } else { + err = kstrtou64(param->string, 0, &msk); + if (err) + return err; + } + /* Setting delegation mount options requires privileges */ + if (msk && !capable(CAP_SYS_ADMIN)) + return -EPERM; + switch (opt) { + case OPT_DELEGATE_CMDS: opts->delegate_cmds |= msk; break; + case OPT_DELEGATE_MAPS: opts->delegate_maps |= msk; break; + case OPT_DELEGATE_PROGS: opts->delegate_progs |= msk; break; + case OPT_DELEGATE_ATTACHS: opts->delegate_attachs |= msk; break; + default: return -EINVAL; + } + break; } return 0; @@ -739,10 +788,14 @@ out: static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { static const struct tree_descr bpf_rfiles[] = { { "" } }; - struct bpf_mount_opts *opts = fc->fs_private; + struct bpf_mount_opts *opts = sb->s_fs_info; struct inode *inode; int ret; + /* Mounting an instance of BPF FS requires privileges */ + if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) + return -EPERM; + ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); if (ret) return ret; @@ -764,7 +817,7 @@ static int bpf_get_tree(struct fs_context *fc) static void bpf_free_fc(struct fs_context *fc) { - kfree(fc->fs_private); + kfree(fc->s_fs_info); } static const struct fs_context_operations bpf_context_ops = { @@ -786,17 +839,32 @@ static int bpf_init_fs_context(struct fs_context *fc) opts->mode = S_IRWXUGO; - fc->fs_private = opts; + /* start out with no BPF token delegation enabled */ + opts->delegate_cmds = 0; + opts->delegate_maps = 0; + opts->delegate_progs = 0; + opts->delegate_attachs = 0; + + fc->s_fs_info = opts; fc->ops = &bpf_context_ops; return 0; } +static void bpf_kill_super(struct super_block *sb) +{ + struct bpf_mount_opts *opts = sb->s_fs_info; + + kill_litter_super(sb); + kfree(opts); +} + static struct file_system_type bpf_fs_type = { .owner = THIS_MODULE, .name = "bpf", .init_fs_context = bpf_init_fs_context, .parameters = bpf_fs_parameters, - .kill_sb = kill_litter_super, + .kill_sb = bpf_kill_super, + .fs_flags = FS_USERNS_MOUNT, }; static int __init bpf_init(void) -- cgit v1.2.3 From 4527358b76861dfd64ee34aba45d81648fbc8a61 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:15 -0800 Subject: bpf: introduce BPF token object Add new kind of BPF kernel object, BPF token. BPF token is meant to allow delegating privileged BPF functionality, like loading a BPF program or creating a BPF map, from privileged process to a *trusted* unprivileged process, all while having a good amount of control over which privileged operations could be performed using provided BPF token. This is achieved through mounting BPF FS instance with extra delegation mount options, which determine what operations are delegatable, and also constraining it to the owning user namespace (as mentioned in the previous patch). BPF token itself is just a derivative from BPF FS and can be created through a new bpf() syscall command, BPF_TOKEN_CREATE, which accepts BPF FS FD, which can be attained through open() API by opening BPF FS mount point. Currently, BPF token "inherits" delegated command, map types, prog type, and attach type bit sets from BPF FS as is. In the future, having an BPF token as a separate object with its own FD, we can allow to further restrict BPF token's allowable set of things either at the creation time or after the fact, allowing the process to guard itself further from unintentionally trying to load undesired kind of BPF programs. But for now we keep things simple and just copy bit sets as is. When BPF token is created from BPF FS mount, we take reference to the BPF super block's owning user namespace, and then use that namespace for checking all the {CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN} capabilities that are normally only checked against init userns (using capable()), but now we check them using ns_capable() instead (if BPF token is provided). See bpf_token_capable() for details. Such setup means that BPF token in itself is not sufficient to grant BPF functionality. User namespaced process has to *also* have necessary combination of capabilities inside that user namespace. So while previously CAP_BPF was useless when granted within user namespace, now it gains a meaning and allows container managers and sys admins to have a flexible control over which processes can and need to use BPF functionality within the user namespace (i.e., container in practice). And BPF FS delegation mount options and derived BPF tokens serve as a per-container "flag" to grant overall ability to use bpf() (plus further restrict on which parts of bpf() syscalls are treated as namespaced). Note also, BPF_TOKEN_CREATE command itself requires ns_capable(CAP_BPF) within the BPF FS owning user namespace, rounding up the ns_capable() story of BPF token. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 41 ++++++++ include/uapi/linux/bpf.h | 37 +++++++ kernel/bpf/Makefile | 2 +- kernel/bpf/inode.c | 12 ++- kernel/bpf/syscall.c | 17 ++++ kernel/bpf/token.c | 214 +++++++++++++++++++++++++++++++++++++++++ tools/include/uapi/linux/bpf.h | 37 +++++++ 7 files changed, 354 insertions(+), 6 deletions(-) create mode 100644 kernel/bpf/token.c (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d3c9acc593ea..aa9cf8e5fab1 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -51,6 +51,10 @@ struct module; struct bpf_func_state; struct ftrace_ops; struct cgroup; +struct bpf_token; +struct user_namespace; +struct super_block; +struct inode; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -1591,6 +1595,13 @@ struct bpf_mount_opts { u64 delegate_attachs; }; +struct bpf_token { + struct work_struct work; + atomic64_t refcnt; + struct user_namespace *userns; + u64 allowed_cmds; +}; + struct bpf_struct_ops_value; struct btf_member; @@ -2048,6 +2059,7 @@ static inline void bpf_enable_instrumentation(void) migrate_enable(); } +extern const struct super_operations bpf_super_ops; extern const struct file_operations bpf_map_fops; extern const struct file_operations bpf_prog_fops; extern const struct file_operations bpf_iter_fops; @@ -2182,6 +2194,8 @@ static inline void bpf_map_dec_elem_count(struct bpf_map *map) extern int sysctl_unprivileged_bpf_disabled; +bool bpf_token_capable(const struct bpf_token *token, int cap); + static inline bool bpf_allow_ptr_leaks(void) { return perfmon_capable(); @@ -2216,8 +2230,17 @@ int bpf_link_new_fd(struct bpf_link *link); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); +void bpf_token_inc(struct bpf_token *token); +void bpf_token_put(struct bpf_token *token); +int bpf_token_create(union bpf_attr *attr); +struct bpf_token *bpf_token_get_from_fd(u32 ufd); + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); + int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); +struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, + umode_t mode); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ @@ -2580,6 +2603,24 @@ static inline int bpf_obj_get_user(const char __user *pathname, int flags) return -EOPNOTSUPP; } +static inline bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); +} + +static inline void bpf_token_inc(struct bpf_token *token) +{ +} + +static inline void bpf_token_put(struct bpf_token *token) +{ +} + +static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static inline void __dev_flush(void) { } diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index e88746ba7d21..d4a567e5bc3c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -1712,6 +1744,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index f526b7573e97..4ce95acfcaa7 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 220fe0f99095..6ce3f9696e72 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -99,9 +99,9 @@ static const struct inode_operations bpf_prog_iops = { }; static const struct inode_operations bpf_map_iops = { }; static const struct inode_operations bpf_link_iops = { }; -static struct inode *bpf_get_inode(struct super_block *sb, - const struct inode *dir, - umode_t mode) +struct inode *bpf_get_inode(struct super_block *sb, + const struct inode *dir, + umode_t mode) { struct inode *inode; @@ -602,11 +602,13 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) { struct bpf_mount_opts *opts = root->d_sb->s_fs_info; umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; + u64 mask; if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); - if (opts->delegate_cmds == ~0ULL) + mask = (1ULL << __MAX_BPF_CMD) - 1; + if ((opts->delegate_cmds & mask) == mask) seq_printf(m, ",delegate_cmds=any"); else if (opts->delegate_cmds) seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); @@ -639,7 +641,7 @@ static void bpf_free_inode(struct inode *inode) free_inode_nonrcu(inode); } -static const struct super_operations bpf_super_ops = { +const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ee33a52abf18..a156d549b356 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5377,6 +5377,20 @@ out_prog_put: return ret; } +#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd + +static int token_create(union bpf_attr *attr) +{ + if (CHECK_ATTR(BPF_TOKEN_CREATE)) + return -EINVAL; + + /* no flags are supported yet */ + if (attr->token_create.flags) + return -EINVAL; + + return bpf_token_create(attr); +} + static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -5510,6 +5524,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_BIND_MAP: err = bpf_prog_bind_map(&attr); break; + case BPF_TOKEN_CREATE: + err = token_create(&attr); + break; default: err = -EINVAL; break; diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c new file mode 100644 index 000000000000..e18aaecc67e9 --- /dev/null +++ b/kernel/bpf/token.c @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +bool bpf_token_capable(const struct bpf_token *token, int cap) +{ + /* BPF token allows ns_capable() level of capabilities, but only if + * token's userns is *exactly* the same as current user's userns + */ + if (token && current_user_ns() == token->userns) { + if (ns_capable(token->userns, cap)) + return true; + if (cap != CAP_SYS_ADMIN && ns_capable(token->userns, CAP_SYS_ADMIN)) + return true; + } + /* otherwise fallback to capable() checks */ + return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); +} + +void bpf_token_inc(struct bpf_token *token) +{ + atomic64_inc(&token->refcnt); +} + +static void bpf_token_free(struct bpf_token *token) +{ + put_user_ns(token->userns); + kvfree(token); +} + +static void bpf_token_put_deferred(struct work_struct *work) +{ + struct bpf_token *token = container_of(work, struct bpf_token, work); + + bpf_token_free(token); +} + +void bpf_token_put(struct bpf_token *token) +{ + if (!token) + return; + + if (!atomic64_dec_and_test(&token->refcnt)) + return; + + INIT_WORK(&token->work, bpf_token_put_deferred); + schedule_work(&token->work); +} + +static int bpf_token_release(struct inode *inode, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + + bpf_token_put(token); + return 0; +} + +static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) +{ + struct bpf_token *token = filp->private_data; + u64 mask; + + BUILD_BUG_ON(__MAX_BPF_CMD >= 64); + mask = (1ULL << __MAX_BPF_CMD) - 1; + if ((token->allowed_cmds & mask) == mask) + seq_printf(m, "allowed_cmds:\tany\n"); + else + seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); +} + +#define BPF_TOKEN_INODE_NAME "bpf-token" + +static const struct inode_operations bpf_token_iops = { }; + +static const struct file_operations bpf_token_fops = { + .release = bpf_token_release, + .show_fdinfo = bpf_token_show_fdinfo, +}; + +int bpf_token_create(union bpf_attr *attr) +{ + struct bpf_mount_opts *mnt_opts; + struct bpf_token *token = NULL; + struct user_namespace *userns; + struct inode *inode; + struct file *file; + struct path path; + struct fd f; + umode_t mode; + int err, fd; + + f = fdget(attr->token_create.bpffs_fd); + if (!f.file) + return -EBADF; + + path = f.file->f_path; + path_get(&path); + fdput(f); + + if (path.dentry != path.mnt->mnt_sb->s_root) { + err = -EINVAL; + goto out_path; + } + if (path.mnt->mnt_sb->s_op != &bpf_super_ops) { + err = -EINVAL; + goto out_path; + } + err = path_permission(&path, MAY_ACCESS); + if (err) + goto out_path; + + userns = path.dentry->d_sb->s_user_ns; + /* + * Enforce that creators of BPF tokens are in the same user + * namespace as the BPF FS instance. This makes reasoning about + * permissions a lot easier and we can always relax this later. + */ + if (current_user_ns() != userns) { + err = -EPERM; + goto out_path; + } + if (!ns_capable(userns, CAP_BPF)) { + err = -EPERM; + goto out_path; + } + + mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask()); + inode = bpf_get_inode(path.mnt->mnt_sb, NULL, mode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_path; + } + + inode->i_op = &bpf_token_iops; + inode->i_fop = &bpf_token_fops; + clear_nlink(inode); /* make sure it is unlinked */ + + file = alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops); + if (IS_ERR(file)) { + iput(inode); + err = PTR_ERR(file); + goto out_path; + } + + token = kvzalloc(sizeof(*token), GFP_USER); + if (!token) { + err = -ENOMEM; + goto out_file; + } + + atomic64_set(&token->refcnt, 1); + + /* remember bpffs owning userns for future ns_capable() checks */ + token->userns = get_user_ns(userns); + + mnt_opts = path.dentry->d_sb->s_fs_info; + token->allowed_cmds = mnt_opts->delegate_cmds; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + err = fd; + goto out_token; + } + + file->private_data = token; + fd_install(fd, file); + + path_put(&path); + return fd; + +out_token: + bpf_token_free(token); +out_file: + fput(file); +out_path: + path_put(&path); + return err; +} + +struct bpf_token *bpf_token_get_from_fd(u32 ufd) +{ + struct fd f = fdget(ufd); + struct bpf_token *token; + + if (!f.file) + return ERR_PTR(-EBADF); + if (f.file->f_op != &bpf_token_fops) { + fdput(f); + return ERR_PTR(-EINVAL); + } + + token = f.file->private_data; + bpf_token_inc(token); + fdput(f); + + return token; +} + +bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + /* BPF token can be used only within exactly the same userns in which + * it was created + */ + if (!token || current_user_ns() != token->userns) + return false; + + return token->allowed_cmds & (1ULL << cmd); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e88746ba7d21..d4a567e5bc3c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -847,6 +847,36 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * + * BPF_TOKEN_CREATE + * Description + * Create BPF token with embedded information about what + * BPF-related functionality it allows: + * - a set of allowed bpf() syscall commands; + * - a set of allowed BPF map types to be created with + * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; + * - a set of allowed BPF program types and BPF program attach + * types to be loaded with BPF_PROG_LOAD command, if + * BPF_PROG_LOAD itself is allowed. + * + * BPF token is created (derived) from an instance of BPF FS, + * assuming it has necessary delegation mount options specified. + * This BPF token can be passed as an extra parameter to various + * bpf() syscall commands to grant BPF subsystem functionality to + * unprivileged processes. + * + * When created, BPF token is "associated" with the owning + * user namespace of BPF FS instance (super block) that it was + * derived from, and subsequent BPF operations performed with + * BPF token would be performing capabilities checks (i.e., + * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within + * that user namespace. Without BPF token, such capabilities + * have to be granted in init user namespace, making bpf() + * syscall incompatible with user namespace, for the most part. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -901,6 +931,8 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_TOKEN_CREATE, + __MAX_BPF_CMD, }; enum bpf_map_type { @@ -1712,6 +1744,11 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; + struct { /* struct used by BPF_TOKEN_CREATE command */ + __u32 flags; + __u32 bpffs_fd; + } token_create; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF -- cgit v1.2.3 From 688b7270b3cb75e8ac78123d719967db40336e5b Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:16 -0800 Subject: bpf: add BPF token support to BPF_MAP_CREATE command Allow providing token_fd for BPF_MAP_CREATE command to allow controlled BPF map creation from unprivileged process through delegated BPF token. Wire through a set of allowed BPF map types to BPF token, derived from BPF FS at BPF token creation time. This, in combination with allowed_cmds allows to create a narrowly-focused BPF token (controlled by privileged agent) with a restrictive set of BPF maps that application can attempt to create. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 + include/uapi/linux/bpf.h | 2 + kernel/bpf/inode.c | 3 +- kernel/bpf/syscall.c | 52 ++++++++++++++++------ kernel/bpf/token.c | 16 +++++++ tools/include/uapi/linux/bpf.h | 2 + .../selftests/bpf/prog_tests/libbpf_probes.c | 2 + .../testing/selftests/bpf/prog_tests/libbpf_str.c | 3 ++ 8 files changed, 67 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index aa9cf8e5fab1..e08e8436df38 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1600,6 +1600,7 @@ struct bpf_token { atomic64_t refcnt; struct user_namespace *userns; u64 allowed_cmds; + u64 allowed_maps; }; struct bpf_struct_ops_value; @@ -2236,6 +2237,7 @@ int bpf_token_create(union bpf_attr *attr); struct bpf_token *bpf_token_get_from_fd(u32 ufd); bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index d4a567e5bc3c..0bba3392b17a 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -983,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1433,6 +1434,7 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; + __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 6ce3f9696e72..9c7865d1c53d 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -613,7 +613,8 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) else if (opts->delegate_cmds) seq_printf(m, ",delegate_cmds=0x%llx", opts->delegate_cmds); - if (opts->delegate_maps == ~0ULL) + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + if ((opts->delegate_maps & mask) == mask) seq_printf(m, ",delegate_maps=any"); else if (opts->delegate_maps) seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a156d549b356..22e14124cd61 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1009,8 +1009,8 @@ int map_check_no_btf(const struct bpf_map *map, return -ENOTSUPP; } -static int map_check_btf(struct bpf_map *map, const struct btf *btf, - u32 btf_key_id, u32 btf_value_id) +static int map_check_btf(struct bpf_map *map, struct bpf_token *token, + const struct btf *btf, u32 btf_key_id, u32 btf_value_id) { const struct btf_type *key_type, *value_type; u32 key_size, value_size; @@ -1038,7 +1038,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, if (!IS_ERR_OR_NULL(map->record)) { int i; - if (!bpf_capable()) { + if (!bpf_token_capable(token, CAP_BPF)) { ret = -EPERM; goto free_map_tab; } @@ -1126,11 +1126,12 @@ static bool bpf_net_capable(void) return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); } -#define BPF_MAP_CREATE_LAST_FIELD map_extra +#define BPF_MAP_CREATE_LAST_FIELD map_token_fd /* called via syscall */ static int map_create(union bpf_attr *attr) { const struct bpf_map_ops *ops; + struct bpf_token *token = NULL; int numa_node = bpf_map_attr_numa_node(attr); u32 map_type = attr->map_type; struct bpf_map *map; @@ -1181,14 +1182,32 @@ static int map_create(union bpf_attr *attr) if (!ops->map_mem_usage) return -EINVAL; + if (attr->map_token_fd) { + token = bpf_token_get_from_fd(attr->map_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + + /* if current token doesn't grant map creation permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) || + !bpf_token_allow_map_type(token, attr->map_type)) { + bpf_token_put(token); + token = NULL; + } + } + + err = -EPERM; + /* Intent here is for unprivileged_bpf_disabled to block BPF map * creation for unprivileged users; other actions depend * on fd availability and access to bpffs, so are dependent on * object creation success. Even with unprivileged BPF disabled, * capability checks are still carried out. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF)) + goto put_token; /* check privileged map type permissions */ switch (map_type) { @@ -1221,25 +1240,27 @@ static int map_create(union bpf_attr *attr) case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_STRUCT_OPS: case BPF_MAP_TYPE_CPUMAP: - if (!bpf_capable()) - return -EPERM; + if (!bpf_token_capable(token, CAP_BPF)) + goto put_token; break; case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_DEVMAP: case BPF_MAP_TYPE_DEVMAP_HASH: case BPF_MAP_TYPE_XSKMAP: - if (!bpf_net_capable()) - return -EPERM; + if (!bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; break; default: WARN(1, "unsupported map type %d", map_type); - return -EPERM; + goto put_token; } map = ops->map_alloc(attr); - if (IS_ERR(map)) - return PTR_ERR(map); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto put_token; + } map->ops = ops; map->map_type = map_type; @@ -1276,7 +1297,7 @@ static int map_create(union bpf_attr *attr) map->btf = btf; if (attr->btf_value_type_id) { - err = map_check_btf(map, btf, attr->btf_key_type_id, + err = map_check_btf(map, token, btf, attr->btf_key_type_id, attr->btf_value_type_id); if (err) goto free_map; @@ -1297,6 +1318,7 @@ static int map_create(union bpf_attr *attr) goto free_map_sec; bpf_map_save_memcg(map); + bpf_token_put(token); err = bpf_map_new_fd(map, f_flags); if (err < 0) { @@ -1317,6 +1339,8 @@ free_map_sec: free_map: btf_put(map->btf); map->ops->map_free(map); +put_token: + bpf_token_put(token); return err; } diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index e18aaecc67e9..06c34dae658e 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -72,6 +72,13 @@ static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) seq_printf(m, "allowed_cmds:\tany\n"); else seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); + + BUILD_BUG_ON(__MAX_BPF_MAP_TYPE >= 64); + mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; + if ((token->allowed_maps & mask) == mask) + seq_printf(m, "allowed_maps:\tany\n"); + else + seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); } #define BPF_TOKEN_INODE_NAME "bpf-token" @@ -161,6 +168,7 @@ int bpf_token_create(union bpf_attr *attr) mnt_opts = path.dentry->d_sb->s_fs_info; token->allowed_cmds = mnt_opts->delegate_cmds; + token->allowed_maps = mnt_opts->delegate_maps; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { @@ -212,3 +220,11 @@ bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) return token->allowed_cmds & (1ULL << cmd); } + +bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) +{ + if (!token || type >= __MAX_BPF_MAP_TYPE) + return false; + + return token->allowed_maps & (1ULL << type); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index d4a567e5bc3c..0bba3392b17a 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -983,6 +983,7 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, + __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1433,6 +1434,7 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; + __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 9f766ddd946a..573249a2814d 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -68,6 +68,8 @@ void test_libbpf_probe_map_types(void) if (map_type == BPF_MAP_TYPE_UNSPEC) continue; + if (strcmp(map_type_name, "__MAX_BPF_MAP_TYPE") == 0) + continue; if (!test__start_subtest(map_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index c440ea3311ed..2a0633f43c73 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -132,6 +132,9 @@ static void test_libbpf_bpf_map_type_str(void) const char *map_type_str; char buf[256]; + if (map_type == __MAX_BPF_MAP_TYPE) + continue; + map_type_name = btf__str_by_offset(btf, e->name_off); map_type_str = libbpf_bpf_map_type_str(map_type); ASSERT_OK_PTR(map_type_str, map_type_name); -- cgit v1.2.3 From e1cef620f598853a90f17701fcb1057a6768f7b8 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:18 -0800 Subject: bpf: add BPF token support to BPF_PROG_LOAD command Add basic support of BPF token to BPF_PROG_LOAD. Wire through a set of allowed BPF program types and attach types, derived from BPF FS at BPF token creation time. Then make sure we perform bpf_token_capable() checks everywhere where it's relevant. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-7-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 6 ++ include/uapi/linux/bpf.h | 2 + kernel/bpf/core.c | 1 + kernel/bpf/inode.c | 6 +- kernel/bpf/syscall.c | 87 ++++++++++++++++------ kernel/bpf/token.c | 27 +++++++ tools/include/uapi/linux/bpf.h | 2 + .../selftests/bpf/prog_tests/libbpf_probes.c | 2 + .../testing/selftests/bpf/prog_tests/libbpf_str.c | 3 + 9 files changed, 110 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e08e8436df38..20af87b59d70 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1461,6 +1461,7 @@ struct bpf_prog_aux { #ifdef CONFIG_SECURITY void *security; #endif + struct bpf_token *token; struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; @@ -1601,6 +1602,8 @@ struct bpf_token { struct user_namespace *userns; u64 allowed_cmds; u64 allowed_maps; + u64 allowed_progs; + u64 allowed_attachs; }; struct bpf_struct_ops_value; @@ -2238,6 +2241,9 @@ struct bpf_token *bpf_token_get_from_fd(u32 ufd); bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type); int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 9f9989e0d062..4df2d025c784 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1028,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1504,6 +1505,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 4b813da8d6c0..47085839af8d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2751,6 +2751,7 @@ void bpf_prog_free(struct bpf_prog *fp) if (aux->dst_prog) bpf_prog_put(aux->dst_prog); + bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 9c7865d1c53d..5359a0929c35 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -619,12 +619,14 @@ static int bpf_show_options(struct seq_file *m, struct dentry *root) else if (opts->delegate_maps) seq_printf(m, ",delegate_maps=0x%llx", opts->delegate_maps); - if (opts->delegate_progs == ~0ULL) + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + if ((opts->delegate_progs & mask) == mask) seq_printf(m, ",delegate_progs=any"); else if (opts->delegate_progs) seq_printf(m, ",delegate_progs=0x%llx", opts->delegate_progs); - if (opts->delegate_attachs == ~0ULL) + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + if ((opts->delegate_attachs & mask) == mask) seq_printf(m, ",delegate_attachs=any"); else if (opts->delegate_attachs) seq_printf(m, ",delegate_attachs=0x%llx", opts->delegate_attachs); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d87c5c27cde3..2c8393c21b8c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2608,13 +2608,15 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD log_true_size +#define BPF_PROG_LOAD_LAST_FIELD prog_token_fd static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) { enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; struct btf *attach_btf = NULL; + struct bpf_token *token = NULL; + bool bpf_cap; int err; char license[128]; @@ -2631,10 +2633,31 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_TEST_REG_INVARIANTS)) return -EINVAL; + bpf_prog_load_fixup_attach_type(attr); + + if (attr->prog_token_fd) { + token = bpf_token_get_from_fd(attr->prog_token_fd); + if (IS_ERR(token)) + return PTR_ERR(token); + /* if current token doesn't grant prog loading permissions, + * then we can't use this token, so ignore it and rely on + * system-wide capabilities checks + */ + if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) || + !bpf_token_allow_prog_type(token, attr->prog_type, + attr->expected_attach_type)) { + bpf_token_put(token); + token = NULL; + } + } + + bpf_cap = bpf_token_capable(token, CAP_BPF); + err = -EPERM; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; /* Intent here is for unprivileged_bpf_disabled to block BPF program * creation for unprivileged users; other actions depend @@ -2643,21 +2666,23 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * capability checks are still carried out for these * and other operations. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) - return -EPERM; + if (sysctl_unprivileged_bpf_disabled && !bpf_cap) + goto put_token; if (attr->insn_cnt == 0 || - attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) - return -E2BIG; + attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { + err = -E2BIG; + goto put_token; + } if (type != BPF_PROG_TYPE_SOCKET_FILTER && type != BPF_PROG_TYPE_CGROUP_SKB && - !bpf_capable()) - return -EPERM; + !bpf_cap) + goto put_token; - if (is_net_admin_prog_type(type) && !bpf_net_capable()) - return -EPERM; - if (is_perfmon_prog_type(type) && !perfmon_capable()) - return -EPERM; + if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; + if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) + goto put_token; /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is @@ -2667,27 +2692,33 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (IS_ERR(dst_prog)) { dst_prog = NULL; attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); - if (IS_ERR(attach_btf)) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = -EINVAL; + goto put_token; + } if (!btf_is_kernel(attach_btf)) { /* attaching through specifying bpf_prog's BTF * objects directly might be supported eventually */ btf_put(attach_btf); - return -ENOTSUPP; + err = -ENOTSUPP; + goto put_token; } } } else if (attr->attach_btf_id) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(attach_btf)) - return PTR_ERR(attach_btf); - if (!attach_btf) - return -EINVAL; + if (IS_ERR(attach_btf)) { + err = PTR_ERR(attach_btf); + goto put_token; + } + if (!attach_btf) { + err = -EINVAL; + goto put_token; + } btf_get(attach_btf); } - bpf_prog_load_fixup_attach_type(attr); if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, dst_prog)) { @@ -2695,7 +2726,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -EINVAL; + err = -EINVAL; + goto put_token; } /* plain bpf_prog allocation */ @@ -2705,7 +2737,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - return -ENOMEM; + err = -EINVAL; + goto put_token; } prog->expected_attach_type = attr->expected_attach_type; @@ -2716,6 +2749,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; + /* move token into prog->aux, reuse taken refcnt */ + prog->aux->token = token; + token = NULL; + err = security_bpf_prog_alloc(prog->aux); if (err) goto free_prog; @@ -2817,6 +2854,8 @@ free_prog: if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); +put_token: + bpf_token_put(token); return err; } @@ -3806,7 +3845,7 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, case BPF_PROG_TYPE_SK_LOOKUP: return attach_type == prog->expected_attach_type ? 0 : -EINVAL; case BPF_PROG_TYPE_CGROUP_SKB: - if (!bpf_net_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) /* cg-skb progs can be loaded by unpriv user. * check permissions at attach time. */ diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index 06c34dae658e..5a51e6b8f6bf 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -79,6 +79,20 @@ static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) seq_printf(m, "allowed_maps:\tany\n"); else seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); + + BUILD_BUG_ON(__MAX_BPF_PROG_TYPE >= 64); + mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; + if ((token->allowed_progs & mask) == mask) + seq_printf(m, "allowed_progs:\tany\n"); + else + seq_printf(m, "allowed_progs:\t0x%llx\n", token->allowed_progs); + + BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE >= 64); + mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; + if ((token->allowed_attachs & mask) == mask) + seq_printf(m, "allowed_attachs:\tany\n"); + else + seq_printf(m, "allowed_attachs:\t0x%llx\n", token->allowed_attachs); } #define BPF_TOKEN_INODE_NAME "bpf-token" @@ -169,6 +183,8 @@ int bpf_token_create(union bpf_attr *attr) mnt_opts = path.dentry->d_sb->s_fs_info; token->allowed_cmds = mnt_opts->delegate_cmds; token->allowed_maps = mnt_opts->delegate_maps; + token->allowed_progs = mnt_opts->delegate_progs; + token->allowed_attachs = mnt_opts->delegate_attachs; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { @@ -228,3 +244,14 @@ bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type t return token->allowed_maps & (1ULL << type); } + +bool bpf_token_allow_prog_type(const struct bpf_token *token, + enum bpf_prog_type prog_type, + enum bpf_attach_type attach_type) +{ + if (!token || prog_type >= __MAX_BPF_PROG_TYPE || attach_type >= __MAX_BPF_ATTACH_TYPE) + return false; + + return (token->allowed_progs & (1ULL << prog_type)) && + (token->allowed_attachs & (1ULL << attach_type)); +} diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 9f9989e0d062..4df2d025c784 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1028,6 +1028,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1504,6 +1505,7 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; + __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 573249a2814d..4ed46ed58a7b 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -30,6 +30,8 @@ void test_libbpf_probe_prog_types(void) if (prog_type == BPF_PROG_TYPE_UNSPEC) continue; + if (strcmp(prog_type_name, "__MAX_BPF_PROG_TYPE") == 0) + continue; if (!test__start_subtest(prog_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index 2a0633f43c73..384bc1f7a65e 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -189,6 +189,9 @@ static void test_libbpf_bpf_prog_type_str(void) const char *prog_type_str; char buf[256]; + if (prog_type == __MAX_BPF_PROG_TYPE) + continue; + prog_type_name = btf__str_by_offset(btf, e->name_off); prog_type_str = libbpf_bpf_prog_type_str(prog_type); ASSERT_OK_PTR(prog_type_str, prog_type_name); -- cgit v1.2.3 From 4cbb270e115bc197ff2046aeb54cc951666b16ec Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:19 -0800 Subject: bpf: take into account BPF token when fetching helper protos Instead of performing unconditional system-wide bpf_capable() and perfmon_capable() calls inside bpf_base_func_proto() function (and other similar ones) to determine eligibility of a given BPF helper for a given program, use previously recorded BPF token during BPF_PROG_LOAD command handling to inform the decision. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-8-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- drivers/media/rc/bpf-lirc.c | 2 +- include/linux/bpf.h | 5 +++-- kernel/bpf/cgroup.c | 6 +++--- kernel/bpf/helpers.c | 6 +++--- kernel/bpf/syscall.c | 5 +++-- kernel/trace/bpf_trace.c | 2 +- net/core/filter.c | 32 ++++++++++++++++---------------- net/ipv4/bpf_tcp_ca.c | 2 +- net/netfilter/nf_bpf_link.c | 2 +- 9 files changed, 32 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index fe17c7f98e81..6d07693c6b9f 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -110,7 +110,7 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_prandom_u32: return &bpf_get_prandom_u32_proto; case BPF_FUNC_trace_printk: - if (perfmon_capable()) + if (bpf_token_capable(prog->aux->token, CAP_PERFMON)) return bpf_get_trace_printk_proto(); fallthrough; default: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 20af87b59d70..2a3ab4f3dd8c 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2492,7 +2492,8 @@ const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); -const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); +const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); @@ -2752,7 +2753,7 @@ static inline int btf_struct_access(struct bpf_verifier_log *log, } static inline const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { return NULL; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 491d20038cbe..98e0e3835b28 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1630,7 +1630,7 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2191,7 +2191,7 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -2348,7 +2348,7 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index ee9bdf29246a..b3be5742d6f1 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1679,7 +1679,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak; const struct bpf_func_proto bpf_task_pt_regs_proto __weak; const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id) +bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_map_lookup_elem: @@ -1730,7 +1730,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return NULL; switch (func_id) { @@ -1788,7 +1788,7 @@ bpf_base_func_proto(enum bpf_func_id func_id) break; } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; switch (func_id) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 2c8393c21b8c..1cc03f08c9cd 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -5712,7 +5712,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { const struct bpf_func_proto * __weak tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } BPF_CALL_1(bpf_sys_close, u32, fd) @@ -5762,7 +5762,8 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_sys_bpf: - return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; + return !bpf_token_capable(prog->aux->token, CAP_PERFMON) + ? NULL : &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; case BPF_FUNC_sys_close: diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 1648bde28f01..774cf476a892 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1626,7 +1626,7 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/net/core/filter.c b/net/core/filter.c index 0adaa4afa35f..0bf2a03d8203 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -87,7 +87,7 @@ #include "dev.h" static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id); +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) { @@ -7841,7 +7841,7 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -7934,7 +7934,7 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return NULL; } default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -7953,7 +7953,7 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_skb_event_output_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8140,7 +8140,7 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8199,7 +8199,7 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } #if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) @@ -8260,7 +8260,7 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_sock_proto; #endif /* CONFIG_INET */ default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8302,7 +8302,7 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_cgroup_classid_curr_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8346,7 +8346,7 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_skc_lookup_tcp_proto; #endif default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8357,7 +8357,7 @@ flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_load_bytes: return &bpf_flow_dissector_load_bytes_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -8384,7 +8384,7 @@ lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_under_cgroup: return &bpf_skb_under_cgroup_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -11215,7 +11215,7 @@ sk_reuseport_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } @@ -11397,7 +11397,7 @@ sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_release: return &bpf_sk_release_proto; default: - return bpf_sk_base_func_proto(func_id); + return bpf_sk_base_func_proto(func_id, prog); } } @@ -11731,7 +11731,7 @@ const struct bpf_func_proto bpf_sock_from_file_proto = { }; static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id) +bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { const struct bpf_func_proto *func; @@ -11760,10 +11760,10 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } - if (!perfmon_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) return NULL; return func; diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 39dcccf0f174..c7bbd8f3c708 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -191,7 +191,7 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } } diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index e502ec00b2fe..1969facac91c 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -314,7 +314,7 @@ static bool nf_is_valid_access(int off, int size, enum bpf_access_type type, static const struct bpf_func_proto * bpf_nf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id); + return bpf_base_func_proto(func_id, prog); } const struct bpf_verifier_ops netfilter_verifier_ops = { -- cgit v1.2.3 From 8062fb12de99b2da33754c6a3be1bfc30d9a35f4 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:20 -0800 Subject: bpf: consistently use BPF token throughout BPF verifier logic Remove remaining direct queries to perfmon_capable() and bpf_capable() in BPF verifier logic and instead use BPF token (if available) to make decisions about privileges. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-9-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 16 ++++++++-------- include/linux/filter.h | 2 +- kernel/bpf/arraymap.c | 2 +- kernel/bpf/core.c | 2 +- kernel/bpf/verifier.c | 13 ++++++------- net/core/filter.c | 4 ++-- 6 files changed, 19 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2a3ab4f3dd8c..435abad3cc61 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2200,24 +2200,24 @@ extern int sysctl_unprivileged_bpf_disabled; bool bpf_token_capable(const struct bpf_token *token, int cap); -static inline bool bpf_allow_ptr_leaks(void) +static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_allow_uninit_stack(void) +static inline bool bpf_allow_uninit_stack(const struct bpf_token *token) { - return perfmon_capable(); + return bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v1(void) +static inline bool bpf_bypass_spec_v1(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } -static inline bool bpf_bypass_spec_v4(void) +static inline bool bpf_bypass_spec_v4(const struct bpf_token *token) { - return cpu_mitigations_off() || perfmon_capable(); + return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } int bpf_map_new_fd(struct bpf_map *map, int flags); diff --git a/include/linux/filter.h b/include/linux/filter.h index a4953fafc8cb..14354605ad26 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1139,7 +1139,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) return false; if (!bpf_jit_harden) return false; - if (bpf_jit_harden == 1 && bpf_capable()) + if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF)) return false; return true; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 4a4a67956e21..8d365bda9a8b 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -82,7 +82,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; int numa_node = bpf_map_attr_numa_node(attr); u32 elem_size, index_mask, max_entries; - bool bypass_spec_v1 = bpf_bypass_spec_v1(); + bool bypass_spec_v1 = bpf_bypass_spec_v1(NULL); u64 array_size, mask64; struct bpf_array *array; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 47085839af8d..ced511f44174 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -675,7 +675,7 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || - !bpf_capable()) + !bpf_token_capable(fp->aux->token, CAP_BPF)) return; bpf_prog_ksym_set_addr(fp); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e5ce530641ba..45e85fb76d82 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20597,7 +20597,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - is_priv = bpf_capable(); + + env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); + env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); + env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); + env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); + env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); bpf_get_btf_vmlinux(); @@ -20629,12 +20634,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) env->strict_alignment = false; - env->allow_ptr_leaks = bpf_allow_ptr_leaks(); - env->allow_uninit_stack = bpf_allow_uninit_stack(); - env->bypass_spec_v1 = bpf_bypass_spec_v1(); - env->bypass_spec_v4 = bpf_bypass_spec_v4(); - env->bpf_capable = bpf_capable(); - if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; diff --git a/net/core/filter.c b/net/core/filter.c index 0bf2a03d8203..adcfc2c25754 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8559,7 +8559,7 @@ static bool cg_skb_is_valid_access(int off, int size, return false; case bpf_ctx_range(struct __sk_buff, data): case bpf_ctx_range(struct __sk_buff, data_end): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; } @@ -8571,7 +8571,7 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): break; case bpf_ctx_range(struct __sk_buff, tstamp): - if (!bpf_capable()) + if (!bpf_token_capable(prog->aux->token, CAP_BPF)) return false; break; default: -- cgit v1.2.3 From c3dd6e94df7193f33f45d33303f5e85afb2a72dc Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:21 -0800 Subject: bpf,lsm: refactor bpf_prog_alloc/bpf_prog_free LSM hooks Based on upstream discussion ([0]), rework existing bpf_prog_alloc_security LSM hook. Rename it to bpf_prog_load and instead of passing bpf_prog_aux, pass proper bpf_prog pointer for a full BPF program struct. Also, we pass bpf_attr union with all the user-provided arguments for BPF_PROG_LOAD command. This will give LSMs as much information as we can basically provide. The hook is also BPF token-aware now, and optional bpf_token struct is passed as a third argument. bpf_prog_load LSM hook is called after a bunch of sanity checks were performed, bpf_prog and bpf_prog_aux were allocated and filled out, but right before performing full-fledged BPF verification step. bpf_prog_free LSM hook is now accepting struct bpf_prog argument, for consistency. SELinux code is adjusted to all new names, types, and signatures. Note, given that bpf_prog_load (previously bpf_prog_alloc) hook can be used by some LSMs to allocate extra security blob, but also by other LSMs to reject BPF program loading, we need to make sure that bpf_prog_free LSM hook is called after bpf_prog_load/bpf_prog_alloc one *even* if the hook itself returned error. If we don't do that, we run the risk of leaking memory. This seems to be possible today when combining SELinux and BPF LSM, as one example, depending on their relative ordering. Also, for BPF LSM setup, add bpf_prog_load and bpf_prog_free to sleepable LSM hooks list, as they are both executed in sleepable context. Also drop bpf_prog_load hook from untrusted, as there is no issue with refcount or anything else anymore, that originally forced us to add it to untrusted list in c0c852dd1876 ("bpf: Do not mark certain LSM hook arguments as trusted"). We now trigger this hook much later and it should not be an issue anymore. [0] https://lore.kernel.org/bpf/9fe88aef7deabbe87d3fc38c4aea3c69.paul@paul-moore.com/ Acked-by: Paul Moore Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-10-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/lsm_hook_defs.h | 5 +++-- include/linux/security.h | 12 +++++++----- kernel/bpf/bpf_lsm.c | 5 +++-- kernel/bpf/syscall.c | 25 +++++++++++++------------ security/security.c | 25 +++++++++++++++---------- security/selinux/hooks.c | 15 ++++++++------- 6 files changed, 49 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index ff217a5ce552..41ec4a7c070e 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -400,8 +400,9 @@ LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) -LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux) -LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free_security, struct bpf_prog_aux *aux) +LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/security.h b/include/linux/security.h index 1d1df326c881..65467eef6678 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2020,15 +2020,16 @@ static inline void securityfs_remove(struct dentry *dentry) union bpf_attr; struct bpf_map; struct bpf_prog; -struct bpf_prog_aux; +struct bpf_token; #ifdef CONFIG_SECURITY extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); extern int security_bpf_map_alloc(struct bpf_map *map); extern void security_bpf_map_free(struct bpf_map *map); -extern int security_bpf_prog_alloc(struct bpf_prog_aux *aux); -extern void security_bpf_prog_free(struct bpf_prog_aux *aux); +extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token); +extern void security_bpf_prog_free(struct bpf_prog *prog); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2054,12 +2055,13 @@ static inline int security_bpf_map_alloc(struct bpf_map *map) static inline void security_bpf_map_free(struct bpf_map *map) { } -static inline int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { return 0; } -static inline void security_bpf_prog_free(struct bpf_prog_aux *aux) +static inline void security_bpf_prog_free(struct bpf_prog *prog) { } #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index e14c822f8911..3e956f6302f3 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -263,6 +263,8 @@ BTF_ID(func, bpf_lsm_bpf_map) BTF_ID(func, bpf_lsm_bpf_map_alloc_security) BTF_ID(func, bpf_lsm_bpf_map_free_security) BTF_ID(func, bpf_lsm_bpf_prog) +BTF_ID(func, bpf_lsm_bpf_prog_load) +BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) @@ -346,8 +348,7 @@ BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) BTF_ID(func, bpf_lsm_bpf_map_free_security) -BTF_ID(func, bpf_lsm_bpf_prog_alloc_security) -BTF_ID(func, bpf_lsm_bpf_prog_free_security) +BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 1cc03f08c9cd..7717c7c7b95d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2162,7 +2162,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); - security_bpf_prog_free(aux); + security_bpf_prog_free(aux->prog); bpf_prog_free(aux->prog); } @@ -2753,10 +2753,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->token = token; token = NULL; - err = security_bpf_prog_alloc(prog->aux); - if (err) - goto free_prog; - prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; @@ -2764,12 +2760,12 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), bpf_prog_insn_size(prog)) != 0) - goto free_prog_sec; + goto free_prog; /* copy eBPF program license from user space */ if (strncpy_from_bpfptr(license, make_bpfptr(attr->license, uattr.is_kernel), sizeof(license) - 1) < 0) - goto free_prog_sec; + goto free_prog; license[sizeof(license) - 1] = 0; /* eBPF programs must be GPL compatible to use GPL-ed functions */ @@ -2783,25 +2779,29 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (bpf_prog_is_dev_bound(prog->aux)) { err = bpf_prog_dev_bound_init(prog, attr); if (err) - goto free_prog_sec; + goto free_prog; } if (type == BPF_PROG_TYPE_EXT && dst_prog && bpf_prog_is_dev_bound(dst_prog->aux)) { err = bpf_prog_dev_bound_inherit(prog, dst_prog); if (err) - goto free_prog_sec; + goto free_prog; } /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) - goto free_prog_sec; + goto free_prog; prog->aux->load_time = ktime_get_boottime_ns(); err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, sizeof(attr->prog_name)); if (err < 0) + goto free_prog; + + err = security_bpf_prog_load(prog, attr, token); + if (err) goto free_prog_sec; /* run eBPF verifier */ @@ -2847,10 +2847,11 @@ free_used_maps: */ __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); return err; + free_prog_sec: - free_uid(prog->aux->user); - security_bpf_prog_free(prog->aux); + security_bpf_prog_free(prog); free_prog: + free_uid(prog->aux->user); if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); diff --git a/security/security.c b/security/security.c index dcb3e7014f9b..c8a1c66cfaad 100644 --- a/security/security.c +++ b/security/security.c @@ -5180,16 +5180,21 @@ int security_bpf_map_alloc(struct bpf_map *map) } /** - * security_bpf_prog_alloc() - Allocate a bpf program LSM blob - * @aux: bpf program aux info struct + * security_bpf_prog_load() - Check if loading of BPF program is allowed + * @prog: BPF program object + * @attr: BPF syscall attributes used to create BPF program + * @token: BPF token used to grant user access to BPF subsystem * - * Initialize the security field inside bpf program. + * Perform an access control check when the kernel loads a BPF program and + * allocates associated BPF program object. This hook is also responsible for + * allocating any required LSM state for the BPF program. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_prog_alloc(struct bpf_prog_aux *aux) +int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { - return call_int_hook(bpf_prog_alloc_security, 0, aux); + return call_int_hook(bpf_prog_load, 0, prog, attr, token); } /** @@ -5204,14 +5209,14 @@ void security_bpf_map_free(struct bpf_map *map) } /** - * security_bpf_prog_free() - Free a bpf program's LSM blob - * @aux: bpf program aux info struct + * security_bpf_prog_free() - Free a BPF program's LSM blob + * @prog: BPF program struct * - * Clean up the security information stored inside bpf prog. + * Clean up the security information stored inside BPF program. */ -void security_bpf_prog_free(struct bpf_prog_aux *aux) +void security_bpf_prog_free(struct bpf_prog *prog) { - call_void_hook(bpf_prog_free_security, aux); + call_void_hook(bpf_prog_free, prog); } #endif /* CONFIG_BPF_SYSCALL */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index feda711c6b7b..eabee39e983c 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6805,7 +6805,8 @@ static void selinux_bpf_map_free(struct bpf_map *map) kfree(bpfsec); } -static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) +static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -6814,16 +6815,16 @@ static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) return -ENOMEM; bpfsec->sid = current_sid(); - aux->security = bpfsec; + prog->aux->security = bpfsec; return 0; } -static void selinux_bpf_prog_free(struct bpf_prog_aux *aux) +static void selinux_bpf_prog_free(struct bpf_prog *prog) { - struct bpf_security_struct *bpfsec = aux->security; + struct bpf_security_struct *bpfsec = prog->aux->security; - aux->security = NULL; + prog->aux->security = NULL; kfree(bpfsec); } #endif @@ -7180,7 +7181,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), - LSM_HOOK_INIT(bpf_prog_free_security, selinux_bpf_prog_free), + LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), #endif #ifdef CONFIG_PERF_EVENTS @@ -7238,7 +7239,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { #endif #ifdef CONFIG_BPF_SYSCALL LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), - LSM_HOOK_INIT(bpf_prog_alloc_security, selinux_bpf_prog_alloc), + LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), #endif #ifdef CONFIG_PERF_EVENTS LSM_HOOK_INIT(perf_event_alloc, selinux_perf_event_alloc), -- cgit v1.2.3 From 66d636d70a79c1d37e3eea67ab50969e6aaef983 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:22 -0800 Subject: bpf,lsm: refactor bpf_map_alloc/bpf_map_free LSM hooks Similarly to bpf_prog_alloc LSM hook, rename and extend bpf_map_alloc hook into bpf_map_create, taking not just struct bpf_map, but also bpf_attr and bpf_token, to give a fuller context to LSMs. Unlike bpf_prog_alloc, there is no need to move the hook around, as it currently is firing right before allocating BPF map ID and FD, which seems to be a sweet spot. But like bpf_prog_alloc/bpf_prog_free combo, make sure that bpf_map_free LSM hook is called even if bpf_map_create hook returned error, as if few LSMs are combined together it could be that one LSM successfully allocated security blob for its needs, while subsequent LSM rejected BPF map creation. The former LSM would still need to free up LSM blob, so we need to ensure security_bpf_map_free() is called regardless of the outcome. Acked-by: Paul Moore Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-11-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/lsm_hook_defs.h | 5 +++-- include/linux/security.h | 6 ++++-- kernel/bpf/bpf_lsm.c | 6 +++--- kernel/bpf/syscall.c | 4 ++-- security/security.c | 16 ++++++++++------ security/selinux/hooks.c | 7 ++++--- 6 files changed, 26 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 41ec4a7c070e..adb25cc63ce3 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -398,8 +398,9 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size) LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) -LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) -LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) +LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) +LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token) LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) diff --git a/include/linux/security.h b/include/linux/security.h index 65467eef6678..08fd777cbe94 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2025,7 +2025,8 @@ struct bpf_token; extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); -extern int security_bpf_map_alloc(struct bpf_map *map); +extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token); extern void security_bpf_map_free(struct bpf_map *map); extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token); @@ -2047,7 +2048,8 @@ static inline int security_bpf_prog(struct bpf_prog *prog) return 0; } -static inline int security_bpf_map_alloc(struct bpf_map *map) +static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { return 0; } diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 3e956f6302f3..9e4e615f11eb 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -260,8 +260,8 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) BTF_SET_START(sleepable_lsm_hooks) BTF_ID(func, bpf_lsm_bpf) BTF_ID(func, bpf_lsm_bpf_map) -BTF_ID(func, bpf_lsm_bpf_map_alloc_security) -BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_map_create) +BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) BTF_ID(func, bpf_lsm_bpf_prog_free) @@ -347,7 +347,7 @@ BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) -BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7717c7c7b95d..aff045eed375 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1309,9 +1309,9 @@ static int map_create(union bpf_attr *attr) attr->btf_vmlinux_value_type_id; } - err = security_bpf_map_alloc(map); + err = security_bpf_map_create(map, attr, token); if (err) - goto free_map; + goto free_map_sec; err = bpf_map_alloc_id(map); if (err) diff --git a/security/security.c b/security/security.c index c8a1c66cfaad..ad24cf36da94 100644 --- a/security/security.c +++ b/security/security.c @@ -5167,16 +5167,20 @@ int security_bpf_prog(struct bpf_prog *prog) } /** - * security_bpf_map_alloc() - Allocate a bpf map LSM blob - * @map: bpf map + * security_bpf_map_create() - Check if BPF map creation is allowed + * @map: BPF map object + * @attr: BPF syscall attributes used to create BPF map + * @token: BPF token used to grant user access * - * Initialize the security field inside bpf map. + * Do a check when the kernel creates a new BPF map. This is also the + * point where LSM blob is allocated for LSMs that need them. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_map_alloc(struct bpf_map *map) +int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { - return call_int_hook(bpf_map_alloc_security, 0, map); + return call_int_hook(bpf_map_create, 0, map, attr, token); } /** @@ -5205,7 +5209,7 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, */ void security_bpf_map_free(struct bpf_map *map) { - call_void_hook(bpf_map_free_security, map); + call_void_hook(bpf_map_free, map); } /** diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index eabee39e983c..002351ab67b7 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6783,7 +6783,8 @@ static int selinux_bpf_prog(struct bpf_prog *prog) BPF__PROG_RUN, NULL); } -static int selinux_bpf_map_alloc(struct bpf_map *map) +static int selinux_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, + struct bpf_token *token) { struct bpf_security_struct *bpfsec; @@ -7180,7 +7181,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf, selinux_bpf), LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), - LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), + LSM_HOOK_INIT(bpf_map_free, selinux_bpf_map_free), LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), #endif @@ -7238,7 +7239,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(audit_rule_init, selinux_audit_rule_init), #endif #ifdef CONFIG_BPF_SYSCALL - LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), + LSM_HOOK_INIT(bpf_map_create, selinux_bpf_map_create), LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), #endif #ifdef CONFIG_PERF_EVENTS -- cgit v1.2.3 From d734ca7b33dbf60eb15dcf7c44f3da7073356777 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 30 Nov 2023 10:52:23 -0800 Subject: bpf,lsm: add BPF token LSM hooks Wire up bpf_token_create and bpf_token_free LSM hooks, which allow to allocate LSM security blob (we add `void *security` field to struct bpf_token for that), but also control who can instantiate BPF token. This follows existing pattern for BPF map and BPF prog. Also add security_bpf_token_allow_cmd() and security_bpf_token_capable() LSM hooks that allow LSM implementation to control and negate (if necessary) BPF token's delegation of a specific bpf_cmd and capability, respectively. Acked-by: Paul Moore Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231130185229.2688956-12-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 3 +++ include/linux/lsm_hook_defs.h | 5 ++++ include/linux/security.h | 25 ++++++++++++++++++ kernel/bpf/bpf_lsm.c | 4 +++ kernel/bpf/token.c | 18 ++++++++----- security/security.c | 60 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 435abad3cc61..7a483f6b6d5f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1604,6 +1604,9 @@ struct bpf_token { u64 allowed_maps; u64 allowed_progs; u64 allowed_attachs; +#ifdef CONFIG_SECURITY + void *security; +#endif }; struct bpf_struct_ops_value; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index adb25cc63ce3..3fdd00b452ac 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -404,6 +404,11 @@ LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token) LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) +LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr, + struct path *path) +LSM_HOOK(void, LSM_RET_VOID, bpf_token_free, struct bpf_token *token) +LSM_HOOK(int, 0, bpf_token_cmd, const struct bpf_token *token, enum bpf_cmd cmd) +LSM_HOOK(int, 0, bpf_token_capable, const struct bpf_token *token, int cap) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/security.h b/include/linux/security.h index 08fd777cbe94..00809d2d5c38 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -32,6 +32,7 @@ #include #include #include +#include struct linux_binprm; struct cred; @@ -2031,6 +2032,11 @@ extern void security_bpf_map_free(struct bpf_map *map); extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token); extern void security_bpf_prog_free(struct bpf_prog *prog); +extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path); +extern void security_bpf_token_free(struct bpf_token *token); +extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd); +extern int security_bpf_token_capable(const struct bpf_token *token, int cap); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2065,6 +2071,25 @@ static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr * static inline void security_bpf_prog_free(struct bpf_prog *prog) { } + +static inline int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + return 0; +} + +static inline void security_bpf_token_free(struct bpf_token *token) +{ } + +static inline int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return 0; +} + +static inline int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return 0; +} #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 9e4e615f11eb..7d2f96413a57 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -265,6 +265,10 @@ BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) BTF_ID(func, bpf_lsm_bpf_prog_free) +BTF_ID(func, bpf_lsm_bpf_token_create) +BTF_ID(func, bpf_lsm_bpf_token_free) +BTF_ID(func, bpf_lsm_bpf_token_cmd) +BTF_ID(func, bpf_lsm_bpf_token_capable) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index 5a51e6b8f6bf..17212efcde60 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -7,6 +7,7 @@ #include #include #include +#include bool bpf_token_capable(const struct bpf_token *token, int cap) { @@ -14,10 +15,9 @@ bool bpf_token_capable(const struct bpf_token *token, int cap) * token's userns is *exactly* the same as current user's userns */ if (token && current_user_ns() == token->userns) { - if (ns_capable(token->userns, cap)) - return true; - if (cap != CAP_SYS_ADMIN && ns_capable(token->userns, CAP_SYS_ADMIN)) - return true; + if (ns_capable(token->userns, cap) || + (cap != CAP_SYS_ADMIN && ns_capable(token->userns, CAP_SYS_ADMIN))) + return security_bpf_token_capable(token, cap) == 0; } /* otherwise fallback to capable() checks */ return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); @@ -30,6 +30,7 @@ void bpf_token_inc(struct bpf_token *token) static void bpf_token_free(struct bpf_token *token) { + security_bpf_token_free(token); put_user_ns(token->userns); kvfree(token); } @@ -186,6 +187,10 @@ int bpf_token_create(union bpf_attr *attr) token->allowed_progs = mnt_opts->delegate_progs; token->allowed_attachs = mnt_opts->delegate_attachs; + err = security_bpf_token_create(token, attr, &path); + if (err) + goto out_token; + fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) { err = fd; @@ -233,8 +238,9 @@ bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) */ if (!token || current_user_ns() != token->userns) return false; - - return token->allowed_cmds & (1ULL << cmd); + if (!(token->allowed_cmds & (1ULL << cmd))) + return false; + return security_bpf_token_cmd(token, cmd) == 0; } bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) diff --git a/security/security.c b/security/security.c index ad24cf36da94..088a79c35c26 100644 --- a/security/security.c +++ b/security/security.c @@ -5201,6 +5201,55 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, return call_int_hook(bpf_prog_load, 0, prog, attr, token); } +/** + * security_bpf_token_create() - Check if creating of BPF token is allowed + * @token: BPF token object + * @attr: BPF syscall attributes used to create BPF token + * @path: path pointing to BPF FS mount point from which BPF token is created + * + * Do a check when the kernel instantiates a new BPF token object from BPF FS + * instance. This is also the point where LSM blob can be allocated for LSMs. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, + struct path *path) +{ + return call_int_hook(bpf_token_create, 0, token, attr, path); +} + +/** + * security_bpf_token_cmd() - Check if BPF token is allowed to delegate + * requested BPF syscall command + * @token: BPF token object + * @cmd: BPF syscall command requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF syscall command. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +{ + return call_int_hook(bpf_token_cmd, 0, token, cmd); +} + +/** + * security_bpf_token_capable() - Check if BPF token is allowed to delegate + * requested BPF-related capability + * @token: BPF token object + * @cap: capabilities requested to be delegated by BPF token + * + * Do a check when the kernel decides whether provided BPF token should allow + * delegation of requested BPF-related capabilities. + * + * Return: Returns 0 on success, error on failure. + */ +int security_bpf_token_capable(const struct bpf_token *token, int cap) +{ + return call_int_hook(bpf_token_capable, 0, token, cap); +} + /** * security_bpf_map_free() - Free a bpf map's LSM blob * @map: bpf map @@ -5222,6 +5271,17 @@ void security_bpf_prog_free(struct bpf_prog *prog) { call_void_hook(bpf_prog_free, prog); } + +/** + * security_bpf_token_free() - Free a BPF token's LSM blob + * @token: BPF token struct + * + * Clean up the security information stored inside BPF token. + */ +void security_bpf_token_free(struct bpf_token *token) +{ + call_void_hook(bpf_token_free, token); +} #endif /* CONFIG_BPF_SYSCALL */ /** -- cgit v1.2.3 From f08a1c658257c73697a819c4ded3a84b6f0ead74 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 6 Dec 2023 14:40:48 -0800 Subject: bpf: Let bpf_prog_pack_free handle any pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, bpf_prog_pack_free only can only free pointer to struct bpf_binary_header, which is not flexible. Add a size argument to bpf_prog_pack_free so that it can handle any pointer. Signed-off-by: Song Liu Acked-by: Ilya Leoshkevich Tested-by: Ilya Leoshkevich # on s390x Reviewed-by: Björn Töpel Acked-by: Jiri Olsa Link: https://lore.kernel.org/r/20231206224054.492250-2-song@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/filter.h | 2 +- kernel/bpf/core.c | 21 ++++++++++----------- kernel/bpf/dispatcher.c | 5 +---- 3 files changed, 12 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 14354605ad26..12d907f17d36 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1067,7 +1067,7 @@ struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); -void bpf_prog_pack_free(struct bpf_binary_header *hdr); +void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) { diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index ced511f44174..c34513d645c4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -928,20 +928,20 @@ out: return ptr; } -void bpf_prog_pack_free(struct bpf_binary_header *hdr) +void bpf_prog_pack_free(void *ptr, u32 size) { struct bpf_prog_pack *pack = NULL, *tmp; unsigned int nbits; unsigned long pos; mutex_lock(&pack_mutex); - if (hdr->size > BPF_PROG_PACK_SIZE) { - bpf_jit_free_exec(hdr); + if (size > BPF_PROG_PACK_SIZE) { + bpf_jit_free_exec(ptr); goto out; } list_for_each_entry(tmp, &pack_list, list) { - if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) { + if (ptr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > ptr) { pack = tmp; break; } @@ -950,10 +950,10 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) if (WARN_ONCE(!pack, "bpf_prog_pack bug\n")) goto out; - nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); - pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; + nbits = BPF_PROG_SIZE_TO_NBITS(size); + pos = ((unsigned long)ptr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; - WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), + WARN_ONCE(bpf_arch_text_invalidate(ptr, size), "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); @@ -1100,8 +1100,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, *rw_header = kvmalloc(size, GFP_KERNEL); if (!*rw_header) { - bpf_arch_text_copy(&ro_header->size, &size, sizeof(size)); - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, size); bpf_jit_uncharge_modmem(size); return NULL; } @@ -1132,7 +1131,7 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, kvfree(rw_header); if (IS_ERR(ptr)) { - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, ro_header->size); return PTR_ERR(ptr); } return 0; @@ -1153,7 +1152,7 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, { u32 size = ro_header->size; - bpf_prog_pack_free(ro_header); + bpf_prog_pack_free(ro_header, size); kvfree(rw_header); bpf_jit_uncharge_modmem(size); } diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index fa3e9225aedc..56760fc10e78 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -150,10 +150,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); if (!d->rw_image) { - u32 size = PAGE_SIZE; - - bpf_arch_text_copy(d->image, &size, sizeof(size)); - bpf_prog_pack_free((struct bpf_binary_header *)d->image); + bpf_prog_pack_free(d->image, PAGE_SIZE); d->image = NULL; goto out; } -- cgit v1.2.3 From 7a3d9a159b178e87306a6e989071ed9a114a1a31 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 6 Dec 2023 14:40:49 -0800 Subject: bpf: Adjust argument names of arch_prepare_bpf_trampoline() We are using "im" for "struct bpf_tramp_image" and "tr" for "struct bpf_trampoline" in most of the code base. The only exception is the prototype and fallback version of arch_prepare_bpf_trampoline(). Update them to match the rest of the code base. We mix "orig_call" and "func_addr" for the argument in different versions of arch_prepare_bpf_trampoline(). s/orig_call/func_addr/g so they match. Signed-off-by: Song Liu Acked-by: Ilya Leoshkevich Tested-by: Ilya Leoshkevich # on s390x Acked-by: Jiri Olsa Link: https://lore.kernel.org/r/20231206224054.492250-3-song@kernel.org Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 10 +++++----- include/linux/bpf.h | 4 ++-- kernel/bpf/trampoline.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 7d4af64e3982..d81b886ea4df 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1828,7 +1828,7 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nregs) * */ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, - struct bpf_tramp_links *tlinks, void *orig_call, + struct bpf_tramp_links *tlinks, void *func_addr, int nregs, u32 flags) { int i; @@ -1926,7 +1926,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_IP_ARG) { /* save ip address of the traced function */ - emit_addr_mov_i64(A64_R(10), (const u64)orig_call, ctx); + emit_addr_mov_i64(A64_R(10), (const u64)func_addr, ctx); emit(A64_STR64I(A64_R(10), A64_SP, ip_off), ctx); } @@ -2029,7 +2029,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call) + void *func_addr) { int i, ret; int nregs = m->nr_args; @@ -2050,7 +2050,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, if (nregs > 8) return -ENOTSUPP; - ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nregs, flags); + ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); if (ret < 0) return ret; @@ -2061,7 +2061,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, ctx.idx = 0; jit_fill_hole(image, (unsigned int)(image_end - image)); - ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nregs, flags); + ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); if (ret > 0 && validate_code(&ctx) < 0) ret = -EINVAL; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7a483f6b6d5f..17eb6d905204 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1098,10 +1098,10 @@ struct bpf_tramp_run_ctx; * fexit = a set of program to run after original function */ struct bpf_tramp_image; -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call); + void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e97aeda3a86b..e114a1c7961e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1032,10 +1032,10 @@ bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) } int __weak -arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, +arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call) + void *func_addr) { return -ENOTSUPP; } -- cgit v1.2.3 From 82583daa2efc2e336962b231a46bad03a280b3e0 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 6 Dec 2023 14:40:50 -0800 Subject: bpf: Add helpers for trampoline image management As BPF trampoline of different archs moves from bpf_jit_[alloc|free]_exec() to bpf_prog_pack_[alloc|free](), we need to use different _alloc, _free for different archs during the transition. Add the following helpers for this transition: void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); void arch_protect_bpf_trampoline(void *image, unsigned int size); void arch_unprotect_bpf_trampoline(void *image, unsigned int size); The fallback version of these helpers require size <= PAGE_SIZE, but they are only called with size == PAGE_SIZE. They will be called with size < PAGE_SIZE when arch_bpf_trampoline_size() helper is introduced later. Signed-off-by: Song Liu Acked-by: Ilya Leoshkevich Tested-by: Ilya Leoshkevich # on s390x Acked-by: Jiri Olsa Link: https://lore.kernel.org/r/20231206224054.492250-4-song@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 5 +++++ kernel/bpf/bpf_struct_ops.c | 12 +++++------ kernel/bpf/trampoline.c | 46 +++++++++++++++++++++++++++++++++++------- net/bpf/bpf_dummy_struct_ops.c | 7 +++---- 4 files changed, 52 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 17eb6d905204..b7fca151cf1b 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1102,6 +1102,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr); +void *arch_alloc_bpf_trampoline(unsigned int size); +void arch_free_bpf_trampoline(void *image, unsigned int size); +void arch_protect_bpf_trampoline(void *image, unsigned int size); +void arch_unprotect_bpf_trampoline(void *image, unsigned int size); + u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index db6176fb64dc..e9e95879bce2 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -515,7 +515,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, if (err) goto reset_unlock; } - set_memory_rox((long)st_map->image, 1); + arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); /* Let bpf_link handle registration & unregistration. * * Pair with smp_load_acquire() during lookup_elem(). @@ -524,7 +524,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, goto unlock; } - set_memory_rox((long)st_map->image, 1); + arch_protect_bpf_trampoline(st_map->image, PAGE_SIZE); err = st_ops->reg(kdata); if (likely(!err)) { /* This refcnt increment on the map here after @@ -547,8 +547,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, * there was a race in registering the struct_ops (under the same name) to * a sub-system through different struct_ops's maps. */ - set_memory_nx((long)st_map->image, 1); - set_memory_rw((long)st_map->image, 1); + arch_unprotect_bpf_trampoline(st_map->image, PAGE_SIZE); reset_unlock: bpf_struct_ops_map_put_progs(st_map); @@ -616,7 +615,7 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_put_progs(st_map); bpf_map_area_free(st_map->links); if (st_map->image) { - bpf_jit_free_exec(st_map->image); + arch_free_bpf_trampoline(st_map->image, PAGE_SIZE); bpf_jit_uncharge_modmem(PAGE_SIZE); } bpf_map_area_free(st_map->uvalue); @@ -691,7 +690,7 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) return ERR_PTR(ret); } - st_map->image = bpf_jit_alloc_exec(PAGE_SIZE); + st_map->image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!st_map->image) { /* __bpf_struct_ops_map_free() uses st_map->image as flag * for "charged or not". In this case, we need to unchange @@ -711,7 +710,6 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) } mutex_init(&st_map->lock); - set_vm_flush_reset_perms(st_map->image); bpf_map_init_from_attr(map, attr); return map; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index e114a1c7961e..affbcbf7e76e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -254,7 +254,7 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); - bpf_jit_free_exec(im->image); + arch_free_bpf_trampoline(im->image, PAGE_SIZE); bpf_jit_uncharge_modmem(PAGE_SIZE); percpu_ref_exit(&im->pcref); kfree_rcu(im, rcu); @@ -365,10 +365,9 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) goto out_free_im; err = -ENOMEM; - im->image = image = bpf_jit_alloc_exec(PAGE_SIZE); + im->image = image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!image) goto out_uncharge; - set_vm_flush_reset_perms(image); err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL); if (err) @@ -381,7 +380,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) return im; out_free_image: - bpf_jit_free_exec(im->image); + arch_free_bpf_trampoline(im->image, PAGE_SIZE); out_uncharge: bpf_jit_uncharge_modmem(PAGE_SIZE); out_free_im: @@ -444,7 +443,7 @@ again: if (err < 0) goto out_free; - set_memory_rox((long)im->image, 1); + arch_protect_bpf_trampoline(im->image, PAGE_SIZE); WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) @@ -465,8 +464,7 @@ again: tr->fops->trampoline = 0; /* reset im->image memory attr for arch_prepare_bpf_trampoline */ - set_memory_nx((long)im->image, 1); - set_memory_rw((long)im->image, 1); + arch_unprotect_bpf_trampoline(im->image, PAGE_SIZE); goto again; } #endif @@ -1040,6 +1038,40 @@ arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image return -ENOTSUPP; } +void * __weak arch_alloc_bpf_trampoline(unsigned int size) +{ + void *image; + + if (WARN_ON_ONCE(size > PAGE_SIZE)) + return NULL; + image = bpf_jit_alloc_exec(PAGE_SIZE); + if (image) + set_vm_flush_reset_perms(image); + return image; +} + +void __weak arch_free_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + /* bpf_jit_free_exec doesn't need "size", but + * bpf_prog_pack_free() needs it. + */ + bpf_jit_free_exec(image); +} + +void __weak arch_protect_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + set_memory_rox((long)image, 1); +} + +void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size) +{ + WARN_ON_ONCE(size > PAGE_SIZE); + set_memory_nx((long)image, 1); + set_memory_rw((long)image, 1); +} + static int __init init_trampolines(void) { int i; diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 5918d1b32e19..2748f9d77b18 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -101,12 +101,11 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, goto out; } - image = bpf_jit_alloc_exec(PAGE_SIZE); + image = arch_alloc_bpf_trampoline(PAGE_SIZE); if (!image) { err = -ENOMEM; goto out; } - set_vm_flush_reset_perms(image); link = kzalloc(sizeof(*link), GFP_USER); if (!link) { @@ -124,7 +123,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, if (err < 0) goto out; - set_memory_rox((long)image, 1); + arch_protect_bpf_trampoline(image, PAGE_SIZE); prog_ret = dummy_ops_call_op(image, args); err = dummy_ops_copy_args(args); @@ -134,7 +133,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -EFAULT; out: kfree(args); - bpf_jit_free_exec(image); + arch_free_bpf_trampoline(image, PAGE_SIZE); if (link) bpf_link_put(&link->link); kfree(tlinks); -- cgit v1.2.3 From 96d1b7c081c0c96cbe8901045f4ff15a2e9974a2 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 6 Dec 2023 14:40:52 -0800 Subject: bpf: Add arch_bpf_trampoline_size() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helper will be used to calculate the size of the trampoline before allocating the memory. arch_prepare_bpf_trampoline() for arm64 and riscv64 can use arch_bpf_trampoline_size() to check the trampoline fits in the image. OTOH, arch_prepare_bpf_trampoline() for s390 has to call the JIT process twice, so it cannot use arch_bpf_trampoline_size(). Signed-off-by: Song Liu Acked-by: Ilya Leoshkevich Tested-by: Ilya Leoshkevich # on s390x Acked-by: Jiri Olsa Acked-by: Björn Töpel Tested-by: Björn Töpel # on riscv Link: https://lore.kernel.org/r/20231206224054.492250-6-song@kernel.org Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 56 ++++++++++++++++++++++++++++++----------- arch/riscv/net/bpf_jit_comp64.c | 22 ++++++++++++---- arch/s390/net/bpf_jit_comp.c | 56 +++++++++++++++++++++++++---------------- arch/x86/net/bpf_jit_comp.c | 40 ++++++++++++++++++++++++++--- include/linux/bpf.h | 2 ++ kernel/bpf/trampoline.c | 6 +++++ 6 files changed, 136 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index d81b886ea4df..a6671253b7ed 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2026,18 +2026,10 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, return ctx->idx; } -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, - void *image_end, const struct btf_func_model *m, - u32 flags, struct bpf_tramp_links *tlinks, - void *func_addr) +static int btf_func_model_nregs(const struct btf_func_model *m) { - int i, ret; int nregs = m->nr_args; - int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; - struct jit_ctx ctx = { - .image = NULL, - .idx = 0, - }; + int i; /* extra registers needed for struct argument */ for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { @@ -2046,19 +2038,53 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, nregs += (m->arg_size[i] + 7) / 8 - 1; } + return nregs; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct jit_ctx ctx = { + .image = NULL, + .idx = 0, + }; + struct bpf_tramp_image im; + int nregs, ret; + + nregs = btf_func_model_nregs(m); /* the first 8 registers are used for arguments */ if (nregs > 8) return -ENOTSUPP; - ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); + ret = prepare_trampoline(&ctx, &im, tlinks, func_addr, nregs, flags); if (ret < 0) return ret; - if (ret > max_insns) - return -EFBIG; + return ret < 0 ? ret : ret * AARCH64_INSN_SIZE; +} - ctx.image = image; - ctx.idx = 0; +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, + void *image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *func_addr) +{ + int ret, nregs; + struct jit_ctx ctx = { + .image = image, + .idx = 0, + }; + + nregs = btf_func_model_nregs(m); + /* the first 8 registers are used for arguments */ + if (nregs > 8) + return -ENOTSUPP; + + ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); + if (ret < 0) + return ret; + + if (ret > ((long)image_end - (long)image)) + return -EFBIG; jit_fill_hole(image, (unsigned int)(image_end - image)); ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 8581693e62d3..35747fafde57 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1029,6 +1029,21 @@ out: return ret; } +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + struct rv_jit_context ctx; + int ret; + + ctx.ninsns = 0; + ctx.insns = NULL; + ctx.ro_insns = NULL; + ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); + + return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, @@ -1037,14 +1052,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, int ret; struct rv_jit_context ctx; - ctx.ninsns = 0; - ctx.insns = NULL; - ctx.ro_insns = NULL; - ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); + ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr); if (ret < 0) return ret; - if (ninsns_rvoff(ret) > (long)image_end - (long)image) + if (ret > (long)image_end - (long)image) return -EFBIG; ctx.ninsns = 0; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index bf06b7283f0c..cc129617480a 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2637,6 +2637,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, return 0; } +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *orig_call) +{ + struct bpf_tramp_image im; + struct bpf_tramp_jit tjit; + int ret; + + memset(&tjit, 0, sizeof(tjit)); + + ret = __arch_prepare_bpf_trampoline(&im, &tjit, m, flags, + tlinks, orig_call); + + return ret < 0 ? ret : tjit.common.prg; +} + int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, @@ -2644,30 +2659,27 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, { struct bpf_tramp_jit tjit; int ret; - int i; - for (i = 0; i < 2; i++) { - if (i == 0) { - /* Compute offsets, check whether the code fits. */ - memset(&tjit, 0, sizeof(tjit)); - } else { - /* Generate the code. */ - tjit.common.prg = 0; - tjit.common.prg_buf = image; - } - ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, - tlinks, func_addr); - if (ret < 0) - return ret; - if (tjit.common.prg > (char *)image_end - (char *)image) - /* - * Use the same error code as for exceeding - * BPF_MAX_TRAMP_LINKS. - */ - return -E2BIG; - } + /* Compute offsets, check whether the code fits. */ + memset(&tjit, 0, sizeof(tjit)); + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, + tlinks, func_addr); + + if (ret < 0) + return ret; + if (tjit.common.prg > (char *)image_end - (char *)image) + /* + * Use the same error code as for exceeding + * BPF_MAX_TRAMP_LINKS. + */ + return -E2BIG; + + tjit.common.prg = 0; + tjit.common.prg_buf = image; + ret = __arch_prepare_bpf_trampoline(im, &tjit, m, flags, + tlinks, func_addr); - return tjit.common.prg; + return ret < 0 ? ret : tjit.common.prg; } bool bpf_jit_supports_subprog_tailcalls(void) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 5f7528cac344..5d75069fdcc2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2422,10 +2422,10 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, * add rsp, 8 // skip eth_type_trans's frame * ret // return to its caller */ -int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, - const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, - void *func_addr) +static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) { int i, ret, nr_regs = m->nr_args, stack_size = 0; int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; @@ -2678,6 +2678,38 @@ cleanup: return ret; } +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, + const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, + void *func_addr) +{ + return __arch_prepare_bpf_trampoline(im, image, image_end, m, flags, tlinks, func_addr); +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + void *image; + int ret; + + /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). + * This will NOT cause fragmentation in direct map, as we do not + * call set_memory_*() on this buffer. + * + * We cannot use kvmalloc here, because we need image to be in + * module memory range. + */ + image = bpf_jit_alloc_exec(PAGE_SIZE); + if (!image) + return -ENOMEM; + + ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, m, flags, + tlinks, func_addr); + bpf_jit_free_exec(image); + return ret; +} + static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) { u8 *jg_reloc, *prog = *pprog; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b7fca151cf1b..2332ddeb396b 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1106,6 +1106,8 @@ void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); void arch_protect_bpf_trampoline(void *image, unsigned int size); void arch_unprotect_bpf_trampoline(void *image, unsigned int size); +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index affbcbf7e76e..b553cbd89e55 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -1072,6 +1072,12 @@ void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size) set_memory_rw((long)image, 1); } +int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + return -ENOTSUPP; +} + static int __init init_trampolines(void) { int i; -- cgit v1.2.3 From 26ef208c209a0e6eed8942a5d191b39dccfa6e38 Mon Sep 17 00:00:00 2001 From: Song Liu Date: Wed, 6 Dec 2023 14:40:53 -0800 Subject: bpf: Use arch_bpf_trampoline_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of blindly allocating PAGE_SIZE for each trampoline, check the size of the trampoline with arch_bpf_trampoline_size(). This size is saved in bpf_tramp_image->size, and used for modmem charge/uncharge. The fallback arch_alloc_bpf_trampoline() still allocates a whole page because we need to use set_memory_* to protect the memory. struct_ops trampoline still uses a whole page for multiple trampolines. With this size check at caller (regular trampoline and struct_ops trampoline), remove arch_bpf_trampoline_size() from arch_prepare_bpf_trampoline() in archs. Also, update bpf_image_ksym_add() to handle symbol of different sizes. Signed-off-by: Song Liu Acked-by: Ilya Leoshkevich Tested-by: Ilya Leoshkevich # on s390x Acked-by: Jiri Olsa Acked-by: Björn Töpel Tested-by: Björn Töpel # on riscv Link: https://lore.kernel.org/r/20231206224054.492250-7-song@kernel.org Signed-off-by: Alexei Starovoitov --- arch/arm64/net/bpf_jit_comp.c | 7 ------ arch/riscv/net/bpf_jit_comp64.c | 7 ------ include/linux/bpf.h | 3 ++- kernel/bpf/bpf_struct_ops.c | 7 ++++++ kernel/bpf/dispatcher.c | 2 +- kernel/bpf/trampoline.c | 55 +++++++++++++++++++++++++---------------- 6 files changed, 44 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index a6671253b7ed..8955da5c47cf 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2079,13 +2079,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, if (nregs > 8) return -ENOTSUPP; - ret = arch_bpf_trampoline_size(m, flags, tlinks, func_addr); - if (ret < 0) - return ret; - - if (ret > ((long)image_end - (long)image)) - return -EFBIG; - jit_fill_hole(image, (unsigned int)(image_end - image)); ret = prepare_trampoline(&ctx, im, tlinks, func_addr, nregs, flags); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 35747fafde57..58dc64dd94a8 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1052,13 +1052,6 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, int ret; struct rv_jit_context ctx; - ret = arch_bpf_trampoline_size(im, m, flags, tlinks, func_addr); - if (ret < 0) - return ret; - - if (ret > (long)image_end - (long)image) - return -EFBIG; - ctx.ninsns = 0; /* * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2332ddeb396b..c1a06263a4f3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1141,6 +1141,7 @@ enum bpf_tramp_prog_type { struct bpf_tramp_image { void *image; + int size; struct bpf_ksym ksym; struct percpu_ref pcref; void *ip_after_call; @@ -1325,7 +1326,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, struct bpf_prog *to); /* Called only from JIT-enabled code, so there's no need for stubs. */ -void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); +void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym); void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index e9e95879bce2..4d53c53fc5aa 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -355,6 +355,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, void *image, void *image_end) { u32 flags; + int size; tlinks[BPF_TRAMP_FENTRY].links[0] = link; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; @@ -362,6 +363,12 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, * and it must be used alone. */ flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; + + size = arch_bpf_trampoline_size(model, flags, tlinks, NULL); + if (size < 0) + return size; + if (size > (unsigned long)image_end - (unsigned long)image) + return -E2BIG; return arch_prepare_bpf_trampoline(NULL, image, image_end, model, flags, tlinks, NULL); } diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index 56760fc10e78..70fb82bf1637 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -154,7 +154,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, d->image = NULL; goto out; } - bpf_image_ksym_add(d->image, &d->ksym); + bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym); } prev_num_progs = d->num_progs; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index b553cbd89e55..d382f5ebe06c 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -115,10 +115,10 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC); } -void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym) +void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym) { ksym->start = (unsigned long) data; - ksym->end = ksym->start + PAGE_SIZE; + ksym->end = ksym->start + size; bpf_ksym_add(ksym); perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start, PAGE_SIZE, false, ksym->name); @@ -254,8 +254,8 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); - arch_free_bpf_trampoline(im->image, PAGE_SIZE); - bpf_jit_uncharge_modmem(PAGE_SIZE); + arch_free_bpf_trampoline(im->image, im->size); + bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); kfree_rcu(im, rcu); } @@ -349,7 +349,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -360,12 +360,13 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) if (!im) goto out; - err = bpf_jit_charge_modmem(PAGE_SIZE); + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; + im->size = size; err = -ENOMEM; - im->image = image = arch_alloc_bpf_trampoline(PAGE_SIZE); + im->image = image = arch_alloc_bpf_trampoline(size); if (!image) goto out_uncharge; @@ -376,13 +377,13 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key) ksym = &im->ksym; INIT_LIST_HEAD_RCU(&ksym->lnode); snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key); - bpf_image_ksym_add(image, ksym); + bpf_image_ksym_add(image, size, ksym); return im; out_free_image: - arch_free_bpf_trampoline(im->image, PAGE_SIZE); + arch_free_bpf_trampoline(im->image, im->size); out_uncharge: - bpf_jit_uncharge_modmem(PAGE_SIZE); + bpf_jit_uncharge_modmem(size); out_free_im: kfree(im); out: @@ -395,7 +396,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut struct bpf_tramp_links *tlinks; u32 orig_flags = tr->flags; bool ip_arg = false; - int err, total; + int err, total, size; tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg); if (IS_ERR(tlinks)) @@ -408,12 +409,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut goto out; } - im = bpf_tramp_image_alloc(tr->key); - if (IS_ERR(im)) { - err = PTR_ERR(im); - goto out; - } - /* clear all bits except SHARE_IPMODIFY and TAIL_CALL_CTX */ tr->flags &= (BPF_TRAMP_F_SHARE_IPMODIFY | BPF_TRAMP_F_TAIL_CALL_CTX); @@ -437,13 +432,31 @@ again: tr->flags |= BPF_TRAMP_F_ORIG_STACK; #endif - err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE, + size = arch_bpf_trampoline_size(&tr->func.model, tr->flags, + tlinks, tr->func.addr); + if (size < 0) { + err = size; + goto out; + } + + if (size > PAGE_SIZE) { + err = -E2BIG; + goto out; + } + + im = bpf_tramp_image_alloc(tr->key, size); + if (IS_ERR(im)) { + err = PTR_ERR(im); + goto out; + } + + err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tlinks, tr->func.addr); if (err < 0) goto out_free; - arch_protect_bpf_trampoline(im->image, PAGE_SIZE); + arch_protect_bpf_trampoline(im->image, im->size); WARN_ON(tr->cur_image && total == 0); if (tr->cur_image) @@ -463,8 +476,8 @@ again: tr->fops->func = NULL; tr->fops->trampoline = 0; - /* reset im->image memory attr for arch_prepare_bpf_trampoline */ - arch_unprotect_bpf_trampoline(im->image, PAGE_SIZE); + /* free im memory and reallocate later */ + bpf_tramp_image_free(im); goto again; } #endif -- cgit v1.2.3 From 3bc05faf37876f99e2a7baffa9c66fdcfb11d1f7 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Tue, 5 Dec 2023 17:42:30 +0100 Subject: net: dsa: microchip: properly support platform_data probing The ksz driver has bits and pieces of platform_data probing support, but it doesn't work. The conventional thing to do is to have an encapsulating structure for struct dsa_chip_data that gets put into dev->platform_data. This driver expects a struct ksz_platform_data, but that doesn't contain a struct dsa_chip_data as first element, which will obviously not work with dsa_switch_probe() -> dsa_switch_parse(). Pointing dev->platform_data to a struct dsa_chip_data directly is in principle possible, but that doesn't work either. The driver has ksz_switch_detect() to read the device ID from hardware, followed by ksz_check_device_id() to compare it against a predetermined expected value. This protects against early errors in the SPI/I2C communication. With platform_data, the mechanism in ksz_check_device_id() doesn't work and even leads to NULL pointer dereferences, since of_device_get_match_data() doesn't work in that probe path. So obviously, the platform_data support is actually missing, and the existing handling of struct ksz_platform_data is bogus. Complete the support by adding a struct dsa_chip_data as first element, and fixing up ksz_check_device_id() to pick up the platform_data instead of the unavailable of_device_get_match_data(). The early dev->chip_id assignment from ksz_switch_register() is also bogus, because ksz_switch_detect() sets it to an initial value. So remove it. Also, ksz_platform_data :: enabled_ports isn't used anywhere, delete it. Link: https://lore.kernel.org/netdev/20231204154315.3906267-1-dd@embedd.com/ Signed-off-by: Vladimir Oltean Signed-off-by: Daniel Danzberger Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/dsa/microchip/ksz_common.c | 21 +++++++++++++-------- include/linux/platform_data/microchip-ksz.h | 4 +++- 2 files changed, 16 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 9545aed905f5..db1bbcf3a5f2 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -1673,15 +1673,23 @@ static const struct ksz_chip_data *ksz_lookup_info(unsigned int prod_num) static int ksz_check_device_id(struct ksz_device *dev) { - const struct ksz_chip_data *dt_chip_data; + const struct ksz_chip_data *expected_chip_data; + u32 expected_chip_id; - dt_chip_data = of_device_get_match_data(dev->dev); + if (dev->pdata) { + expected_chip_id = dev->pdata->chip_id; + expected_chip_data = ksz_lookup_info(expected_chip_id); + if (WARN_ON(!expected_chip_data)) + return -ENODEV; + } else { + expected_chip_data = of_device_get_match_data(dev->dev); + expected_chip_id = expected_chip_data->chip_id; + } - /* Check for Device Tree and Chip ID */ - if (dt_chip_data->chip_id != dev->chip_id) { + if (expected_chip_id != dev->chip_id) { dev_err(dev->dev, "Device tree specifies chip %s but found %s, please fix it!\n", - dt_chip_data->dev_name, dev->info->dev_name); + expected_chip_data->dev_name, dev->info->dev_name); return -ENODEV; } @@ -4156,9 +4164,6 @@ int ksz_switch_register(struct ksz_device *dev) int ret; int i; - if (dev->pdata) - dev->chip_id = dev->pdata->chip_id; - dev->reset_gpio = devm_gpiod_get_optional(dev->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(dev->reset_gpio)) diff --git a/include/linux/platform_data/microchip-ksz.h b/include/linux/platform_data/microchip-ksz.h index ea1cc6d829e9..6480bf4af0fb 100644 --- a/include/linux/platform_data/microchip-ksz.h +++ b/include/linux/platform_data/microchip-ksz.h @@ -20,10 +20,12 @@ #define __MICROCHIP_KSZ_H #include +#include struct ksz_platform_data { + /* Must be first such that dsa_register_switch() can access it */ + struct dsa_chip_data cd; u32 chip_id; - u16 enabled_ports; }; #endif -- cgit v1.2.3 From d16f1096b320d42e41ad9dee4d4098afd140d3e1 Mon Sep 17 00:00:00 2001 From: Daniel Danzberger Date: Tue, 5 Dec 2023 17:42:31 +0100 Subject: net: dsa: microchip: move ksz_chip_id enum to platform include With the ksz_chip_id enums moved to the platform include file for ksz switches, platform code that instantiates a device can now use these to set ksz_platform_data::chip_id. Signed-off-by: Daniel Danzberger Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/dsa/microchip/ksz_common.h | 20 +------------------- include/linux/platform_data/microchip-ksz.h | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index d1a1c876ba0d..15612101a155 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "ksz_ptp.h" @@ -203,25 +204,6 @@ enum ksz_model { LAN9374, }; -enum ksz_chip_id { - KSZ8563_CHIP_ID = 0x8563, - KSZ8795_CHIP_ID = 0x8795, - KSZ8794_CHIP_ID = 0x8794, - KSZ8765_CHIP_ID = 0x8765, - KSZ8830_CHIP_ID = 0x8830, - KSZ9477_CHIP_ID = 0x00947700, - KSZ9896_CHIP_ID = 0x00989600, - KSZ9897_CHIP_ID = 0x00989700, - KSZ9893_CHIP_ID = 0x00989300, - KSZ9563_CHIP_ID = 0x00956300, - KSZ9567_CHIP_ID = 0x00956700, - LAN9370_CHIP_ID = 0x00937000, - LAN9371_CHIP_ID = 0x00937100, - LAN9372_CHIP_ID = 0x00937200, - LAN9373_CHIP_ID = 0x00937300, - LAN9374_CHIP_ID = 0x00937400, -}; - enum ksz_regs { REG_SW_MAC_ADDR, REG_IND_CTRL_0, diff --git a/include/linux/platform_data/microchip-ksz.h b/include/linux/platform_data/microchip-ksz.h index 6480bf4af0fb..f177416635a2 100644 --- a/include/linux/platform_data/microchip-ksz.h +++ b/include/linux/platform_data/microchip-ksz.h @@ -22,6 +22,25 @@ #include #include +enum ksz_chip_id { + KSZ8563_CHIP_ID = 0x8563, + KSZ8795_CHIP_ID = 0x8795, + KSZ8794_CHIP_ID = 0x8794, + KSZ8765_CHIP_ID = 0x8765, + KSZ8830_CHIP_ID = 0x8830, + KSZ9477_CHIP_ID = 0x00947700, + KSZ9896_CHIP_ID = 0x00989600, + KSZ9897_CHIP_ID = 0x00989700, + KSZ9893_CHIP_ID = 0x00989300, + KSZ9563_CHIP_ID = 0x00956300, + KSZ9567_CHIP_ID = 0x00956700, + LAN9370_CHIP_ID = 0x00937000, + LAN9371_CHIP_ID = 0x00937100, + LAN9372_CHIP_ID = 0x00937200, + LAN9373_CHIP_ID = 0x00937300, + LAN9374_CHIP_ID = 0x00937400, +}; + struct ksz_platform_data { /* Must be first such that dsa_register_switch() can access it */ struct dsa_chip_data cd; -- cgit v1.2.3 From 2a48c635fd9a48699805bbfeee1e4b94b8fe819d Mon Sep 17 00:00:00 2001 From: "justinstitt@google.com" Date: Wed, 6 Dec 2023 23:16:10 +0000 Subject: ethtool: Implement ethtool_puts() Use strscpy() to implement ethtool_puts(). Functionally the same as ethtool_sprintf() when it's used with two arguments or with just "%s" format specifier. Signed-off-by: Justin Stitt Reviewed-by: Przemek Kitszel Reviewed-by: Andrew Lunn Reviewed-by: Madhuri Sripada Signed-off-by: David S. Miller --- include/linux/ethtool.h | 13 +++++++++++++ net/ethtool/ioctl.c | 7 +++++++ 2 files changed, 20 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index c2bb74143eda..deb683d3360f 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -1061,6 +1061,19 @@ int ethtool_get_ts_info_by_layer(struct net_device *dev, struct ethtool_ts_info */ extern __printf(2, 3) void ethtool_sprintf(u8 **data, const char *fmt, ...); +/** + * ethtool_puts - Write string to ethtool string data + * @data: Pointer to a pointer to the start of string to update + * @str: String to write + * + * Write string to *data without a trailing newline. Update *data + * to point at start of next string. + * + * Prefer this function to ethtool_sprintf() when given only + * two arguments or if @fmt is just "%s". + */ +extern void ethtool_puts(u8 **data, const char *str); + /* Link mode to forced speed capabilities maps */ struct ethtool_forced_speed_map { u32 speed; diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index a977f8903467..755a4be01bef 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1994,6 +1994,13 @@ __printf(2, 3) void ethtool_sprintf(u8 **data, const char *fmt, ...) } EXPORT_SYMBOL(ethtool_sprintf); +void ethtool_puts(u8 **data, const char *str) +{ + strscpy(*data, str, ETH_GSTRING_LEN); + *data += ETH_GSTRING_LEN; +} +EXPORT_SYMBOL(ethtool_puts); + static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) { struct ethtool_value id; -- cgit v1.2.3 From 92e1567ee3e3f6f160e320890ac77eec50bf8e7d Mon Sep 17 00:00:00 2001 From: Andrei Matei Date: Thu, 7 Dec 2023 22:25:17 -0500 Subject: bpf: Add some comments to stack representation Add comments to the datastructure tracking the stack state, as the mapping between each stack slot and where its state is stored is not entirely obvious. Signed-off-by: Andrei Matei Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20231208032519.260451-2-andreimatei1@gmail.com --- include/linux/bpf_verifier.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bada59812e00..314b679fb494 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -321,7 +321,17 @@ struct bpf_func_state { /* The following fields should be last. See copy_func_state() */ int acquired_refs; struct bpf_reference_state *refs; + /* The state of the stack. Each element of the array describes BPF_REG_SIZE + * (i.e. 8) bytes worth of stack memory. + * stack[0] represents bytes [*(r10-8)..*(r10-1)] + * stack[1] represents bytes [*(r10-16)..*(r10-9)] + * ... + * stack[allocated_stack/8 - 1] represents [*(r10-allocated_stack)..*(r10-allocated_stack+7)] + */ struct bpf_stack_state *stack; + /* Size of the current stack, in bytes. The stack state is tracked below, in + * `stack`. allocated_stack is always a multiple of BPF_REG_SIZE. + */ int allocated_stack; }; @@ -658,6 +668,10 @@ struct bpf_verifier_env { int exception_callback_subprog; bool explore_alu_limits; bool allow_ptr_leaks; + /* Allow access to uninitialized stack memory. Writes with fixed offset are + * always allowed, so this refers to reads (with fixed or variable offset), + * to writes with variable offset and to indirect (helper) accesses. + */ bool allow_uninit_stack; bool bpf_capable; bool bypass_spec_v1; -- cgit v1.2.3 From 2ebe81c814355d000fe49d9c4213983844dcb32b Mon Sep 17 00:00:00 2001 From: Aleksander Lobakin Date: Wed, 6 Dec 2023 21:59:19 +0100 Subject: net, xdp: Allow metadata > 32 32 bytes may be not enough for some custom metadata. Relax the restriction, allow metadata larger than 32 bytes and make __skb_metadata_differs() work with bigger lengths. Now size of metadata is only limited by the fact it is stored as u8 in skb_shared_info, so maximum possible value is 255. Size still has to be aligned to 4, so the actual upper limit becomes 252. Most driver implementations will offer less, none can offer more. Other important conditions, such as having enough space for xdp_frame building, are already checked in bpf_xdp_adjust_meta(). Signed-off-by: Aleksander Lobakin Signed-off-by: Larysa Zaremba Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/eb87653c-8ff8-447d-a7a1-25961f60518a@kernel.org Link: https://lore.kernel.org/bpf/20231206205919.404415-3-larysa.zaremba@intel.com --- include/linux/skbuff.h | 13 ++++++++----- include/net/xdp.h | 7 ++++++- 2 files changed, 14 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b370eb8d70f7..df6ef42639d8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4247,10 +4247,13 @@ static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, { const void *a = skb_metadata_end(skb_a); const void *b = skb_metadata_end(skb_b); - /* Using more efficient varaiant than plain call to memcmp(). */ -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 u64 diffs = 0; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || + BITS_PER_LONG != 64) + goto slow; + + /* Using more efficient variant than plain call to memcmp(). */ switch (meta_len) { #define __it(x, op) (x -= sizeof(u##op)) #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op)) @@ -4270,11 +4273,11 @@ static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, fallthrough; case 4: diffs |= __it_diff(a, b, 32); break; + default: +slow: + return memcmp(a - meta_len, b - meta_len, meta_len); } return diffs; -#else - return memcmp(a - meta_len, b - meta_len, meta_len); -#endif } static inline bool skb_metadata_differs(const struct sk_buff *skb_a, diff --git a/include/net/xdp.h b/include/net/xdp.h index 349c36fb5fd8..5d3673afc037 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -369,7 +369,12 @@ xdp_data_meta_unsupported(const struct xdp_buff *xdp) static inline bool xdp_metalen_invalid(unsigned long metalen) { - return (metalen & (sizeof(__u32) - 1)) || (metalen > 32); + unsigned long meta_max; + + meta_max = type_max(typeof_member(struct skb_shared_info, meta_len)); + BUILD_BUG_ON(!__builtin_constant_p(meta_max)); + + return !IS_ALIGNED(metalen, sizeof(u32)) || metalen > meta_max; } struct xdp_attachment_info { -- cgit v1.2.3 From c5e2a973448d958feb7881e4d875eac59fdeff3d Mon Sep 17 00:00:00 2001 From: Jamal Hadi Salim Date: Fri, 8 Dec 2023 16:28:41 -0300 Subject: rtnl: add helper to check if rtnl group has listeners As of today, rtnl code creates a new skb and unconditionally fills and broadcasts it to the relevant group. For most operations this is okay and doesn't waste resources in general. When operations are done without the rtnl_lock, as in tc-flower, such skb allocation, message fill and no-op broadcasting can happen in all cores of the system, which contributes to system pressure and wastes precious cpu cycles when no one will receive the built message. Introduce this helper so rtnetlink operations can simply check if someone is listening and then proceed if necessary. Reviewed-by: Jiri Pirko Reviewed-by: Simon Horman Signed-off-by: Jamal Hadi Salim Signed-off-by: Victor Nogueira Signed-off-by: Pedro Tammela Link: https://lore.kernel.org/r/20231208192847.714940-2-pctammela@mojatatu.com Signed-off-by: Jakub Kicinski --- include/linux/rtnetlink.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 3d6cf306cd55..a7d757e96c55 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -130,4 +130,11 @@ extern int ndo_dflt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, extern void rtnl_offload_xstats_notify(struct net_device *dev); +static inline int rtnl_has_listeners(const struct net *net, u32 group) +{ + struct sock *rtnl = net->rtnl; + + return netlink_has_listeners(rtnl, group); +} + #endif /* __LINUX_RTNETLINK_H */ -- cgit v1.2.3 From 8439109b76a3c405808383bf9dd532fc4b9c2dbd Mon Sep 17 00:00:00 2001 From: Victor Nogueira Date: Fri, 8 Dec 2023 16:28:42 -0300 Subject: rtnl: add helper to check if a notification is needed Building on the rtnl_has_listeners helper, add the rtnl_notify_needed helper to check if we can bail out early in the notification routines. Reviewed-by: Jiri Pirko Reviewed-by: Simon Horman Signed-off-by: Victor Nogueira Signed-off-by: Pedro Tammela Link: https://lore.kernel.org/r/20231208192847.714940-3-pctammela@mojatatu.com Signed-off-by: Jakub Kicinski --- include/linux/rtnetlink.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index a7d757e96c55..0cbbbded0331 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -137,4 +137,19 @@ static inline int rtnl_has_listeners(const struct net *net, u32 group) return netlink_has_listeners(rtnl, group); } +/** + * rtnl_notify_needed - check if notification is needed + * @net: Pointer to the net namespace + * @nlflags: netlink ingress message flags + * @group: rtnl group + * + * Based on the ingress message flags and rtnl group, returns true + * if a notification is needed, false otherwise. + */ +static inline bool +rtnl_notify_needed(const struct net *net, u16 nlflags, u32 group) +{ + return (nlflags & NLM_F_ECHO) || rtnl_has_listeners(net, group); +} + #endif /* __LINUX_RTNETLINK_H */ -- cgit v1.2.3 From ddb6b284bdc32b6e218b3d90b5a745ea26620812 Mon Sep 17 00:00:00 2001 From: Pedro Tammela Date: Fri, 8 Dec 2023 16:28:43 -0300 Subject: rtnl: add helper to send if skb is not null This is a convenience helper for routines handling conditional rtnl events, that is code that might send a notification depending on rtnl_has_listeners/rtnl_notify_needed. Instead of: if (skb) rtnetlink_send(...) Use: rtnetlink_maybe_send(...) Reviewed-by: Jiri Pirko Reviewed-by: Simon Horman Signed-off-by: Pedro Tammela Link: https://lore.kernel.org/r/20231208192847.714940-4-pctammela@mojatatu.com Signed-off-by: Jakub Kicinski --- include/linux/rtnetlink.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 0cbbbded0331..6a8543b34e2c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -10,6 +10,13 @@ #include extern int rtnetlink_send(struct sk_buff *skb, struct net *net, u32 pid, u32 group, int echo); + +static inline int rtnetlink_maybe_send(struct sk_buff *skb, struct net *net, + u32 pid, u32 group, int echo) +{ + return !skb ? 0 : rtnetlink_send(skb, net, pid, group, echo); +} + extern int rtnl_unicast(struct sk_buff *skb, struct net *net, u32 pid); extern void rtnl_notify(struct sk_buff *skb, struct net *net, u32 pid, u32 group, const struct nlmsghdr *nlh, gfp_t flags); -- cgit v1.2.3 From 1a1ad782dcbbacd9e8d4e2e7ff1bf14d1db80727 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 4 Dec 2023 15:39:21 -0800 Subject: bpf: tidy up exception callback management a bit Use the fact that we are passing subprog index around and have a corresponding struct bpf_subprog_info in bpf_verifier_env for each subprogram. We don't need to separately pass around a flag whether subprog is exception callback or not, each relevant verifier function can determine this using provided subprog index if we maintain bpf_subprog_info properly. Also move out exception callback-specific logic from btf_prepare_func_args(), keeping it generic. We can enforce all these restriction right before exception callback verification pass. We add out parameter, arg_cnt, for now, but this will be unnecessary with subsequent refactoring and will be removed. Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20231204233931.49758-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 +- kernel/bpf/btf.c | 11 ++--------- kernel/bpf/verifier.c | 52 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 42 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c1a06263a4f3..0bd4889e917a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2494,7 +2494,7 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *reg, bool is_ex_cb); + struct bpf_reg_state *reg, u32 *nargs); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 63cf4128fc05..d56433bf8aba 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6956,7 +6956,7 @@ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, * (either PTR_TO_CTX or SCALAR_VALUE). */ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs, bool is_ex_cb) + struct bpf_reg_state *regs, u32 *arg_cnt) { struct bpf_verifier_log *log = &env->log; struct bpf_prog *prog = env->prog; @@ -7013,6 +7013,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, tname, nargs, MAX_BPF_FUNC_REG_ARGS); return -EINVAL; } + *arg_cnt = nargs; /* check that function returns int, exception cb also requires this */ t = btf_type_by_id(btf, t->type); while (btf_type_is_modifier(t)) @@ -7062,14 +7063,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, i, btf_type_str(t), tname); return -EINVAL; } - /* We have already ensured that the callback returns an integer, just - * like all global subprogs. We need to determine it only has a single - * scalar argument. - */ - if (is_ex_cb && (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE)) { - bpf_log(log, "exception cb only supports single integer argument\n"); - return -EINVAL; - } return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 727a59e4a647..d1755db1b503 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -442,6 +442,25 @@ static struct bpf_func_info_aux *subprog_aux(const struct bpf_verifier_env *env, return &env->prog->aux->func_info_aux[subprog]; } +static struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog) +{ + return &env->subprog_info[subprog]; +} + +static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_subprog_info *info = subprog_info(env, subprog); + + info->is_cb = true; + info->is_async_cb = true; + info->is_exception_cb = true; +} + +static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->is_exception_cb; +} + static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) { return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); @@ -2892,6 +2911,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) if (env->subprog_info[i].start != ex_cb_insn) continue; env->exception_callback_subprog = i; + mark_subprog_exc_cb(env, i); break; } } @@ -19166,9 +19186,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env) env->exception_callback_subprog = env->subprog_cnt - 1; /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ - env->subprog_info[env->exception_callback_subprog].is_cb = true; - env->subprog_info[env->exception_callback_subprog].is_async_cb = true; - env->subprog_info[env->exception_callback_subprog].is_exception_cb = true; + mark_subprog_exc_cb(env, env->exception_callback_subprog); } for (i = 0; i < insn_cnt; i++, insn++) { @@ -19868,7 +19886,7 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex_cb) +static int do_check_common(struct bpf_verifier_env *env, int subprog) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_verifier_state *state; @@ -19899,9 +19917,23 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - ret = btf_prepare_func_args(env, subprog, regs, is_ex_cb); + u32 nargs; + + ret = btf_prepare_func_args(env, subprog, regs, &nargs); if (ret) goto out; + if (subprog_is_exc_cb(env, subprog)) { + state->frame[0]->in_exception_callback_fn = true; + /* We have already ensured that the callback returns an integer, just + * like all global subprogs. We need to determine it only has a single + * scalar argument. + */ + if (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE) { + verbose(env, "exception cb only supports single integer argument\n"); + ret = -EINVAL; + goto out; + } + } for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (regs[i].type == PTR_TO_CTX) mark_reg_known_zero(env, regs, i); @@ -19915,12 +19947,6 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_ex regs[i].id = ++env->id_gen; } } - if (is_ex_cb) { - state->frame[0]->in_exception_callback_fn = true; - env->subprog_info[subprog].is_cb = true; - env->subprog_info[subprog].is_async_cb = true; - env->subprog_info[subprog].is_exception_cb = true; - } } else { /* 1st arg to a function */ regs[BPF_REG_1].type = PTR_TO_CTX; @@ -20000,7 +20026,7 @@ again: env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i, env->exception_callback_subprog == i); + ret = do_check_common(env, i); if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { @@ -20030,7 +20056,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0, false); + ret = do_check_common(env, 0); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; -- cgit v1.2.3 From 406a6fa44bfbc8563f0612b08d43df2fa65e8bc5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 4 Dec 2023 15:39:22 -0800 Subject: bpf: use bitfields for simple per-subprog bool flags We have a bunch of bool flags for each subprog. Instead of wasting bytes for them, use bitfields instead. Signed-off-by: Andrii Nakryiko Acked-by: Eduard Zingerman Link: https://lore.kernel.org/r/20231204233931.49758-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 314b679fb494..c2819a6579a5 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -611,12 +611,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u16 stack_depth; /* max. stack depth used by this function */ - bool has_tail_call; - bool tail_call_reachable; - bool has_ld_abs; - bool is_cb; - bool is_async_cb; - bool is_exception_cb; + bool has_tail_call: 1; + bool tail_call_reachable: 1; + bool has_ld_abs: 1; + bool is_cb: 1; + bool is_async_cb: 1; + bool is_exception_cb: 1; }; struct bpf_verifier_env; -- cgit v1.2.3 From 82c944d05b1a24c76948ee9d6bb1d7de1ebb8b3a Mon Sep 17 00:00:00 2001 From: Herve Codina Date: Tue, 28 Nov 2023 14:25:30 +0100 Subject: net: wan: Add framer framework support A framer is a component in charge of an E1/T1 line interface. Connected usually to a TDM bus, it converts TDM frames to/from E1/T1 frames. It also provides information related to the E1/T1 line. The framer framework provides a set of APIs for the framer drivers (framer provider) to create/destroy a framer and APIs for the framer users (framer consumer) to obtain a reference to the framer, and use the framer. This basic implementation provides a framer abstraction for: - power on/off the framer - get the framer status (line state) - be notified on framer status changes - get/set the framer configuration Signed-off-by: Herve Codina Reviewed-by: Christophe Leroy Acked-by: Jakub Kicinski Link: https://lore.kernel.org/r/20231128132534.258459-2-herve.codina@bootlin.com Signed-off-by: Linus Walleij --- drivers/net/wan/Kconfig | 2 + drivers/net/wan/Makefile | 2 + drivers/net/wan/framer/Kconfig | 25 + drivers/net/wan/framer/Makefile | 6 + drivers/net/wan/framer/framer-core.c | 882 +++++++++++++++++++++++++++++++++ include/linux/framer/framer-provider.h | 194 ++++++++ include/linux/framer/framer.h | 205 ++++++++ 7 files changed, 1316 insertions(+) create mode 100644 drivers/net/wan/framer/Kconfig create mode 100644 drivers/net/wan/framer/Makefile create mode 100644 drivers/net/wan/framer/framer-core.c create mode 100644 include/linux/framer/framer-provider.h create mode 100644 include/linux/framer/framer.h (limited to 'include/linux') diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig index dcb069dde66b..7dda87756d3f 100644 --- a/drivers/net/wan/Kconfig +++ b/drivers/net/wan/Kconfig @@ -95,6 +95,8 @@ config HDLC_X25 comment "X.25/LAPB support is disabled" depends on HDLC && (LAPB!=m || HDLC!=m) && LAPB!=y +source "drivers/net/wan/framer/Kconfig" + config PCI200SYN tristate "Goramo PCI200SYN support" depends on HDLC && PCI diff --git a/drivers/net/wan/Makefile b/drivers/net/wan/Makefile index 5bec8fae47f8..8119b49d1da9 100644 --- a/drivers/net/wan/Makefile +++ b/drivers/net/wan/Makefile @@ -14,6 +14,8 @@ obj-$(CONFIG_HDLC_FR) += hdlc_fr.o obj-$(CONFIG_HDLC_PPP) += hdlc_ppp.o obj-$(CONFIG_HDLC_X25) += hdlc_x25.o +obj-y += framer/ + obj-$(CONFIG_FARSYNC) += farsync.o obj-$(CONFIG_LAPBETHER) += lapbether.o diff --git a/drivers/net/wan/framer/Kconfig b/drivers/net/wan/framer/Kconfig new file mode 100644 index 000000000000..57fe7ba6aa37 --- /dev/null +++ b/drivers/net/wan/framer/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# FRAMER +# + +menuconfig FRAMER + tristate "Framer Subsystem" + help + A framer is a component in charge of an E1/T1 line interface. + Connected usually to a TDM bus, it converts TDM frames to/from E1/T1 + frames. It also provides information related to the E1/T1 line. + Used with HDLC, the network can be reached through the E1/T1 line. + + This framework is designed to provide a generic interface for framer + devices present in the kernel. This layer will have the generic + API by which framer drivers can create framer using the framer + framework and framer users can obtain reference to the framer. + All the users of this framework should select this config. + +if FRAMER + +config GENERIC_FRAMER + bool + +endif # FRAMER diff --git a/drivers/net/wan/framer/Makefile b/drivers/net/wan/framer/Makefile new file mode 100644 index 000000000000..78dbd8e563d0 --- /dev/null +++ b/drivers/net/wan/framer/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the framer drivers. +# + +obj-$(CONFIG_GENERIC_FRAMER) += framer-core.o diff --git a/drivers/net/wan/framer/framer-core.c b/drivers/net/wan/framer/framer-core.c new file mode 100644 index 000000000000..c04dc88bda6c --- /dev/null +++ b/drivers/net/wan/framer/framer-core.c @@ -0,0 +1,882 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Generic Framer framework. + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct class *framer_class; +static DEFINE_MUTEX(framer_provider_mutex); +static LIST_HEAD(framer_provider_list); +static DEFINE_IDA(framer_ida); + +#define dev_to_framer(a) (container_of((a), struct framer, dev)) + +int framer_pm_runtime_get(struct framer *framer) +{ + int ret; + + if (!pm_runtime_enabled(&framer->dev)) + return -EOPNOTSUPP; + + ret = pm_runtime_get(&framer->dev); + if (ret < 0 && ret != -EINPROGRESS) + pm_runtime_put_noidle(&framer->dev); + + return ret; +} +EXPORT_SYMBOL_GPL(framer_pm_runtime_get); + +int framer_pm_runtime_get_sync(struct framer *framer) +{ + int ret; + + if (!pm_runtime_enabled(&framer->dev)) + return -EOPNOTSUPP; + + ret = pm_runtime_get_sync(&framer->dev); + if (ret < 0) + pm_runtime_put_sync(&framer->dev); + + return ret; +} +EXPORT_SYMBOL_GPL(framer_pm_runtime_get_sync); + +int framer_pm_runtime_put(struct framer *framer) +{ + if (!pm_runtime_enabled(&framer->dev)) + return -EOPNOTSUPP; + + return pm_runtime_put(&framer->dev); +} +EXPORT_SYMBOL_GPL(framer_pm_runtime_put); + +int framer_pm_runtime_put_sync(struct framer *framer) +{ + if (!pm_runtime_enabled(&framer->dev)) + return -EOPNOTSUPP; + + return pm_runtime_put_sync(&framer->dev); +} +EXPORT_SYMBOL_GPL(framer_pm_runtime_put_sync); + +/** + * framer_init - framer internal initialization before framer operation + * @framer: the framer returned by framer_get() + * + * Used to allow framer's driver to perform framer internal initialization, + * such as PLL block powering, clock initialization or anything that's + * is required by the framer to perform the start of operation. + * Must be called before framer_power_on(). + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_init(struct framer *framer) +{ + bool start_polling = false; + int ret; + + ret = framer_pm_runtime_get_sync(framer); + if (ret < 0 && ret != -EOPNOTSUPP) + return ret; + ret = 0; /* Override possible ret == -EOPNOTSUPP */ + + mutex_lock(&framer->mutex); + if (framer->power_count > framer->init_count) + dev_warn(&framer->dev, "framer_power_on was called before framer init\n"); + + if (framer->init_count == 0) { + if (framer->ops->init) { + ret = framer->ops->init(framer); + if (ret < 0) { + dev_err(&framer->dev, "framer init failed --> %d\n", ret); + goto out; + } + } + if (framer->ops->flags & FRAMER_FLAG_POLL_STATUS) + start_polling = true; + } + ++framer->init_count; + +out: + mutex_unlock(&framer->mutex); + + if (!ret && start_polling) { + ret = framer_get_status(framer, &framer->prev_status); + if (ret < 0) { + dev_warn(&framer->dev, "framer get status failed --> %d\n", ret); + /* Will be retried on polling_work */ + ret = 0; + } + queue_delayed_work(system_power_efficient_wq, &framer->polling_work, 1 * HZ); + } + + framer_pm_runtime_put(framer); + return ret; +} +EXPORT_SYMBOL_GPL(framer_init); + +/** + * framer_exit - Framer internal un-initialization + * @framer: the framer returned by framer_get() + * + * Must be called after framer_power_off(). + */ +int framer_exit(struct framer *framer) +{ + int ret; + + ret = framer_pm_runtime_get_sync(framer); + if (ret < 0 && ret != -EOPNOTSUPP) + return ret; + ret = 0; /* Override possible ret == -EOPNOTSUPP */ + + mutex_lock(&framer->mutex); + --framer->init_count; + if (framer->init_count == 0) { + if (framer->ops->flags & FRAMER_FLAG_POLL_STATUS) { + mutex_unlock(&framer->mutex); + cancel_delayed_work_sync(&framer->polling_work); + mutex_lock(&framer->mutex); + } + + if (framer->ops->exit) + framer->ops->exit(framer); + } + + mutex_unlock(&framer->mutex); + framer_pm_runtime_put(framer); + return ret; +} +EXPORT_SYMBOL_GPL(framer_exit); + +/** + * framer_power_on - Enable the framer and enter proper operation + * @framer: the framer returned by framer_get() + * + * Must be called after framer_init(). + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_power_on(struct framer *framer) +{ + int ret; + + if (framer->pwr) { + ret = regulator_enable(framer->pwr); + if (ret) + return ret; + } + + ret = framer_pm_runtime_get_sync(framer); + if (ret < 0 && ret != -EOPNOTSUPP) + goto err_pm_sync; + + mutex_lock(&framer->mutex); + if (framer->power_count == 0 && framer->ops->power_on) { + ret = framer->ops->power_on(framer); + if (ret < 0) { + dev_err(&framer->dev, "framer poweron failed --> %d\n", ret); + goto err_pwr_on; + } + } + ++framer->power_count; + mutex_unlock(&framer->mutex); + return 0; + +err_pwr_on: + mutex_unlock(&framer->mutex); + framer_pm_runtime_put_sync(framer); +err_pm_sync: + if (framer->pwr) + regulator_disable(framer->pwr); + return ret; +} +EXPORT_SYMBOL_GPL(framer_power_on); + +/** + * framer_power_off - Disable the framer. + * @framer: the framer returned by framer_get() + * + * Must be called before framer_exit(). + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_power_off(struct framer *framer) +{ + int ret; + + mutex_lock(&framer->mutex); + if (framer->power_count == 1 && framer->ops->power_off) { + ret = framer->ops->power_off(framer); + if (ret < 0) { + dev_err(&framer->dev, "framer poweroff failed --> %d\n", ret); + mutex_unlock(&framer->mutex); + return ret; + } + } + --framer->power_count; + mutex_unlock(&framer->mutex); + framer_pm_runtime_put(framer); + + if (framer->pwr) + regulator_disable(framer->pwr); + + return 0; +} +EXPORT_SYMBOL_GPL(framer_power_off); + +/** + * framer_get_status() - Gets the framer status + * @framer: the framer returned by framer_get() + * @status: the status to retrieve + * + * Used to get the framer status. framer_init() must have been called + * on the framer. + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_get_status(struct framer *framer, struct framer_status *status) +{ + int ret; + + if (!framer->ops->get_status) + return -EOPNOTSUPP; + + /* Be sure to have known values (struct padding and future extensions) */ + memset(status, 0, sizeof(*status)); + + mutex_lock(&framer->mutex); + ret = framer->ops->get_status(framer, status); + mutex_unlock(&framer->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(framer_get_status); + +/** + * framer_set_config() - Sets the framer configuration + * @framer: the framer returned by framer_get() + * @config: the configuration to set + * + * Used to set the framer configuration. framer_init() must have been called + * on the framer. + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_set_config(struct framer *framer, const struct framer_config *config) +{ + int ret; + + if (!framer->ops->set_config) + return -EOPNOTSUPP; + + mutex_lock(&framer->mutex); + ret = framer->ops->set_config(framer, config); + mutex_unlock(&framer->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(framer_set_config); + +/** + * framer_get_config() - Gets the framer configuration + * @framer: the framer returned by framer_get() + * @config: the configuration to retrieve + * + * Used to get the framer configuration. framer_init() must have been called + * on the framer. + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_get_config(struct framer *framer, struct framer_config *config) +{ + int ret; + + if (!framer->ops->get_config) + return -EOPNOTSUPP; + + mutex_lock(&framer->mutex); + ret = framer->ops->get_config(framer, config); + mutex_unlock(&framer->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(framer_get_config); + +static void framer_polling_work(struct work_struct *work) +{ + struct framer *framer = container_of(work, struct framer, polling_work.work); + struct framer_status status; + int ret; + + ret = framer_get_status(framer, &status); + if (ret) { + dev_err(&framer->dev, "polling, get status failed (%d)\n", ret); + goto end; + } + if (memcmp(&framer->prev_status, &status, sizeof(status))) { + blocking_notifier_call_chain(&framer->notifier_list, + FRAMER_EVENT_STATUS, NULL); + memcpy(&framer->prev_status, &status, sizeof(status)); + } + +end: + /* Re-schedule task in 1 sec */ + queue_delayed_work(system_power_efficient_wq, &framer->polling_work, 1 * HZ); +} + +/** + * framer_notifier_register() - Registers a notifier + * @framer: the framer returned by framer_get() + * @nb: the notifier block to register + * + * Used to register a notifier block on framer events. framer_init() must have + * been called on the framer. + * The available framer events are present in enum framer_events. + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_notifier_register(struct framer *framer, struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&framer->notifier_list, nb); +} +EXPORT_SYMBOL_GPL(framer_notifier_register); + +/** + * framer_notifier_unregister() - Unregisters a notifier + * @framer: the framer returned by framer_get() + * @nb: the notifier block to unregister + * + * Used to unregister a notifier block. framer_init() must have + * been called on the framer. + * + * Return: %0 if successful, a negative error code otherwise + */ +int framer_notifier_unregister(struct framer *framer, struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&framer->notifier_list, nb); +} +EXPORT_SYMBOL_GPL(framer_notifier_unregister); + +static struct framer_provider *framer_provider_of_lookup(const struct device_node *node) +{ + struct framer_provider *framer_provider; + + list_for_each_entry(framer_provider, &framer_provider_list, list) { + if (device_match_of_node(framer_provider->dev, node)) + return framer_provider; + } + + return ERR_PTR(-EPROBE_DEFER); +} + +static struct framer *framer_of_get_from_provider(struct of_phandle_args *args) +{ + struct framer_provider *framer_provider; + struct framer *framer; + + mutex_lock(&framer_provider_mutex); + framer_provider = framer_provider_of_lookup(args->np); + if (IS_ERR(framer_provider) || !try_module_get(framer_provider->owner)) { + framer = ERR_PTR(-EPROBE_DEFER); + goto end; + } + + framer = framer_provider->of_xlate(framer_provider->dev, args); + + module_put(framer_provider->owner); + +end: + mutex_unlock(&framer_provider_mutex); + + return framer; +} + +static struct framer *framer_of_get_byphandle(struct device_node *np, const char *propname, + int index) +{ + struct of_phandle_args args; + struct framer *framer; + int ret; + + ret = of_parse_phandle_with_optional_args(np, propname, "#framer-cells", index, &args); + if (ret) + return ERR_PTR(-ENODEV); + + if (!of_device_is_available(args.np)) { + framer = ERR_PTR(-ENODEV); + goto out_node_put; + } + + framer = framer_of_get_from_provider(&args); + +out_node_put: + of_node_put(args.np); + + return framer; +} + +static struct framer *framer_of_get_byparent(struct device_node *np, int index) +{ + struct of_phandle_args args; + struct framer *framer; + + args.np = of_get_parent(np); + args.args_count = 1; + args.args[0] = index; + + while (args.np) { + framer = framer_of_get_from_provider(&args); + if (IS_ERR(framer) && PTR_ERR(framer) != -EPROBE_DEFER) { + args.np = of_get_next_parent(args.np); + continue; + } + of_node_put(args.np); + return framer; + } + + return ERR_PTR(-ENODEV); +} + +/** + * framer_get() - lookup and obtain a reference to a framer. + * @dev: device that requests the framer + * @con_id: name of the framer from device's point of view + * + * Returns the framer driver, after getting a refcount to it; or + * -ENODEV if there is no such framer. The caller is responsible for + * calling framer_put() to release that count. + */ +struct framer *framer_get(struct device *dev, const char *con_id) +{ + struct framer *framer = ERR_PTR(-ENODEV); + struct device_link *link; + int ret; + + if (dev->of_node) { + if (con_id) + framer = framer_of_get_byphandle(dev->of_node, con_id, 0); + else + framer = framer_of_get_byparent(dev->of_node, 0); + } + + if (IS_ERR(framer)) + return framer; + + get_device(&framer->dev); + + if (!try_module_get(framer->ops->owner)) { + ret = -EPROBE_DEFER; + goto err_put_device; + } + + link = device_link_add(dev, &framer->dev, DL_FLAG_STATELESS); + if (!link) { + dev_err(dev, "failed to create device_link to %s\n", dev_name(&framer->dev)); + ret = -EPROBE_DEFER; + goto err_module_put; + } + + return framer; + +err_module_put: + module_put(framer->ops->owner); +err_put_device: + put_device(&framer->dev); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(framer_get); + +/** + * framer_put() - release the framer + * @dev: device that wants to release this framer + * @framer: the framer returned by framer_get() + * + * Releases a refcount the caller received from framer_get(). + */ +void framer_put(struct device *dev, struct framer *framer) +{ + device_link_remove(dev, &framer->dev); + + module_put(framer->ops->owner); + put_device(&framer->dev); +} +EXPORT_SYMBOL_GPL(framer_put); + +static void devm_framer_put(struct device *dev, void *res) +{ + struct framer *framer = *(struct framer **)res; + + framer_put(dev, framer); +} + +/** + * devm_framer_get() - lookup and obtain a reference to a framer. + * @dev: device that requests this framer + * @con_id: name of the framer from device's point of view + * + * Gets the framer using framer_get(), and associates a device with it using + * devres. On driver detach, framer_put() function is invoked on the devres + * data, then, devres data is freed. + */ +struct framer *devm_framer_get(struct device *dev, const char *con_id) +{ + struct framer **ptr, *framer; + + ptr = devres_alloc(devm_framer_put, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + framer = framer_get(dev, con_id); + if (!IS_ERR(framer)) { + *ptr = framer; + devres_add(dev, ptr); + } else { + devres_free(ptr); + return framer; + } + + return framer; +} +EXPORT_SYMBOL_GPL(devm_framer_get); + +/** + * devm_framer_optional_get() - lookup and obtain a reference to an optional + * framer. + * @dev: device that requests this framer + * @con_id: name of the framer from device's point of view + * + * Same as devm_framer_get() except that if the framer does not exist, it is not + * considered an error and -ENODEV will not be returned. Instead the NULL framer + * is returned. + */ +struct framer *devm_framer_optional_get(struct device *dev, const char *con_id) +{ + struct framer *framer = devm_framer_get(dev, con_id); + + if (PTR_ERR(framer) == -ENODEV) + framer = NULL; + + return framer; +} +EXPORT_SYMBOL_GPL(devm_framer_optional_get); + +static void framer_notify_status_work(struct work_struct *work) +{ + struct framer *framer = container_of(work, struct framer, notify_status_work); + + blocking_notifier_call_chain(&framer->notifier_list, FRAMER_EVENT_STATUS, NULL); +} + +void framer_notify_status_change(struct framer *framer) +{ + /* Can be called from atomic context -> just schedule a task to call + * blocking notifiers + */ + queue_work(system_power_efficient_wq, &framer->notify_status_work); +} +EXPORT_SYMBOL_GPL(framer_notify_status_change); + +/** + * framer_create() - create a new framer + * @dev: device that is creating the new framer + * @node: device node of the framer. default to dev->of_node. + * @ops: function pointers for performing framer operations + * + * Called to create a framer using framer framework. + */ +struct framer *framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops) +{ + struct framer *framer; + int ret; + int id; + + /* get_status() is mandatory if the provider ask for polling status */ + if (WARN_ON((ops->flags & FRAMER_FLAG_POLL_STATUS) && !ops->get_status)) + return ERR_PTR(-EINVAL); + + framer = kzalloc(sizeof(*framer), GFP_KERNEL); + if (!framer) + return ERR_PTR(-ENOMEM); + + id = ida_alloc(&framer_ida, GFP_KERNEL); + if (id < 0) { + dev_err(dev, "unable to get id\n"); + ret = id; + goto free_framer; + } + + device_initialize(&framer->dev); + mutex_init(&framer->mutex); + INIT_WORK(&framer->notify_status_work, framer_notify_status_work); + INIT_DELAYED_WORK(&framer->polling_work, framer_polling_work); + BLOCKING_INIT_NOTIFIER_HEAD(&framer->notifier_list); + + framer->dev.class = framer_class; + framer->dev.parent = dev; + framer->dev.of_node = node ? node : dev->of_node; + framer->id = id; + framer->ops = ops; + + ret = dev_set_name(&framer->dev, "framer-%s.%d", dev_name(dev), id); + if (ret) + goto put_dev; + + /* framer-supply */ + framer->pwr = regulator_get_optional(&framer->dev, "framer"); + if (IS_ERR(framer->pwr)) { + ret = PTR_ERR(framer->pwr); + if (ret == -EPROBE_DEFER) + goto put_dev; + + framer->pwr = NULL; + } + + ret = device_add(&framer->dev); + if (ret) + goto put_dev; + + if (pm_runtime_enabled(dev)) { + pm_runtime_enable(&framer->dev); + pm_runtime_no_callbacks(&framer->dev); + } + + return framer; + +put_dev: + put_device(&framer->dev); /* calls framer_release() which frees resources */ + return ERR_PTR(ret); + +free_framer: + kfree(framer); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(framer_create); + +/** + * framer_destroy() - destroy the framer + * @framer: the framer to be destroyed + * + * Called to destroy the framer. + */ +void framer_destroy(struct framer *framer) +{ + /* polling_work should already be stopped but if framer_exit() was not + * called (bug), here it's the last time to do that ... + */ + cancel_delayed_work_sync(&framer->polling_work); + cancel_work_sync(&framer->notify_status_work); + pm_runtime_disable(&framer->dev); + device_unregister(&framer->dev); /* calls framer_release() which frees resources */ +} +EXPORT_SYMBOL_GPL(framer_destroy); + +static void devm_framer_destroy(struct device *dev, void *res) +{ + struct framer *framer = *(struct framer **)res; + + framer_destroy(framer); +} + +/** + * devm_framer_create() - create a new framer + * @dev: device that is creating the new framer + * @node: device node of the framer + * @ops: function pointers for performing framer operations + * + * Creates a new framer device adding it to the framer class. + * While at that, it also associates the device with the framer using devres. + * On driver detach, release function is invoked on the devres data, + * then, devres data is freed. + */ +struct framer *devm_framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops) +{ + struct framer **ptr, *framer; + + ptr = devres_alloc(devm_framer_destroy, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + framer = framer_create(dev, node, ops); + if (!IS_ERR(framer)) { + *ptr = framer; + devres_add(dev, ptr); + } else { + devres_free(ptr); + } + + return framer; +} +EXPORT_SYMBOL_GPL(devm_framer_create); + +/** + * framer_provider_simple_of_xlate() - returns the framer instance from framer provider + * @dev: the framer provider device + * @args: of_phandle_args (not used here) + * + * Intended to be used by framer provider for the common case where #framer-cells is + * 0. For other cases where #framer-cells is greater than '0', the framer provider + * should provide a custom of_xlate function that reads the *args* and returns + * the appropriate framer. + */ +struct framer *framer_provider_simple_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + struct class_dev_iter iter; + struct framer *framer; + + class_dev_iter_init(&iter, framer_class, NULL, NULL); + while ((dev = class_dev_iter_next(&iter))) { + framer = dev_to_framer(dev); + if (args->np != framer->dev.of_node) + continue; + + class_dev_iter_exit(&iter); + return framer; + } + + class_dev_iter_exit(&iter); + return ERR_PTR(-ENODEV); +} +EXPORT_SYMBOL_GPL(framer_provider_simple_of_xlate); + +/** + * __framer_provider_of_register() - create/register framer provider with the framework + * @dev: struct device of the framer provider + * @owner: the module owner containing of_xlate + * @of_xlate: function pointer to obtain framer instance from framer provider + * + * Creates struct framer_provider from dev and of_xlate function pointer. + * This is used in the case of dt boot for finding the framer instance from + * framer provider. + */ +struct framer_provider * +__framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)) +{ + struct framer_provider *framer_provider; + + framer_provider = kzalloc(sizeof(*framer_provider), GFP_KERNEL); + if (!framer_provider) + return ERR_PTR(-ENOMEM); + + framer_provider->dev = dev; + framer_provider->owner = owner; + framer_provider->of_xlate = of_xlate; + + of_node_get(framer_provider->dev->of_node); + + mutex_lock(&framer_provider_mutex); + list_add_tail(&framer_provider->list, &framer_provider_list); + mutex_unlock(&framer_provider_mutex); + + return framer_provider; +} +EXPORT_SYMBOL_GPL(__framer_provider_of_register); + +/** + * framer_provider_of_unregister() - unregister framer provider from the framework + * @framer_provider: framer provider returned by framer_provider_of_register() + * + * Removes the framer_provider created using framer_provider_of_register(). + */ +void framer_provider_of_unregister(struct framer_provider *framer_provider) +{ + mutex_lock(&framer_provider_mutex); + list_del(&framer_provider->list); + mutex_unlock(&framer_provider_mutex); + + of_node_put(framer_provider->dev->of_node); + kfree(framer_provider); +} +EXPORT_SYMBOL_GPL(framer_provider_of_unregister); + +static void devm_framer_provider_of_unregister(struct device *dev, void *res) +{ + struct framer_provider *framer_provider = *(struct framer_provider **)res; + + framer_provider_of_unregister(framer_provider); +} + +/** + * __devm_framer_provider_of_register() - create/register framer provider with + * the framework + * @dev: struct device of the framer provider + * @owner: the module owner containing of_xlate + * @of_xlate: function pointer to obtain framer instance from framer provider + * + * Creates struct framer_provider from dev and of_xlate function pointer. + * This is used in the case of dt boot for finding the framer instance from + * framer provider. While at that, it also associates the device with the + * framer provider using devres. On driver detach, release function is invoked + * on the devres data, then, devres data is freed. + */ +struct framer_provider * +__devm_framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)) +{ + struct framer_provider **ptr, *framer_provider; + + ptr = devres_alloc(devm_framer_provider_of_unregister, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + framer_provider = __framer_provider_of_register(dev, owner, of_xlate); + if (!IS_ERR(framer_provider)) { + *ptr = framer_provider; + devres_add(dev, ptr); + } else { + devres_free(ptr); + } + + return framer_provider; +} +EXPORT_SYMBOL_GPL(__devm_framer_provider_of_register); + +/** + * framer_release() - release the framer + * @dev: the dev member within framer + * + * When the last reference to the device is removed, it is called + * from the embedded kobject as release method. + */ +static void framer_release(struct device *dev) +{ + struct framer *framer; + + framer = dev_to_framer(dev); + regulator_put(framer->pwr); + ida_free(&framer_ida, framer->id); + kfree(framer); +} + +static int __init framer_core_init(void) +{ + framer_class = class_create("framer"); + if (IS_ERR(framer_class)) { + pr_err("failed to create framer class (%pe)\n", framer_class); + return PTR_ERR(framer_class); + } + + framer_class->dev_release = framer_release; + + return 0; +} +device_initcall(framer_core_init); diff --git a/include/linux/framer/framer-provider.h b/include/linux/framer/framer-provider.h new file mode 100644 index 000000000000..782cd5fc83d5 --- /dev/null +++ b/include/linux/framer/framer-provider.h @@ -0,0 +1,194 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Generic framer profider header file + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ + +#ifndef __DRIVERS_PROVIDER_FRAMER_H +#define __DRIVERS_PROVIDER_FRAMER_H + +#include +#include +#include + +#define FRAMER_FLAG_POLL_STATUS BIT(0) + +/** + * struct framer_ops - set of function pointers for performing framer operations + * @init: operation to be performed for initializing the framer + * @exit: operation to be performed while exiting + * @power_on: powering on the framer + * @power_off: powering off the framer + * @flags: OR-ed flags (FRAMER_FLAG_*) to ask for core functionality + * - @FRAMER_FLAG_POLL_STATUS: + * Ask the core to perform a polling to get the framer status and + * notify consumers on change. + * The framer should call @framer_notify_status_change() when it + * detects a status change. This is usually done using interrupts. + * If the framer cannot detect this change, it can ask the core for + * a status polling. The core will call @get_status() periodically + * and, on change detected, it will notify the consumer. + * the @get_status() + * @owner: the module owner containing the ops + */ +struct framer_ops { + int (*init)(struct framer *framer); + void (*exit)(struct framer *framer); + int (*power_on)(struct framer *framer); + int (*power_off)(struct framer *framer); + + /** + * @get_status: + * + * Optional. + * + * Used to get the framer status. framer_init() must have + * been called on the framer. + * + * Returns: 0 if successful, an negative error code otherwise + */ + int (*get_status)(struct framer *framer, struct framer_status *status); + + /** + * @set_config: + * + * Optional. + * + * Used to set the framer configuration. framer_init() must have + * been called on the framer. + * + * Returns: 0 if successful, an negative error code otherwise + */ + int (*set_config)(struct framer *framer, const struct framer_config *config); + + /** + * @get_config: + * + * Optional. + * + * Used to get the framer configuration. framer_init() must have + * been called on the framer. + * + * Returns: 0 if successful, an negative error code otherwise + */ + int (*get_config)(struct framer *framer, struct framer_config *config); + + u32 flags; + struct module *owner; +}; + +/** + * struct framer_provider - represents the framer provider + * @dev: framer provider device + * @children: can be used to override the default (dev->of_node) child node + * @owner: the module owner having of_xlate + * @list: to maintain a linked list of framer providers + * @of_xlate: function pointer to obtain framer instance from framer pointer + */ +struct framer_provider { + struct device *dev; + struct module *owner; + struct list_head list; + struct framer * (*of_xlate)(struct device *dev, + struct of_phandle_args *args); +}; + +static inline void framer_set_drvdata(struct framer *framer, void *data) +{ + dev_set_drvdata(&framer->dev, data); +} + +static inline void *framer_get_drvdata(struct framer *framer) +{ + return dev_get_drvdata(&framer->dev); +} + +#if IS_ENABLED(CONFIG_GENERIC_FRAMER) + +/* Create and destroy a framer */ +struct framer *framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops); +void framer_destroy(struct framer *framer); + +/* devm version */ +struct framer *devm_framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops); + +struct framer *framer_provider_simple_of_xlate(struct device *dev, + struct of_phandle_args *args); + +struct framer_provider * +__framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)); + +void framer_provider_of_unregister(struct framer_provider *framer_provider); + +struct framer_provider * +__devm_framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)); + +void framer_notify_status_change(struct framer *framer); + +#else /* IS_ENABLED(CONFIG_GENERIC_FRAMER) */ + +static inline struct framer *framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops) +{ + return ERR_PTR(-ENOSYS); +} + +static inline void framer_destroy(struct framer *framer) +{ +} + +/* devm version */ +static inline struct framer *devm_framer_create(struct device *dev, struct device_node *node, + const struct framer_ops *ops) +{ + return ERR_PTR(-ENOSYS); +} + +static inline struct framer *framer_provider_simple_of_xlate(struct device *dev, + struct of_phandle_args *args) +{ + return ERR_PTR(-ENOSYS); +} + +static inline struct framer_provider * +__framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)) +{ + return ERR_PTR(-ENOSYS); +} + +void framer_provider_of_unregister(struct framer_provider *framer_provider) +{ +} + +static inline struct framer_provider * +__devm_framer_provider_of_register(struct device *dev, struct module *owner, + struct framer *(*of_xlate)(struct device *dev, + struct of_phandle_args *args)) +{ + return ERR_PTR(-ENOSYS); +} + +void framer_notify_status_change(struct framer *framer) +{ +} + +#endif /* IS_ENABLED(CONFIG_GENERIC_FRAMER) */ + +#define framer_provider_of_register(dev, xlate) \ + __framer_provider_of_register((dev), THIS_MODULE, (xlate)) + +#define devm_framer_provider_of_register(dev, xlate) \ + __devm_framer_provider_of_register((dev), THIS_MODULE, (xlate)) + +#endif /* __DRIVERS_PROVIDER_FRAMER_H */ diff --git a/include/linux/framer/framer.h b/include/linux/framer/framer.h new file mode 100644 index 000000000000..9a9b88962c29 --- /dev/null +++ b/include/linux/framer/framer.h @@ -0,0 +1,205 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Generic framer header file + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ + +#ifndef __DRIVERS_FRAMER_H +#define __DRIVERS_FRAMER_H + +#include +#include +#include +#include +#include +#include + +/** + * enum framer_iface - Framer interface + * @FRAMER_IFACE_E1: E1 interface + * @FRAMER_IFACE_T1: T1 interface + */ +enum framer_iface { + FRAMER_IFACE_E1, + FRAMER_IFACE_T1, +}; + +/** + * enum framer_clock_type - Framer clock type + * @FRAMER_CLOCK_EXT: External clock + * @FRAMER_CLOCK_INT: Internal clock + */ +enum framer_clock_type { + FRAMER_CLOCK_EXT, + FRAMER_CLOCK_INT, +}; + +/** + * struct framer_config - Framer configuration + * @iface: Framer line interface + * @clock_type: Framer clock type + * @line_clock_rate: Framer line clock rate + */ +struct framer_config { + enum framer_iface iface; + enum framer_clock_type clock_type; + unsigned long line_clock_rate; +}; + +/** + * struct framer_status - Framer status + * @link_is_on: Framer link state. true, the link is on, false, the link is off. + */ +struct framer_status { + bool link_is_on; +}; + +/** + * enum framer_event - Event available for notification + * @FRAMER_EVENT_STATUS: Event notified on framer_status changes + */ +enum framer_event { + FRAMER_EVENT_STATUS, +}; + +/** + * struct framer - represents the framer device + * @dev: framer device + * @id: id of the framer device + * @ops: function pointers for performing framer operations + * @mutex: mutex to protect framer_ops + * @init_count: used to protect when the framer is used by multiple consumers + * @power_count: used to protect when the framer is used by multiple consumers + * @pwr: power regulator associated with the framer + * @notify_status_work: work structure used for status notifications + * @notifier_list: notifier list used for notifications + * @polling_work: delayed work structure used for the polling task + * @prev_status: previous read status used by the polling task to detect changes + */ +struct framer { + struct device dev; + int id; + const struct framer_ops *ops; + struct mutex mutex; /* Protect framer */ + int init_count; + int power_count; + struct regulator *pwr; + struct work_struct notify_status_work; + struct blocking_notifier_head notifier_list; + struct delayed_work polling_work; + struct framer_status prev_status; +}; + +#if IS_ENABLED(CONFIG_GENERIC_FRAMER) +int framer_pm_runtime_get(struct framer *framer); +int framer_pm_runtime_get_sync(struct framer *framer); +int framer_pm_runtime_put(struct framer *framer); +int framer_pm_runtime_put_sync(struct framer *framer); +int framer_init(struct framer *framer); +int framer_exit(struct framer *framer); +int framer_power_on(struct framer *framer); +int framer_power_off(struct framer *framer); +int framer_get_status(struct framer *framer, struct framer_status *status); +int framer_get_config(struct framer *framer, struct framer_config *config); +int framer_set_config(struct framer *framer, const struct framer_config *config); +int framer_notifier_register(struct framer *framer, struct notifier_block *nb); +int framer_notifier_unregister(struct framer *framer, struct notifier_block *nb); + +struct framer *framer_get(struct device *dev, const char *con_id); +void framer_put(struct device *dev, struct framer *framer); + +struct framer *devm_framer_get(struct device *dev, const char *con_id); +struct framer *devm_framer_optional_get(struct device *dev, const char *con_id); +#else +static inline int framer_pm_runtime_get(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_pm_runtime_get_sync(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_pm_runtime_put(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_pm_runtime_put_sync(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_init(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_exit(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_power_on(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_power_off(struct framer *framer) +{ + return -ENOSYS; +} + +static inline int framer_get_status(struct framer *framer, struct framer_status *status) +{ + return -ENOSYS; +} + +static inline int framer_get_config(struct framer *framer, struct framer_config *config) +{ + return -ENOSYS; +} + +static inline int framer_set_config(struct framer *framer, const struct framer_config *config) +{ + return -ENOSYS; +} + +static inline int framer_notifier_register(struct framer *framer, + struct notifier_block *nb) +{ + return -ENOSYS; +} + +static inline int framer_notifier_unregister(struct framer *framer, + struct notifier_block *nb) +{ + return -ENOSYS; +} + +struct framer *framer_get(struct device *dev, const char *con_id) +{ + return ERR_PTR(-ENOSYS); +} + +void framer_put(struct device *dev, struct framer *framer) +{ +} + +static inline struct framer *devm_framer_get(struct device *dev, const char *con_id) +{ + return ERR_PTR(-ENOSYS); +} + +static inline struct framer *devm_framer_optional_get(struct device *dev, const char *con_id) +{ + return NULL; +} + +#endif + +#endif /* __DRIVERS_FRAMER_H */ -- cgit v1.2.3 From c96e976d9a05d559f4ac4f617ea0f798c75a1799 Mon Sep 17 00:00:00 2001 From: Herve Codina Date: Tue, 28 Nov 2023 14:25:32 +0100 Subject: net: wan: framer: Add support for the Lantiq PEF2256 framer The Lantiq PEF2256 is a framer and line interface component designed to fulfill all required interfacing between an analog E1/T1/J1 line and the digital PCM system highway/H.100 bus. Signed-off-by: Herve Codina Reviewed-by: Christophe Leroy Reviewed-by: Linus Walleij Acked-by: Jakub Kicinski Link: https://lore.kernel.org/r/20231128132534.258459-4-herve.codina@bootlin.com Signed-off-by: Linus Walleij --- drivers/net/wan/framer/Kconfig | 17 + drivers/net/wan/framer/Makefile | 1 + drivers/net/wan/framer/pef2256/Makefile | 8 + drivers/net/wan/framer/pef2256/pef2256-regs.h | 250 ++++++++ drivers/net/wan/framer/pef2256/pef2256.c | 880 ++++++++++++++++++++++++++ include/linux/framer/pef2256.h | 31 + 6 files changed, 1187 insertions(+) create mode 100644 drivers/net/wan/framer/pef2256/Makefile create mode 100644 drivers/net/wan/framer/pef2256/pef2256-regs.h create mode 100644 drivers/net/wan/framer/pef2256/pef2256.c create mode 100644 include/linux/framer/pef2256.h (limited to 'include/linux') diff --git a/drivers/net/wan/framer/Kconfig b/drivers/net/wan/framer/Kconfig index 57fe7ba6aa37..dc83caef9485 100644 --- a/drivers/net/wan/framer/Kconfig +++ b/drivers/net/wan/framer/Kconfig @@ -22,4 +22,21 @@ if FRAMER config GENERIC_FRAMER bool +config FRAMER_PEF2256 + tristate "Lantiq PEF2256" + depends on OF + depends on HAS_IOMEM + select GENERIC_FRAMER + select MFD_CORE + select REGMAP_MMIO + help + Enable support for the Lantiq PEF2256 (FALC56) framer. + The PEF2256 is a framer and line interface between analog E1/T1/J1 + line and a digital PCM bus. + + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called framer-pef2256. + endif # FRAMER diff --git a/drivers/net/wan/framer/Makefile b/drivers/net/wan/framer/Makefile index 78dbd8e563d0..3403f2b14534 100644 --- a/drivers/net/wan/framer/Makefile +++ b/drivers/net/wan/framer/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_GENERIC_FRAMER) += framer-core.o +obj-$(CONFIG_FRAMER_PEF2256) += pef2256/ diff --git a/drivers/net/wan/framer/pef2256/Makefile b/drivers/net/wan/framer/pef2256/Makefile new file mode 100644 index 000000000000..f4d1208dd8a4 --- /dev/null +++ b/drivers/net/wan/framer/pef2256/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the pef2256 driver. +# + +obj-$(CONFIG_FRAMER_PEF2256) += framer-pef2256.o + +framer-pef2256-objs := pef2256.o diff --git a/drivers/net/wan/framer/pef2256/pef2256-regs.h b/drivers/net/wan/framer/pef2256/pef2256-regs.h new file mode 100644 index 000000000000..5d3183c91714 --- /dev/null +++ b/drivers/net/wan/framer/pef2256/pef2256-regs.h @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PEF2256 registers definition + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ +#ifndef __PEF2256_REGS_H__ +#define __PEF2256_REGS_H__ + +#include "linux/bitfield.h" + +/* Command Register */ +#define PEF2256_CMDR 0x02 +#define PEF2256_CMDR_RRES BIT(6) +#define PEF2256_CMDR_XRES BIT(4) +#define PEF2256_CMDR_SRES BIT(0) + +/* Interrupt Mask Register 0..5 */ +#define PEF2256_IMR0 0x14 +#define PEF2256_IMR1 0x15 +#define PEF2256_IMR2 0x16 +#define PEF2256_IMR3 0x17 +#define PEF2256_IMR4 0x18 +#define PEF2256_IMR5 0x19 + +/* Framer Mode Register 0 */ +#define PEF2256_FMR0 0x1C +#define PEF2256_FMR0_XC_MASK GENMASK(7, 6) +#define PEF2256_FMR0_XC_NRZ FIELD_PREP_CONST(PEF2256_FMR0_XC_MASK, 0x0) +#define PEF2256_FMR0_XC_CMI FIELD_PREP_CONST(PEF2256_FMR0_XC_MASK, 0x1) +#define PEF2256_FMR0_XC_AMI FIELD_PREP_CONST(PEF2256_FMR0_XC_MASK, 0x2) +#define PEF2256_FMR0_XC_HDB3 FIELD_PREP_CONST(PEF2256_FMR0_XC_MASK, 0x3) +#define PEF2256_FMR0_RC_MASK GENMASK(5, 4) +#define PEF2256_FMR0_RC_NRZ FIELD_PREP_CONST(PEF2256_FMR0_RC_MASK, 0x0) +#define PEF2256_FMR0_RC_CMI FIELD_PREP_CONST(PEF2256_FMR0_RC_MASK, 0x1) +#define PEF2256_FMR0_RC_AMI FIELD_PREP_CONST(PEF2256_FMR0_RC_MASK, 0x2) +#define PEF2256_FMR0_RC_HDB3 FIELD_PREP_CONST(PEF2256_FMR0_RC_MASK, 0x3) + +/* Framer Mode Register 1 */ +#define PEF2256_FMR1 0x1D +#define PEF2256_FMR1_XFS BIT(3) +#define PEF2256_FMR1_ECM BIT(2) +/* SSD is defined on 2 bits. The other bit is on SIC1 register */ +#define PEF2256_FMR1_SSD_MASK GENMASK(1, 1) +#define PEF2256_FMR1_SSD_2048 FIELD_PREP_CONST(PEF2256_FMR1_SSD_MASK, 0x0) +#define PEF2256_FMR1_SSD_4096 FIELD_PREP_CONST(PEF2256_FMR1_SSD_MASK, 0x1) +#define PEF2256_FMR1_SSD_8192 FIELD_PREP_CONST(PEF2256_FMR1_SSD_MASK, 0x0) +#define PEF2256_FMR1_SSD_16384 FIELD_PREP_CONST(PEF2256_FMR1_SSD_MASK, 0x1) + +/* Framer Mode Register 2 */ +#define PEF2256_FMR2 0x1E +#define PEF2256_FMR2_RFS_MASK GENMASK(7, 6) +#define PEF2256_FMR2_RFS_DOUBLEFRAME FIELD_PREP_CONST(PEF2256_FMR2_RFS_MASK, 0x0) +#define PEF2256_FMR2_RFS_CRC4_MULTIFRAME FIELD_PREP_CONST(PEF2256_FMR2_RFS_MASK, 0x2) +#define PEF2256_FMR2_RFS_AUTO_MULTIFRAME FIELD_PREP_CONST(PEF2256_FMR2_RFS_MASK, 0x3) +#define PEF2256_FMR2_AXRA BIT(1) + +/* Transmit Service Word */ +#define PEF2256_XSW 0x20 +#define PEF2256_XSW_XSIS BIT(7) +#define PEF2256_XSW_XTM BIT(6) +#define PEF2256_XSW_XY_MASK GENMASK(5, 0) +#define PEF2256_XSW_XY(_v) FIELD_PREP(PEF2256_XSW_XY_MASK, _v) + +/* Transmit Spare Bits */ +#define PEF2256_XSP 0x21 +#define PEF2256_XSP_XSIF BIT(2) + +/* Transmit Control 0..1 */ +#define PEF2256_XC0 0x22 +#define PEF2256_XC1 0x23 + +/* Receive Control 0 */ +#define PEF2256_RC0 0x24 +#define PEF2256_RC0_SWD BIT(7) +#define PEF2256_RC0_ASY4 BIT(6) + +/* Receive Control 1 */ +#define PEF2256_RC1 0x25 + +/* Transmit Pulse Mask 0..1 */ +#define PEF2256_XPM0 0x26 +#define PEF2256_XPM1 0x27 + +/* Transmit Pulse Mask 2 */ +#define PEF2256_XPM2 0x28 +#define PEF2256_XPM2_XLT BIT(6) + +/* Transparent Service Word Mask */ +#define PEF2256_TSWM 0x29 + +/* Line Interface Mode 0 */ +#define PEF2256_LIM0 0x36 +#define PEF2256_2X_LIM0_BIT3 BIT(3) /* v2.x, described as a forced '1' bit */ +#define PEF2256_LIM0_MAS BIT(0) + +/* Line Interface Mode 1 */ +#define PEF2256_LIM1 0x37 +#define PEF2256_12_LIM1_RIL_MASK GENMASK(6, 4) +#define PEF2256_12_LIM1_RIL_910 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x0) +#define PEF2256_12_LIM1_RIL_740 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x1) +#define PEF2256_12_LIM1_RIL_590 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x2) +#define PEF2256_12_LIM1_RIL_420 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x3) +#define PEF2256_12_LIM1_RIL_320 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x4) +#define PEF2256_12_LIM1_RIL_210 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x5) +#define PEF2256_12_LIM1_RIL_160 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x6) +#define PEF2256_12_LIM1_RIL_100 FIELD_PREP_CONST(PEF2256_12_LIM1_RIL_MASK, 0x7) +#define PEF2256_2X_LIM1_RIL_MASK GENMASK(6, 4) +#define PEF2256_2X_LIM1_RIL_2250 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x0) +#define PEF2256_2X_LIM1_RIL_1100 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x1) +#define PEF2256_2X_LIM1_RIL_600 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x2) +#define PEF2256_2X_LIM1_RIL_350 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x3) +#define PEF2256_2X_LIM1_RIL_210 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x4) +#define PEF2256_2X_LIM1_RIL_140 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x5) +#define PEF2256_2X_LIM1_RIL_100 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x6) +#define PEF2256_2X_LIM1_RIL_50 FIELD_PREP_CONST(PEF2256_2X_LIM1_RIL_MASK, 0x7) + +/* Pulse Count Detection */ +#define PEF2256_PCD 0x38 + + /* Pulse Count Recovery */ +#define PEF2256_PCR 0x39 + + /* Line Interface Mode 2 */ +#define PEF2256_LIM2 0x3A +#define PEF2256_LIM2_SLT_MASK GENMASK(5, 4) +#define PEF2256_LIM2_SLT_THR55 FIELD_PREP_CONST(PEF2256_LIM2_SLT_MASK, 0x0) +#define PEF2256_LIM2_SLT_THR67 FIELD_PREP_CONST(PEF2256_LIM2_SLT_MASK, 0x1) +#define PEF2256_LIM2_SLT_THR50 FIELD_PREP_CONST(PEF2256_LIM2_SLT_MASK, 0x2) +#define PEF2256_LIM2_SLT_THR45 FIELD_PREP_CONST(PEF2256_LIM2_SLT_MASK, 0x3) +#define PEF2256_LIM2_ELT BIT(2) + +/* System Interface Control 1 */ +#define PEF2256_SIC1 0x3E +#define PEF2256_SIC1_SSC_MASK (BIT(7) | BIT(3)) +#define PEF2256_SIC1_SSC_2048 (0) +#define PEF2256_SIC1_SSC_4096 BIT(3) +#define PEF2256_SIC1_SSC_8192 BIT(7) +#define PEF2256_SIC1_SSC_16384 (BIT(7) | BIT(3)) +/* SSD is defined on 2 bits. The other bit is on FMR1 register */ +#define PEF2256_SIC1_SSD_MASK GENMASK(6, 6) +#define PEF2256_SIC1_SSD_2048 FIELD_PREP_CONST(PEF2256_SIC1_SSD_MASK, 0x0) +#define PEF2256_SIC1_SSD_4096 FIELD_PREP_CONST(PEF2256_SIC1_SSD_MASK, 0x0) +#define PEF2256_SIC1_SSD_8192 FIELD_PREP_CONST(PEF2256_SIC1_SSD_MASK, 0x1) +#define PEF2256_SIC1_SSD_16384 FIELD_PREP_CONST(PEF2256_SIC1_SSD_MASK, 0x1) +#define PEF2256_SIC1_RBS_MASK GENMASK(5, 4) +#define PEF2256_SIC1_RBS_2FRAMES FIELD_PREP_CONST(PEF2256_SIC1_RBS_MASK, 0x0) +#define PEF2256_SIC1_RBS_1FRAME FIELD_PREP_CONST(PEF2256_SIC1_RBS_MASK, 0x1) +#define PEF2256_SIC1_RBS_96BITS FIELD_PREP_CONST(PEF2256_SIC1_RBS_MASK, 0x2) +#define PEF2256_SIC1_RBS_BYPASS FIELD_PREP_CONST(PEF2256_SIC1_RBS_MASK, 0x3) +#define PEF2256_SIC1_XBS_MASK GENMASK(1, 0) +#define PEF2256_SIC1_XBS_BYPASS FIELD_PREP_CONST(PEF2256_SIC1_XBS_MASK, 0x0) +#define PEF2256_SIC1_XBS_1FRAME FIELD_PREP_CONST(PEF2256_SIC1_XBS_MASK, 0x1) +#define PEF2256_SIC1_XBS_2FRAMES FIELD_PREP_CONST(PEF2256_SIC1_XBS_MASK, 0x2) +#define PEF2256_SIC1_XBS_96BITS FIELD_PREP_CONST(PEF2256_SIC1_XBS_MASK, 0x3) + +/* System Interface Control 2 */ +#define PEF2256_SIC2 0x3F +#define PEF2256_SIC2_SICS_MASK GENMASK(3, 1) +#define PEF2256_SIC2_SICS(_v) FIELD_PREP(PEF2256_SIC2_SICS_MASK, _v) + +/* System Interface Control 3 */ +#define PEF2256_SIC3 0x40 +#define PEF2256_SIC3_RTRI BIT(5) +#define PEF2256_SIC3_RESX BIT(3) +#define PEF2256_SIC3_RESR BIT(2) + +/* Clock Mode Register 1 */ +#define PEF2256_CMR1 0x44 +#define PEF2256_CMR1_RS_MASK GENMASK(5, 4) +#define PEF2256_CMR1_RS_DPLL FIELD_PREP_CONST(PEF2256_CMR1_RS_MASK, 0x0) +#define PEF2256_CMR1_RS_DPLL_LOS_HIGH FIELD_PREP_CONST(PEF2256_CMR1_RS_MASK, 0x1) +#define PEF2256_CMR1_RS_DCOR_2048 FIELD_PREP_CONST(PEF2256_CMR1_RS_MASK, 0x2) +#define PEF2256_CMR1_RS_DCOR_8192 FIELD_PREP_CONST(PEF2256_CMR1_RS_MASK, 0x3) +#define PEF2256_CMR1_DCS BIT(3) + +/* Clock Mode Register 2 */ +#define PEF2256_CMR2 0x45 +#define PEF2256_CMR2_DCOXC BIT(5) + +/* Global Configuration Register */ +#define PEF2256_GCR 0x46 +#define PEF2256_GCR_SCI BIT(6) +#define PEF2256_GCR_ECMC BIT(4) + +/* Port Configuration 5 */ +#define PEF2256_PC5 0x84 +#define PEF2256_PC5_CRP BIT(0) + +/* Global Port Configuration 1 */ +#define PEF2256_GPC1 0x85 +#define PEF2256_GPC1_CSFP_MASK GENMASK(7, 5) +#define PEF2256_GPC1_CSFP_SEC_IN_HIGH FIELD_PREP_CONST(PEF2256_GPC1_CSFP_MASK, 0x0) +#define PEF2256_GPC1_CSFP_SEC_OUT_HIGH FIELD_PREP_CONST(PEF2256_GPC1_CSFP_MASK, 0x1) +#define PEF2256_GPC1_CSFP_FSC_OUT_HIGH FIELD_PREP_CONST(PEF2256_GPC1_CSFP_MASK, 0x2) +#define PEF2256_GPC1_CSFP_FSC_OUT_LOW FIELD_PREP_CONST(PEF2256_GPC1_CSFP_MASK, 0x3) + +/* Port Configuration 6 */ +#define PEF2256_PC6 0x86 + +/* Global Counter Mode n=1..8 */ +#define PEF2256_GCM(_n) (0x92 + (_n) - 1) +#define PEF2256_GCM1 0x92 +#define PEF2256_GCM2 0x93 +#define PEF2256_GCM3 0x94 +#define PEF2256_GCM4 0x95 +#define PEF2256_GCM5 0x96 +#define PEF2256_GCM6 0x97 +#define PEF2256_GCM7 0x98 +#define PEF2256_GCM8 0x99 + +/* Version Status Register */ +#define PEF2256_VSTR 0x4A +#define PEF2256_VSTR_VERSION_12 0x00 +#define PEF2256_VSTR_VERSION_21 0x10 +#define PEF2256_VSTR_VERSION_2x 0x05 + +/* Framer Receive Status 0 */ +#define PEF2256_FRS0 0x4C +#define PEF2256_FRS0_LOS BIT(7) +#define PEF2256_FRS0_AIS BIT(6) + +/* Interrupt Status Register 0..5 */ +#define PEF2256_ISR(_n) (0x68 + (_n)) +#define PEF2256_ISR0 0x68 +#define PEF2256_ISR1 0x69 +#define PEF2256_ISR2 0x6A +#define PEF2256_ISR3 0x6B +#define PEF2256_ISR4 0x6C +#define PEF2256_ISR5 0x6D + +/* Global Interrupt Status */ +#define PEF2256_GIS 0x6E +#define PEF2256_GIS_ISR(_n) BIT(_n) + +/* Wafer Identification Register */ +#define PEF2256_WID 0xEC +#define PEF2256_12_WID_MASK GENMASK(1, 0) +#define PEF2256_12_WID_VERSION_12 FIELD_PREP_CONST(PEF2256_12_WID_MASK, 0x3) +#define PEF2256_2X_WID_MASK GENMASK(7, 6) +#define PEF2256_2X_WID_VERSION_21 FIELD_PREP_CONST(PEF2256_2X_WID_MASK, 0x0) +#define PEF2256_2X_WID_VERSION_22 FIELD_PREP_CONST(PEF2256_2X_WID_MASK, 0x1) + +/* IMR2/ISR2 Interrupts common bits */ +#define PEF2256_INT2_AIS BIT(3) +#define PEF2256_INT2_LOS BIT(2) + +#endif /* __PEF2256_REGS_H__ */ diff --git a/drivers/net/wan/framer/pef2256/pef2256.c b/drivers/net/wan/framer/pef2256/pef2256.c new file mode 100644 index 000000000000..4f81053ee4f0 --- /dev/null +++ b/drivers/net/wan/framer/pef2256/pef2256.c @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PEF2256 also known as FALC56 driver + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pef2256-regs.h" + +enum pef2256_frame_type { + PEF2256_FRAME_E1_DOUBLEFRAME, + PEF2256_FRAME_E1_CRC4_MULTIFRAME, + PEF2256_FRAME_E1_AUTO_MULTIFRAME, + PEF2256_FRAME_T1J1_4FRAME, + PEF2256_FRAME_T1J1_12FRAME, + PEF2256_FRAME_T1J1_24FRAME, + PEF2256_FRAME_T1J1_72FRAME, +}; + +struct pef2256 { + struct device *dev; + struct regmap *regmap; + enum pef2256_version version; + struct clk *mclk; + struct clk *sclkr; + struct clk *sclkx; + struct gpio_desc *reset_gpio; + unsigned long sysclk_rate; + u32 data_rate; + bool is_tx_falling_edge; + bool is_subordinate; + enum pef2256_frame_type frame_type; + u8 channel_phase; + atomic_t carrier; + struct framer *framer; +}; + +static u8 pef2256_read8(struct pef2256 *pef2256, int offset) +{ + int val; + + regmap_read(pef2256->regmap, offset, &val); + return val; +} + +static void pef2256_write8(struct pef2256 *pef2256, int offset, u8 val) +{ + regmap_write(pef2256->regmap, offset, val); +} + +static void pef2256_clrbits8(struct pef2256 *pef2256, int offset, u8 clr) +{ + regmap_clear_bits(pef2256->regmap, offset, clr); +} + +static void pef2256_setbits8(struct pef2256 *pef2256, int offset, u8 set) +{ + regmap_set_bits(pef2256->regmap, offset, set); +} + +static void pef2256_clrsetbits8(struct pef2256 *pef2256, int offset, u8 clr, u8 set) +{ + regmap_update_bits(pef2256->regmap, offset, clr | set, set); +} + +enum pef2256_version pef2256_get_version(struct pef2256 *pef2256) +{ + enum pef2256_version version = PEF2256_VERSION_UNKNOWN; + u8 vstr, wid; + + vstr = pef2256_read8(pef2256, PEF2256_VSTR); + wid = pef2256_read8(pef2256, PEF2256_WID); + + switch (vstr) { + case PEF2256_VSTR_VERSION_12: + if ((wid & PEF2256_12_WID_MASK) == PEF2256_12_WID_VERSION_12) + version = PEF2256_VERSION_1_2; + break; + case PEF2256_VSTR_VERSION_2x: + switch (wid & PEF2256_2X_WID_MASK) { + case PEF2256_2X_WID_VERSION_21: + version = PEF2256_VERSION_2_1; + break; + case PEF2256_2X_WID_VERSION_22: + version = PEF2256_VERSION_2_2; + break; + } + break; + case PEF2256_VSTR_VERSION_21: + version = PEF2256_VERSION_2_1; + break; + } + + if (version == PEF2256_VERSION_UNKNOWN) + dev_err(pef2256->dev, "Unknown version (0x%02x, 0x%02x)\n", vstr, wid); + + return version; +} +EXPORT_SYMBOL_GPL(pef2256_get_version); + +enum pef2256_gcm_config_item { + PEF2256_GCM_CONFIG_1544000 = 0, + PEF2256_GCM_CONFIG_2048000, + PEF2256_GCM_CONFIG_8192000, + PEF2256_GCM_CONFIG_10000000, + PEF2256_GCM_CONFIG_12352000, + PEF2256_GCM_CONFIG_16384000, +}; + +struct pef2256_gcm_config { + u8 gcm_12[6]; + u8 gcm_2x[8]; +}; + +static const struct pef2256_gcm_config pef2256_gcm_configs[] = { + [PEF2256_GCM_CONFIG_1544000] = { + .gcm_12 = {0xF0, 0x51, 0x00, 0x80, 0x00, 0x15}, + .gcm_2x = {0x00, 0x15, 0x00, 0x08, 0x00, 0x3F, 0x9C, 0xDF}, + }, + [PEF2256_GCM_CONFIG_2048000] = { + .gcm_12 = {0x00, 0x58, 0xD2, 0xC2, 0x00, 0x10}, + .gcm_2x = {0x00, 0x18, 0xFB, 0x0B, 0x00, 0x2F, 0xDB, 0xDF}, + }, + [PEF2256_GCM_CONFIG_8192000] = { + .gcm_12 = {0x00, 0x58, 0xD2, 0xC2, 0x03, 0x10}, + .gcm_2x = {0x00, 0x18, 0xFB, 0x0B, 0x00, 0x0B, 0xDB, 0xDF}, + }, + [PEF2256_GCM_CONFIG_10000000] = { + .gcm_12 = {0x90, 0x51, 0x81, 0x8F, 0x04, 0x10}, + .gcm_2x = {0x40, 0x1B, 0x3D, 0x0A, 0x00, 0x07, 0xC9, 0xDC}, + }, + [PEF2256_GCM_CONFIG_12352000] = { + .gcm_12 = {0xF0, 0x51, 0x00, 0x80, 0x07, 0x15}, + .gcm_2x = {0x00, 0x19, 0x00, 0x08, 0x01, 0x0A, 0x98, 0xDA}, + }, + [PEF2256_GCM_CONFIG_16384000] = { + .gcm_12 = {0x00, 0x58, 0xD2, 0xC2, 0x07, 0x10}, + .gcm_2x = {0x00, 0x18, 0xFB, 0x0B, 0x01, 0x0B, 0xDB, 0xDF}, + }, +}; + +static int pef2256_setup_gcm(struct pef2256 *pef2256) +{ + enum pef2256_gcm_config_item item; + unsigned long mclk_rate; + const u8 *gcm; + int i, count; + + mclk_rate = clk_get_rate(pef2256->mclk); + switch (mclk_rate) { + case 1544000: + item = PEF2256_GCM_CONFIG_1544000; + break; + case 2048000: + item = PEF2256_GCM_CONFIG_2048000; + break; + case 8192000: + item = PEF2256_GCM_CONFIG_8192000; + break; + case 10000000: + item = PEF2256_GCM_CONFIG_10000000; + break; + case 12352000: + item = PEF2256_GCM_CONFIG_12352000; + break; + case 16384000: + item = PEF2256_GCM_CONFIG_16384000; + break; + default: + dev_err(pef2256->dev, "Unsupported v2.x MCLK rate %lu\n", mclk_rate); + return -EINVAL; + } + + BUILD_BUG_ON(item >= ARRAY_SIZE(pef2256_gcm_configs)); + + if (pef2256->version == PEF2256_VERSION_1_2) { + gcm = pef2256_gcm_configs[item].gcm_12; + count = ARRAY_SIZE(pef2256_gcm_configs[item].gcm_12); + } else { + gcm = pef2256_gcm_configs[item].gcm_2x; + count = ARRAY_SIZE(pef2256_gcm_configs[item].gcm_2x); + } + + for (i = 0; i < count; i++) + pef2256_write8(pef2256, PEF2256_GCM(i + 1), *(gcm + i)); + + return 0; +} + +static int pef2256_setup_e1_line(struct pef2256 *pef2256) +{ + u8 fmr1, fmr2; + + /* RCLK output : DPLL clock, DCO-X enabled, DCO-X internal ref clock */ + pef2256_write8(pef2256, PEF2256_CMR1, 0x00); + + /* SCLKR selected, SCLKX selected, + * receive synchro pulse sourced by SYPR, + * transmit synchro pulse sourced by SYPX, + * DCO-X center frequency enabled + */ + pef2256_write8(pef2256, PEF2256_CMR2, PEF2256_CMR2_DCOXC); + + if (pef2256->is_subordinate) { + /* select RCLK source = 2M, disable switching from RCLK to SYNC */ + pef2256_clrsetbits8(pef2256, PEF2256_CMR1, PEF2256_CMR1_RS_MASK, + PEF2256_CMR1_RS_DCOR_2048 | PEF2256_CMR1_DCS); + } + + /* slave mode, local loop off, mode short-haul + * In v2.x, bit3 is a forced 1 bit in the datasheet -> Need to be set. + */ + if (pef2256->version == PEF2256_VERSION_1_2) + pef2256_write8(pef2256, PEF2256_LIM0, 0x00); + else + pef2256_write8(pef2256, PEF2256_LIM0, PEF2256_2X_LIM0_BIT3); + + /* "master" mode */ + if (!pef2256->is_subordinate) + pef2256_setbits8(pef2256, PEF2256_LIM0, PEF2256_LIM0_MAS); + + /* analog interface selected, remote loop off */ + pef2256_write8(pef2256, PEF2256_LIM1, 0x00); + + /* receive input threshold = 0,21V */ + if (pef2256->version == PEF2256_VERSION_1_2) + pef2256_clrsetbits8(pef2256, PEF2256_LIM1, PEF2256_12_LIM1_RIL_MASK, + PEF2256_12_LIM1_RIL_210); + else + pef2256_clrsetbits8(pef2256, PEF2256_LIM1, PEF2256_2X_LIM1_RIL_MASK, + PEF2256_2X_LIM1_RIL_210); + + /* transmit pulse mask, default value from datasheet + * transmit line in normal operation + */ + if (pef2256->version == PEF2256_VERSION_1_2) + pef2256_write8(pef2256, PEF2256_XPM0, 0x7B); + else + pef2256_write8(pef2256, PEF2256_XPM0, 0x9C); + pef2256_write8(pef2256, PEF2256_XPM1, 0x03); + pef2256_write8(pef2256, PEF2256_XPM2, 0x00); + + /* HDB3 coding, no alarm simulation */ + pef2256_write8(pef2256, PEF2256_FMR0, PEF2256_FMR0_XC_HDB3 | PEF2256_FMR0_RC_HDB3); + + /* E1, frame format, 2 Mbit/s system data rate, no AIS + * transmission to remote end or system interface, payload loop + * off, transmit remote alarm on + */ + fmr1 = 0x00; + fmr2 = PEF2256_FMR2_AXRA; + switch (pef2256->frame_type) { + case PEF2256_FRAME_E1_DOUBLEFRAME: + fmr2 |= PEF2256_FMR2_RFS_DOUBLEFRAME; + break; + case PEF2256_FRAME_E1_CRC4_MULTIFRAME: + fmr1 |= PEF2256_FMR1_XFS; + fmr2 |= PEF2256_FMR2_RFS_CRC4_MULTIFRAME; + break; + case PEF2256_FRAME_E1_AUTO_MULTIFRAME: + fmr1 |= PEF2256_FMR1_XFS; + fmr2 |= PEF2256_FMR2_RFS_AUTO_MULTIFRAME; + break; + default: + dev_err(pef2256->dev, "Unsupported frame type %d\n", pef2256->frame_type); + return -EINVAL; + } + pef2256_clrsetbits8(pef2256, PEF2256_FMR1, PEF2256_FMR1_XFS, fmr1); + pef2256_write8(pef2256, PEF2256_FMR2, fmr2); + + if (!pef2256->is_subordinate) { + /* SEC input, active high */ + pef2256_write8(pef2256, PEF2256_GPC1, PEF2256_GPC1_CSFP_SEC_IN_HIGH); + } else { + /* FSC output, active high */ + pef2256_write8(pef2256, PEF2256_GPC1, PEF2256_GPC1_CSFP_FSC_OUT_HIGH); + } + + /* SCLKR, SCLKX, RCLK configured to inputs, + * XFMS active low, CLK1 and CLK2 pin configuration + */ + pef2256_write8(pef2256, PEF2256_PC5, 0x00); + pef2256_write8(pef2256, PEF2256_PC6, 0x00); + + /* port RCLK is output */ + pef2256_setbits8(pef2256, PEF2256_PC5, PEF2256_PC5_CRP); + + return 0; +} + +static void pef2256_setup_e1_los(struct pef2256 *pef2256) +{ + /* detection of LOS alarm = 176 pulses (ie (10 + 1) * 16) */ + pef2256_write8(pef2256, PEF2256_PCD, 10); + /* recovery of LOS alarm = 22 pulses (ie 21 + 1) */ + pef2256_write8(pef2256, PEF2256_PCR, 21); + /* E1 default for the receive slicer threshold */ + pef2256_write8(pef2256, PEF2256_LIM2, PEF2256_LIM2_SLT_THR50); + if (pef2256->is_subordinate) { + /* Loop-timed */ + pef2256_setbits8(pef2256, PEF2256_LIM2, PEF2256_LIM2_ELT); + } +} + +static int pef2256_setup_e1_system(struct pef2256 *pef2256) +{ + u8 sic1, fmr1; + + /* 2.048 MHz system clocking rate, receive buffer 2 frames, transmit + * buffer bypass, data sampled and transmitted on the falling edge of + * SCLKR/X, automatic freeze signaling, data is active in the first + * channel phase + */ + pef2256_write8(pef2256, PEF2256_SIC1, 0x00); + pef2256_write8(pef2256, PEF2256_SIC2, 0x00); + pef2256_write8(pef2256, PEF2256_SIC3, 0x00); + + if (pef2256->is_subordinate) { + /* transmit buffer size = 2 frames, transparent mode */ + pef2256_clrsetbits8(pef2256, PEF2256_SIC1, PEF2256_SIC1_XBS_MASK, + PEF2256_SIC1_XBS_2FRAMES); + } + + if (pef2256->version != PEF2256_VERSION_1_2) { + /* during inactive channel phase switch RDO/RSIG into tri-state */ + pef2256_setbits8(pef2256, PEF2256_SIC3, PEF2256_SIC3_RTRI); + } + + if (pef2256->is_tx_falling_edge) { + /* falling edge sync pulse transmit, rising edge sync pulse receive */ + pef2256_clrsetbits8(pef2256, PEF2256_SIC3, PEF2256_SIC3_RESX, PEF2256_SIC3_RESR); + } else { + /* rising edge sync pulse transmit, falling edge sync pulse receive */ + pef2256_clrsetbits8(pef2256, PEF2256_SIC3, PEF2256_SIC3_RESR, PEF2256_SIC3_RESX); + } + + /* transmit offset counter (XCO10..0) = 4 */ + pef2256_write8(pef2256, PEF2256_XC0, 0); + pef2256_write8(pef2256, PEF2256_XC1, 4); + /* receive offset counter (RCO10..0) = 4 */ + pef2256_write8(pef2256, PEF2256_RC0, 0); + pef2256_write8(pef2256, PEF2256_RC1, 4); + + /* system clock rate */ + switch (pef2256->sysclk_rate) { + case 2048000: + sic1 = PEF2256_SIC1_SSC_2048; + break; + case 4096000: + sic1 = PEF2256_SIC1_SSC_4096; + break; + case 8192000: + sic1 = PEF2256_SIC1_SSC_8192; + break; + case 16384000: + sic1 = PEF2256_SIC1_SSC_16384; + break; + default: + dev_err(pef2256->dev, "Unsupported sysclk rate %lu\n", pef2256->sysclk_rate); + return -EINVAL; + } + pef2256_clrsetbits8(pef2256, PEF2256_SIC1, PEF2256_SIC1_SSC_MASK, sic1); + + /* data clock rate */ + switch (pef2256->data_rate) { + case 2048000: + fmr1 = PEF2256_FMR1_SSD_2048; + sic1 = PEF2256_SIC1_SSD_2048; + break; + case 4096000: + fmr1 = PEF2256_FMR1_SSD_4096; + sic1 = PEF2256_SIC1_SSD_4096; + break; + case 8192000: + fmr1 = PEF2256_FMR1_SSD_8192; + sic1 = PEF2256_SIC1_SSD_8192; + break; + case 16384000: + fmr1 = PEF2256_FMR1_SSD_16384; + sic1 = PEF2256_SIC1_SSD_16384; + break; + default: + dev_err(pef2256->dev, "Unsupported data rate %u\n", pef2256->data_rate); + return -EINVAL; + } + pef2256_clrsetbits8(pef2256, PEF2256_FMR1, PEF2256_FMR1_SSD_MASK, fmr1); + pef2256_clrsetbits8(pef2256, PEF2256_SIC1, PEF2256_SIC1_SSD_MASK, sic1); + + /* channel phase */ + pef2256_clrsetbits8(pef2256, PEF2256_SIC2, PEF2256_SIC2_SICS_MASK, + PEF2256_SIC2_SICS(pef2256->channel_phase)); + + return 0; +} + +static void pef2256_setup_e1_signaling(struct pef2256 *pef2256) +{ + /* All bits of the transmitted service word are cleared */ + pef2256_write8(pef2256, PEF2256_XSW, PEF2256_XSW_XY(0x1F)); + + /* CAS disabled and clear spare bit values */ + pef2256_write8(pef2256, PEF2256_XSP, 0x00); + + if (pef2256->is_subordinate) { + /* transparent mode */ + pef2256_setbits8(pef2256, PEF2256_XSW, PEF2256_XSW_XTM); + } + + /* Si-Bit, Spare bit For International, FAS word */ + pef2256_setbits8(pef2256, PEF2256_XSW, PEF2256_XSW_XSIS); + pef2256_setbits8(pef2256, PEF2256_XSP, PEF2256_XSP_XSIF); + + /* no transparent mode active */ + pef2256_write8(pef2256, PEF2256_TSWM, 0x00); +} + +static void pef2256_setup_e1_errors(struct pef2256 *pef2256) +{ + /* error counter latched every 1s */ + pef2256_setbits8(pef2256, PEF2256_FMR1, PEF2256_FMR1_ECM); + + /* error counter mode COFA */ + pef2256_setbits8(pef2256, PEF2256_GCR, PEF2256_GCR_ECMC); + + /* errors in service words have no influence */ + pef2256_setbits8(pef2256, PEF2256_RC0, PEF2256_RC0_SWD); + + /* 4 consecutive incorrect FAS causes loss of sync */ + pef2256_setbits8(pef2256, PEF2256_RC0, PEF2256_RC0_ASY4); +} + +static int pef2256_setup_e1(struct pef2256 *pef2256) +{ + int ret; + + /* Setup, Master clocking mode (GCM8..1) */ + ret = pef2256_setup_gcm(pef2256); + if (ret) + return ret; + + /* Select E1 mode */ + pef2256_write8(pef2256, PEF2256_FMR1, 0x00); + + /* internal second timer, power on */ + pef2256_write8(pef2256, PEF2256_GCR, 0x00); + + /* Setup line interface */ + ret = pef2256_setup_e1_line(pef2256); + if (ret) + return ret; + + /* Setup Loss-of-signal detection and recovery */ + pef2256_setup_e1_los(pef2256); + + /* Setup system interface */ + ret = pef2256_setup_e1_system(pef2256); + if (ret) + return ret; + + /* Setup signaling */ + pef2256_setup_e1_signaling(pef2256); + + /* Setup errors counters and condition */ + pef2256_setup_e1_errors(pef2256); + + /* status changed interrupt at both up and down */ + pef2256_setbits8(pef2256, PEF2256_GCR, PEF2256_GCR_SCI); + + /* Clear any ISR2 pending interrupts and unmask needed interrupts */ + pef2256_read8(pef2256, PEF2256_ISR2); + pef2256_clrbits8(pef2256, PEF2256_IMR2, PEF2256_INT2_LOS | PEF2256_INT2_AIS); + + /* reset lines */ + pef2256_write8(pef2256, PEF2256_CMDR, PEF2256_CMDR_RRES | PEF2256_CMDR_XRES); + return 0; +} + +static void pef2256_isr_default_handler(struct pef2256 *pef2256, u8 nbr, u8 isr) +{ + dev_warn_ratelimited(pef2256->dev, "ISR%u: 0x%02x not handled\n", nbr, isr); +} + +static bool pef2256_is_carrier_on(struct pef2256 *pef2256) +{ + u8 frs0; + + frs0 = pef2256_read8(pef2256, PEF2256_FRS0); + return !(frs0 & (PEF2256_FRS0_LOS | PEF2256_FRS0_AIS)); +} + +static void pef2256_isr2_handler(struct pef2256 *pef2256, u8 nbr, u8 isr) +{ + bool carrier; + + if (isr & (PEF2256_INT2_LOS | PEF2256_INT2_AIS)) { + carrier = pef2256_is_carrier_on(pef2256); + if (atomic_xchg(&pef2256->carrier, carrier) != carrier) + framer_notify_status_change(pef2256->framer); + } +} + +static irqreturn_t pef2256_irq_handler(int irq, void *priv) +{ + static void (*pef2256_isr_handler[])(struct pef2256 *, u8, u8) = { + [0] = pef2256_isr_default_handler, + [1] = pef2256_isr_default_handler, + [2] = pef2256_isr2_handler, + [3] = pef2256_isr_default_handler, + [4] = pef2256_isr_default_handler, + [5] = pef2256_isr_default_handler + }; + struct pef2256 *pef2256 = (struct pef2256 *)priv; + u8 gis; + u8 isr; + u8 n; + + gis = pef2256_read8(pef2256, PEF2256_GIS); + + for (n = 0; n < ARRAY_SIZE(pef2256_isr_handler); n++) { + if (gis & PEF2256_GIS_ISR(n)) { + isr = pef2256_read8(pef2256, PEF2256_ISR(n)); + pef2256_isr_handler[n](pef2256, n, isr); + } + } + + return IRQ_HANDLED; +} + +static int pef2256_check_rates(struct pef2256 *pef2256, unsigned long sysclk_rate, + unsigned long data_rate) +{ + unsigned long rate; + + switch (sysclk_rate) { + case 2048000: + case 4096000: + case 8192000: + case 16384000: + break; + default: + dev_err(pef2256->dev, "Unsupported system clock rate %lu\n", sysclk_rate); + return -EINVAL; + } + + for (rate = data_rate; rate <= data_rate * 4; rate *= 2) { + if (rate == sysclk_rate) + return 0; + } + dev_err(pef2256->dev, "Unsupported data rate %lu with system clock rate %lu\n", + data_rate, sysclk_rate); + return -EINVAL; +} + +static int pef2556_of_parse(struct pef2256 *pef2256, struct device_node *np) +{ + int ret; + + pef2256->data_rate = 2048000; + ret = of_property_read_u32(np, "lantiq,data-rate-bps", &pef2256->data_rate); + if (ret && ret != -EINVAL) { + dev_err(pef2256->dev, "%pOF: failed to read lantiq,data-rate-bps\n", np); + return ret; + } + + ret = pef2256_check_rates(pef2256, pef2256->sysclk_rate, pef2256->data_rate); + if (ret) + return ret; + + pef2256->is_tx_falling_edge = of_property_read_bool(np, "lantiq,clock-falling-edge"); + + pef2256->channel_phase = 0; + ret = of_property_read_u8(np, "lantiq,channel-phase", &pef2256->channel_phase); + if (ret && ret != -EINVAL) { + dev_err(pef2256->dev, "%pOF: failed to read lantiq,channel-phase\n", + np); + return ret; + } + if (pef2256->channel_phase >= pef2256->sysclk_rate / pef2256->data_rate) { + dev_err(pef2256->dev, "%pOF: Invalid lantiq,channel-phase %u\n", + np, pef2256->channel_phase); + return -EINVAL; + } + + return 0; +} + +static const struct regmap_config pef2256_regmap_config = { + .reg_bits = 32, + .val_bits = 8, + .max_register = 0xff, +}; + +static const struct mfd_cell pef2256_devs[] = { + { .name = "lantiq-pef2256-pinctrl", }, +}; + +static int pef2256_add_audio_devices(struct pef2256 *pef2256) +{ + const char *compatible = "lantiq,pef2256-codec"; + struct mfd_cell *audio_devs; + struct device_node *np; + unsigned int count = 0; + unsigned int i; + int ret; + + for_each_available_child_of_node(pef2256->dev->of_node, np) { + if (of_device_is_compatible(np, compatible)) + count++; + } + + if (!count) + return 0; + + audio_devs = kcalloc(count, sizeof(*audio_devs), GFP_KERNEL); + if (!audio_devs) + return -ENOMEM; + + for (i = 0; i < count; i++) { + audio_devs[i].name = "framer-codec"; + audio_devs[i].of_compatible = compatible; + audio_devs[i].id = i; + } + + ret = mfd_add_devices(pef2256->dev, 0, audio_devs, count, NULL, 0, NULL); + kfree(audio_devs); + return ret; +} + +static int pef2256_framer_get_status(struct framer *framer, struct framer_status *status) +{ + struct pef2256 *pef2256 = framer_get_drvdata(framer); + + status->link_is_on = !!atomic_read(&pef2256->carrier); + return 0; +} + +static int pef2256_framer_set_config(struct framer *framer, const struct framer_config *config) +{ + struct pef2256 *pef2256 = framer_get_drvdata(framer); + + if (config->iface != FRAMER_IFACE_E1) { + dev_err(pef2256->dev, "Only E1 line is currently supported\n"); + return -EOPNOTSUPP; + } + + switch (config->clock_type) { + case FRAMER_CLOCK_EXT: + pef2256->is_subordinate = true; + break; + case FRAMER_CLOCK_INT: + pef2256->is_subordinate = false; + break; + default: + return -EINVAL; + } + + /* Apply the new settings */ + return pef2256_setup_e1(pef2256); +} + +static int pef2256_framer_get_config(struct framer *framer, struct framer_config *config) +{ + struct pef2256 *pef2256 = framer_get_drvdata(framer); + + config->iface = FRAMER_IFACE_E1; + config->clock_type = pef2256->is_subordinate ? FRAMER_CLOCK_EXT : FRAMER_CLOCK_INT; + config->line_clock_rate = 2048000; + return 0; +} + +static const struct framer_ops pef2256_framer_ops = { + .owner = THIS_MODULE, + .get_status = pef2256_framer_get_status, + .get_config = pef2256_framer_get_config, + .set_config = pef2256_framer_set_config, +}; + +static int pef2256_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + unsigned long sclkr_rate, sclkx_rate; + struct framer_provider *framer_provider; + struct pef2256 *pef2256; + const char *version_txt; + void __iomem *iomem; + int ret; + int irq; + + pef2256 = devm_kzalloc(&pdev->dev, sizeof(*pef2256), GFP_KERNEL); + if (!pef2256) + return -ENOMEM; + + pef2256->dev = &pdev->dev; + atomic_set(&pef2256->carrier, 0); + + pef2256->is_subordinate = true; + pef2256->frame_type = PEF2256_FRAME_E1_DOUBLEFRAME; + + iomem = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(iomem)) + return PTR_ERR(iomem); + + pef2256->regmap = devm_regmap_init_mmio(&pdev->dev, iomem, + &pef2256_regmap_config); + if (IS_ERR(pef2256->regmap)) { + dev_err(&pdev->dev, "Failed to initialise Regmap (%ld)\n", + PTR_ERR(pef2256->regmap)); + return PTR_ERR(pef2256->regmap); + } + + pef2256->mclk = devm_clk_get_enabled(&pdev->dev, "mclk"); + if (IS_ERR(pef2256->mclk)) + return PTR_ERR(pef2256->mclk); + + pef2256->sclkr = devm_clk_get_enabled(&pdev->dev, "sclkr"); + if (IS_ERR(pef2256->sclkr)) + return PTR_ERR(pef2256->sclkr); + + pef2256->sclkx = devm_clk_get_enabled(&pdev->dev, "sclkx"); + if (IS_ERR(pef2256->sclkx)) + return PTR_ERR(pef2256->sclkx); + + /* Both SCLKR (receive) and SCLKX (transmit) must have the same rate, + * stored as sysclk_rate. + * The exact value will be checked at pef2256_check_rates() + */ + sclkr_rate = clk_get_rate(pef2256->sclkr); + sclkx_rate = clk_get_rate(pef2256->sclkx); + if (sclkr_rate != sclkx_rate) { + dev_err(pef2256->dev, "clk rate mismatch. sclkr %lu Hz, sclkx %lu Hz\n", + sclkr_rate, sclkx_rate); + return -EINVAL; + } + pef2256->sysclk_rate = sclkr_rate; + + /* Reset the component. The MCLK clock must be active during reset */ + pef2256->reset_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(pef2256->reset_gpio)) + return PTR_ERR(pef2256->reset_gpio); + if (pef2256->reset_gpio) { + gpiod_set_value_cansleep(pef2256->reset_gpio, 1); + usleep_range(10, 20); + gpiod_set_value_cansleep(pef2256->reset_gpio, 0); + usleep_range(10, 20); + } + + pef2256->version = pef2256_get_version(pef2256); + switch (pef2256->version) { + case PEF2256_VERSION_1_2: + version_txt = "1.2"; + break; + case PEF2256_VERSION_2_1: + version_txt = "2.1"; + break; + case PEF2256_VERSION_2_2: + version_txt = "2.2"; + break; + default: + return -ENODEV; + } + dev_info(pef2256->dev, "Version %s detected\n", version_txt); + + ret = pef2556_of_parse(pef2256, np); + if (ret) + return ret; + + /* Create the framer. It can be used on interrupts */ + pef2256->framer = devm_framer_create(pef2256->dev, NULL, &pef2256_framer_ops); + if (IS_ERR(pef2256->framer)) + return PTR_ERR(pef2256->framer); + + framer_set_drvdata(pef2256->framer, pef2256); + + /* Disable interrupts */ + pef2256_write8(pef2256, PEF2256_IMR0, 0xff); + pef2256_write8(pef2256, PEF2256_IMR1, 0xff); + pef2256_write8(pef2256, PEF2256_IMR2, 0xff); + pef2256_write8(pef2256, PEF2256_IMR3, 0xff); + pef2256_write8(pef2256, PEF2256_IMR4, 0xff); + pef2256_write8(pef2256, PEF2256_IMR5, 0xff); + + /* Clear any pending interrupts */ + pef2256_read8(pef2256, PEF2256_ISR0); + pef2256_read8(pef2256, PEF2256_ISR1); + pef2256_read8(pef2256, PEF2256_ISR2); + pef2256_read8(pef2256, PEF2256_ISR3); + pef2256_read8(pef2256, PEF2256_ISR4); + pef2256_read8(pef2256, PEF2256_ISR5); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + ret = devm_request_irq(pef2256->dev, irq, pef2256_irq_handler, 0, "pef2256", pef2256); + if (ret < 0) + return ret; + + platform_set_drvdata(pdev, pef2256); + + ret = mfd_add_devices(pef2256->dev, 0, pef2256_devs, + ARRAY_SIZE(pef2256_devs), NULL, 0, NULL); + if (ret) { + dev_err(pef2256->dev, "add devices failed (%d)\n", ret); + return ret; + } + + ret = pef2256_setup_e1(pef2256); + if (ret) + return ret; + + framer_provider = devm_framer_provider_of_register(pef2256->dev, + framer_provider_simple_of_xlate); + if (IS_ERR(framer_provider)) + return PTR_ERR(framer_provider); + + /* Add audio devices */ + ret = pef2256_add_audio_devices(pef2256); + if (ret < 0) { + dev_err(pef2256->dev, "add audio devices failed (%d)\n", ret); + return ret; + } + + return 0; +} + +static int pef2256_remove(struct platform_device *pdev) +{ + struct pef2256 *pef2256 = platform_get_drvdata(pdev); + + /* Disable interrupts */ + pef2256_write8(pef2256, PEF2256_IMR0, 0xff); + pef2256_write8(pef2256, PEF2256_IMR1, 0xff); + pef2256_write8(pef2256, PEF2256_IMR2, 0xff); + pef2256_write8(pef2256, PEF2256_IMR3, 0xff); + pef2256_write8(pef2256, PEF2256_IMR4, 0xff); + pef2256_write8(pef2256, PEF2256_IMR5, 0xff); + + return 0; +} + +static const struct of_device_id pef2256_id_table[] = { + { .compatible = "lantiq,pef2256" }, + {} /* sentinel */ +}; +MODULE_DEVICE_TABLE(of, pef2256_id_table); + +static struct platform_driver pef2256_driver = { + .driver = { + .name = "lantiq-pef2256", + .of_match_table = pef2256_id_table, + }, + .probe = pef2256_probe, + .remove = pef2256_remove, +}; +module_platform_driver(pef2256_driver); + +struct regmap *pef2256_get_regmap(struct pef2256 *pef2256) +{ + return pef2256->regmap; +} +EXPORT_SYMBOL_GPL(pef2256_get_regmap); + +MODULE_AUTHOR("Herve Codina "); +MODULE_DESCRIPTION("PEF2256 driver"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/framer/pef2256.h b/include/linux/framer/pef2256.h new file mode 100644 index 000000000000..71d80af58c40 --- /dev/null +++ b/include/linux/framer/pef2256.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * PEF2256 consumer API + * + * Copyright 2023 CS GROUP France + * + * Author: Herve Codina + */ +#ifndef __PEF2256_H__ +#define __PEF2256_H__ + +#include + +struct pef2256; +struct regmap; + +/* Retrieve the PEF2256 regmap */ +struct regmap *pef2256_get_regmap(struct pef2256 *pef2256); + +/* PEF2256 hardware versions */ +enum pef2256_version { + PEF2256_VERSION_UNKNOWN, + PEF2256_VERSION_1_2, + PEF2256_VERSION_2_1, + PEF2256_VERSION_2_2, +}; + +/* Get the PEF2256 hardware version */ +enum pef2256_version pef2256_get_version(struct pef2256 *pef2256); + +#endif /* __PEF2256_H__ */ -- cgit v1.2.3 From 4f7aa122bc9219baca0bfface5917062d6c45ee8 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Thu, 7 Dec 2023 16:12:04 +0100 Subject: dpll: remove leftover mode_supported() op and use mode_get() instead Mode supported is currently reported to the user exactly the same, as the current mode. That's because mode changing is not implemented. Remove the leftover mode_supported() op and use mode_get() to fill up the supported mode exposed to user. One, if even, mode changing is going to be introduced, this could be very easily taken back. In the meantime, prevent drivers form implementing this in wrong way (as for example recent netdevsim implementation attempt intended to do). Signed-off-by: Jiri Pirko Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/dpll/dpll_netlink.c | 16 ++++++++++------ drivers/net/ethernet/intel/ice/ice_dpll.c | 26 -------------------------- drivers/net/ethernet/mellanox/mlx5/core/dpll.c | 9 --------- drivers/ptp/ptp_ocp.c | 8 -------- include/linux/dpll.h | 3 --- 5 files changed, 10 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index 442a0ebeb953..e1a4737500f5 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -101,13 +101,17 @@ dpll_msg_add_mode_supported(struct sk_buff *msg, struct dpll_device *dpll, { const struct dpll_device_ops *ops = dpll_device_ops(dpll); enum dpll_mode mode; + int ret; - if (!ops->mode_supported) - return 0; - for (mode = DPLL_MODE_MANUAL; mode <= DPLL_MODE_MAX; mode++) - if (ops->mode_supported(dpll, dpll_priv(dpll), mode, extack)) - if (nla_put_u32(msg, DPLL_A_MODE_SUPPORTED, mode)) - return -EMSGSIZE; + /* No mode change is supported now, so the only supported mode is the + * one obtained by mode_get(). + */ + + ret = ops->mode_get(dpll, dpll_priv(dpll), &mode, extack); + if (ret) + return ret; + if (nla_put_u32(msg, DPLL_A_MODE_SUPPORTED, mode)) + return -EMSGSIZE; return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index 86b180cb32a0..b9c5eced6326 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -512,31 +512,6 @@ ice_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv, return 0; } -/** - * ice_dpll_mode_supported - check if dpll's working mode is supported - * @dpll: registered dpll pointer - * @dpll_priv: private data pointer passed on dpll registration - * @mode: mode to be checked for support - * @extack: error reporting - * - * Dpll subsystem callback. Provides information if working mode is supported - * by dpll. - * - * Return: - * * true - mode is supported - * * false - mode is not supported - */ -static bool ice_dpll_mode_supported(const struct dpll_device *dpll, - void *dpll_priv, - enum dpll_mode mode, - struct netlink_ext_ack *extack) -{ - if (mode == DPLL_MODE_AUTOMATIC) - return true; - - return false; -} - /** * ice_dpll_mode_get - get dpll's working mode * @dpll: registered dpll pointer @@ -1197,7 +1172,6 @@ static const struct dpll_pin_ops ice_dpll_output_ops = { static const struct dpll_device_ops ice_dpll_ops = { .lock_status_get = ice_dpll_lock_status_get, - .mode_supported = ice_dpll_mode_supported, .mode_get = ice_dpll_mode_get, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c index 2cd81bb32c66..a7ffd61fe248 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c @@ -128,18 +128,9 @@ static int mlx5_dpll_device_mode_get(const struct dpll_device *dpll, return 0; } -static bool mlx5_dpll_device_mode_supported(const struct dpll_device *dpll, - void *priv, - enum dpll_mode mode, - struct netlink_ext_ack *extack) -{ - return mode == DPLL_MODE_MANUAL; -} - static const struct dpll_device_ops mlx5_dpll_device_ops = { .lock_status_get = mlx5_dpll_device_lock_status_get, .mode_get = mlx5_dpll_device_mode_get, - .mode_supported = mlx5_dpll_device_mode_supported, }; static int mlx5_dpll_pin_direction_get(const struct dpll_pin *pin, diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index 4021d3d325f9..b022af3d20fe 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -4260,13 +4260,6 @@ static int ptp_ocp_dpll_mode_get(const struct dpll_device *dpll, void *priv, return 0; } -static bool ptp_ocp_dpll_mode_supported(const struct dpll_device *dpll, - void *priv, const enum dpll_mode mode, - struct netlink_ext_ack *extack) -{ - return mode == DPLL_MODE_AUTOMATIC; -} - static int ptp_ocp_dpll_direction_get(const struct dpll_pin *pin, void *pin_priv, const struct dpll_device *dpll, @@ -4350,7 +4343,6 @@ static int ptp_ocp_dpll_frequency_get(const struct dpll_pin *pin, static const struct dpll_device_ops dpll_ops = { .lock_status_get = ptp_ocp_dpll_lock_status_get, .mode_get = ptp_ocp_dpll_mode_get, - .mode_supported = ptp_ocp_dpll_mode_supported, }; static const struct dpll_pin_ops dpll_pins_ops = { diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 578fc5fa3750..b1a5f9ca8ee5 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -17,9 +17,6 @@ struct dpll_pin; struct dpll_device_ops { int (*mode_get)(const struct dpll_device *dpll, void *dpll_priv, enum dpll_mode *mode, struct netlink_ext_ack *extack); - bool (*mode_supported)(const struct dpll_device *dpll, void *dpll_priv, - const enum dpll_mode mode, - struct netlink_ext_ack *extack); int (*lock_status_get)(const struct dpll_device *dpll, void *dpll_priv, enum dpll_lock_status *status, struct netlink_ext_ack *extack); -- cgit v1.2.3 From 750e785796bb72423b97cac21ecd0fa3b3b65610 Mon Sep 17 00:00:00 2001 From: Jie Jiang Date: Tue, 12 Dec 2023 09:39:23 +0000 Subject: bpf: Support uid and gid when mounting bpffs Parse uid and gid in bpf_parse_param() so that they can be passed in as the `data` parameter when mount() bpffs. This will be useful when we want to control which user/group has the control to the mounted bpffs, otherwise a separate chown() call will be needed. Signed-off-by: Jie Jiang Signed-off-by: Andrii Nakryiko Acked-by: Mike Frysinger Acked-by: Christian Brauner Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20231212093923.497838-1-jiejiang@chromium.org --- include/linux/bpf.h | 2 ++ kernel/bpf/inode.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0bd4889e917a..c87c608a3689 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1595,6 +1595,8 @@ struct bpf_link_primer { }; struct bpf_mount_opts { + kuid_t uid; + kgid_t gid; umode_t mode; /* BPF token-related delegation options */ diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 5359a0929c35..0a8e1188ea46 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -601,9 +601,16 @@ EXPORT_SYMBOL(bpf_prog_get_type_path); static int bpf_show_options(struct seq_file *m, struct dentry *root) { struct bpf_mount_opts *opts = root->d_sb->s_fs_info; - umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; + struct inode *inode = d_inode(root); + umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; u64 mask; + if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, inode->i_uid)); + if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, inode->i_gid)); if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); @@ -652,6 +659,8 @@ const struct super_operations bpf_super_ops = { }; enum { + OPT_UID, + OPT_GID, OPT_MODE, OPT_DELEGATE_CMDS, OPT_DELEGATE_MAPS, @@ -660,6 +669,8 @@ enum { }; static const struct fs_parameter_spec bpf_fs_parameters[] = { + fsparam_u32 ("uid", OPT_UID), + fsparam_u32 ("gid", OPT_GID), fsparam_u32oct ("mode", OPT_MODE), fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), @@ -672,6 +683,8 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { struct bpf_mount_opts *opts = fc->s_fs_info; struct fs_parse_result result; + kuid_t uid; + kgid_t gid; int opt, err; u64 msk; @@ -694,6 +707,34 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) } switch (opt) { + case OPT_UID: + uid = make_kuid(current_user_ns(), result.uint_32); + if (!uid_valid(uid)) + goto bad_value; + + /* + * The requested uid must be representable in the + * filesystem's idmapping. + */ + if (!kuid_has_mapping(fc->user_ns, uid)) + goto bad_value; + + opts->uid = uid; + break; + case OPT_GID: + gid = make_kgid(current_user_ns(), result.uint_32); + if (!gid_valid(gid)) + goto bad_value; + + /* + * The requested gid must be representable in the + * filesystem's idmapping. + */ + if (!kgid_has_mapping(fc->user_ns, gid)) + goto bad_value; + + opts->gid = gid; + break; case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; @@ -722,6 +763,9 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) } return 0; + +bad_value: + return invalfc(fc, "Bad value for '%s'", param->key); } struct bpf_preload_ops *bpf_preload_ops; @@ -808,6 +852,8 @@ static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &bpf_super_ops; inode = sb->s_root->d_inode; + inode->i_uid = opts->uid; + inode->i_gid = opts->gid; inode->i_op = &bpf_dir_iops; inode->i_mode &= ~S_IALLUGO; populate_bpffs(sb->s_root); @@ -843,6 +889,8 @@ static int bpf_init_fs_context(struct fs_context *fc) return -ENOMEM; opts->mode = S_IRWXUGO; + opts->uid = current_fsuid(); + opts->gid = current_fsgid(); /* start out with no BPF token delegation enabled */ opts->delegate_cmds = 0; -- cgit v1.2.3 From 537fec0733c4a72e2a2b69fee365459c5b75d92e Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 5 Dec 2023 22:08:42 +0100 Subject: net: make vlan_get_tag() return -ENODATA instead of -EINVAL __vlan_hwaccel_get_tag() is used in veth XDP hints implementation, its return value (-EINVAL if skb is not VLAN tagged) is passed to bpf code, but XDP hints specification requires drivers to return -ENODATA, if a hint cannot be provided for a particular packet. Solve this inconsistency by changing error return value of __vlan_hwaccel_get_tag() from -EINVAL to -ENODATA, do the same thing to __vlan_get_tag(), because this function is supposed to follow the same convention. This, in turn, makes -ENODATA the only non-zero value vlan_get_tag() can return. We can do this with no side effects, because none of the users of the 3 above-mentioned functions rely on the exact value. Suggested-by: Jesper Dangaard Brouer Acked-by: Stanislav Fomichev Signed-off-by: Larysa Zaremba Link: https://lore.kernel.org/r/20231205210847.28460-14-larysa.zaremba@intel.com Signed-off-by: Alexei Starovoitov --- include/linux/if_vlan.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 3028af87716e..c1645c86eed9 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -540,7 +540,7 @@ static inline int __vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci) struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); if (!eth_type_vlan(veth->h_vlan_proto)) - return -EINVAL; + return -ENODATA; *vlan_tci = ntohs(veth->h_vlan_TCI); return 0; @@ -561,7 +561,7 @@ static inline int __vlan_hwaccel_get_tag(const struct sk_buff *skb, return 0; } else { *vlan_tci = 0; - return -EINVAL; + return -ENODATA; } } -- cgit v1.2.3 From 7978bad4b6b9265a1e808a5f679ee428d1dd6523 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 5 Dec 2023 22:08:43 +0100 Subject: mlx5: implement VLAN tag XDP hint Implement the newly added .xmo_rx_vlan_tag() hint function. Reviewed-by: Tariq Toukan Signed-off-by: Larysa Zaremba Acked-by: Jesper Dangaard Brouer Link: https://lore.kernel.org/r/20231205210847.28460-15-larysa.zaremba@intel.com Signed-off-by: Alexei Starovoitov --- drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c | 15 +++++++++++++++ include/linux/mlx5/device.h | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index e2e7d82cfca4..9e695ed122ee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -256,9 +256,24 @@ static int mlx5e_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, return 0; } +static int mlx5e_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, + u16 *vlan_tci) +{ + const struct mlx5e_xdp_buff *_ctx = (void *)ctx; + const struct mlx5_cqe64 *cqe = _ctx->cqe; + + if (!cqe_has_vlan(cqe)) + return -ENODATA; + + *vlan_proto = htons(ETH_P_8021Q); + *vlan_tci = be16_to_cpu(cqe->vlan_info); + return 0; +} + const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = { .xmo_rx_timestamp = mlx5e_xdp_rx_timestamp, .xmo_rx_hash = mlx5e_xdp_rx_hash, + .xmo_rx_vlan_tag = mlx5e_xdp_rx_vlan_tag, }; struct mlx5e_xsk_tx_complete { diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 820bca965fb6..01275c6e8468 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -918,7 +918,7 @@ static inline u8 get_cqe_tls_offload(struct mlx5_cqe64 *cqe) return (cqe->tls_outer_l3_tunneled >> 3) & 0x3; } -static inline bool cqe_has_vlan(struct mlx5_cqe64 *cqe) +static inline bool cqe_has_vlan(const struct mlx5_cqe64 *cqe) { return cqe->l4_l3_hdr_type & 0x1; } -- cgit v1.2.3 From 0c476157085fe2ad13b9bec70ea672e86647fa1a Mon Sep 17 00:00:00 2001 From: Oleksij Rempel Date: Tue, 12 Dec 2023 06:41:43 +0100 Subject: net: phy: c45: add genphy_c45_pma_read_ext_abilities() function Move part of the genphy_c45_pma_read_abilities() code to a separate function. Some PHYs do not implement PMA/PMD status 2 register (Register 1.8) but do implement PMA/PMD extended ability register (Register 1.11). To make use of it, we need to be able to access this part of code separately. Signed-off-by: Oleksij Rempel Reviewed-by: Andrew Lunn Reviewed-by: Russell King (Oracle) Link: https://lore.kernel.org/r/20231212054144.87527-2-o.rempel@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/phy/phy-c45.c | 129 ++++++++++++++++++++++++++-------------------- include/linux/phy.h | 1 + 2 files changed, 75 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c index 8e6fd4962c48..747d14bf152c 100644 --- a/drivers/net/phy/phy-c45.c +++ b/drivers/net/phy/phy-c45.c @@ -919,6 +919,79 @@ int genphy_c45_pma_baset1_read_abilities(struct phy_device *phydev) } EXPORT_SYMBOL_GPL(genphy_c45_pma_baset1_read_abilities); +/** + * genphy_c45_pma_read_ext_abilities - read supported link modes from PMA + * @phydev: target phy_device struct + * + * Read the supported link modes from the PMA/PMD extended ability register + * (Register 1.11). + */ +int genphy_c45_pma_read_ext_abilities(struct phy_device *phydev) +{ + int val; + + val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_EXTABLE); + if (val < 0) + return val; + + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10GBLRM); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10GBT); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10GBKX4); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10GBKR); + linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_1000BT); + linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_1000BKX); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_100BTX); + linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_100BTX); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10BT); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, + phydev->supported, + val & MDIO_PMA_EXTABLE_10BT); + + if (val & MDIO_PMA_EXTABLE_NBT) { + val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, + MDIO_PMA_NG_EXTABLE); + if (val < 0) + return val; + + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_NG_EXTABLE_2_5GBT); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + phydev->supported, + val & MDIO_PMA_NG_EXTABLE_5GBT); + } + + if (val & MDIO_PMA_EXTABLE_BT1) { + val = genphy_c45_pma_baset1_read_abilities(phydev); + if (val < 0) + return val; + } + + return 0; +} +EXPORT_SYMBOL_GPL(genphy_c45_pma_read_ext_abilities); + /** * genphy_c45_pma_read_abilities - read supported link modes from PMA * @phydev: target phy_device struct @@ -962,63 +1035,9 @@ int genphy_c45_pma_read_abilities(struct phy_device *phydev) val & MDIO_PMA_STAT2_10GBER); if (val & MDIO_PMA_STAT2_EXTABLE) { - val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, MDIO_PMA_EXTABLE); + val = genphy_c45_pma_read_ext_abilities(phydev); if (val < 0) return val; - - linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10GBLRM); - linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10GBT); - linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10GBKX4); - linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10GBKR); - linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_1000BT); - linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_1000BKX); - - linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_100BTX); - linkmode_mod_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_100BTX); - - linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10BT); - linkmode_mod_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, - phydev->supported, - val & MDIO_PMA_EXTABLE_10BT); - - if (val & MDIO_PMA_EXTABLE_NBT) { - val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD, - MDIO_PMA_NG_EXTABLE); - if (val < 0) - return val; - - linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_NG_EXTABLE_2_5GBT); - - linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, - phydev->supported, - val & MDIO_PMA_NG_EXTABLE_5GBT); - } - - if (val & MDIO_PMA_EXTABLE_BT1) { - val = genphy_c45_pma_baset1_read_abilities(phydev); - if (val < 0) - return val; - } } /* This is optional functionality. If not supported, we may get an error diff --git a/include/linux/phy.h b/include/linux/phy.h index 6e7ebcc50b85..dbb5e13e3e1b 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1866,6 +1866,7 @@ int genphy_c45_an_config_aneg(struct phy_device *phydev); int genphy_c45_an_disable_aneg(struct phy_device *phydev); int genphy_c45_read_mdix(struct phy_device *phydev); int genphy_c45_pma_read_abilities(struct phy_device *phydev); +int genphy_c45_pma_read_ext_abilities(struct phy_device *phydev); int genphy_c45_pma_baset1_read_abilities(struct phy_device *phydev); int genphy_c45_read_eee_abilities(struct phy_device *phydev); int genphy_c45_pma_baset1_read_master_slave(struct phy_device *phydev); -- cgit v1.2.3 From 13049408a4bd29c92227ca2d6befab80dbb96663 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Sun, 7 May 2023 16:47:42 +0300 Subject: net/mlx5: Add mlx5_ifc bits used for supporting single netdev Socket-Direct Multiple device caps and features are required to support single netdev Socket-Direct. Add them here in preparation for the feature implementation. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index ce2e71cd6d2a..405d141b4a08 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -435,7 +435,7 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 flow_table_modify[0x1]; u8 reformat[0x1]; u8 decap[0x1]; - u8 reserved_at_9[0x1]; + u8 reset_root_to_default[0x1]; u8 pop_vlan[0x1]; u8 push_vlan[0x1]; u8 reserved_at_c[0x1]; @@ -1801,7 +1801,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 disable_local_lb_uc[0x1]; u8 disable_local_lb_mc[0x1]; u8 log_min_hairpin_wq_data_sz[0x5]; - u8 reserved_at_3e8[0x2]; + u8 reserved_at_3e8[0x1]; + u8 silent_mode[0x1]; u8 vhca_state[0x1]; u8 log_max_vlan_list[0x5]; u8 reserved_at_3f0[0x3]; @@ -1818,7 +1819,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_460[0x1]; u8 ats[0x1]; - u8 reserved_at_462[0x1]; + u8 cross_vhca_rqt[0x1]; u8 log_max_uctx[0x5]; u8 reserved_at_468[0x1]; u8 crypto[0x1]; @@ -1943,6 +1944,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { enum { MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_TO_REMOTE_FLOW_TABLE_MISS = 0x80000, + MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE = (1ULL << 20), }; enum { @@ -1992,7 +1994,11 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 reserved_at_260[0x120]; u8 reserved_at_380[0x10]; u8 ec_vf_vport_base[0x10]; - u8 reserved_at_3a0[0x460]; + + u8 reserved_at_3a0[0x10]; + u8 max_rqt_vhca_id[0x10]; + + u8 reserved_at_3c0[0x440]; }; enum mlx5_ifc_flow_destination_type { @@ -2151,6 +2157,13 @@ struct mlx5_ifc_rq_num_bits { u8 rq_num[0x18]; }; +struct mlx5_ifc_rq_vhca_bits { + u8 reserved_at_0[0x8]; + u8 rq_num[0x18]; + u8 reserved_at_20[0x10]; + u8 rq_vhca_id[0x10]; +}; + struct mlx5_ifc_mac_address_layout_bits { u8 reserved_at_0[0x10]; u8 mac_addr_47_32[0x10]; @@ -3901,7 +3914,10 @@ struct mlx5_ifc_rqtc_bits { u8 reserved_at_e0[0x6a0]; - struct mlx5_ifc_rq_num_bits rq_num[]; + union { + DECLARE_FLEX_ARRAY(struct mlx5_ifc_rq_num_bits, rq_num); + DECLARE_FLEX_ARRAY(struct mlx5_ifc_rq_vhca_bits, rq_vhca); + }; }; enum { @@ -4744,7 +4760,10 @@ struct mlx5_ifc_set_l2_table_entry_in_bits { u8 reserved_at_c0[0x20]; - u8 reserved_at_e0[0x13]; + u8 reserved_at_e0[0x10]; + u8 silent_mode_valid[0x1]; + u8 silent_mode[0x1]; + u8 reserved_at_f2[0x1]; u8 vlan_valid[0x1]; u8 vlan[0xc]; -- cgit v1.2.3 From f5e956329960903d908668d7a20bbc08e0a8b92b Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 7 Aug 2023 09:05:34 +0300 Subject: net/mlx5: Expose Management PCIe Index Register (MPIR) MPIR register allows to query the PCIe indexes and Socket-Direct related parameters. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/port.c | 10 ++++++++++ include/linux/mlx5/driver.h | 1 + include/linux/mlx5/mlx5_ifc.h | 14 ++++++++++++++ 4 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 6b14e347d914..a79b7959361b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -243,6 +243,7 @@ int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcap, u8 feature_group, u8 access_reg_group); int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam, u8 feature_group, u8 access_reg_group); +int mlx5_query_mpir_reg(struct mlx5_core_dev *dev, u32 *mpir); void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 7d8c732818f2..7fba1c46e2ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -1206,3 +1206,13 @@ int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed) *speed = max_speed; return 0; } + +int mlx5_query_mpir_reg(struct mlx5_core_dev *dev, u32 *mpir) +{ + u32 in[MLX5_ST_SZ_DW(mpir_reg)] = {}; + int sz = MLX5_ST_SZ_BYTES(mpir_reg); + + MLX5_SET(mpir_reg, in, local_port, 1); + + return mlx5_core_access_reg(dev, in, sz, mpir, sz, MLX5_REG_MPIR, 0, 0); +} diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index d2b8d4a74a30..2f67cec1a898 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -150,6 +150,7 @@ enum { MLX5_REG_MTPPSE = 0x9054, MLX5_REG_MTUTC = 0x9055, MLX5_REG_MPEGC = 0x9056, + MLX5_REG_MPIR = 0x9059, MLX5_REG_MCQS = 0x9060, MLX5_REG_MCQI = 0x9061, MLX5_REG_MCC = 0x9062, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 405d141b4a08..828938368fb7 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -10108,6 +10108,20 @@ struct mlx5_ifc_mpegc_reg_bits { u8 reserved_at_60[0x100]; }; +struct mlx5_ifc_mpir_reg_bits { + u8 sdm[0x1]; + u8 reserved_at_1[0x1b]; + u8 host_buses[0x4]; + + u8 reserved_at_20[0x20]; + + u8 local_port[0x8]; + u8 reserved_at_28[0x15]; + u8 sd_group[0x3]; + + u8 reserved_at_60[0x20]; +}; + enum { MLX5_MTUTC_FREQ_ADJ_UNITS_PPB = 0x0, MLX5_MTUTC_FREQ_ADJ_UNITS_SCALED_PPM = 0x1, -- cgit v1.2.3 From b25bd37c859f32e50a436ab9d2078b76e433008e Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Sun, 6 Aug 2023 14:01:10 +0300 Subject: net/mlx5: Move TISes from priv to mdev HW resources The transport interface send (TIS) object is responsible for performing all transport related operations of the transmit side. Messages from Send Queues get segmented and transmitted by the TIS including all transport required implications, e.g. in the case of large send offload, the TIS is responsible for the segmentation. These are stateless objects and can be used by multiple netdevs (e.g. representors) who share the same core device. Providing the TISes as a service from the core layer to the netdev layer reduces the number of replecated TIS objects (in case of multiple netdevs), and will ease the transition to netdev with multiple mdevs. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 19 ++-- drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c | 6 +- drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en/qos.c | 7 +- .../net/ethernet/mellanox/mlx5/core/en_common.c | 74 +++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 102 ++++----------------- drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 10 +- .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c | 19 +++- .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h | 2 + .../ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c | 7 +- include/linux/mlx5/driver.h | 2 + 11 files changed, 140 insertions(+), 110 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 43f027bf2da3..6808e0d82944 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -72,7 +72,6 @@ struct page_pool; #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu)) #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu)) -#define MLX5E_MAX_NUM_TC 8 #define MLX5E_MAX_NUM_MQPRIO_CH_TC TC_QOPT_MAX_QUEUE #define MLX5_RX_HEADROOM NET_SKB_PAD @@ -758,7 +757,7 @@ struct mlx5e_channel { /* data path */ struct mlx5e_rq rq; struct mlx5e_xdpsq rq_xdpsq; - struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC]; + struct mlx5e_txqsq sq[MLX5_MAX_NUM_TC]; struct mlx5e_icosq icosq; /* internal control operations */ struct mlx5e_txqsq __rcu * __rcu *qos_sqs; bool xdp; @@ -808,7 +807,7 @@ struct mlx5e_channels { struct mlx5e_channel_stats { struct mlx5e_ch_stats ch; - struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC]; + struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC]; struct mlx5e_rq_stats rq; struct mlx5e_rq_stats xskrq; struct mlx5e_xdpsq_stats rq_xdpsq; @@ -818,8 +817,8 @@ struct mlx5e_channel_stats { struct mlx5e_ptp_stats { struct mlx5e_ch_stats ch; - struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC]; - struct mlx5e_ptp_cq_stats cq[MLX5E_MAX_NUM_TC]; + struct mlx5e_sq_stats sq[MLX5_MAX_NUM_TC]; + struct mlx5e_ptp_cq_stats cq[MLX5_MAX_NUM_TC]; struct mlx5e_rq_stats rq; } ____cacheline_aligned_in_smp; @@ -886,7 +885,6 @@ struct mlx5e_priv { struct mlx5e_rq drop_rq; struct mlx5e_channels channels; - u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC]; struct mlx5e_rx_res *rx_res; u32 *tx_rates; @@ -984,6 +982,8 @@ struct mlx5e_profile { void (*update_stats)(struct mlx5e_priv *priv); void (*update_carrier)(struct mlx5e_priv *priv); int (*max_nch_limit)(struct mlx5_core_dev *mdev); + u32 (*get_tisn)(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, + u8 lag_port, u8 tc); unsigned int (*stats_grps_num)(struct mlx5e_priv *priv); mlx5e_stats_grp_t *stats_grps; const struct mlx5e_rx_handlers *rx_handlers; @@ -991,6 +991,11 @@ struct mlx5e_profile { u32 features; }; +u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, + struct mlx5e_priv *priv, + const struct mlx5e_profile *profile, + u8 lag_port, u8 tc); + #define mlx5e_profile_feature_cap(profile, feature) \ ((profile)->features & BIT(MLX5E_PROFILE_FEATURE_##feature)) @@ -1132,8 +1137,6 @@ void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq); int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn); void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn); -int mlx5e_create_tises(struct mlx5e_priv *priv); -void mlx5e_destroy_tises(struct mlx5e_priv *priv); int mlx5e_update_nic_rx(struct mlx5e_priv *priv); void mlx5e_update_carrier(struct mlx5e_priv *priv); int mlx5e_close(struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index af3928eddafd..04cec76c1ac4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -518,9 +518,11 @@ static int mlx5e_ptp_open_txqsqs(struct mlx5e_ptp *c, for (tc = 0; tc < num_tc; tc++) { int txq_ix = ix_base + tc; + u32 tisn; - err = mlx5e_ptp_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, - cparams, tc, &c->ptpsq[tc]); + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, tc); + err = mlx5e_ptp_open_txqsq(c, tisn, txq_ix, cparams, tc, &c->ptpsq[tc]); if (err) goto close_txqsq; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index 7b700d0f956a..86f1854698b4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -49,7 +49,7 @@ enum { struct mlx5e_ptp { /* data path */ - struct mlx5e_ptpsq ptpsq[MLX5E_MAX_NUM_TC]; + struct mlx5e_ptpsq ptpsq[MLX5_MAX_NUM_TC]; struct mlx5e_rq rq; struct napi_struct napi; struct device *pdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 244bc15a42ab..9e2211f0c3a4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -77,6 +77,7 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, struct mlx5e_params *params; struct mlx5e_channel *c; struct mlx5e_txqsq *sq; + u32 tisn; params = &chs->params; @@ -126,8 +127,10 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); if (err) goto err_free_sq; - err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, - ¶m_sq, sq, 0, hw_id, + + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, 0); + err = mlx5e_open_txqsq(c, tisn, txq_ix, params, ¶m_sq, sq, 0, hw_id, priv->htb_qos_sq_stats[node_qid]); if (err) goto err_close_cq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 41c396e76457..67f546683e85 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -74,6 +74,72 @@ int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey) return err; } +int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) +{ + void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); + + if (mlx5_lag_is_lacp_owner(mdev)) + MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); + + return mlx5_core_create_tis(mdev, in, tisn); +} + +void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) +{ + mlx5_core_destroy_tis(mdev, tisn); +} + +static void mlx5e_destroy_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]) +{ + int tc, i; + + for (i = 0; i < MLX5_MAX_PORTS; i++) + for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) + mlx5e_destroy_tis(mdev, tisn[i][tc]); +} + +static bool mlx5_lag_should_assign_affinity(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; +} + +static int mlx5e_create_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]) +{ + int tc, i; + int err; + + for (i = 0; i < MLX5_MAX_PORTS; i++) { + for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) { + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + void *tisc; + + tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + MLX5_SET(tisc, tisc, prio, tc << 1); + + if (mlx5_lag_should_assign_affinity(mdev)) + MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); + + err = mlx5e_create_tis(mdev, in, &tisn[i][tc]); + if (err) + goto err_close_tises; + } + } + + return 0; + +err_close_tises: + for (; i >= 0; i--) { + for (tc--; tc >= 0; tc--) + mlx5e_destroy_tis(mdev, tisn[i][tc]); + tc = MLX5_MAX_NUM_TC; + } + + return err; +} + int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; @@ -103,6 +169,11 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) goto err_destroy_mkey; } + err = mlx5e_create_tises(mdev, res->tisn); + if (err) { + mlx5_core_err(mdev, "alloc tises failed, %d\n", err); + goto err_destroy_bfreg; + } INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); @@ -115,6 +186,8 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) return 0; +err_destroy_bfreg: + mlx5_free_bfreg(mdev, &res->bfreg); err_destroy_mkey: mlx5_core_destroy_mkey(mdev, res->mkey); err_dealloc_transport_domain: @@ -130,6 +203,7 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); mdev->mlx5e_res.dek_priv = NULL; + mlx5e_destroy_tises(mdev, res->tisn); mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b49b7c28863c..ecb40950ec8d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1352,6 +1352,17 @@ void mlx5e_close_rq(struct mlx5e_rq *rq) mlx5e_free_rq(rq); } +u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, + struct mlx5e_priv *priv, + const struct mlx5e_profile *profile, + u8 lag_port, u8 tc) +{ + if (profile->get_tisn) + return profile->get_tisn(mdev, priv, lag_port, tc); + + return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; +} + static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) { kvfree(sq->db.xdpi_fifo.xi); @@ -1920,7 +1931,8 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, return err; csp.tis_lst_sz = 1; - csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */ + csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, 0); /* tc = 0 */ csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; @@ -2204,12 +2216,15 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { int txq_ix = c->ix + tc * params->num_channels; u32 qos_queue_group_id; + u32 tisn; + tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, + c->lag_port, tc); err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); if (err) goto err_close_sqs; - err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, + err = mlx5e_open_txqsq(c, tisn, txq_ix, params, &cparam->txq_sq, &c->sq[tc], tc, qos_queue_group_id, &c->priv->channel_stats[c->ix]->sq[tc]); @@ -3351,72 +3366,6 @@ void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) mlx5e_free_cq(&drop_rq->cq); } -int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) -{ - void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); - - if (mlx5_lag_is_lacp_owner(mdev)) - MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); - - return mlx5_core_create_tis(mdev, in, tisn); -} - -void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) -{ - mlx5_core_destroy_tis(mdev, tisn); -} - -void mlx5e_destroy_tises(struct mlx5e_priv *priv) -{ - int tc, i; - - for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) - for (tc = 0; tc < priv->profile->max_tc; tc++) - mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); -} - -static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev) -{ - return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; -} - -int mlx5e_create_tises(struct mlx5e_priv *priv) -{ - int tc, i; - int err; - - for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) { - for (tc = 0; tc < priv->profile->max_tc; tc++) { - u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); - - MLX5_SET(tisc, tisc, prio, tc << 1); - - if (mlx5e_lag_should_assign_affinity(priv->mdev)) - MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); - - err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]); - if (err) - goto err_close_tises; - } - } - - return 0; - -err_close_tises: - for (; i >= 0; i--) { - for (tc--; tc >= 0; tc--) - mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); - tc = priv->profile->max_tc; - } - - return err; -} - static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) { if (priv->mqprio_rl) { @@ -3425,7 +3374,6 @@ static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) priv->mqprio_rl = NULL; } mlx5e_accel_cleanup_tx(priv); - mlx5e_destroy_tises(priv); } static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) @@ -3527,7 +3475,7 @@ static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; - if (tc && tc != MLX5E_MAX_NUM_TC) + if (tc && tc != MLX5_MAX_NUM_TC) return -EINVAL; new_params = priv->channels.params; @@ -5482,23 +5430,13 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) { int err; - err = mlx5e_create_tises(priv); - if (err) { - mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); - return err; - } - err = mlx5e_accel_init_tx(priv); if (err) - goto err_destroy_tises; + return err; mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; - -err_destroy_tises: - mlx5e_destroy_tises(priv); - return err; } static void mlx5e_nic_enable(struct mlx5e_priv *priv) @@ -5593,7 +5531,7 @@ static const struct mlx5e_profile mlx5e_nic_profile = { .update_stats = mlx5e_stats_update_ndo_stats, .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_nic, - .max_tc = MLX5E_MAX_NUM_TC, + .max_tc = MLX5_MAX_NUM_TC, .stats_grps = mlx5e_nic_stats_grps, .stats_grps_num = mlx5e_nic_stats_grps_num, .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index fe0726c7b847..e3018a141b6f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -1180,12 +1180,6 @@ static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) struct mlx5e_rep_priv *rpriv = priv->ppriv; int err; - err = mlx5e_create_tises(priv); - if (err) { - mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); - return err; - } - err = mlx5e_rep_neigh_init(rpriv); if (err) goto err_neigh_init; @@ -1208,7 +1202,6 @@ err_ht_init: err_init_tx: mlx5e_rep_neigh_cleanup(rpriv); err_neigh_init: - mlx5e_destroy_tises(priv); return err; } @@ -1222,7 +1215,6 @@ static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) mlx5e_cleanup_uplink_rep_tx(rpriv); mlx5e_rep_neigh_cleanup(rpriv); - mlx5e_destroy_tises(priv); } static void mlx5e_rep_enable(struct mlx5e_priv *priv) @@ -1452,7 +1444,7 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { .update_stats = mlx5e_stats_update_ndo_stats, .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_rep, - .max_tc = MLX5E_MAX_NUM_TC, + .max_tc = MLX5_MAX_NUM_TC, .stats_grps = mlx5e_ul_rep_stats_grps, .stats_grps_num = mlx5e_ul_rep_stats_grps_num, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 2bf77a5251b4..58845121954c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -339,7 +339,7 @@ static int mlx5i_init_tx(struct mlx5e_priv *priv) return err; } - err = mlx5i_create_tis(priv->mdev, ipriv->qpn, &priv->tisn[0][0]); + err = mlx5i_create_tis(priv->mdev, ipriv->qpn, &ipriv->tisn); if (err) { mlx5_core_warn(priv->mdev, "create tis failed, %d\n", err); goto err_destroy_underlay_qp; @@ -356,7 +356,7 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv) { struct mlx5i_priv *ipriv = priv->ppriv; - mlx5e_destroy_tis(priv->mdev, priv->tisn[0][0]); + mlx5e_destroy_tis(priv->mdev, ipriv->tisn); mlx5i_destroy_underlay_qp(priv->mdev, ipriv->qpn); } @@ -483,6 +483,18 @@ static unsigned int mlx5i_stats_grps_num(struct mlx5e_priv *priv) return ARRAY_SIZE(mlx5i_stats_grps); } +u32 mlx5i_get_tisn(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, u8 lag_port, u8 tc) +{ + struct mlx5i_priv *ipriv = priv->ppriv; + + if (WARN(lag_port || tc, + "IPoIB unexpected non-zero value: lag_port (%u), tc (%u)\n", + lag_port, tc)) + return 0; + + return ipriv->tisn; +} + static const struct mlx5e_profile mlx5i_nic_profile = { .init = mlx5i_init, .cleanup = mlx5i_cleanup, @@ -499,6 +511,7 @@ static const struct mlx5e_profile mlx5i_nic_profile = { .max_tc = MLX5I_MAX_NUM_TC, .stats_grps = mlx5i_stats_grps, .stats_grps_num = mlx5i_stats_grps_num, + .get_tisn = mlx5i_get_tisn, }; /* mlx5i netdev NDos */ @@ -829,7 +842,7 @@ int mlx5_rdma_rn_get_params(struct mlx5_core_dev *mdev, *params = (struct rdma_netdev_alloc_params){ .sizeof_priv = sizeof(struct mlx5i_priv) + sizeof(struct mlx5e_priv), - .txqs = nch * MLX5E_MAX_NUM_TC, + .txqs = nch * MLX5_MAX_NUM_TC, .rxqs = nch, .param = mdev, .initialize_rdma_netdev = mlx5_rdma_setup_rn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h index f3f2af972020..2ab6437a1c49 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h @@ -53,6 +53,7 @@ extern const struct mlx5e_rx_handlers mlx5i_rx_handlers; struct mlx5i_priv { struct rdma_netdev rn; /* keep this first */ u32 qpn; + u32 tisn; bool sub_interface; u32 num_sub_interfaces; u32 qkey; @@ -63,6 +64,7 @@ struct mlx5i_priv { }; int mlx5i_create_tis(struct mlx5_core_dev *mdev, u32 underlay_qpn, u32 *tisn); +u32 mlx5i_get_tisn(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, u8 lag_port, u8 tc); /* Underlay QP create/destroy functions */ int mlx5i_create_underlay_qp(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c index 03e681297937..f87471306f6b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c @@ -218,7 +218,7 @@ static int mlx5i_pkey_open(struct net_device *netdev) goto err_unint_underlay_qp; } - err = mlx5i_create_tis(mdev, ipriv->qpn, &epriv->tisn[0][0]); + err = mlx5i_create_tis(mdev, ipriv->qpn, &ipriv->tisn); if (err) { mlx5_core_warn(mdev, "create child tis failed, %d\n", err); goto err_remove_rx_uderlay_qp; @@ -240,7 +240,7 @@ static int mlx5i_pkey_open(struct net_device *netdev) err_close_channels: mlx5e_close_channels(&epriv->channels); err_clear_state_opened_flag: - mlx5e_destroy_tis(mdev, epriv->tisn[0][0]); + mlx5e_destroy_tis(mdev, ipriv->tisn); err_remove_rx_uderlay_qp: mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qpn); err_unint_underlay_qp: @@ -269,7 +269,7 @@ static int mlx5i_pkey_close(struct net_device *netdev) mlx5i_uninit_underlay_qp(priv); mlx5e_deactivate_priv_channels(priv); mlx5e_close_channels(&priv->channels); - mlx5e_destroy_tis(mdev, priv->tisn[0][0]); + mlx5e_destroy_tis(mdev, ipriv->tisn); unlock: mutex_unlock(&priv->state_lock); return 0; @@ -361,6 +361,7 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = { .update_stats = NULL, .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, + .get_tisn = mlx5i_get_tisn, }; const struct mlx5e_profile *mlx5i_pkey_get_profile(void) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 2f67cec1a898..7ee5b79ff3d6 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -679,6 +679,8 @@ struct mlx5e_resources { struct mlx5_td td; u32 mkey; struct mlx5_sq_bfreg bfreg; +#define MLX5_MAX_NUM_TC 8 + u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]; } hw_objs; struct net_device *uplink_netdev; struct mutex uplink_netdev_lock; -- cgit v1.2.3 From 50d73710715de7d1a2c88194562f520816af9c2a Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 12 Dec 2023 15:27:51 +0100 Subject: ethtool: add SET for TCP_DATA_SPLIT ringparam Follow up commit 9690ae604290 ("ethtool: add header/data split indication") and add the set part of Ethtool's header split, i.e. ability to enable/disable header split via the Ethtool Netlink interface. This might be helpful to optimize the setup for particular workloads, for example, to avoid XDP frags, and so on. A driver should advertise ``ETHTOOL_RING_USE_TCP_DATA_SPLIT`` in its ops->supported_ring_params to allow doing that. "Unknown" passed from the userspace when the header split is supported means the driver is free to choose the preferred state. Reviewed-by: Przemek Kitszel Signed-off-by: Alexander Lobakin Link: https://lore.kernel.org/r/20231212142752.935000-2-aleksander.lobakin@intel.com Signed-off-by: Jakub Kicinski --- include/linux/ethtool.h | 2 ++ net/ethtool/rings.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index deb683d3360f..67b30940234b 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -95,6 +95,7 @@ struct kernel_ethtool_ringparam { * @ETHTOOL_RING_USE_TX_PUSH: capture for setting tx_push * @ETHTOOL_RING_USE_RX_PUSH: capture for setting rx_push * @ETHTOOL_RING_USE_TX_PUSH_BUF_LEN: capture for setting tx_push_buf_len + * @ETHTOOL_RING_USE_TCP_DATA_SPLIT: capture for setting tcp_data_split */ enum ethtool_supported_ring_param { ETHTOOL_RING_USE_RX_BUF_LEN = BIT(0), @@ -102,6 +103,7 @@ enum ethtool_supported_ring_param { ETHTOOL_RING_USE_TX_PUSH = BIT(2), ETHTOOL_RING_USE_RX_PUSH = BIT(3), ETHTOOL_RING_USE_TX_PUSH_BUF_LEN = BIT(4), + ETHTOOL_RING_USE_TCP_DATA_SPLIT = BIT(5), }; #define __ETH_RSS_HASH_BIT(bit) ((u32)1 << (bit)) diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c index fb09f774ea01..b7865a14fdf8 100644 --- a/net/ethtool/rings.c +++ b/net/ethtool/rings.c @@ -124,6 +124,8 @@ const struct nla_policy ethnl_rings_set_policy[] = { [ETHTOOL_A_RINGS_RX_JUMBO] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_TX] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_RX_BUF_LEN] = NLA_POLICY_MIN(NLA_U32, 1), + [ETHTOOL_A_RINGS_TCP_DATA_SPLIT] = + NLA_POLICY_MAX(NLA_U8, ETHTOOL_TCP_DATA_SPLIT_ENABLED), [ETHTOOL_A_RINGS_CQE_SIZE] = NLA_POLICY_MIN(NLA_U32, 1), [ETHTOOL_A_RINGS_TX_PUSH] = NLA_POLICY_MAX(NLA_U8, 1), [ETHTOOL_A_RINGS_RX_PUSH] = NLA_POLICY_MAX(NLA_U8, 1), @@ -145,6 +147,14 @@ ethnl_set_rings_validate(struct ethnl_req_info *req_info, return -EOPNOTSUPP; } + if (tb[ETHTOOL_A_RINGS_TCP_DATA_SPLIT] && + !(ops->supported_ring_params & ETHTOOL_RING_USE_TCP_DATA_SPLIT)) { + NL_SET_ERR_MSG_ATTR(info->extack, + tb[ETHTOOL_A_RINGS_TCP_DATA_SPLIT], + "setting TCP data split is not supported"); + return -EOPNOTSUPP; + } + if (tb[ETHTOOL_A_RINGS_CQE_SIZE] && !(ops->supported_ring_params & ETHTOOL_RING_USE_CQE_SIZE)) { NL_SET_ERR_MSG_ATTR(info->extack, @@ -202,6 +212,8 @@ ethnl_set_rings(struct ethnl_req_info *req_info, struct genl_info *info) ethnl_update_u32(&ringparam.tx_pending, tb[ETHTOOL_A_RINGS_TX], &mod); ethnl_update_u32(&kernel_ringparam.rx_buf_len, tb[ETHTOOL_A_RINGS_RX_BUF_LEN], &mod); + ethnl_update_u8(&kernel_ringparam.tcp_data_split, + tb[ETHTOOL_A_RINGS_TCP_DATA_SPLIT], &mod); ethnl_update_u32(&kernel_ringparam.cqe_size, tb[ETHTOOL_A_RINGS_CQE_SIZE], &mod); ethnl_update_u8(&kernel_ringparam.tx_push, -- cgit v1.2.3 From 0a149ab78ee220c75eef797abea7a29f4490e226 Mon Sep 17 00:00:00 2001 From: Liang Chen Date: Tue, 12 Dec 2023 12:46:11 +0800 Subject: page_pool: transition to reference count management after page draining To support multiple users referencing the same fragment, 'pp_frag_count' is renamed to 'pp_ref_count', transitioning pp pages from fragment management to reference count management after draining based on the suggestion from [1]. The idea is that the concept of fragmenting exists before the page is drained, and all related functions retain their current names. However, once the page is drained, its management shifts to being governed by 'pp_ref_count'. Therefore, all functions associated with that lifecycle stage of a pp page are renamed. [1] http://lore.kernel.org/netdev/f71d9448-70c8-8793-dc9a-0eb48a570300@huawei.com Signed-off-by: Liang Chen Reviewed-by: Yunsheng Lin Reviewed-by: Ilias Apalodimas Reviewed-by: Mina Almasry Link: https://lore.kernel.org/r/20231212044614.42733-2-liangchen.linux@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 4 +- include/linux/mm_types.h | 2 +- include/net/page_pool/helpers.h | 60 ++++++++++++++----------- include/net/page_pool/types.h | 6 +-- net/core/page_pool.c | 12 ++--- 5 files changed, 46 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 8d9743a5e42c..98d33ac7ec64 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -298,8 +298,8 @@ static void mlx5e_page_release_fragmented(struct mlx5e_rq *rq, u16 drain_count = MLX5E_PAGECNT_BIAS_MAX - frag_page->frags; struct page *page = frag_page->page; - if (page_pool_defrag_page(page, drain_count) == 0) - page_pool_put_defragged_page(rq->page_pool, page, -1, true); + if (page_pool_unref_page(page, drain_count) == 0) + page_pool_put_unrefed_page(rq->page_pool, page, -1, true); } static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 957ce38768b2..64e4572ef06d 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -125,7 +125,7 @@ struct page { struct page_pool *pp; unsigned long _pp_mapping_pad; unsigned long dma_addr; - atomic_long_t pp_frag_count; + atomic_long_t pp_ref_count; }; struct { /* Tail pages of compound page */ unsigned long compound_head; /* Bit zero is set */ diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index 7dc65774cde5..841e0a930bd7 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -29,7 +29,7 @@ * page allocated from page pool. Page splitting enables memory saving and thus * avoids TLB/cache miss for data access, but there also is some cost to * implement page splitting, mainly some cache line dirtying/bouncing for - * 'struct page' and atomic operation for page->pp_frag_count. + * 'struct page' and atomic operation for page->pp_ref_count. * * The API keeps track of in-flight pages, in order to let API users know when * it is safe to free a page_pool object, the API users must call @@ -210,69 +210,77 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) return pool->p.dma_dir; } -/* pp_frag_count represents the number of writers who can update the page - * either by updating skb->data or via DMA mappings for the device. - * We can't rely on the page refcnt for that as we don't know who might be - * holding page references and we can't reliably destroy or sync DMA mappings - * of the fragments. +/** + * page_pool_fragment_page() - split a fresh page into fragments + * @page: page to split + * @nr: references to set + * + * pp_ref_count represents the number of outstanding references to the page, + * which will be freed using page_pool APIs (rather than page allocator APIs + * like put_page()). Such references are usually held by page_pool-aware + * objects like skbs marked for page pool recycling. * - * When pp_frag_count reaches 0 we can either recycle the page if the page - * refcnt is 1 or return it back to the memory allocator and destroy any - * mappings we have. + * This helper allows the caller to take (set) multiple references to a + * freshly allocated page. The page must be freshly allocated (have a + * pp_ref_count of 1). This is commonly done by drivers and + * "fragment allocators" to save atomic operations - either when they know + * upfront how many references they will need; or to take MAX references and + * return the unused ones with a single atomic dec(), instead of performing + * multiple atomic inc() operations. */ static inline void page_pool_fragment_page(struct page *page, long nr) { - atomic_long_set(&page->pp_frag_count, nr); + atomic_long_set(&page->pp_ref_count, nr); } -static inline long page_pool_defrag_page(struct page *page, long nr) +static inline long page_pool_unref_page(struct page *page, long nr) { long ret; - /* If nr == pp_frag_count then we have cleared all remaining + /* If nr == pp_ref_count then we have cleared all remaining * references to the page: * 1. 'n == 1': no need to actually overwrite it. * 2. 'n != 1': overwrite it with one, which is the rare case - * for pp_frag_count draining. + * for pp_ref_count draining. * * The main advantage to doing this is that not only we avoid a atomic * update, as an atomic_read is generally a much cheaper operation than * an atomic update, especially when dealing with a page that may be - * partitioned into only 2 or 3 pieces; but also unify the pp_frag_count + * referenced by only 2 or 3 users; but also unify the pp_ref_count * handling by ensuring all pages have partitioned into only 1 piece * initially, and only overwrite it when the page is partitioned into * more than one piece. */ - if (atomic_long_read(&page->pp_frag_count) == nr) { + if (atomic_long_read(&page->pp_ref_count) == nr) { /* As we have ensured nr is always one for constant case using * the BUILD_BUG_ON(), only need to handle the non-constant case - * here for pp_frag_count draining, which is a rare case. + * here for pp_ref_count draining, which is a rare case. */ BUILD_BUG_ON(__builtin_constant_p(nr) && nr != 1); if (!__builtin_constant_p(nr)) - atomic_long_set(&page->pp_frag_count, 1); + atomic_long_set(&page->pp_ref_count, 1); return 0; } - ret = atomic_long_sub_return(nr, &page->pp_frag_count); + ret = atomic_long_sub_return(nr, &page->pp_ref_count); WARN_ON(ret < 0); - /* We are the last user here too, reset pp_frag_count back to 1 to + /* We are the last user here too, reset pp_ref_count back to 1 to * ensure all pages have been partitioned into 1 piece initially, * this should be the rare case when the last two fragment users call - * page_pool_defrag_page() currently. + * page_pool_unref_page() currently. */ if (unlikely(!ret)) - atomic_long_set(&page->pp_frag_count, 1); + atomic_long_set(&page->pp_ref_count, 1); return ret; } -static inline bool page_pool_is_last_frag(struct page *page) +static inline bool page_pool_is_last_ref(struct page *page) { - /* If page_pool_defrag_page() returns 0, we were the last user */ - return page_pool_defrag_page(page, 1) == 0; + /* If page_pool_unref_page() returns 0, we were the last user */ + return page_pool_unref_page(page, 1) == 0; } /** @@ -297,10 +305,10 @@ static inline void page_pool_put_page(struct page_pool *pool, * allow registering MEM_TYPE_PAGE_POOL, but shield linker. */ #ifdef CONFIG_PAGE_POOL - if (!page_pool_is_last_frag(page)) + if (!page_pool_is_last_ref(page)) return; - page_pool_put_defragged_page(pool, page, dma_sync_size, allow_direct); + page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct); #endif } diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index ac286ea8ce2d..76481c465375 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -234,9 +234,9 @@ static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, } #endif -void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, - unsigned int dma_sync_size, - bool allow_direct); +void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, + unsigned int dma_sync_size, + bool allow_direct); static inline bool is_page_pool_compiled_in(void) { diff --git a/net/core/page_pool.c b/net/core/page_pool.c index c2e7c9a6efbe..c92b757369f2 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -677,8 +677,8 @@ __page_pool_put_page(struct page_pool *pool, struct page *page, return NULL; } -void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, - unsigned int dma_sync_size, bool allow_direct) +void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, + unsigned int dma_sync_size, bool allow_direct) { page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct); if (page && !page_pool_recycle_in_ring(pool, page)) { @@ -687,7 +687,7 @@ void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, page_pool_return_page(pool, page); } } -EXPORT_SYMBOL(page_pool_put_defragged_page); +EXPORT_SYMBOL(page_pool_put_unrefed_page); /** * page_pool_put_page_bulk() - release references on multiple pages @@ -714,7 +714,7 @@ void page_pool_put_page_bulk(struct page_pool *pool, void **data, struct page *page = virt_to_head_page(data[i]); /* It is not the last user for the page frag case */ - if (!page_pool_is_last_frag(page)) + if (!page_pool_is_last_ref(page)) continue; page = __page_pool_put_page(pool, page, -1, false); @@ -756,7 +756,7 @@ static struct page *page_pool_drain_frag(struct page_pool *pool, long drain_count = BIAS_MAX - pool->frag_users; /* Some user is still using the page frag */ - if (likely(page_pool_defrag_page(page, drain_count))) + if (likely(page_pool_unref_page(page, drain_count))) return NULL; if (page_ref_count(page) == 1 && !page_is_pfmemalloc(page)) { @@ -777,7 +777,7 @@ static void page_pool_free_frag(struct page_pool *pool) pool->frag_page = NULL; - if (!page || page_pool_defrag_page(page, drain_count)) + if (!page || page_pool_unref_page(page, drain_count)) return; page_pool_return_page(pool, page); -- cgit v1.2.3 From fb6e30a72539ce28c1323aef4190d35aac106f6f Mon Sep 17 00:00:00 2001 From: Ahmed Zaki Date: Tue, 12 Dec 2023 17:33:14 -0700 Subject: net: ethtool: pass a pointer to parameters to get/set_rxfh ethtool ops The get/set_rxfh ethtool ops currently takes the rxfh (RSS) parameters as direct function arguments. This will force us to change the API (and all drivers' functions) every time some new parameters are added. This is part 1/2 of the fix, as suggested in [1]: - First simplify the code by always providing a pointer to all params (indir, key and func); the fact that some of them may be NULL seems like a weird historic thing or a premature optimization. It will simplify the drivers if all pointers are always present. - Then make the functions take a dev pointer, and a pointer to a single struct wrapping all arguments. The set_* should also take an extack. Link: https://lore.kernel.org/netdev/20231121152906.2dd5f487@kernel.org/ [1] Suggested-by: Jakub Kicinski Suggested-by: Jacob Keller Signed-off-by: Ahmed Zaki Link: https://lore.kernel.org/r/20231213003321.605376-2-ahmed.zaki@intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amazon/ena/ena_ethtool.c | 28 ++--- drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c | 33 +++--- .../net/ethernet/aquantia/atlantic/aq_ethtool.c | 31 ++--- .../net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c | 25 ++-- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 28 ++--- drivers/net/ethernet/broadcom/tg3.c | 22 ++-- .../net/ethernet/cavium/thunder/nicvf_ethtool.c | 31 ++--- drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c | 24 ++-- drivers/net/ethernet/cisco/enic/enic_ethtool.c | 25 ++-- drivers/net/ethernet/emulex/benet/be_ethtool.c | 28 ++--- .../net/ethernet/freescale/enetc/enetc_ethtool.c | 31 ++--- .../net/ethernet/fungible/funeth/funeth_ethtool.c | 40 +++---- drivers/net/ethernet/hisilicon/hns/hns_ethtool.c | 17 +-- drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c | 23 ++-- drivers/net/ethernet/huawei/hinic/hinic_ethtool.c | 40 +++---- drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c | 26 +++-- drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 38 +++---- drivers/net/ethernet/intel/iavf/iavf_ethtool.c | 42 ++++--- drivers/net/ethernet/intel/ice/ice_ethtool.c | 44 ++++--- drivers/net/ethernet/intel/idpf/idpf_ethtool.c | 40 ++++--- drivers/net/ethernet/intel/igb/igb_ethtool.c | 27 ++--- drivers/net/ethernet/intel/igc/igc_ethtool.c | 27 ++--- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 35 +++--- drivers/net/ethernet/intel/ixgbevf/ethtool.c | 27 ++--- drivers/net/ethernet/marvell/mvneta.c | 25 ++-- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 50 ++++---- .../ethernet/marvell/octeontx2/nic/otx2_ethtool.c | 41 +++---- drivers/net/ethernet/mellanox/mlx4/en_ethtool.c | 40 +++---- drivers/net/ethernet/mellanox/mlx5/core/en.h | 6 +- .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 28 +++-- drivers/net/ethernet/microchip/lan743x_ethtool.c | 34 +++--- drivers/net/ethernet/microsoft/mana/mana_ethtool.c | 33 +++--- .../net/ethernet/netronome/nfp/nfp_net_ethtool.c | 38 +++---- .../net/ethernet/pensando/ionic/ionic_ethtool.c | 26 +++-- drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 32 +++--- drivers/net/ethernet/sfc/ethtool_common.c | 53 +++++---- drivers/net/ethernet/sfc/ethtool_common.h | 17 +-- drivers/net/ethernet/sfc/falcon/ethtool.c | 26 +++-- drivers/net/ethernet/sfc/siena/ethtool_common.c | 53 +++++---- drivers/net/ethernet/sfc/siena/ethtool_common.h | 17 +-- .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c | 31 ++--- drivers/net/hyperv/netvsc_drv.c | 32 +++--- drivers/net/virtio_net.c | 29 ++--- drivers/net/vmxnet3/vmxnet3_ethtool.c | 22 ++-- include/linux/ethtool.h | 38 +++++-- net/ethtool/common.c | 12 +- net/ethtool/ioctl.c | 126 +++++++++++---------- net/ethtool/rss.c | 16 +-- 48 files changed, 827 insertions(+), 730 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index e3ef081aa42b..2d8c5a2841b8 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -802,15 +802,15 @@ static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir) return rc; } -static int ena_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int ena_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct ena_adapter *adapter = netdev_priv(netdev); enum ena_admin_hash_functions ena_func; u8 func; int rc; - rc = ena_indirection_table_get(adapter, indir); + rc = ena_indirection_table_get(adapter, rxfh->indir); if (rc) return rc; @@ -825,7 +825,7 @@ static int ena_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, return rc; } - rc = ena_com_get_hash_key(adapter->ena_dev, key); + rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key); if (rc) return rc; @@ -842,27 +842,27 @@ static int ena_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, return -EOPNOTSUPP; } - if (hfunc) - *hfunc = func; + rxfh->hfunc = func; return 0; } -static int ena_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int ena_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct ena_adapter *adapter = netdev_priv(netdev); struct ena_com_dev *ena_dev = adapter->ena_dev; enum ena_admin_hash_functions func = 0; int rc; - if (indir) { - rc = ena_indirection_table_set(adapter, indir); + if (rxfh->indir) { + rc = ena_indirection_table_set(adapter, rxfh->indir); if (rc) return rc; } - switch (hfunc) { + switch (rxfh->hfunc) { case ETH_RSS_HASH_NO_CHANGE: func = ena_com_get_current_hash_function(ena_dev); break; @@ -874,12 +874,12 @@ static int ena_set_rxfh(struct net_device *netdev, const u32 *indir, break; default: netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n", - hfunc); + rxfh->hfunc); return -EOPNOTSUPP; } - if (key || func) { - rc = ena_com_fill_hash_function(ena_dev, func, key, + if (rxfh->key || func) { + rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key, ENA_HASH_KEY_SIZE, 0xFFFFFFFF); if (unlikely(rc)) { diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 32fab5e77246..58e7e88aae5b 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -527,47 +527,48 @@ static u32 xgbe_get_rxfh_indir_size(struct net_device *netdev) return ARRAY_SIZE(pdata->rss_table); } -static int xgbe_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int xgbe_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct xgbe_prv_data *pdata = netdev_priv(netdev); unsigned int i; - if (indir) { + if (rxfh->indir) { for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) - indir[i] = XGMAC_GET_BITS(pdata->rss_table[i], - MAC_RSSDR, DMCH); + rxfh->indir[i] = XGMAC_GET_BITS(pdata->rss_table[i], + MAC_RSSDR, DMCH); } - if (key) - memcpy(key, pdata->rss_key, sizeof(pdata->rss_key)); + if (rxfh->key) + memcpy(rxfh->key, pdata->rss_key, sizeof(pdata->rss_key)); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int xgbe_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int xgbe_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; unsigned int ret; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) { + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) { netdev_err(netdev, "unsupported hash function\n"); return -EOPNOTSUPP; } - if (indir) { - ret = hw_if->set_rss_lookup_table(pdata, indir); + if (rxfh->indir) { + ret = hw_if->set_rss_lookup_table(pdata, rxfh->indir); if (ret) return ret; } - if (key) { - ret = hw_if->set_rss_hash_key(pdata, key); + if (rxfh->key) { + ret = hw_if->set_rss_hash_key(pdata, rxfh->key); if (ret) return ret; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index ac4ea93bd8dd..18a6c8d99fa0 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -447,8 +447,8 @@ static u32 aq_ethtool_get_rss_key_size(struct net_device *ndev) return sizeof(cfg->aq_rss.hash_secret_key); } -static int aq_ethtool_get_rss(struct net_device *ndev, u32 *indir, u8 *key, - u8 *hfunc) +static int aq_ethtool_get_rss(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh) { struct aq_nic_s *aq_nic = netdev_priv(ndev); struct aq_nic_cfg_s *cfg; @@ -456,21 +456,21 @@ static int aq_ethtool_get_rss(struct net_device *ndev, u32 *indir, u8 *key, cfg = aq_nic_get_cfg(aq_nic); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ - if (indir) { + rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ + if (rxfh->indir) { for (i = 0; i < AQ_CFG_RSS_INDIRECTION_TABLE_MAX; i++) - indir[i] = cfg->aq_rss.indirection_table[i]; + rxfh->indir[i] = cfg->aq_rss.indirection_table[i]; } - if (key) - memcpy(key, cfg->aq_rss.hash_secret_key, + if (rxfh->key) + memcpy(rxfh->key, cfg->aq_rss.hash_secret_key, sizeof(cfg->aq_rss.hash_secret_key)); return 0; } -static int aq_ethtool_set_rss(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int aq_ethtool_set_rss(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct aq_nic_s *aq_nic = netdev_priv(netdev); struct aq_nic_cfg_s *cfg; @@ -482,16 +482,17 @@ static int aq_ethtool_set_rss(struct net_device *netdev, const u32 *indir, rss_entries = cfg->aq_rss.indirection_table_size; /* We do not allow change in unsupported parameters */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; /* Fill out the redirection table */ - if (indir) + if (rxfh->indir) for (i = 0; i < rss_entries; i++) - cfg->aq_rss.indirection_table[i] = indir[i]; + cfg->aq_rss.indirection_table[i] = rxfh->indir[i]; /* Fill out the rss hash key */ - if (key) { - memcpy(cfg->aq_rss.hash_secret_key, key, + if (rxfh->key) { + memcpy(cfg->aq_rss.hash_secret_key, rxfh->key, sizeof(cfg->aq_rss.hash_secret_key)); err = aq_nic->aq_hw_ops->hw_rss_hash_set(aq_nic->aq_hw, &cfg->aq_rss); diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index bda3ccc28eca..81d232e6d05f 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -3486,16 +3486,15 @@ static u32 bnx2x_get_rxfh_indir_size(struct net_device *dev) return T_ETH_INDIRECTION_TABLE_SIZE; } -static int bnx2x_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int bnx2x_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct bnx2x *bp = netdev_priv(dev); u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0}; size_t i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!indir) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; /* Get the current configuration of the RSS indirection table */ @@ -3511,13 +3510,14 @@ static int bnx2x_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, * queue. */ for (i = 0; i < T_ETH_INDIRECTION_TABLE_SIZE; i++) - indir[i] = ind_table[i] - bp->fp->cl_id; + rxfh->indir[i] = ind_table[i] - bp->fp->cl_id; return 0; } -static int bnx2x_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int bnx2x_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct bnx2x *bp = netdev_priv(dev); size_t i; @@ -3525,11 +3525,12 @@ static int bnx2x_set_rxfh(struct net_device *dev, const u32 *indir, /* We require at least one supported parameter to be changed and no * change in any of the unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; for (i = 0; i < T_ETH_INDIRECTION_TABLE_SIZE; i++) { @@ -3542,7 +3543,7 @@ static int bnx2x_set_rxfh(struct net_device *dev, const u32 *indir, * align the received table to the Client ID of the leading RSS * queue */ - bp->rss_conf_obj.ind_table[i] = indir[i] + bp->fp->cl_id; + bp->rss_conf_obj.ind_table[i] = rxfh->indir[i] + bp->fp->cl_id; } if (bp->state == BNX2X_STATE_OPEN) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 45ce7e2e3662..2742d9decc73 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -1333,49 +1333,49 @@ static u32 bnxt_get_rxfh_key_size(struct net_device *dev) return HW_HASH_KEY_SIZE; } -static int bnxt_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int bnxt_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct bnxt *bp = netdev_priv(dev); struct bnxt_vnic_info *vnic; u32 i, tbl_size; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; if (!bp->vnic_info) return 0; vnic = &bp->vnic_info[0]; - if (indir && bp->rss_indir_tbl) { + if (rxfh->indir && bp->rss_indir_tbl) { tbl_size = bnxt_get_rxfh_indir_size(dev); for (i = 0; i < tbl_size; i++) - indir[i] = bp->rss_indir_tbl[i]; + rxfh->indir[i] = bp->rss_indir_tbl[i]; } - if (key && vnic->rss_hash_key) - memcpy(key, vnic->rss_hash_key, HW_HASH_KEY_SIZE); + if (rxfh->key && vnic->rss_hash_key) + memcpy(rxfh->key, vnic->rss_hash_key, HW_HASH_KEY_SIZE); return 0; } -static int bnxt_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int bnxt_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct bnxt *bp = netdev_priv(dev); int rc = 0; - if (hfunc && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc && rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (key) + if (rxfh->key) return -EOPNOTSUPP; - if (indir) { + if (rxfh->indir) { u32 i, pad, tbl_size = bnxt_get_rxfh_indir_size(dev); for (i = 0; i < tbl_size; i++) - bp->rss_indir_tbl[i] = indir[i]; + bp->rss_indir_tbl[i] = rxfh->indir[i]; pad = bp->rss_indir_tbl_entries - tbl_size; if (pad) memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16)); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index f52830dfb26a..04964bbe08cf 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -12745,24 +12745,23 @@ static u32 tg3_get_rxfh_indir_size(struct net_device *dev) return size; } -static int tg3_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) +static int tg3_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh) { struct tg3 *tp = netdev_priv(dev); int i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!indir) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) - indir[i] = tp->rss_ind_tbl[i]; + rxfh->indir[i] = tp->rss_ind_tbl[i]; return 0; } -static int tg3_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, - const u8 hfunc) +static int tg3_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct tg3 *tp = netdev_priv(dev); size_t i; @@ -12770,15 +12769,16 @@ static int tg3_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, /* We require at least one supported parameter to be changed and no * change in any of the unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) - tp->rss_ind_tbl[i] = indir[i]; + tp->rss_ind_tbl[i] = rxfh->indir[i]; if (!netif_running(dev) || !tg3_flag(tp, ENABLE_RSS)) return 0; diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c index d8d71bf97983..34125b8cd935 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c @@ -653,35 +653,36 @@ static u32 nicvf_get_rxfh_indir_size(struct net_device *dev) return nic->rss_info.rss_size; } -static int nicvf_get_rxfh(struct net_device *dev, u32 *indir, u8 *hkey, - u8 *hfunc) +static int nicvf_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct nicvf *nic = netdev_priv(dev); struct nicvf_rss_info *rss = &nic->rss_info; int idx; - if (indir) { + if (rxfh->indir) { for (idx = 0; idx < rss->rss_size; idx++) - indir[idx] = rss->ind_tbl[idx]; + rxfh->indir[idx] = rss->ind_tbl[idx]; } - if (hkey) - memcpy(hkey, rss->key, RSS_HASH_KEY_SIZE * sizeof(u64)); + if (rxfh->key) + memcpy(rxfh->key, rss->key, RSS_HASH_KEY_SIZE * sizeof(u64)); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int nicvf_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *hkey, const u8 hfunc) +static int nicvf_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct nicvf *nic = netdev_priv(dev); struct nicvf_rss_info *rss = &nic->rss_info; int idx; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (!rss->enable) { @@ -690,13 +691,13 @@ static int nicvf_set_rxfh(struct net_device *dev, const u32 *indir, return -EIO; } - if (indir) { + if (rxfh->indir) { for (idx = 0; idx < rss->rss_size; idx++) - rss->ind_tbl[idx] = indir[idx]; + rss->ind_tbl[idx] = rxfh->indir[idx]; } - if (hkey) { - memcpy(rss->key, hkey, RSS_HASH_KEY_SIZE * sizeof(u64)); + if (rxfh->key) { + memcpy(rss->key, rxfh->key, RSS_HASH_KEY_SIZE * sizeof(u64)); nicvf_set_rss_key(nic); } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 8477a93cee6b..47eecde36285 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -1588,22 +1588,23 @@ static u32 get_rss_table_size(struct net_device *dev) return pi->rss_size; } -static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc) +static int get_rss_table(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { const struct port_info *pi = netdev_priv(dev); unsigned int n = pi->rss_size; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!p) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; while (n--) - p[n] = pi->rss[n]; + rxfh->indir[n] = pi->rss[n]; return 0; } -static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key, - const u8 hfunc) +static int set_rss_table(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { unsigned int i; struct port_info *pi = netdev_priv(dev); @@ -1611,16 +1612,17 @@ static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key, /* We require at least one supported parameter to be changed and no * change in any of the unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!p) + if (!rxfh->indir) return 0; /* Interface must be brought up atleast once */ if (pi->adapter->flags & CXGB4_FULL_INIT_DONE) { for (i = 0; i < pi->rss_size; i++) - pi->rss[i] = p[i]; + pi->rss[i] = rxfh->indir[i]; return cxgb4_write_rss(pi, pi->rss); } diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index 08b7cc0a1809..241906697019 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -568,31 +568,32 @@ static u32 enic_get_rxfh_key_size(struct net_device *netdev) return ENIC_RSS_LEN; } -static int enic_get_rxfh(struct net_device *netdev, u32 *indir, u8 *hkey, - u8 *hfunc) +static int enic_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct enic *enic = netdev_priv(netdev); - if (hkey) - memcpy(hkey, enic->rss_key, ENIC_RSS_LEN); + if (rxfh->key) + memcpy(rxfh->key, enic->rss_key, ENIC_RSS_LEN); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int enic_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *hkey, const u8 hfunc) +static int enic_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct enic *enic = netdev_priv(netdev); - if ((hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) || - indir) + if (rxfh->indir || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EINVAL; - if (hkey) - memcpy(enic->rss_key, hkey, ENIC_RSS_LEN); + if (rxfh->key) + memcpy(enic->rss_key, rxfh->key, ENIC_RSS_LEN); return __enic_set_rsskey(enic); } diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index a29de29bdf23..f001a649f58f 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -1271,43 +1271,45 @@ static u32 be_get_rxfh_key_size(struct net_device *netdev) return RSS_HASH_KEY_LEN; } -static int be_get_rxfh(struct net_device *netdev, u32 *indir, u8 *hkey, - u8 *hfunc) +static int be_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct be_adapter *adapter = netdev_priv(netdev); int i; struct rss_info *rss = &adapter->rss_info; - if (indir) { + if (rxfh->indir) { for (i = 0; i < RSS_INDIR_TABLE_LEN; i++) - indir[i] = rss->rss_queue[i]; + rxfh->indir[i] = rss->rss_queue[i]; } - if (hkey) - memcpy(hkey, rss->rss_hkey, RSS_HASH_KEY_LEN); + if (rxfh->key) + memcpy(rxfh->key, rss->rss_hkey, RSS_HASH_KEY_LEN); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int be_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *hkey, const u8 hfunc) +static int be_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { int rc = 0, i, j; struct be_adapter *adapter = netdev_priv(netdev); + u8 *hkey = rxfh->key; u8 rsstable[RSS_INDIR_TABLE_LEN]; /* We do not allow change in unsupported parameters */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (indir) { + if (rxfh->indir) { struct be_rx_obj *rxo; for (i = 0; i < RSS_INDIR_TABLE_LEN; i++) { - j = indir[i]; + j = rxfh->indir[i]; rxo = &adapter->rx_obj[j]; rsstable[i] = rxo->rss_id; adapter->rss_info.rss_queue[i] = j; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index e993ed04ab57..f7753ea5b57e 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -690,25 +690,26 @@ static u32 enetc_get_rxfh_indir_size(struct net_device *ndev) return priv->si->num_rss; } -static int enetc_get_rxfh(struct net_device *ndev, u32 *indir, u8 *key, - u8 *hfunc) +static int enetc_get_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct enetc_hw *hw = &priv->si->hw; int err = 0, i; /* return hash function */ - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; /* return hash key */ - if (key && hw->port) + if (rxfh->key && hw->port) for (i = 0; i < ENETC_RSSHASH_KEY_SIZE / 4; i++) - ((u32 *)key)[i] = enetc_port_rd(hw, ENETC_PRSSK(i)); + ((u32 *)rxfh->key)[i] = enetc_port_rd(hw, + ENETC_PRSSK(i)); /* return RSS table */ - if (indir) - err = enetc_get_rss_table(priv->si, indir, priv->si->num_rss); + if (rxfh->indir) + err = enetc_get_rss_table(priv->si, rxfh->indir, + priv->si->num_rss); return err; } @@ -722,20 +723,22 @@ void enetc_set_rss_key(struct enetc_hw *hw, const u8 *bytes) } EXPORT_SYMBOL_GPL(enetc_set_rss_key); -static int enetc_set_rxfh(struct net_device *ndev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int enetc_set_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct enetc_hw *hw = &priv->si->hw; int err = 0; /* set hash key, if PF */ - if (key && hw->port) - enetc_set_rss_key(hw, key); + if (rxfh->key && hw->port) + enetc_set_rss_key(hw, rxfh->key); /* set RSS table */ - if (indir) - err = enetc_set_rss_table(priv->si, indir, priv->si->num_rss); + if (rxfh->indir) + err = enetc_set_rss_table(priv->si, rxfh->indir, + priv->si->num_rss); return err; } diff --git a/drivers/net/ethernet/fungible/funeth/funeth_ethtool.c b/drivers/net/ethernet/fungible/funeth/funeth_ethtool.c index 091c93bd7587..4edd0adfc6c7 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_ethtool.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_ethtool.c @@ -977,44 +977,44 @@ static u32 fun_get_rxfh_key_size(struct net_device *netdev) return sizeof(fp->rss_key); } -static int fun_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int fun_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { const struct funeth_priv *fp = netdev_priv(netdev); if (!fp->rss_cfg) return -EOPNOTSUPP; - if (indir) - memcpy(indir, fp->indir_table, + if (rxfh->indir) + memcpy(rxfh->indir, fp->indir_table, sizeof(u32) * fp->indir_table_nentries); - if (key) - memcpy(key, fp->rss_key, sizeof(fp->rss_key)); + if (rxfh->key) + memcpy(rxfh->key, fp->rss_key, sizeof(fp->rss_key)); - if (hfunc) - *hfunc = fp->hash_algo == FUN_ETH_RSS_ALG_TOEPLITZ ? - ETH_RSS_HASH_TOP : ETH_RSS_HASH_CRC32; + rxfh->hfunc = fp->hash_algo == FUN_ETH_RSS_ALG_TOEPLITZ ? + ETH_RSS_HASH_TOP : ETH_RSS_HASH_CRC32; return 0; } -static int fun_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int fun_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct funeth_priv *fp = netdev_priv(netdev); - const u32 *rss_indir = indir ? indir : fp->indir_table; - const u8 *rss_key = key ? key : fp->rss_key; + const u32 *rss_indir = rxfh->indir ? rxfh->indir : fp->indir_table; + const u8 *rss_key = rxfh->key ? rxfh->key : fp->rss_key; enum fun_eth_hash_alg algo; if (!fp->rss_cfg) return -EOPNOTSUPP; - if (hfunc == ETH_RSS_HASH_NO_CHANGE) + if (rxfh->hfunc == ETH_RSS_HASH_NO_CHANGE) algo = fp->hash_algo; - else if (hfunc == ETH_RSS_HASH_CRC32) + else if (rxfh->hfunc == ETH_RSS_HASH_CRC32) algo = FUN_ETH_RSS_ALG_CRC32; - else if (hfunc == ETH_RSS_HASH_TOP) + else if (rxfh->hfunc == ETH_RSS_HASH_TOP) algo = FUN_ETH_RSS_ALG_TOEPLITZ; else return -EINVAL; @@ -1031,10 +1031,10 @@ static int fun_set_rxfh(struct net_device *netdev, const u32 *indir, } fp->hash_algo = algo; - if (key) - memcpy(fp->rss_key, key, sizeof(fp->rss_key)); - if (indir) - memcpy(fp->indir_table, indir, + if (rxfh->key) + memcpy(fp->rss_key, rxfh->key, sizeof(fp->rss_key)); + if (rxfh->indir) + memcpy(fp->indir_table, rxfh->indir, sizeof(u32) * fp->indir_table_nentries); return 0; } diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c index fe40cceb0f79..a5bb306b2cf1 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c @@ -1186,7 +1186,7 @@ hns_get_rss_indir_size(struct net_device *netdev) } static int -hns_get_rss(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc) +hns_get_rss(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { struct hns_nic_priv *priv = netdev_priv(netdev); struct hnae_ae_ops *ops; @@ -1199,15 +1199,16 @@ hns_get_rss(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc) ops = priv->ae_handle->dev->ops; - if (!indir) + if (!rxfh->indir) return 0; - return ops->get_rss(priv->ae_handle, indir, key, hfunc); + return ops->get_rss(priv->ae_handle, + rxfh->indir, rxfh->key, &rxfh->hfunc); } static int -hns_set_rss(struct net_device *netdev, const u32 *indir, const u8 *key, - const u8 hfunc) +hns_set_rss(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct hns_nic_priv *priv = netdev_priv(netdev); struct hnae_ae_ops *ops; @@ -1220,12 +1221,14 @@ hns_set_rss(struct net_device *netdev, const u32 *indir, const u8 *key, ops = priv->ae_handle->dev->ops; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) { + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) { netdev_err(netdev, "Invalid hfunc!\n"); return -EOPNOTSUPP; } - return ops->set_rss(priv->ae_handle, indir, key, hfunc); + return ops->set_rss(priv->ae_handle, + rxfh->indir, rxfh->key, rxfh->hfunc); } static int hns_get_rxnfc(struct net_device *netdev, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 682239f33082..999a0ee162a6 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -941,19 +941,21 @@ static u32 hns3_get_rss_indir_size(struct net_device *netdev) return ae_dev->dev_specs.rss_ind_tbl_size; } -static int hns3_get_rss(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int hns3_get_rss(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct hnae3_handle *h = hns3_get_handle(netdev); if (!h->ae_algo->ops->get_rss) return -EOPNOTSUPP; - return h->ae_algo->ops->get_rss(h, indir, key, hfunc); + return h->ae_algo->ops->get_rss(h, rxfh->indir, rxfh->key, + &rxfh->hfunc); } -static int hns3_set_rss(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int hns3_set_rss(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct hnae3_handle *h = hns3_get_handle(netdev); struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); @@ -962,19 +964,22 @@ static int hns3_set_rss(struct net_device *netdev, const u32 *indir, return -EOPNOTSUPP; if ((ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2 && - hfunc != ETH_RSS_HASH_TOP) || (hfunc != ETH_RSS_HASH_NO_CHANGE && - hfunc != ETH_RSS_HASH_TOP && hfunc != ETH_RSS_HASH_XOR)) { + rxfh->hfunc != ETH_RSS_HASH_TOP) || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP && + rxfh->hfunc != ETH_RSS_HASH_XOR)) { netdev_err(netdev, "hash func not supported\n"); return -EOPNOTSUPP; } - if (!indir) { + if (!rxfh->indir) { netdev_err(netdev, "set rss failed for indir is empty\n"); return -EOPNOTSUPP; } - return h->ae_algo->ops->set_rss(h, indir, key, hfunc); + return h->ae_algo->ops->set_rss(h, rxfh->indir, rxfh->key, + rxfh->hfunc); } static int hns3_get_rxnfc(struct net_device *netdev, diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index f4b680286911..0304f03d4093 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -1137,7 +1137,7 @@ static int hinic_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) } static int hinic_get_rxfh(struct net_device *netdev, - u32 *indir, u8 *key, u8 *hfunc) + struct ethtool_rxfh_param *rxfh) { struct hinic_dev *nic_dev = netdev_priv(netdev); u8 hash_engine_type = 0; @@ -1146,32 +1146,33 @@ static int hinic_get_rxfh(struct net_device *netdev, if (!(nic_dev->flags & HINIC_RSS_ENABLE)) return -EOPNOTSUPP; - if (hfunc) { - err = hinic_rss_get_hash_engine(nic_dev, - nic_dev->rss_tmpl_idx, - &hash_engine_type); - if (err) - return -EFAULT; + err = hinic_rss_get_hash_engine(nic_dev, + nic_dev->rss_tmpl_idx, + &hash_engine_type); + if (err) + return -EFAULT; - *hfunc = hash_engine_type ? ETH_RSS_HASH_TOP : ETH_RSS_HASH_XOR; - } + rxfh->hfunc = hash_engine_type ? ETH_RSS_HASH_TOP : ETH_RSS_HASH_XOR; - if (indir) { + if (rxfh->indir) { err = hinic_rss_get_indir_tbl(nic_dev, - nic_dev->rss_tmpl_idx, indir); + nic_dev->rss_tmpl_idx, + rxfh->indir); if (err) return -EFAULT; } - if (key) + if (rxfh->key) err = hinic_rss_get_template_tbl(nic_dev, - nic_dev->rss_tmpl_idx, key); + nic_dev->rss_tmpl_idx, + rxfh->key); return err; } -static int hinic_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int hinic_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct hinic_dev *nic_dev = netdev_priv(netdev); int err = 0; @@ -1179,11 +1180,12 @@ static int hinic_set_rxfh(struct net_device *netdev, const u32 *indir, if (!(nic_dev->flags & HINIC_RSS_ENABLE)) return -EOPNOTSUPP; - if (hfunc != ETH_RSS_HASH_NO_CHANGE) { - if (hfunc != ETH_RSS_HASH_TOP && hfunc != ETH_RSS_HASH_XOR) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE) { + if (rxfh->hfunc != ETH_RSS_HASH_TOP && + rxfh->hfunc != ETH_RSS_HASH_XOR) return -EOPNOTSUPP; - nic_dev->rss_hash_engine = (hfunc == ETH_RSS_HASH_XOR) ? + nic_dev->rss_hash_engine = (rxfh->hfunc == ETH_RSS_HASH_XOR) ? HINIC_RSS_HASH_ENGINE_TYPE_XOR : HINIC_RSS_HASH_ENGINE_TYPE_TOEP; err = hinic_rss_set_hash_engine @@ -1193,7 +1195,7 @@ static int hinic_set_rxfh(struct net_device *netdev, const u32 *indir, return -EFAULT; } - err = __set_rss_rxfh(netdev, indir, key); + err = __set_rss_rxfh(netdev, rxfh->indir, rxfh->key); return err; } diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c index 13a05604dcc0..1bc5b6c0b897 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c @@ -1057,16 +1057,16 @@ static u32 fm10k_get_rssrk_size(struct net_device __always_unused *netdev) return FM10K_RSSRK_SIZE * FM10K_RSSRK_ENTRIES_PER_REG; } -static int fm10k_get_rssh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int fm10k_get_rssh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct fm10k_intfc *interface = netdev_priv(netdev); + u8 *key = rxfh->key; int i, err; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - err = fm10k_get_reta(netdev, indir); + err = fm10k_get_reta(netdev, rxfh->indir); if (err || !key) return err; @@ -1076,23 +1076,25 @@ static int fm10k_get_rssh(struct net_device *netdev, u32 *indir, u8 *key, return 0; } -static int fm10k_set_rssh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int fm10k_set_rssh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct fm10k_intfc *interface = netdev_priv(netdev); struct fm10k_hw *hw = &interface->hw; int i, err; /* We do not allow change in unsupported parameters */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - err = fm10k_set_reta(netdev, indir); - if (err || !key) + err = fm10k_set_reta(netdev, rxfh->indir); + if (err || !rxfh->key) return err; - for (i = 0; i < FM10K_RSSRK_SIZE; i++, key += 4) { - u32 rssrk = le32_to_cpu(*(__le32 *)key); + for (i = 0; i < FM10K_RSSRK_SIZE; i++, rxfh->key += 4) { + u32 rssrk = le32_to_cpu(*(__le32 *)rxfh->key); if (interface->rssrk[i] == rssrk) continue; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index a0b10230268d..53820d63e6fa 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -5120,15 +5120,13 @@ static u32 i40e_get_rxfh_indir_size(struct net_device *netdev) /** * i40e_get_rxfh - get the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function + * @rxfh: pointer to param struct (indir, key, hfunc) * * Reads the indirection table directly from the hardware. Returns 0 on * success. **/ -static int i40e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int i40e_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_vsi *vsi = np->vsi; @@ -5136,13 +5134,12 @@ static int i40e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, int ret; u16 i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (!indir) + if (!rxfh->indir) return 0; - seed = key; + seed = rxfh->key; lut = kzalloc(I40E_HLUT_ARRAY_SIZE, GFP_KERNEL); if (!lut) return -ENOMEM; @@ -5150,7 +5147,7 @@ static int i40e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, if (ret) goto out; for (i = 0; i < I40E_HLUT_ARRAY_SIZE; i++) - indir[i] = (u32)(lut[i]); + rxfh->indir[i] = (u32)(lut[i]); out: kfree(lut); @@ -5161,15 +5158,15 @@ out: /** * i40e_set_rxfh - set the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function to use + * @rxfh: pointer to param struct (indir, key, hfunc) + * @extack: extended ACK from the Netlink message * * Returns -EINVAL if the table specifies an invalid queue id, otherwise * returns 0 after programming the table. **/ -static int i40e_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int i40e_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_vsi *vsi = np->vsi; @@ -5177,17 +5174,18 @@ static int i40e_set_rxfh(struct net_device *netdev, const u32 *indir, u8 *seed = NULL; u16 i; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (key) { + if (rxfh->key) { if (!vsi->rss_hkey_user) { vsi->rss_hkey_user = kzalloc(I40E_HKEY_ARRAY_SIZE, GFP_KERNEL); if (!vsi->rss_hkey_user) return -ENOMEM; } - memcpy(vsi->rss_hkey_user, key, I40E_HKEY_ARRAY_SIZE); + memcpy(vsi->rss_hkey_user, rxfh->key, I40E_HKEY_ARRAY_SIZE); seed = vsi->rss_hkey_user; } if (!vsi->rss_lut_user) { @@ -5197,9 +5195,9 @@ static int i40e_set_rxfh(struct net_device *netdev, const u32 *indir, } /* Each 32 bits pointed by 'indir' is stored with a lut entry */ - if (indir) + if (rxfh->indir) for (i = 0; i < I40E_HLUT_ARRAY_SIZE; i++) - vsi->rss_lut_user[i] = (u8)(indir[i]); + vsi->rss_lut_user[i] = (u8)(rxfh->indir[i]); else i40e_fill_rss_lut(pf, vsi->rss_lut_user, I40E_HLUT_ARRAY_SIZE, vsi->rss_size); diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 631c7cfdc814..c376a489e4c2 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -1894,27 +1894,24 @@ static u32 iavf_get_rxfh_indir_size(struct net_device *netdev) /** * iavf_get_rxfh - get the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function in use + * @rxfh: pointer to param struct (indir, key, hfunc) * * Reads the indirection table directly from the hardware. Always returns 0. **/ -static int iavf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int iavf_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct iavf_adapter *adapter = netdev_priv(netdev); u16 i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (key) - memcpy(key, adapter->rss_key, adapter->rss_key_size); + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->key) + memcpy(rxfh->key, adapter->rss_key, adapter->rss_key_size); - if (indir) + if (rxfh->indir) /* Each 32 bits pointed by 'indir' is stored with a lut entry */ for (i = 0; i < adapter->rss_lut_size; i++) - indir[i] = (u32)adapter->rss_lut[i]; + rxfh->indir[i] = (u32)adapter->rss_lut[i]; return 0; } @@ -1922,33 +1919,34 @@ static int iavf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, /** * iavf_set_rxfh - set the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function to use + * @rxfh: pointer to param struct (indir, key, hfunc) + * @extack: extended ACK from the Netlink message * * Returns -EINVAL if the table specifies an invalid queue id, otherwise * returns 0 after programming the table. **/ -static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int iavf_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct iavf_adapter *adapter = netdev_priv(netdev); u16 i; /* Only support toeplitz hash function */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (!key && !indir) + if (!rxfh->key && !rxfh->indir) return 0; - if (key) - memcpy(adapter->rss_key, key, adapter->rss_key_size); + if (rxfh->key) + memcpy(adapter->rss_key, rxfh->key, adapter->rss_key_size); - if (indir) { + if (rxfh->indir) { /* Each 32 bits pointed by 'indir' is stored with a lut entry */ for (i = 0; i < adapter->rss_lut_size; i++) - adapter->rss_lut[i] = (u8)(indir[i]); + adapter->rss_lut[i] = (u8)(rxfh->indir[i]); } return iavf_config_rss(adapter); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 98c9317581e0..70cbd8092aea 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3196,8 +3196,8 @@ static u32 ice_get_rxfh_indir_size(struct net_device *netdev) } static int -ice_get_rxfh_context(struct net_device *netdev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context) +ice_get_rxfh_context(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, u32 rss_context) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; @@ -3230,17 +3230,16 @@ ice_get_rxfh_context(struct net_device *netdev, u32 *indir, vsi = vsi->tc_map_vsi[rss_context]; } - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (!indir) + if (!rxfh->indir) return 0; lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); if (!lut) return -ENOMEM; - err = ice_get_rss_key(vsi, key); + err = ice_get_rss_key(vsi, rxfh->key); if (err) goto out; @@ -3250,12 +3249,12 @@ ice_get_rxfh_context(struct net_device *netdev, u32 *indir, if (ice_is_adq_active(pf)) { for (i = 0; i < vsi->rss_table_size; i++) - indir[i] = offset + lut[i] % qcount; + rxfh->indir[i] = offset + lut[i] % qcount; goto out; } for (i = 0; i < vsi->rss_table_size; i++) - indir[i] = lut[i]; + rxfh->indir[i] = lut[i]; out: kfree(lut); @@ -3265,31 +3264,28 @@ out: /** * ice_get_rxfh - get the Rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function + * @rxfh: pointer to param struct (indir, key, hfunc) * * Reads the indirection table directly from the hardware. */ static int -ice_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc) +ice_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { - return ice_get_rxfh_context(netdev, indir, key, hfunc, 0); + return ice_get_rxfh_context(netdev, rxfh, 0); } /** * ice_set_rxfh - set the Rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function + * @rxfh: pointer to param struct (indir, key, hfunc) + * @extack: extended ACK from the Netlink message * * Returns -EINVAL if the table specifies an invalid queue ID, otherwise * returns 0 after programming the table. */ static int -ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, - const u8 hfunc) +ice_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; @@ -3298,7 +3294,8 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, int err; dev = ice_pf_to_dev(pf); - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { @@ -3312,7 +3309,7 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, return -EOPNOTSUPP; } - if (key) { + if (rxfh->key) { if (!vsi->rss_hkey_user) { vsi->rss_hkey_user = devm_kzalloc(dev, ICE_VSIQF_HKEY_ARRAY_SIZE, @@ -3320,7 +3317,8 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, if (!vsi->rss_hkey_user) return -ENOMEM; } - memcpy(vsi->rss_hkey_user, key, ICE_VSIQF_HKEY_ARRAY_SIZE); + memcpy(vsi->rss_hkey_user, rxfh->key, + ICE_VSIQF_HKEY_ARRAY_SIZE); err = ice_set_rss_key(vsi, vsi->rss_hkey_user); if (err) @@ -3335,11 +3333,11 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, } /* Each 32 bits pointed by 'indir' is stored with a lut entry */ - if (indir) { + if (rxfh->indir) { int i; for (i = 0; i < vsi->rss_table_size; i++) - vsi->rss_lut_user[i] = (u8)(indir[i]); + vsi->rss_lut_user[i] = (u8)(rxfh->indir[i]); } else { ice_fill_rss_lut(vsi->rss_lut_user, vsi->rss_table_size, vsi->rss_size); diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index d549d3c5910d..986d429d1175 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -75,14 +75,12 @@ static u32 idpf_get_rxfh_indir_size(struct net_device *netdev) /** * idpf_get_rxfh - get the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function in use + * @rxfh: pointer to param struct (indir, key, hfunc) * * Reads the indirection table directly from the hardware. Always returns 0. */ -static int idpf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int idpf_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct idpf_netdev_priv *np = netdev_priv(netdev); struct idpf_rss_data *rss_data; @@ -103,15 +101,14 @@ static int idpf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, if (np->state != __IDPF_VPORT_UP) goto unlock_mutex; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (key) - memcpy(key, rss_data->rss_key, rss_data->rss_key_size); + if (rxfh->key) + memcpy(rxfh->key, rss_data->rss_key, rss_data->rss_key_size); - if (indir) { + if (rxfh->indir) { for (i = 0; i < rss_data->rss_lut_size; i++) - indir[i] = rss_data->rss_lut[i]; + rxfh->indir[i] = rss_data->rss_lut[i]; } unlock_mutex: @@ -123,15 +120,15 @@ unlock_mutex: /** * idpf_set_rxfh - set the rx flow hash indirection table * @netdev: network interface device structure - * @indir: indirection table - * @key: hash key - * @hfunc: hash function to use + * @rxfh: pointer to param struct (indir, key, hfunc) + * @extack: extended ACK from the Netlink message * * Returns -EINVAL if the table specifies an invalid queue id, otherwise * returns 0 after programming the table. */ -static int idpf_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int idpf_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct idpf_netdev_priv *np = netdev_priv(netdev); struct idpf_rss_data *rss_data; @@ -154,17 +151,18 @@ static int idpf_set_rxfh(struct net_device *netdev, const u32 *indir, if (np->state != __IDPF_VPORT_UP) goto unlock_mutex; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) { + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) { err = -EOPNOTSUPP; goto unlock_mutex; } - if (key) - memcpy(rss_data->rss_key, key, rss_data->rss_key_size); + if (rxfh->key) + memcpy(rss_data->rss_key, rxfh->key, rss_data->rss_key_size); - if (indir) { + if (rxfh->indir) { for (lut = 0; lut < rss_data->rss_lut_size; lut++) - rss_data->rss_lut[lut] = indir[lut]; + rss_data->rss_lut[lut] = rxfh->indir[lut]; } err = idpf_config_rss(vport); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 89dac7b215e5..d868a70732f4 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -3280,18 +3280,17 @@ static u32 igb_get_rxfh_indir_size(struct net_device *netdev) return IGB_RETA_SIZE; } -static int igb_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int igb_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct igb_adapter *adapter = netdev_priv(netdev); int i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!indir) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; for (i = 0; i < IGB_RETA_SIZE; i++) - indir[i] = adapter->rss_indir_tbl[i]; + rxfh->indir[i] = adapter->rss_indir_tbl[i]; return 0; } @@ -3331,8 +3330,9 @@ void igb_write_rss_indir_tbl(struct igb_adapter *adapter) } } -static int igb_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int igb_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct igb_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -3340,10 +3340,11 @@ static int igb_set_rxfh(struct net_device *netdev, const u32 *indir, u32 num_queues; /* We do not allow change in unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; num_queues = adapter->rss_queues; @@ -3360,12 +3361,12 @@ static int igb_set_rxfh(struct net_device *netdev, const u32 *indir, /* Verify user input. */ for (i = 0; i < IGB_RETA_SIZE; i++) - if (indir[i] >= num_queues) + if (rxfh->indir[i] >= num_queues) return -EINVAL; for (i = 0; i < IGB_RETA_SIZE; i++) - adapter->rss_indir_tbl[i] = indir[i]; + adapter->rss_indir_tbl[i] = rxfh->indir[i]; igb_write_rss_indir_tbl(adapter); diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index 2ed92bf34059..684f6f2a0572 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -1426,45 +1426,46 @@ static u32 igc_ethtool_get_rxfh_indir_size(struct net_device *netdev) return IGC_RETA_SIZE; } -static int igc_ethtool_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int igc_ethtool_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct igc_adapter *adapter = netdev_priv(netdev); int i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!indir) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; for (i = 0; i < IGC_RETA_SIZE; i++) - indir[i] = adapter->rss_indir_tbl[i]; + rxfh->indir[i] = adapter->rss_indir_tbl[i]; return 0; } -static int igc_ethtool_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int igc_ethtool_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); u32 num_queues; int i; /* We do not allow change in unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; num_queues = adapter->rss_queues; /* Verify user input. */ for (i = 0; i < IGC_RETA_SIZE; i++) - if (indir[i] >= num_queues) + if (rxfh->indir[i] >= num_queues) return -EINVAL; for (i = 0; i < IGC_RETA_SIZE; i++) - adapter->rss_indir_tbl[i] = indir[i]; + adapter->rss_indir_tbl[i] = rxfh->indir[i]; igc_write_rss_indir_tbl(adapter); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index dd722b0381e0..af55f19d5bba 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -3107,35 +3107,37 @@ static void ixgbe_get_reta(struct ixgbe_adapter *adapter, u32 *indir) indir[i] = adapter->rss_indir_tbl[i] & rss_m; } -static int ixgbe_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int ixgbe_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct ixgbe_adapter *adapter = netdev_priv(netdev); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (indir) - ixgbe_get_reta(adapter, indir); + if (rxfh->indir) + ixgbe_get_reta(adapter, rxfh->indir); - if (key) - memcpy(key, adapter->rss_key, ixgbe_get_rxfh_key_size(netdev)); + if (rxfh->key) + memcpy(rxfh->key, adapter->rss_key, + ixgbe_get_rxfh_key_size(netdev)); return 0; } -static int ixgbe_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int ixgbe_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = netdev_priv(netdev); int i; u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter); - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; /* Fill out the redirection table */ - if (indir) { + if (rxfh->indir) { int max_queues = min_t(int, adapter->num_rx_queues, ixgbe_rss_indir_tbl_max(adapter)); @@ -3146,18 +3148,19 @@ static int ixgbe_set_rxfh(struct net_device *netdev, const u32 *indir, /* Verify user input. */ for (i = 0; i < reta_entries; i++) - if (indir[i] >= max_queues) + if (rxfh->indir[i] >= max_queues) return -EINVAL; for (i = 0; i < reta_entries; i++) - adapter->rss_indir_tbl[i] = indir[i]; + adapter->rss_indir_tbl[i] = rxfh->indir[i]; ixgbe_store_reta(adapter); } /* Fill out the rss hash key */ - if (key) { - memcpy(adapter->rss_key, key, ixgbe_get_rxfh_key_size(netdev)); + if (rxfh->key) { + memcpy(adapter->rss_key, rxfh->key, + ixgbe_get_rxfh_key_size(netdev)); ixgbe_store_key(adapter); } diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 296915414a7c..7ac53171b041 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -897,40 +897,41 @@ static u32 ixgbevf_get_rxfh_key_size(struct net_device *netdev) return IXGBEVF_RSS_HASH_KEY_SIZE; } -static int ixgbevf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int ixgbevf_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); int err = 0; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) { - if (key) - memcpy(key, adapter->rss_key, + if (rxfh->key) + memcpy(rxfh->key, adapter->rss_key, ixgbevf_get_rxfh_key_size(netdev)); - if (indir) { + if (rxfh->indir) { int i; for (i = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++) - indir[i] = adapter->rss_indir_tbl[i]; + rxfh->indir[i] = adapter->rss_indir_tbl[i]; } } else { /* If neither indirection table nor hash key was requested * - just return a success avoiding taking any locks. */ - if (!indir && !key) + if (!rxfh->indir && !rxfh->key) return 0; spin_lock_bh(&adapter->mbx_lock); - if (indir) - err = ixgbevf_get_reta_locked(&adapter->hw, indir, + if (rxfh->indir) + err = ixgbevf_get_reta_locked(&adapter->hw, + rxfh->indir, adapter->num_rx_queues); - if (!err && key) - err = ixgbevf_get_rss_key_locked(&adapter->hw, key); + if (!err && rxfh->key) + err = ixgbevf_get_rss_key_locked(&adapter->hw, + rxfh->key); spin_unlock_bh(&adapter->mbx_lock); } diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 29aac327574d..a641b3534ca3 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -5030,8 +5030,9 @@ static int mvneta_config_rss(struct mvneta_port *pp) return 0; } -static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int mvneta_ethtool_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(dev); @@ -5042,20 +5043,21 @@ static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, /* We require at least one supported parameter to be changed * and no change in any of the unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; - memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE); + memcpy(pp->indir, rxfh->indir, MVNETA_RSS_LU_TABLE_SIZE); return mvneta_config_rss(pp); } -static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int mvneta_ethtool_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct mvneta_port *pp = netdev_priv(dev); @@ -5063,13 +5065,12 @@ static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, if (pp->neta_armada3700) return -EOPNOTSUPP; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (!indir) + if (!rxfh->indir) return 0; - memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE); + memcpy(rxfh->indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE); return 0; } diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 93137606869e..ceef48ddd26e 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5634,8 +5634,8 @@ static u32 mvpp2_ethtool_get_rxfh_indir_size(struct net_device *dev) return mvpp22_rss_is_supported(port) ? MVPP22_RSS_TABLE_ENTRIES : 0; } -static int mvpp2_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int mvpp2_ethtool_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct mvpp2_port *port = netdev_priv(dev); int ret = 0; @@ -5643,17 +5643,17 @@ static int mvpp2_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; - if (indir) - ret = mvpp22_port_rss_ctx_indir_get(port, 0, indir); + if (rxfh->indir) + ret = mvpp22_port_rss_ctx_indir_get(port, 0, rxfh->indir); - if (hfunc) - *hfunc = ETH_RSS_HASH_CRC32; + rxfh->hfunc = ETH_RSS_HASH_CRC32; return ret; } -static int mvpp2_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int mvpp2_ethtool_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); int ret = 0; @@ -5661,20 +5661,22 @@ static int mvpp2_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_CRC32) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_CRC32) return -EOPNOTSUPP; - if (key) + if (rxfh->key) return -EOPNOTSUPP; - if (indir) - ret = mvpp22_port_rss_ctx_indir_set(port, 0, indir); + if (rxfh->indir) + ret = mvpp22_port_rss_ctx_indir_set(port, 0, rxfh->indir); return ret; } -static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context) +static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context) { struct mvpp2_port *port = netdev_priv(dev); int ret = 0; @@ -5684,19 +5686,18 @@ static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, u32 *indir, if (rss_context >= MVPP22_N_RSS_TABLES) return -EINVAL; - if (hfunc) - *hfunc = ETH_RSS_HASH_CRC32; + rxfh->hfunc = ETH_RSS_HASH_CRC32; - if (indir) - ret = mvpp22_port_rss_ctx_indir_get(port, rss_context, indir); + if (rxfh->indir) + ret = mvpp22_port_rss_ctx_indir_get(port, rss_context, + rxfh->indir); return ret; } static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, - const u32 *indir, const u8 *key, - const u8 hfunc, u32 *rss_context, - bool delete) + struct ethtool_rxfh_param *rxfh, + u32 *rss_context, bool delete) { struct mvpp2_port *port = netdev_priv(dev); int ret; @@ -5704,10 +5705,11 @@ static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_CRC32) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_CRC32) return -EOPNOTSUPP; - if (key) + if (rxfh->key) return -EOPNOTSUPP; if (delete) @@ -5719,7 +5721,7 @@ static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, return ret; } - return mvpp22_port_rss_ctx_indir_set(port, *rss_context, indir); + return mvpp22_port_rss_ctx_indir_set(port, *rss_context, rxfh->indir); } /* Device ops */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 53f6258a973c..a2c182d7bfc4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -836,8 +836,8 @@ static int otx2_rss_ctx_create(struct otx2_nic *pfvf, } /* RSS context configuration */ -static int otx2_set_rxfh_context(struct net_device *dev, const u32 *indir, - const u8 *hkey, const u8 hfunc, +static int otx2_set_rxfh_context(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, u32 *rss_context, bool delete) { struct otx2_nic *pfvf = netdev_priv(dev); @@ -845,7 +845,8 @@ static int otx2_set_rxfh_context(struct net_device *dev, const u32 *indir, struct otx2_rss_info *rss; int ret, idx; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (*rss_context != ETH_RXFH_CONTEXT_ALLOC && @@ -859,8 +860,8 @@ static int otx2_set_rxfh_context(struct net_device *dev, const u32 *indir, return -EIO; } - if (hkey) { - memcpy(rss->key, hkey, sizeof(rss->key)); + if (rxfh->key) { + memcpy(rss->key, rxfh->key, sizeof(rss->key)); otx2_set_rss_key(pfvf); } if (delete) @@ -871,28 +872,29 @@ static int otx2_set_rxfh_context(struct net_device *dev, const u32 *indir, if (ret) return ret; } - if (indir) { + if (rxfh->indir) { rss_ctx = rss->rss_ctx[*rss_context]; for (idx = 0; idx < rss->rss_size; idx++) - rss_ctx->ind_tbl[idx] = indir[idx]; + rss_ctx->ind_tbl[idx] = rxfh->indir[idx]; } otx2_set_rss_table(pfvf, *rss_context); return 0; } -static int otx2_get_rxfh_context(struct net_device *dev, u32 *indir, - u8 *hkey, u8 *hfunc, u32 rss_context) +static int otx2_get_rxfh_context(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context) { struct otx2_nic *pfvf = netdev_priv(dev); struct otx2_rss_ctx *rss_ctx; struct otx2_rss_info *rss; + u32 *indir = rxfh->indir; int idx, rx_queues; rss = &pfvf->hw.rss_info; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; if (!indir) return 0; @@ -914,28 +916,29 @@ static int otx2_get_rxfh_context(struct net_device *dev, u32 *indir, for (idx = 0; idx < rss->rss_size; idx++) indir[idx] = rss_ctx->ind_tbl[idx]; } - if (hkey) - memcpy(hkey, rss->key, sizeof(rss->key)); + if (rxfh->key) + memcpy(rxfh->key, rss->key, sizeof(rss->key)); return 0; } /* Get RSS configuration */ -static int otx2_get_rxfh(struct net_device *dev, u32 *indir, - u8 *hkey, u8 *hfunc) +static int otx2_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { - return otx2_get_rxfh_context(dev, indir, hkey, hfunc, + return otx2_get_rxfh_context(dev, rxfh, DEFAULT_RSS_CONTEXT_GROUP); } /* Configure RSS table and hash key */ -static int otx2_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *hkey, const u8 hfunc) +static int otx2_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { u32 rss_context = DEFAULT_RSS_CONTEXT_GROUP; - return otx2_set_rxfh_context(dev, indir, hkey, hfunc, &rss_context, 0); + return otx2_set_rxfh_context(dev, rxfh, &rss_context, 0); } static u32 otx2_get_msglevel(struct net_device *netdev) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 164a13272faa..619e1c3ef7f9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -1258,8 +1258,8 @@ static int mlx4_en_check_rxfh_func(struct net_device *dev, u8 hfunc) return -EINVAL; } -static int mlx4_en_get_rxfh(struct net_device *dev, u32 *ring_index, u8 *key, - u8 *hfunc) +static int mlx4_en_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct mlx4_en_priv *priv = netdev_priv(dev); u32 n = mlx4_en_get_rxfh_indir_size(dev); @@ -1269,19 +1269,19 @@ static int mlx4_en_get_rxfh(struct net_device *dev, u32 *ring_index, u8 *key, rss_rings = rounddown_pow_of_two(rss_rings); for (i = 0; i < n; i++) { - if (!ring_index) + if (!rxfh->indir) break; - ring_index[i] = i % rss_rings; + rxfh->indir[i] = i % rss_rings; } - if (key) - memcpy(key, priv->rss_key, MLX4_EN_RSS_KEY_SIZE); - if (hfunc) - *hfunc = priv->rss_hash_fn; + if (rxfh->key) + memcpy(rxfh->key, priv->rss_key, MLX4_EN_RSS_KEY_SIZE); + rxfh->hfunc = priv->rss_hash_fn; return 0; } -static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index, - const u8 *key, const u8 hfunc) +static int mlx4_en_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mlx4_en_priv *priv = netdev_priv(dev); u32 n = mlx4_en_get_rxfh_indir_size(dev); @@ -1295,12 +1295,12 @@ static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index, * between rings */ for (i = 0; i < n; i++) { - if (!ring_index) + if (!rxfh->indir) break; - if (i > 0 && !ring_index[i] && !rss_rings) + if (i > 0 && !rxfh->indir[i] && !rss_rings) rss_rings = i; - if (ring_index[i] != (i % (rss_rings ?: n))) + if (rxfh->indir[i] != (i % (rss_rings ?: n))) return -EINVAL; } @@ -1311,8 +1311,8 @@ static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index, if (!is_power_of_2(rss_rings)) return -EINVAL; - if (hfunc != ETH_RSS_HASH_NO_CHANGE) { - err = mlx4_en_check_rxfh_func(dev, hfunc); + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE) { + err = mlx4_en_check_rxfh_func(dev, rxfh->hfunc); if (err) return err; } @@ -1323,12 +1323,12 @@ static int mlx4_en_set_rxfh(struct net_device *dev, const u32 *ring_index, mlx4_en_stop_port(dev, 1); } - if (ring_index) + if (rxfh->indir) priv->prof->rss_rings = rss_rings; - if (key) - memcpy(priv->rss_key, key, MLX4_EN_RSS_KEY_SIZE); - if (hfunc != ETH_RSS_HASH_NO_CHANGE) - priv->rss_hash_fn = hfunc; + if (rxfh->key) + memcpy(priv->rss_key, rxfh->key, MLX4_EN_RSS_KEY_SIZE); + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE) + priv->rss_hash_fn = rxfh->hfunc; if (port_up) { err = mlx4_en_start_port(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 43f027bf2da3..2cb47268bac6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1175,9 +1175,9 @@ int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv, struct ethtool_link_ksettings *link_ksettings); int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv, const struct ethtool_link_ksettings *link_ksettings); -int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc); -int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, - const u8 hfunc); +int mlx5e_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh); +int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack); u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv); u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv); int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 792a0ea544cd..110cb7973eae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1262,23 +1262,26 @@ static u32 mlx5e_get_rxfh_indir_size(struct net_device *netdev) return mlx5e_ethtool_get_rxfh_indir_size(priv); } -static int mlx5e_get_rxfh_context(struct net_device *dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context) +static int mlx5e_get_rxfh_context(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context) { struct mlx5e_priv *priv = netdev_priv(dev); int err; mutex_lock(&priv->state_lock); - err = mlx5e_rx_res_rss_get_rxfh(priv->rx_res, rss_context, indir, key, hfunc); + err = mlx5e_rx_res_rss_get_rxfh(priv->rx_res, rss_context, + rxfh->indir, rxfh->key, &rxfh->hfunc); mutex_unlock(&priv->state_lock); return err; } -static int mlx5e_set_rxfh_context(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc, +static int mlx5e_set_rxfh_context(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, u32 *rss_context, bool delete) { struct mlx5e_priv *priv = netdev_priv(dev); + u8 hfunc = rxfh->hfunc; int err; mutex_lock(&priv->state_lock); @@ -1295,7 +1298,8 @@ static int mlx5e_set_rxfh_context(struct net_device *dev, const u32 *indir, goto unlock; } - err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, *rss_context, indir, key, + err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, *rss_context, + rxfh->indir, rxfh->key, hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc); unlock: @@ -1303,20 +1307,20 @@ unlock: return err; } -int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { - return mlx5e_get_rxfh_context(netdev, indir, key, hfunc, 0); + return mlx5e_get_rxfh_context(netdev, rxfh, 0); } -int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); + u8 hfunc = rxfh->hfunc; int err; mutex_lock(&priv->state_lock); - err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, 0, indir, key, + err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, 0, rxfh->indir, rxfh->key, hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc); mutex_unlock(&priv->state_lock); return err; diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index 6961cfc55fb9..8c4a2bb6a537 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -934,11 +934,11 @@ static u32 lan743x_ethtool_get_rxfh_indir_size(struct net_device *netdev) } static int lan743x_ethtool_get_rxfh(struct net_device *netdev, - u32 *indir, u8 *key, u8 *hfunc) + struct ethtool_rxfh_param *rxfh) { struct lan743x_adapter *adapter = netdev_priv(netdev); - if (indir) { + if (rxfh->indir) { int dw_index; int byte_index = 0; @@ -947,17 +947,17 @@ static int lan743x_ethtool_get_rxfh(struct net_device *netdev, lan743x_csr_read(adapter, RFE_INDX(dw_index)); byte_index = dw_index << 2; - indir[byte_index + 0] = + rxfh->indir[byte_index + 0] = ((four_entries >> 0) & 0x000000FF); - indir[byte_index + 1] = + rxfh->indir[byte_index + 1] = ((four_entries >> 8) & 0x000000FF); - indir[byte_index + 2] = + rxfh->indir[byte_index + 2] = ((four_entries >> 16) & 0x000000FF); - indir[byte_index + 3] = + rxfh->indir[byte_index + 3] = ((four_entries >> 24) & 0x000000FF); } } - if (key) { + if (rxfh->key) { int dword_index; int byte_index = 0; @@ -967,28 +967,30 @@ static int lan743x_ethtool_get_rxfh(struct net_device *netdev, RFE_HASH_KEY(dword_index)); byte_index = dword_index << 2; - key[byte_index + 0] = + rxfh->key[byte_index + 0] = ((four_entries >> 0) & 0x000000FF); - key[byte_index + 1] = + rxfh->key[byte_index + 1] = ((four_entries >> 8) & 0x000000FF); - key[byte_index + 2] = + rxfh->key[byte_index + 2] = ((four_entries >> 16) & 0x000000FF); - key[byte_index + 3] = + rxfh->key[byte_index + 3] = ((four_entries >> 24) & 0x000000FF); } } - if (hfunc) - (*hfunc) = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } static int lan743x_ethtool_set_rxfh(struct net_device *netdev, - const u32 *indir, const u8 *key, - const u8 hfunc) + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct lan743x_adapter *adapter = netdev_priv(netdev); + u32 *indir = rxfh->indir; + u8 *key = rxfh->key; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (indir) { diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c index 777e65b8223d..ab2413d71f6c 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c +++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c @@ -248,28 +248,28 @@ static u32 mana_rss_indir_size(struct net_device *ndev) return MANA_INDIRECT_TABLE_SIZE; } -static int mana_get_rxfh(struct net_device *ndev, u32 *indir, u8 *key, - u8 *hfunc) +static int mana_get_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh) { struct mana_port_context *apc = netdev_priv(ndev); int i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ + rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ - if (indir) { + if (rxfh->indir) { for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) - indir[i] = apc->indir_table[i]; + rxfh->indir[i] = apc->indir_table[i]; } - if (key) - memcpy(key, apc->hashkey, MANA_HASH_KEY_SIZE); + if (rxfh->key) + memcpy(rxfh->key, apc->hashkey, MANA_HASH_KEY_SIZE); return 0; } -static int mana_set_rxfh(struct net_device *ndev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int mana_set_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mana_port_context *apc = netdev_priv(ndev); bool update_hash = false, update_table = false; @@ -280,25 +280,26 @@ static int mana_set_rxfh(struct net_device *ndev, const u32 *indir, if (!apc->port_is_up) return -EOPNOTSUPP; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (indir) { + if (rxfh->indir) { for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) - if (indir[i] >= apc->num_queues) + if (rxfh->indir[i] >= apc->num_queues) return -EINVAL; update_table = true; for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) { save_table[i] = apc->indir_table[i]; - apc->indir_table[i] = indir[i]; + apc->indir_table[i] = rxfh->indir[i]; } } - if (key) { + if (rxfh->key) { update_hash = true; memcpy(save_key, apc->hashkey, MANA_HASH_KEY_SIZE); - memcpy(apc->hashkey, key, MANA_HASH_KEY_SIZE); + memcpy(apc->hashkey, rxfh->key, MANA_HASH_KEY_SIZE); } err = mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index ff9f39969215..fbca8d0efd85 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1794,8 +1794,8 @@ static u32 nfp_net_get_rxfh_key_size(struct net_device *netdev) return nfp_net_rss_key_sz(nn); } -static int nfp_net_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int nfp_net_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct nfp_net *nn = netdev_priv(netdev); int i; @@ -1803,41 +1803,41 @@ static int nfp_net_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, if (!(nn->cap & NFP_NET_CFG_CTRL_RSS_ANY)) return -EOPNOTSUPP; - if (indir) + if (rxfh->indir) for (i = 0; i < ARRAY_SIZE(nn->rss_itbl); i++) - indir[i] = nn->rss_itbl[i]; - if (key) - memcpy(key, nn->rss_key, nfp_net_rss_key_sz(nn)); - if (hfunc) { - *hfunc = nn->rss_hfunc; - if (*hfunc >= 1 << ETH_RSS_HASH_FUNCS_COUNT) - *hfunc = ETH_RSS_HASH_UNKNOWN; - } + rxfh->indir[i] = nn->rss_itbl[i]; + if (rxfh->key) + memcpy(rxfh->key, nn->rss_key, nfp_net_rss_key_sz(nn)); + + rxfh->hfunc = nn->rss_hfunc; + if (rxfh->hfunc >= 1 << ETH_RSS_HASH_FUNCS_COUNT) + rxfh->hfunc = ETH_RSS_HASH_UNKNOWN; return 0; } static int nfp_net_set_rxfh(struct net_device *netdev, - const u32 *indir, const u8 *key, - const u8 hfunc) + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct nfp_net *nn = netdev_priv(netdev); int i; if (!(nn->cap & NFP_NET_CFG_CTRL_RSS_ANY) || - !(hfunc == ETH_RSS_HASH_NO_CHANGE || hfunc == nn->rss_hfunc)) + !(rxfh->hfunc == ETH_RSS_HASH_NO_CHANGE || + rxfh->hfunc == nn->rss_hfunc)) return -EOPNOTSUPP; - if (!key && !indir) + if (!rxfh->key && !rxfh->indir) return 0; - if (key) { - memcpy(nn->rss_key, key, nfp_net_rss_key_sz(nn)); + if (rxfh->key) { + memcpy(nn->rss_key, rxfh->key, nfp_net_rss_key_sz(nn)); nfp_net_rss_write_key(nn); } - if (indir) { + if (rxfh->indir) { for (i = 0; i < ARRAY_SIZE(nn->rss_itbl); i++) - nn->rss_itbl[i] = indir[i]; + nn->rss_itbl[i] = rxfh->indir[i]; nfp_net_rss_write_itbl(nn); } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 3a6b0a9bc241..cd3c0b01402e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -823,36 +823,38 @@ static u32 ionic_get_rxfh_key_size(struct net_device *netdev) return IONIC_RSS_HASH_KEY_SIZE; } -static int ionic_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, - u8 *hfunc) +static int ionic_get_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh) { struct ionic_lif *lif = netdev_priv(netdev); unsigned int i, tbl_sz; - if (indir) { + if (rxfh->indir) { tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz); for (i = 0; i < tbl_sz; i++) - indir[i] = lif->rss_ind_tbl[i]; + rxfh->indir[i] = lif->rss_ind_tbl[i]; } - if (key) - memcpy(key, lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE); + if (rxfh->key) + memcpy(rxfh->key, lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int ionic_set_rxfh(struct net_device *netdev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int ionic_set_rxfh(struct net_device *netdev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct ionic_lif *lif = netdev_priv(netdev); - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - return ionic_lif_rss_config(lif, lif->rss_types, key, indir); + return ionic_lif_rss_config(lif, lif->rss_types, + rxfh->key, rxfh->indir); } static int ionic_set_tunable(struct net_device *dev, diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index b6b849a079ed..0e240b5ab8d4 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -1370,28 +1370,29 @@ static u32 qede_get_rxfh_key_size(struct net_device *dev) return sizeof(edev->rss_key); } -static int qede_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) +static int qede_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct qede_dev *edev = netdev_priv(dev); int i; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; - if (!indir) + if (!rxfh->indir) return 0; for (i = 0; i < QED_RSS_IND_TABLE_SIZE; i++) - indir[i] = edev->rss_ind_table[i]; + rxfh->indir[i] = edev->rss_ind_table[i]; - if (key) - memcpy(key, edev->rss_key, qede_get_rxfh_key_size(dev)); + if (rxfh->key) + memcpy(rxfh->key, edev->rss_key, qede_get_rxfh_key_size(dev)); return 0; } -static int qede_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int qede_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct qed_update_vport_params *vport_update_params; struct qede_dev *edev = netdev_priv(dev); @@ -1403,20 +1404,21 @@ static int qede_set_rxfh(struct net_device *dev, const u32 *indir, return -EOPNOTSUPP; } - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (!indir && !key) + if (!rxfh->indir && !rxfh->key) return 0; - if (indir) { + if (rxfh->indir) { for (i = 0; i < QED_RSS_IND_TABLE_SIZE; i++) - edev->rss_ind_table[i] = indir[i]; + edev->rss_ind_table[i] = rxfh->indir[i]; edev->rss_params_inited |= QEDE_RSS_INDIR_INITED; } - if (key) { - memcpy(&edev->rss_key, key, qede_get_rxfh_key_size(dev)); + if (rxfh->key) { + memcpy(&edev->rss_key, rxfh->key, qede_get_rxfh_key_size(dev)); edev->rss_params_inited |= QEDE_RSS_KEY_INITED; } diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index a8cbceeb301b..e32ae9038d9e 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -1163,8 +1163,8 @@ u32 efx_ethtool_get_rxfh_key_size(struct net_device *net_dev) return efx->type->rx_hash_key_size; } -int efx_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, - u8 *hfunc) +int efx_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = efx_netdev_priv(net_dev); int rc; @@ -1173,24 +1173,27 @@ int efx_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, if (rc) return rc; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (indir) - memcpy(indir, efx->rss_context.rx_indir_table, + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, efx->rss_context.rx_indir_table, sizeof(efx->rss_context.rx_indir_table)); - if (key) - memcpy(key, efx->rss_context.rx_hash_key, + if (rxfh->key) + memcpy(rxfh->key, efx->rss_context.rx_hash_key, efx->type->rx_hash_key_size); return 0; } -int efx_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir, - const u8 *key, const u8 hfunc) +int efx_ethtool_set_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct efx_nic *efx = efx_netdev_priv(net_dev); + u32 *indir = rxfh->indir; + u8 *key = rxfh->key; /* Hash function is Toeplitz, cannot be changed */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (!indir && !key) return 0; @@ -1203,8 +1206,9 @@ int efx_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir, return efx->type->rx_push_rss_config(efx, true, indir, key); } -int efx_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context) +int efx_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context) { struct efx_nic *efx = efx_netdev_priv(net_dev); struct efx_rss_context *ctx; @@ -1223,31 +1227,34 @@ int efx_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, if (rc) goto out_unlock; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (indir) - memcpy(indir, ctx->rx_indir_table, sizeof(ctx->rx_indir_table)); - if (key) - memcpy(key, ctx->rx_hash_key, efx->type->rx_hash_key_size); + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, ctx->rx_indir_table, + sizeof(ctx->rx_indir_table)); + if (rxfh->key) + memcpy(rxfh->key, ctx->rx_hash_key, + efx->type->rx_hash_key_size); out_unlock: mutex_unlock(&efx->rss_lock); return rc; } int efx_ethtool_set_rxfh_context(struct net_device *net_dev, - const u32 *indir, const u8 *key, - const u8 hfunc, u32 *rss_context, - bool delete) + struct ethtool_rxfh_param *rxfh, + u32 *rss_context, bool delete) { struct efx_nic *efx = efx_netdev_priv(net_dev); struct efx_rss_context *ctx; + u32 *indir = rxfh->indir; bool allocated = false; + u8 *key = rxfh->key; int rc; if (!efx->type->rx_push_rss_context_config) return -EOPNOTSUPP; /* Hash function is Toeplitz, cannot be changed */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h index 659491932101..384318ee1127 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.h +++ b/drivers/net/ethernet/sfc/ethtool_common.h @@ -44,16 +44,17 @@ int efx_ethtool_set_rxnfc(struct net_device *net_dev, struct ethtool_rxnfc *info); u32 efx_ethtool_get_rxfh_indir_size(struct net_device *net_dev); u32 efx_ethtool_get_rxfh_key_size(struct net_device *net_dev); -int efx_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, - u8 *hfunc); +int efx_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh); int efx_ethtool_set_rxfh(struct net_device *net_dev, - const u32 *indir, const u8 *key, const u8 hfunc); -int efx_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context); + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack); +int efx_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context); int efx_ethtool_set_rxfh_context(struct net_device *net_dev, - const u32 *indir, const u8 *key, - const u8 hfunc, u32 *rss_context, - bool delete); + struct ethtool_rxfh_param *rxfh, + u32 *rss_context, bool delete); int efx_ethtool_reset(struct net_device *net_dev, u32 *flags); int efx_ethtool_get_module_eeprom(struct net_device *net_dev, struct ethtool_eeprom *ee, diff --git a/drivers/net/ethernet/sfc/falcon/ethtool.c b/drivers/net/ethernet/sfc/falcon/ethtool.c index 3976a333f7e3..f4db683b80f7 100644 --- a/drivers/net/ethernet/sfc/falcon/ethtool.c +++ b/drivers/net/ethernet/sfc/falcon/ethtool.c @@ -1257,31 +1257,33 @@ static u32 ef4_ethtool_get_rxfh_indir_size(struct net_device *net_dev) 0 : ARRAY_SIZE(efx->rx_indir_table)); } -static int ef4_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, - u8 *hfunc) +static int ef4_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct ef4_nic *efx = netdev_priv(net_dev); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (indir) - memcpy(indir, efx->rx_indir_table, sizeof(efx->rx_indir_table)); + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, efx->rx_indir_table, + sizeof(efx->rx_indir_table)); return 0; } -static int ef4_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int ef4_ethtool_set_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct ef4_nic *efx = netdev_priv(net_dev); /* We do not allow change in unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!indir) + if (!rxfh->indir) return 0; - return efx->type->rx_push_rss_config(efx, true, indir); + return efx->type->rx_push_rss_config(efx, true, rxfh->indir); } static int ef4_ethtool_get_module_eeprom(struct net_device *net_dev, diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.c b/drivers/net/ethernet/sfc/siena/ethtool_common.c index f590e87e5a23..20b64e3521cb 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool_common.c +++ b/drivers/net/ethernet/sfc/siena/ethtool_common.c @@ -1164,8 +1164,8 @@ u32 efx_siena_ethtool_get_rxfh_key_size(struct net_device *net_dev) return efx->type->rx_hash_key_size; } -int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, - u8 *hfunc) +int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = netdev_priv(net_dev); int rc; @@ -1174,24 +1174,27 @@ int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, if (rc) return rc; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (indir) - memcpy(indir, efx->rss_context.rx_indir_table, + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, efx->rss_context.rx_indir_table, sizeof(efx->rss_context.rx_indir_table)); - if (key) - memcpy(key, efx->rss_context.rx_hash_key, + if (rxfh->key) + memcpy(rxfh->key, efx->rss_context.rx_hash_key, efx->type->rx_hash_key_size); return 0; } -int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir, - const u8 *key, const u8 hfunc) +int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); + u32 *indir = rxfh->indir; + u8 *key = rxfh->key; /* Hash function is Toeplitz, cannot be changed */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; if (!indir && !key) return 0; @@ -1204,8 +1207,9 @@ int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, const u32 *indir, return efx->type->rx_push_rss_config(efx, true, indir, key); } -int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context) +int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context) { struct efx_nic *efx = netdev_priv(net_dev); struct efx_rss_context *ctx; @@ -1224,31 +1228,34 @@ int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, if (rc) goto out_unlock; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (indir) - memcpy(indir, ctx->rx_indir_table, sizeof(ctx->rx_indir_table)); - if (key) - memcpy(key, ctx->rx_hash_key, efx->type->rx_hash_key_size); + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, ctx->rx_indir_table, + sizeof(ctx->rx_indir_table)); + if (rxfh->key) + memcpy(rxfh->key, ctx->rx_hash_key, + efx->type->rx_hash_key_size); out_unlock: mutex_unlock(&efx->rss_lock); return rc; } int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, - const u32 *indir, const u8 *key, - const u8 hfunc, u32 *rss_context, - bool delete) + struct ethtool_rxfh_param *rxfh, + u32 *rss_context, bool delete) { struct efx_nic *efx = netdev_priv(net_dev); struct efx_rss_context *ctx; + u32 *indir = rxfh->indir; bool allocated = false; + u8 *key = rxfh->key; int rc; if (!efx->type->rx_push_rss_context_config) return -EOPNOTSUPP; /* Hash function is Toeplitz, cannot be changed */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.h b/drivers/net/ethernet/sfc/siena/ethtool_common.h index 04b375dc6800..2f892f8eff70 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool_common.h +++ b/drivers/net/ethernet/sfc/siena/ethtool_common.h @@ -41,16 +41,17 @@ int efx_siena_ethtool_set_rxnfc(struct net_device *net_dev, struct ethtool_rxnfc *info); u32 efx_siena_ethtool_get_rxfh_indir_size(struct net_device *net_dev); u32 efx_siena_ethtool_get_rxfh_key_size(struct net_device *net_dev); -int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, u32 *indir, u8 *key, - u8 *hfunc); +int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh); int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, - const u32 *indir, const u8 *key, const u8 hfunc); -int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, u32 *indir, - u8 *key, u8 *hfunc, u32 rss_context); + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack); +int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + u32 rss_context); int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, - const u32 *indir, const u8 *key, - const u8 hfunc, u32 *rss_context, - bool delete); + struct ethtool_rxfh_param *rxfh, + u32 *rss_context, bool delete); int efx_siena_ethtool_reset(struct net_device *net_dev, u32 *flags); int efx_siena_ethtool_get_module_eeprom(struct net_device *net_dev, struct ethtool_eeprom *ee, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index b9b80e0fceeb..8105ce47c6ad 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -1087,41 +1087,42 @@ static u32 stmmac_get_rxfh_indir_size(struct net_device *dev) return ARRAY_SIZE(priv->rss.table); } -static int stmmac_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int stmmac_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct stmmac_priv *priv = netdev_priv(dev); int i; - if (indir) { + if (rxfh->indir) { for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) - indir[i] = priv->rss.table[i]; + rxfh->indir[i] = priv->rss.table[i]; } - if (key) - memcpy(key, priv->rss.key, sizeof(priv->rss.key)); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + if (rxfh->key) + memcpy(rxfh->key, priv->rss.key, sizeof(priv->rss.key)); + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int stmmac_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int stmmac_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct stmmac_priv *priv = netdev_priv(dev); int i; - if ((hfunc != ETH_RSS_HASH_NO_CHANGE) && (hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (indir) { + if (rxfh->indir) { for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) - priv->rss.table[i] = indir[i]; + priv->rss.table[i] = rxfh->indir[i]; } - if (key) - memcpy(priv->rss.key, key, sizeof(priv->rss.key)); + if (rxfh->key) + memcpy(priv->rss.key, rxfh->key, sizeof(priv->rss.key)); return stmmac_rss_configure(priv, priv->hw, &priv->rss, priv->plat->rx_queues_to_use); diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 1e2fa0ec2327..4406427d4617 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -1752,8 +1752,8 @@ static u32 netvsc_rss_indir_size(struct net_device *dev) return ndc->rx_table_sz; } -static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) +static int netvsc_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct net_device_context *ndc = netdev_priv(dev); struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); @@ -1763,47 +1763,49 @@ static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, if (!ndev) return -ENODEV; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ + rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ rndis_dev = ndev->extension; - if (indir) { + if (rxfh->indir) { for (i = 0; i < ndc->rx_table_sz; i++) - indir[i] = ndc->rx_table[i]; + rxfh->indir[i] = ndc->rx_table[i]; } - if (key) - memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN); + if (rxfh->key) + memcpy(rxfh->key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN); return 0; } -static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) +static int netvsc_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct net_device_context *ndc = netdev_priv(dev); struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev); struct rndis_device *rndis_dev; + u8 *key = rxfh->key; int i; if (!ndev) return -ENODEV; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; rndis_dev = ndev->extension; - if (indir) { + if (rxfh->indir) { for (i = 0; i < ndc->rx_table_sz; i++) - if (indir[i] >= ndev->num_chn) + if (rxfh->indir[i] >= ndev->num_chn) return -EINVAL; for (i = 0; i < ndc->rx_table_sz; i++) - ndc->rx_table[i] = indir[i]; + ndc->rx_table[i] = rxfh->indir[i]; } if (!key) { - if (!indir) + if (!rxfh->indir) return 0; key = rndis_dev->rss_key; diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 10614e9f7cad..25cf44ce95dd 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3731,39 +3731,42 @@ static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; } -static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc) +static int virtnet_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { struct virtnet_info *vi = netdev_priv(dev); int i; - if (indir) { + if (rxfh->indir) { for (i = 0; i < vi->rss_indir_table_size; ++i) - indir[i] = vi->ctrl->rss.indirection_table[i]; + rxfh->indir[i] = vi->ctrl->rss.indirection_table[i]; } - if (key) - memcpy(key, vi->ctrl->rss.key, vi->rss_key_size); + if (rxfh->key) + memcpy(rxfh->key, vi->ctrl->rss.key, vi->rss_key_size); - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; + rxfh->hfunc = ETH_RSS_HASH_TOP; return 0; } -static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc) +static int virtnet_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); int i; - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (indir) { + if (rxfh->indir) { for (i = 0; i < vi->rss_indir_table_size; ++i) - vi->ctrl->rss.indirection_table[i] = indir[i]; + vi->ctrl->rss.indirection_table[i] = rxfh->indir[i]; } - if (key) - memcpy(vi->ctrl->rss.key, key, vi->rss_key_size); + if (rxfh->key) + memcpy(vi->ctrl->rss.key, rxfh->key, vi->rss_key_size); virtnet_commit_rss_command(vi); diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c index 8f5f202cde39..7e8008d5378a 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethtool.c +++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c @@ -1136,27 +1136,26 @@ vmxnet3_get_rss_indir_size(struct net_device *netdev) } static int -vmxnet3_get_rss(struct net_device *netdev, u32 *p, u8 *key, u8 *hfunc) +vmxnet3_get_rss(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); struct UPT1_RSSConf *rssConf = adapter->rss_conf; unsigned int n = rssConf->indTableSize; - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - if (!p) + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (!rxfh->indir) return 0; if (n > UPT1_RSS_MAX_IND_TABLE_SIZE) return 0; while (n--) - p[n] = rssConf->indTable[n]; + rxfh->indir[n] = rssConf->indTable[n]; return 0; } static int -vmxnet3_set_rss(struct net_device *netdev, const u32 *p, const u8 *key, - const u8 hfunc) +vmxnet3_set_rss(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { unsigned int i; unsigned long flags; @@ -1164,13 +1163,14 @@ vmxnet3_set_rss(struct net_device *netdev, const u32 *p, const u8 *key, struct UPT1_RSSConf *rssConf = adapter->rss_conf; /* We do not allow change in unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + if (rxfh->key || + (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP)) return -EOPNOTSUPP; - if (!p) + if (!rxfh->indir) return 0; for (i = 0; i < rssConf->indTableSize; i++) - rssConf->indTable[i] = p[i]; + rssConf->indTable[i] = rxfh->indir[i]; spin_lock_irqsave(&adapter->cmd_lock, flags); VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 67b30940234b..3ab2b6a90419 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -596,6 +596,28 @@ struct ethtool_mm_stats { u64 MACMergeHoldCount; }; +/** + * struct ethtool_rxfh_param - RXFH (RSS) parameters + * @hfunc: Defines the current RSS hash function used by HW (or to be set to). + * Valid values are one of the %ETH_RSS_HASH_*. + * @indir_size: On SET, the array size of the user buffer for the + * indirection table, which may be zero, or + * %ETH_RXFH_INDIR_NO_CHANGE. On GET (read from the driver), + * the array size of the hardware indirection table. + * @indir: The indirection table of size @indir_size entries. + * @key_size: On SET, the array size of the user buffer for the hash key, + * which may be zero. On GET (read from the driver), the size of the + * hardware hash key. + * @key: The hash key of size @key_size bytes. + */ +struct ethtool_rxfh_param { + u8 hfunc; + u32 indir_size; + u32 *indir; + u32 key_size; + u8 *key; +}; + /** * struct ethtool_ops - optional netdev operations * @cap_link_lanes_supported: indicates if the driver supports lanes @@ -846,14 +868,14 @@ struct ethtool_ops { int (*reset)(struct net_device *, u32 *); u32 (*get_rxfh_key_size)(struct net_device *); u32 (*get_rxfh_indir_size)(struct net_device *); - int (*get_rxfh)(struct net_device *, u32 *indir, u8 *key, - u8 *hfunc); - int (*set_rxfh)(struct net_device *, const u32 *indir, - const u8 *key, const u8 hfunc); - int (*get_rxfh_context)(struct net_device *, u32 *indir, u8 *key, - u8 *hfunc, u32 rss_context); - int (*set_rxfh_context)(struct net_device *, const u32 *indir, - const u8 *key, const u8 hfunc, + int (*get_rxfh)(struct net_device *, struct ethtool_rxfh_param *); + int (*set_rxfh)(struct net_device *, struct ethtool_rxfh_param *, + struct netlink_ext_ack *extack); + int (*get_rxfh_context)(struct net_device *, + struct ethtool_rxfh_param *, + u32 rss_context); + int (*set_rxfh_context)(struct net_device *, + struct ethtool_rxfh_param *, u32 *rss_context, bool delete); void (*get_channels)(struct net_device *, struct ethtool_channels *); int (*set_channels)(struct net_device *, struct ethtool_channels *); diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 11d8797f63f6..6b2a360dcdf0 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -589,8 +589,8 @@ err_free_info: int ethtool_get_max_rxfh_channel(struct net_device *dev, u32 *max) { + struct ethtool_rxfh_param rxfh = {}; u32 dev_size, current_max = 0; - u32 *indir; int ret; if (!dev->ethtool_ops->get_rxfh_indir_size || @@ -600,21 +600,21 @@ int ethtool_get_max_rxfh_channel(struct net_device *dev, u32 *max) if (dev_size == 0) return -EOPNOTSUPP; - indir = kcalloc(dev_size, sizeof(indir[0]), GFP_USER); - if (!indir) + rxfh.indir = kcalloc(dev_size, sizeof(rxfh.indir[0]), GFP_USER); + if (!rxfh.indir) return -ENOMEM; - ret = dev->ethtool_ops->get_rxfh(dev, indir, NULL, NULL); + ret = dev->ethtool_ops->get_rxfh(dev, &rxfh); if (ret) goto out; while (dev_size--) - current_max = max(current_max, indir[dev_size]); + current_max = max(current_max, rxfh.indir[dev_size]); *max = current_max; out: - kfree(indir); + kfree(rxfh.indir); return ret; } diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 755a4be01bef..5f49a105fc73 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1061,15 +1061,15 @@ EXPORT_SYMBOL(netdev_rss_key_fill); static noinline_for_stack int ethtool_get_rxfh_indir(struct net_device *dev, void __user *useraddr) { - u32 user_size, dev_size; - u32 *indir; + struct ethtool_rxfh_param rxfh = {}; + u32 user_size; int ret; if (!dev->ethtool_ops->get_rxfh_indir_size || !dev->ethtool_ops->get_rxfh) return -EOPNOTSUPP; - dev_size = dev->ethtool_ops->get_rxfh_indir_size(dev); - if (dev_size == 0) + rxfh.indir_size = dev->ethtool_ops->get_rxfh_indir_size(dev); + if (rxfh.indir_size == 0) return -EOPNOTSUPP; if (copy_from_user(&user_size, @@ -1078,41 +1078,41 @@ static noinline_for_stack int ethtool_get_rxfh_indir(struct net_device *dev, return -EFAULT; if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh_indir, size), - &dev_size, sizeof(dev_size))) + &rxfh.indir_size, sizeof(rxfh.indir_size))) return -EFAULT; /* If the user buffer size is 0, this is just a query for the * device table size. Otherwise, if it's smaller than the * device table size it's an error. */ - if (user_size < dev_size) + if (user_size < rxfh.indir_size) return user_size == 0 ? 0 : -EINVAL; - indir = kcalloc(dev_size, sizeof(indir[0]), GFP_USER); - if (!indir) + rxfh.indir = kcalloc(rxfh.indir_size, sizeof(rxfh.indir[0]), GFP_USER); + if (!rxfh.indir) return -ENOMEM; - ret = dev->ethtool_ops->get_rxfh(dev, indir, NULL, NULL); + ret = dev->ethtool_ops->get_rxfh(dev, &rxfh); if (ret) goto out; - if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh_indir, ring_index[0]), - indir, dev_size * sizeof(indir[0]))) + rxfh.indir, rxfh.indir_size * sizeof(*rxfh.indir))) ret = -EFAULT; out: - kfree(indir); + kfree(rxfh.indir); return ret; } static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev, void __user *useraddr) { - struct ethtool_rxnfc rx_rings; - u32 user_size, dev_size, i; - u32 *indir; const struct ethtool_ops *ops = dev->ethtool_ops; + struct ethtool_rxfh_param rxfh_dev = {}; + struct netlink_ext_ack *extack = NULL; + struct ethtool_rxnfc rx_rings; + u32 user_size, i; int ret; u32 ringidx_offset = offsetof(struct ethtool_rxfh_indir, ring_index[0]); @@ -1120,8 +1120,8 @@ static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev, !ops->get_rxnfc) return -EOPNOTSUPP; - dev_size = ops->get_rxfh_indir_size(dev); - if (dev_size == 0) + rxfh_dev.indir_size = ops->get_rxfh_indir_size(dev); + if (rxfh_dev.indir_size == 0) return -EOPNOTSUPP; if (copy_from_user(&user_size, @@ -1129,11 +1129,12 @@ static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev, sizeof(user_size))) return -EFAULT; - if (user_size != 0 && user_size != dev_size) + if (user_size != 0 && user_size != rxfh_dev.indir_size) return -EINVAL; - indir = kcalloc(dev_size, sizeof(indir[0]), GFP_USER); - if (!indir) + rxfh_dev.indir = kcalloc(rxfh_dev.indir_size, + sizeof(rxfh_dev.indir[0]), GFP_USER); + if (!rxfh_dev.indir) return -ENOMEM; rx_rings.cmd = ETHTOOL_GRXRINGS; @@ -1142,18 +1143,21 @@ static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev, goto out; if (user_size == 0) { - for (i = 0; i < dev_size; i++) + u32 *indir = rxfh_dev.indir; + + for (i = 0; i < rxfh_dev.indir_size; i++) indir[i] = ethtool_rxfh_indir_default(i, rx_rings.data); } else { - ret = ethtool_copy_validate_indir(indir, + ret = ethtool_copy_validate_indir(rxfh_dev.indir, useraddr + ringidx_offset, &rx_rings, - dev_size); + rxfh_dev.indir_size); if (ret) goto out; } - ret = ops->set_rxfh(dev, indir, NULL, ETH_RSS_HASH_NO_CHANGE); + rxfh_dev.hfunc = ETH_RSS_HASH_NO_CHANGE; + ret = ops->set_rxfh(dev, &rxfh_dev, extack); if (ret) goto out; @@ -1164,32 +1168,29 @@ static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev, dev->priv_flags |= IFF_RXFH_CONFIGURED; out: - kfree(indir); + kfree(rxfh_dev.indir); return ret; } static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev, void __user *useraddr) { - int ret; const struct ethtool_ops *ops = dev->ethtool_ops; + struct ethtool_rxfh_param rxfh_dev = {}; u32 user_indir_size, user_key_size; - u32 dev_indir_size = 0, dev_key_size = 0; struct ethtool_rxfh rxfh; - u32 total_size; u32 indir_bytes; - u32 *indir = NULL; - u8 dev_hfunc = 0; - u8 *hkey = NULL; u8 *rss_config; + u32 total_size; + int ret; if (!ops->get_rxfh) return -EOPNOTSUPP; if (ops->get_rxfh_indir_size) - dev_indir_size = ops->get_rxfh_indir_size(dev); + rxfh_dev.indir_size = ops->get_rxfh_indir_size(dev); if (ops->get_rxfh_key_size) - dev_key_size = ops->get_rxfh_key_size(dev); + rxfh_dev.key_size = ops->get_rxfh_key_size(dev); if (copy_from_user(&rxfh, useraddr, sizeof(rxfh))) return -EFAULT; @@ -1203,38 +1204,37 @@ static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev, if (rxfh.rss_context && !ops->get_rxfh_context) return -EOPNOTSUPP; - rxfh.indir_size = dev_indir_size; - rxfh.key_size = dev_key_size; + rxfh.indir_size = rxfh_dev.indir_size; + rxfh.key_size = rxfh_dev.key_size; if (copy_to_user(useraddr, &rxfh, sizeof(rxfh))) return -EFAULT; - if ((user_indir_size && (user_indir_size != dev_indir_size)) || - (user_key_size && (user_key_size != dev_key_size))) + if ((user_indir_size && user_indir_size != rxfh_dev.indir_size) || + (user_key_size && user_key_size != rxfh_dev.key_size)) return -EINVAL; - indir_bytes = user_indir_size * sizeof(indir[0]); + indir_bytes = user_indir_size * sizeof(rxfh_dev.indir[0]); total_size = indir_bytes + user_key_size; rss_config = kzalloc(total_size, GFP_USER); if (!rss_config) return -ENOMEM; if (user_indir_size) - indir = (u32 *)rss_config; + rxfh_dev.indir = (u32 *)rss_config; if (user_key_size) - hkey = rss_config + indir_bytes; + rxfh_dev.key = rss_config + indir_bytes; if (rxfh.rss_context) - ret = dev->ethtool_ops->get_rxfh_context(dev, indir, hkey, - &dev_hfunc, + ret = dev->ethtool_ops->get_rxfh_context(dev, &rxfh_dev, rxfh.rss_context); else - ret = dev->ethtool_ops->get_rxfh(dev, indir, hkey, &dev_hfunc); + ret = dev->ethtool_ops->get_rxfh(dev, &rxfh_dev); if (ret) goto out; if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh, hfunc), - &dev_hfunc, sizeof(rxfh.hfunc))) { + &rxfh_dev.hfunc, sizeof(rxfh.hfunc))) { ret = -EFAULT; } else if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh, rss_config[0]), @@ -1250,16 +1250,17 @@ out: static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, void __user *useraddr) { - int ret; + u32 rss_cfg_offset = offsetof(struct ethtool_rxfh, rss_config[0]); const struct ethtool_ops *ops = dev->ethtool_ops; + u32 dev_indir_size = 0, dev_key_size = 0, i; + struct ethtool_rxfh_param rxfh_dev = {}; + struct netlink_ext_ack *extack = NULL; struct ethtool_rxnfc rx_rings; struct ethtool_rxfh rxfh; - u32 dev_indir_size = 0, dev_key_size = 0, i; - u32 *indir = NULL, indir_bytes = 0; - u8 *hkey = NULL; - u8 *rss_config; - u32 rss_cfg_offset = offsetof(struct ethtool_rxfh, rss_config[0]); bool delete = false; + u32 indir_bytes = 0; + u8 *rss_config; + int ret; if (!ops->get_rxnfc || !ops->set_rxfh) return -EOPNOTSUPP; @@ -1291,7 +1292,7 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, return -EINVAL; if (rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE) - indir_bytes = dev_indir_size * sizeof(indir[0]); + indir_bytes = dev_indir_size * sizeof(rxfh_dev.indir[0]); rss_config = kzalloc(indir_bytes + rxfh.key_size, GFP_USER); if (!rss_config) @@ -1308,8 +1309,9 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, */ if (rxfh.indir_size && rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE) { - indir = (u32 *)rss_config; - ret = ethtool_copy_validate_indir(indir, + rxfh_dev.indir = (u32 *)rss_config; + rxfh_dev.indir_size = dev_indir_size; + ret = ethtool_copy_validate_indir(rxfh_dev.indir, useraddr + rss_cfg_offset, &rx_rings, rxfh.indir_size); @@ -1317,7 +1319,11 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, goto out; } else if (rxfh.indir_size == 0) { if (rxfh.rss_context == 0) { - indir = (u32 *)rss_config; + u32 *indir; + + rxfh_dev.indir = (u32 *)rss_config; + rxfh_dev.indir_size = dev_indir_size; + indir = rxfh_dev.indir; for (i = 0; i < dev_indir_size; i++) indir[i] = ethtool_rxfh_indir_default(i, rx_rings.data); } else { @@ -1326,8 +1332,9 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, } if (rxfh.key_size) { - hkey = rss_config + indir_bytes; - if (copy_from_user(hkey, + rxfh_dev.key_size = dev_key_size; + rxfh_dev.key = rss_config + indir_bytes; + if (copy_from_user(rxfh_dev.key, useraddr + rss_cfg_offset + indir_bytes, rxfh.key_size)) { ret = -EFAULT; @@ -1335,11 +1342,14 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, } } + rxfh_dev.hfunc = rxfh.hfunc; + if (rxfh.rss_context) - ret = ops->set_rxfh_context(dev, indir, hkey, rxfh.hfunc, + ret = ops->set_rxfh_context(dev, &rxfh_dev, &rxfh.rss_context, delete); else - ret = ops->set_rxfh(dev, indir, hkey, rxfh.hfunc); + ret = ops->set_rxfh(dev, &rxfh_dev, extack); + if (ret) goto out; diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index 5764202e6cb6..56fae7d5c0f7 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -48,9 +48,9 @@ rss_prepare_data(const struct ethnl_req_info *req_base, struct rss_reply_data *data = RSS_REPDATA(reply_base); struct rss_req_info *request = RSS_REQINFO(req_base); struct net_device *dev = reply_base->dev; + struct ethtool_rxfh_param rxfh = {}; const struct ethtool_ops *ops; u32 total_size, indir_bytes; - u8 dev_hfunc = 0; u8 *rss_config; int ret; @@ -83,21 +83,23 @@ rss_prepare_data(const struct ethnl_req_info *req_base, if (data->indir_size) data->indir_table = (u32 *)rss_config; - if (data->hkey_size) data->hkey = rss_config + indir_bytes; + rxfh.indir_size = data->indir_size; + rxfh.indir = data->indir_table; + rxfh.key_size = data->hkey_size; + rxfh.key = data->hkey; + if (request->rss_context) - ret = ops->get_rxfh_context(dev, data->indir_table, data->hkey, - &dev_hfunc, request->rss_context); + ret = ops->get_rxfh_context(dev, &rxfh, request->rss_context); else - ret = ops->get_rxfh(dev, data->indir_table, data->hkey, - &dev_hfunc); + ret = ops->get_rxfh(dev, &rxfh); if (ret) goto out_ops; - data->hfunc = dev_hfunc; + data->hfunc = rxfh.hfunc; out_ops: ethnl_ops_complete(dev); return ret; -- cgit v1.2.3 From dcd8dbf9e734eb334113ea43186c1c26e9f497bb Mon Sep 17 00:00:00 2001 From: Ahmed Zaki Date: Tue, 12 Dec 2023 17:33:15 -0700 Subject: net: ethtool: get rid of get/set_rxfh_context functions Add the RSS context parameters to struct ethtool_rxfh_param and use the get/set_rxfh to handle the RSS contexts as well. This is part 2/2 of the fix suggested in [1]: - Add a rss_context member to the argument struct and a capability like cap_link_lanes_supported to indicate whether driver supports rss contexts, then you can remove *et_rxfh_context functions, and instead call *et_rxfh() with a non-zero rss_context. Link: https://lore.kernel.org/netdev/20231121152906.2dd5f487@kernel.org/ [1] CC: Jesse Brandeburg CC: Tony Nguyen CC: Marcin Wojtas CC: Russell King CC: Sunil Goutham CC: Geetha sowjanya CC: Subbaraya Sundeep CC: hariprasad CC: Saeed Mahameed CC: Leon Romanovsky CC: Edward Cree CC: Martin Habets Suggested-by: Jakub Kicinski Signed-off-by: Ahmed Zaki Link: https://lore.kernel.org/r/20231213003321.605376-3-ahmed.zaki@intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/ice/ice_ethtool.c | 29 +++--- drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c | 63 +++-------- .../ethernet/marvell/octeontx2/nic/otx2_ethtool.c | 63 +++++------ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 37 ++----- drivers/net/ethernet/sfc/ef100_ethtool.c | 3 +- drivers/net/ethernet/sfc/ethtool.c | 3 +- drivers/net/ethernet/sfc/ethtool_common.c | 115 +++++++++++---------- drivers/net/ethernet/sfc/ethtool_common.h | 6 -- drivers/net/ethernet/sfc/siena/ethtool.c | 3 +- drivers/net/ethernet/sfc/siena/ethtool_common.c | 115 +++++++++++---------- drivers/net/ethernet/sfc/siena/ethtool_common.h | 6 -- include/linux/ethtool.h | 26 ++--- net/ethtool/ioctl.c | 27 ++--- net/ethtool/rss.c | 9 +- 14 files changed, 205 insertions(+), 300 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 70cbd8092aea..c77605b42833 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3195,11 +3195,18 @@ static u32 ice_get_rxfh_indir_size(struct net_device *netdev) return np->vsi->rss_table_size; } +/** + * ice_get_rxfh - get the Rx flow hash indirection table + * @netdev: network interface device structure + * @rxfh: pointer to param struct (indir, key, hfunc) + * + * Reads the indirection table directly from the hardware. + */ static int -ice_get_rxfh_context(struct net_device *netdev, - struct ethtool_rxfh_param *rxfh, u32 rss_context) +ice_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { struct ice_netdev_priv *np = netdev_priv(netdev); + u32 rss_context = rxfh->rss_context; struct ice_vsi *vsi = np->vsi; struct ice_pf *pf = vsi->back; u16 qcount, offset; @@ -3261,19 +3268,6 @@ out: return err; } -/** - * ice_get_rxfh - get the Rx flow hash indirection table - * @netdev: network interface device structure - * @rxfh: pointer to param struct (indir, key, hfunc) - * - * Reads the indirection table directly from the hardware. - */ -static int -ice_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) -{ - return ice_get_rxfh_context(netdev, rxfh, 0); -} - /** * ice_set_rxfh - set the Rx flow hash indirection table * @netdev: network interface device structure @@ -3298,6 +3292,9 @@ ice_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; + if (rxfh->rss_context) + return -EOPNOTSUPP; + if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { /* RSS not supported return error here */ netdev_warn(netdev, "RSS is not configured on this VSI!\n"); @@ -4215,6 +4212,7 @@ ice_get_module_eeprom(struct net_device *netdev, } static const struct ethtool_ops ice_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE | ETHTOOL_COALESCE_RX_USECS_HIGH, @@ -4248,7 +4246,6 @@ static const struct ethtool_ops ice_ethtool_ops = { .set_pauseparam = ice_set_pauseparam, .get_rxfh_key_size = ice_get_rxfh_key_size, .get_rxfh_indir_size = ice_get_rxfh_indir_size, - .get_rxfh_context = ice_get_rxfh_context, .get_rxfh = ice_get_rxfh, .set_rxfh = ice_set_rxfh, .get_channels = ice_get_channels, diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index ceef48ddd26e..9193cf61de26 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5638,47 +5638,7 @@ static int mvpp2_ethtool_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh) { struct mvpp2_port *port = netdev_priv(dev); - int ret = 0; - - if (!mvpp22_rss_is_supported(port)) - return -EOPNOTSUPP; - - if (rxfh->indir) - ret = mvpp22_port_rss_ctx_indir_get(port, 0, rxfh->indir); - - rxfh->hfunc = ETH_RSS_HASH_CRC32; - - return ret; -} - -static int mvpp2_ethtool_set_rxfh(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - struct mvpp2_port *port = netdev_priv(dev); - int ret = 0; - - if (!mvpp22_rss_is_supported(port)) - return -EOPNOTSUPP; - - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && - rxfh->hfunc != ETH_RSS_HASH_CRC32) - return -EOPNOTSUPP; - - if (rxfh->key) - return -EOPNOTSUPP; - - if (rxfh->indir) - ret = mvpp22_port_rss_ctx_indir_set(port, 0, rxfh->indir); - - return ret; -} - -static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context) -{ - struct mvpp2_port *port = netdev_priv(dev); + u32 rss_context = rxfh->rss_context; int ret = 0; if (!mvpp22_rss_is_supported(port)) @@ -5695,12 +5655,13 @@ static int mvpp2_ethtool_get_rxfh_context(struct net_device *dev, return ret; } -static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete) +static int mvpp2_ethtool_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); - int ret; + u32 *rss_context = &rxfh->rss_context; + int ret = 0; if (!mvpp22_rss_is_supported(port)) return -EOPNOTSUPP; @@ -5712,7 +5673,7 @@ static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, if (rxfh->key) return -EOPNOTSUPP; - if (delete) + if (*rss_context && rxfh->rss_delete) return mvpp22_port_rss_ctx_delete(port, *rss_context); if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) { @@ -5721,8 +5682,13 @@ static int mvpp2_ethtool_set_rxfh_context(struct net_device *dev, return ret; } - return mvpp22_port_rss_ctx_indir_set(port, *rss_context, rxfh->indir); + if (rxfh->indir) + ret = mvpp22_port_rss_ctx_indir_set(port, *rss_context, + rxfh->indir); + + return ret; } + /* Device ops */ static const struct net_device_ops mvpp2_netdev_ops = { @@ -5742,6 +5708,7 @@ static const struct net_device_ops mvpp2_netdev_ops = { }; static const struct ethtool_ops mvpp2_eth_tool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES, .nway_reset = mvpp2_ethtool_nway_reset, @@ -5764,8 +5731,6 @@ static const struct ethtool_ops mvpp2_eth_tool_ops = { .get_rxfh_indir_size = mvpp2_ethtool_get_rxfh_indir_size, .get_rxfh = mvpp2_ethtool_get_rxfh, .set_rxfh = mvpp2_ethtool_set_rxfh, - .get_rxfh_context = mvpp2_ethtool_get_rxfh_context, - .set_rxfh_context = mvpp2_ethtool_set_rxfh_context, }; /* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index a2c182d7bfc4..2928898c7f8d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -835,11 +835,12 @@ static int otx2_rss_ctx_create(struct otx2_nic *pfvf, return 0; } -/* RSS context configuration */ -static int otx2_set_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete) +/* Configure RSS table and hash key */ +static int otx2_set_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { + u32 rss_context = DEFAULT_RSS_CONTEXT_GROUP; struct otx2_nic *pfvf = netdev_priv(dev); struct otx2_rss_ctx *rss_ctx; struct otx2_rss_info *rss; @@ -849,8 +850,11 @@ static int otx2_set_rxfh_context(struct net_device *dev, rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (*rss_context != ETH_RXFH_CONTEXT_ALLOC && - *rss_context >= MAX_RSS_GROUPS) + if (rxfh->rss_context) + rss_context = rxfh->rss_context; + + if (rss_context != ETH_RXFH_CONTEXT_ALLOC && + rss_context >= MAX_RSS_GROUPS) return -EINVAL; rss = &pfvf->hw.rss_info; @@ -864,28 +868,30 @@ static int otx2_set_rxfh_context(struct net_device *dev, memcpy(rss->key, rxfh->key, sizeof(rss->key)); otx2_set_rss_key(pfvf); } - if (delete) - return otx2_rss_ctx_delete(pfvf, *rss_context); + if (rxfh->rss_delete) + return otx2_rss_ctx_delete(pfvf, rss_context); - if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) { - ret = otx2_rss_ctx_create(pfvf, rss_context); + if (rss_context == ETH_RXFH_CONTEXT_ALLOC) { + ret = otx2_rss_ctx_create(pfvf, &rss_context); + rxfh->rss_context = rss_context; if (ret) return ret; } if (rxfh->indir) { - rss_ctx = rss->rss_ctx[*rss_context]; + rss_ctx = rss->rss_ctx[rss_context]; for (idx = 0; idx < rss->rss_size; idx++) rss_ctx->ind_tbl[idx] = rxfh->indir[idx]; } - otx2_set_rss_table(pfvf, *rss_context); + otx2_set_rss_table(pfvf, rss_context); return 0; } -static int otx2_get_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context) +/* Get RSS configuration */ +static int otx2_get_rxfh(struct net_device *dev, + struct ethtool_rxfh_param *rxfh) { + u32 rss_context = DEFAULT_RSS_CONTEXT_GROUP; struct otx2_nic *pfvf = netdev_priv(dev); struct otx2_rss_ctx *rss_ctx; struct otx2_rss_info *rss; @@ -895,6 +901,8 @@ static int otx2_get_rxfh_context(struct net_device *dev, rss = &pfvf->hw.rss_info; rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->rss_context) + rss_context = rxfh->rss_context; if (!indir) return 0; @@ -922,25 +930,6 @@ static int otx2_get_rxfh_context(struct net_device *dev, return 0; } -/* Get RSS configuration */ -static int otx2_get_rxfh(struct net_device *dev, - struct ethtool_rxfh_param *rxfh) -{ - return otx2_get_rxfh_context(dev, rxfh, - DEFAULT_RSS_CONTEXT_GROUP); -} - -/* Configure RSS table and hash key */ -static int otx2_set_rxfh(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - - u32 rss_context = DEFAULT_RSS_CONTEXT_GROUP; - - return otx2_set_rxfh_context(dev, rxfh, &rss_context, 0); -} - static u32 otx2_get_msglevel(struct net_device *netdev) { struct otx2_nic *pfvf = netdev_priv(netdev); @@ -1321,6 +1310,7 @@ static void otx2_get_fec_stats(struct net_device *netdev, } static const struct ethtool_ops otx2_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE, @@ -1343,8 +1333,6 @@ static const struct ethtool_ops otx2_ethtool_ops = { .get_rxfh_indir_size = otx2_get_rxfh_indir_size, .get_rxfh = otx2_get_rxfh, .set_rxfh = otx2_set_rxfh, - .get_rxfh_context = otx2_get_rxfh_context, - .set_rxfh_context = otx2_set_rxfh_context, .get_msglevel = otx2_get_msglevel, .set_msglevel = otx2_set_msglevel, .get_pauseparam = otx2_get_pauseparam, @@ -1444,6 +1432,7 @@ static int otx2vf_get_link_ksettings(struct net_device *netdev, } static const struct ethtool_ops otx2vf_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE, @@ -1462,8 +1451,6 @@ static const struct ethtool_ops otx2vf_ethtool_ops = { .get_rxfh_indir_size = otx2_get_rxfh_indir_size, .get_rxfh = otx2_get_rxfh, .set_rxfh = otx2_set_rxfh, - .get_rxfh_context = otx2_get_rxfh_context, - .set_rxfh_context = otx2_set_rxfh_context, .get_ringparam = otx2_get_ringparam, .set_ringparam = otx2_set_ringparam, .get_coalesce = otx2_get_coalesce, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 110cb7973eae..0fe7ea88d567 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1262,11 +1262,10 @@ static u32 mlx5e_get_rxfh_indir_size(struct net_device *netdev) return mlx5e_ethtool_get_rxfh_indir_size(priv); } -static int mlx5e_get_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context) +int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { - struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_priv *priv = netdev_priv(netdev); + u32 rss_context = rxfh->rss_context; int err; mutex_lock(&priv->state_lock); @@ -1276,16 +1275,16 @@ static int mlx5e_get_rxfh_context(struct net_device *dev, return err; } -static int mlx5e_set_rxfh_context(struct net_device *dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete) +int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); + u32 *rss_context = &rxfh->rss_context; u8 hfunc = rxfh->hfunc; int err; mutex_lock(&priv->state_lock); - if (delete) { + if (*rss_context && rxfh->rss_delete) { err = mlx5e_rx_res_rss_destroy(priv->rx_res, *rss_context); goto unlock; } @@ -1307,25 +1306,6 @@ unlock: return err; } -int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) -{ - return mlx5e_get_rxfh_context(netdev, rxfh, 0); -} - -int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - struct mlx5e_priv *priv = netdev_priv(dev); - u8 hfunc = rxfh->hfunc; - int err; - - mutex_lock(&priv->state_lock); - err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, 0, rxfh->indir, rxfh->key, - hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc); - mutex_unlock(&priv->state_lock); - return err; -} - #define MLX5E_PFC_PREVEN_AUTO_TOUT_MSEC 100 #define MLX5E_PFC_PREVEN_TOUT_MAX_MSEC 8000 #define MLX5E_PFC_PREVEN_MINOR_PRECENT 85 @@ -2402,6 +2382,7 @@ static void mlx5e_get_rmon_stats(struct net_device *netdev, } const struct ethtool_ops mlx5e_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE | @@ -2424,8 +2405,6 @@ const struct ethtool_ops mlx5e_ethtool_ops = { .get_rxfh_indir_size = mlx5e_get_rxfh_indir_size, .get_rxfh = mlx5e_get_rxfh, .set_rxfh = mlx5e_set_rxfh, - .get_rxfh_context = mlx5e_get_rxfh_context, - .set_rxfh_context = mlx5e_set_rxfh_context, .get_rxnfc = mlx5e_get_rxnfc, .set_rxnfc = mlx5e_set_rxnfc, .get_tunable = mlx5e_get_tunable, diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c index 702abbe59b76..cf55202b3a7b 100644 --- a/drivers/net/ethernet/sfc/ef100_ethtool.c +++ b/drivers/net/ethernet/sfc/ef100_ethtool.c @@ -37,6 +37,7 @@ ef100_ethtool_get_ringparam(struct net_device *net_dev, /* Ethtool options available */ const struct ethtool_ops ef100_ethtool_ops = { + .cap_rss_ctx_supported = true, .get_drvinfo = efx_ethtool_get_drvinfo, .get_msglevel = efx_ethtool_get_msglevel, .set_msglevel = efx_ethtool_set_msglevel, @@ -60,8 +61,6 @@ const struct ethtool_ops ef100_ethtool_ops = { .get_rxfh_key_size = efx_ethtool_get_rxfh_key_size, .get_rxfh = efx_ethtool_get_rxfh, .set_rxfh = efx_ethtool_set_rxfh, - .get_rxfh_context = efx_ethtool_get_rxfh_context, - .set_rxfh_context = efx_ethtool_set_rxfh_context, .get_module_info = efx_ethtool_get_module_info, .get_module_eeprom = efx_ethtool_get_module_eeprom, diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c index 364323599f7b..37c69c8d90b1 100644 --- a/drivers/net/ethernet/sfc/ethtool.c +++ b/drivers/net/ethernet/sfc/ethtool.c @@ -240,6 +240,7 @@ static int efx_ethtool_get_ts_info(struct net_device *net_dev, } const struct ethtool_ops efx_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USECS_IRQ | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, @@ -269,8 +270,6 @@ const struct ethtool_ops efx_ethtool_ops = { .get_rxfh_key_size = efx_ethtool_get_rxfh_key_size, .get_rxfh = efx_ethtool_get_rxfh, .set_rxfh = efx_ethtool_set_rxfh, - .get_rxfh_context = efx_ethtool_get_rxfh_context, - .set_rxfh_context = efx_ethtool_set_rxfh_context, .get_ts_info = efx_ethtool_get_ts_info, .get_module_info = efx_ethtool_get_module_info, .get_module_eeprom = efx_ethtool_get_module_eeprom, diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index e32ae9038d9e..7d5e5db4eac5 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -1163,52 +1163,8 @@ u32 efx_ethtool_get_rxfh_key_size(struct net_device *net_dev) return efx->type->rx_hash_key_size; } -int efx_ethtool_get_rxfh(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh) -{ - struct efx_nic *efx = efx_netdev_priv(net_dev); - int rc; - - rc = efx->type->rx_pull_rss_config(efx); - if (rc) - return rc; - - rxfh->hfunc = ETH_RSS_HASH_TOP; - if (rxfh->indir) - memcpy(rxfh->indir, efx->rss_context.rx_indir_table, - sizeof(efx->rss_context.rx_indir_table)); - if (rxfh->key) - memcpy(rxfh->key, efx->rss_context.rx_hash_key, - efx->type->rx_hash_key_size); - return 0; -} - -int efx_ethtool_set_rxfh(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - struct efx_nic *efx = efx_netdev_priv(net_dev); - u32 *indir = rxfh->indir; - u8 *key = rxfh->key; - - /* Hash function is Toeplitz, cannot be changed */ - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && - rxfh->hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; - if (!indir && !key) - return 0; - - if (!key) - key = efx->rss_context.rx_hash_key; - if (!indir) - indir = efx->rss_context.rx_indir_table; - - return efx->type->rx_push_rss_config(efx, true, indir, key); -} - -int efx_ethtool_get_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context) +static int efx_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = efx_netdev_priv(net_dev); struct efx_rss_context *ctx; @@ -1218,7 +1174,7 @@ int efx_ethtool_get_rxfh_context(struct net_device *net_dev, return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); - ctx = efx_find_rss_context_entry(efx, rss_context); + ctx = efx_find_rss_context_entry(efx, rxfh->rss_context); if (!ctx) { rc = -ENOENT; goto out_unlock; @@ -1239,11 +1195,35 @@ out_unlock: return rc; } -int efx_ethtool_set_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete) +int efx_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = efx_netdev_priv(net_dev); + int rc; + + if (rxfh->rss_context) + return efx_ethtool_get_rxfh_context(net_dev, rxfh); + + rc = efx->type->rx_pull_rss_config(efx); + if (rc) + return rc; + + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, efx->rss_context.rx_indir_table, + sizeof(efx->rss_context.rx_indir_table)); + if (rxfh->key) + memcpy(rxfh->key, efx->rss_context.rx_hash_key, + efx->type->rx_hash_key_size); + return 0; +} + +static int efx_ethtool_set_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + u32 *rss_context = &rxfh->rss_context; struct efx_rss_context *ctx; u32 *indir = rxfh->indir; bool allocated = false; @@ -1252,15 +1232,11 @@ int efx_ethtool_set_rxfh_context(struct net_device *net_dev, if (!efx->type->rx_push_rss_context_config) return -EOPNOTSUPP; - /* Hash function is Toeplitz, cannot be changed */ - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && - rxfh->hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) { - if (delete) { + if (rxfh->rss_delete) { /* alloc + delete == Nothing to do */ rc = -EINVAL; goto out_unlock; @@ -1283,7 +1259,7 @@ int efx_ethtool_set_rxfh_context(struct net_device *net_dev, } } - if (delete) { + if (rxfh->rss_delete) { /* delete this context */ rc = efx->type->rx_push_rss_context_config(efx, ctx, NULL, NULL); if (!rc) @@ -1306,6 +1282,33 @@ out_unlock: return rc; } +int efx_ethtool_set_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + u32 *indir = rxfh->indir; + u8 *key = rxfh->key; + + /* Hash function is Toeplitz, cannot be changed */ + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + if (rxfh->rss_context) + return efx_ethtool_set_rxfh_context(net_dev, rxfh, extack); + + if (!indir && !key) + return 0; + + if (!key) + key = efx->rss_context.rx_hash_key; + if (!indir) + indir = efx->rss_context.rx_indir_table; + + return efx->type->rx_push_rss_config(efx, true, indir, key); +} + int efx_ethtool_reset(struct net_device *net_dev, u32 *flags) { struct efx_nic *efx = efx_netdev_priv(net_dev); diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h index 384318ee1127..a680e5980213 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.h +++ b/drivers/net/ethernet/sfc/ethtool_common.h @@ -49,12 +49,6 @@ int efx_ethtool_get_rxfh(struct net_device *net_dev, int efx_ethtool_set_rxfh(struct net_device *net_dev, struct ethtool_rxfh_param *rxfh, struct netlink_ext_ack *extack); -int efx_ethtool_get_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context); -int efx_ethtool_set_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete); int efx_ethtool_reset(struct net_device *net_dev, u32 *flags); int efx_ethtool_get_module_eeprom(struct net_device *net_dev, struct ethtool_eeprom *ee, diff --git a/drivers/net/ethernet/sfc/siena/ethtool.c b/drivers/net/ethernet/sfc/siena/ethtool.c index e4ec589216c1..14dd3893bdef 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool.c +++ b/drivers/net/ethernet/sfc/siena/ethtool.c @@ -240,6 +240,7 @@ static int efx_ethtool_get_ts_info(struct net_device *net_dev, } const struct ethtool_ops efx_siena_ethtool_ops = { + .cap_rss_ctx_supported = true, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USECS_IRQ | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, @@ -269,8 +270,6 @@ const struct ethtool_ops efx_siena_ethtool_ops = { .get_rxfh_key_size = efx_siena_ethtool_get_rxfh_key_size, .get_rxfh = efx_siena_ethtool_get_rxfh, .set_rxfh = efx_siena_ethtool_set_rxfh, - .get_rxfh_context = efx_siena_ethtool_get_rxfh_context, - .set_rxfh_context = efx_siena_ethtool_set_rxfh_context, .get_ts_info = efx_ethtool_get_ts_info, .get_module_info = efx_siena_ethtool_get_module_info, .get_module_eeprom = efx_siena_ethtool_get_module_eeprom, diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.c b/drivers/net/ethernet/sfc/siena/ethtool_common.c index 20b64e3521cb..5f0a8127e967 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool_common.c +++ b/drivers/net/ethernet/sfc/siena/ethtool_common.c @@ -1164,52 +1164,8 @@ u32 efx_siena_ethtool_get_rxfh_key_size(struct net_device *net_dev) return efx->type->rx_hash_key_size; } -int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh) -{ - struct efx_nic *efx = netdev_priv(net_dev); - int rc; - - rc = efx->type->rx_pull_rss_config(efx); - if (rc) - return rc; - - rxfh->hfunc = ETH_RSS_HASH_TOP; - if (rxfh->indir) - memcpy(rxfh->indir, efx->rss_context.rx_indir_table, - sizeof(efx->rss_context.rx_indir_table)); - if (rxfh->key) - memcpy(rxfh->key, efx->rss_context.rx_hash_key, - efx->type->rx_hash_key_size); - return 0; -} - -int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - struct efx_nic *efx = netdev_priv(net_dev); - u32 *indir = rxfh->indir; - u8 *key = rxfh->key; - - /* Hash function is Toeplitz, cannot be changed */ - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && - rxfh->hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; - if (!indir && !key) - return 0; - - if (!key) - key = efx->rss_context.rx_hash_key; - if (!indir) - indir = efx->rss_context.rx_indir_table; - - return efx->type->rx_push_rss_config(efx, true, indir, key); -} - -int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context) +static int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = netdev_priv(net_dev); struct efx_rss_context *ctx; @@ -1219,7 +1175,7 @@ int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); - ctx = efx_siena_find_rss_context_entry(efx, rss_context); + ctx = efx_siena_find_rss_context_entry(efx, rxfh->rss_context); if (!ctx) { rc = -ENOENT; goto out_unlock; @@ -1240,11 +1196,35 @@ out_unlock: return rc; } -int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete) +int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh) { struct efx_nic *efx = netdev_priv(net_dev); + int rc; + + if (rxfh->rss_context) + return efx_siena_ethtool_get_rxfh_context(net_dev, rxfh); + + rc = efx->type->rx_pull_rss_config(efx); + if (rc) + return rc; + + rxfh->hfunc = ETH_RSS_HASH_TOP; + if (rxfh->indir) + memcpy(rxfh->indir, efx->rss_context.rx_indir_table, + sizeof(efx->rss_context.rx_indir_table)); + if (rxfh->key) + memcpy(rxfh->key, efx->rss_context.rx_hash_key, + efx->type->rx_hash_key_size); + return 0; +} + +static int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct efx_nic *efx = netdev_priv(net_dev); + u32 *rss_context = &rxfh->rss_context; struct efx_rss_context *ctx; u32 *indir = rxfh->indir; bool allocated = false; @@ -1253,15 +1233,11 @@ int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, if (!efx->type->rx_push_rss_context_config) return -EOPNOTSUPP; - /* Hash function is Toeplitz, cannot be changed */ - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && - rxfh->hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; mutex_lock(&efx->rss_lock); if (*rss_context == ETH_RXFH_CONTEXT_ALLOC) { - if (delete) { + if (rxfh->rss_delete) { /* alloc + delete == Nothing to do */ rc = -EINVAL; goto out_unlock; @@ -1284,7 +1260,7 @@ int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, } } - if (delete) { + if (rxfh->rss_delete) { /* delete this context */ rc = efx->type->rx_push_rss_context_config(efx, ctx, NULL, NULL); if (!rc) @@ -1307,6 +1283,33 @@ out_unlock: return rc; } +int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct efx_nic *efx = netdev_priv(net_dev); + u32 *indir = rxfh->indir; + u8 *key = rxfh->key; + + /* Hash function is Toeplitz, cannot be changed */ + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + if (rxfh->rss_context) + efx_siena_ethtool_set_rxfh_context(net_dev, rxfh, extack); + + if (!indir && !key) + return 0; + + if (!key) + key = efx->rss_context.rx_hash_key; + if (!indir) + indir = efx->rss_context.rx_indir_table; + + return efx->type->rx_push_rss_config(efx, true, indir, key); +} + int efx_siena_ethtool_reset(struct net_device *net_dev, u32 *flags) { struct efx_nic *efx = netdev_priv(net_dev); diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.h b/drivers/net/ethernet/sfc/siena/ethtool_common.h index 2f892f8eff70..d674bab0f65b 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool_common.h +++ b/drivers/net/ethernet/sfc/siena/ethtool_common.h @@ -46,12 +46,6 @@ int efx_siena_ethtool_get_rxfh(struct net_device *net_dev, int efx_siena_ethtool_set_rxfh(struct net_device *net_dev, struct ethtool_rxfh_param *rxfh, struct netlink_ext_ack *extack); -int efx_siena_ethtool_get_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 rss_context); -int efx_siena_ethtool_set_rxfh_context(struct net_device *net_dev, - struct ethtool_rxfh_param *rxfh, - u32 *rss_context, bool delete); int efx_siena_ethtool_reset(struct net_device *net_dev, u32 *flags); int efx_siena_ethtool_get_module_eeprom(struct net_device *net_dev, struct ethtool_eeprom *ee, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 3ab2b6a90419..66fe254c3e51 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -609,6 +609,12 @@ struct ethtool_mm_stats { * which may be zero. On GET (read from the driver), the size of the * hardware hash key. * @key: The hash key of size @key_size bytes. + * @rss_context: RSS context identifier. Context 0 is the default for normal + * traffic; other contexts can be referenced as the destination for RX flow + * classification rules. On SET, %ETH_RXFH_CONTEXT_ALLOC is used + * to allocate a new RSS context; on return this field will + * contain the ID of the newly allocated context. + * @rss_delete: Set to non-ZERO to remove the @rss_context context. */ struct ethtool_rxfh_param { u8 hfunc; @@ -616,12 +622,16 @@ struct ethtool_rxfh_param { u32 *indir; u32 key_size; u8 *key; + u32 rss_context; + u8 rss_delete; }; /** * struct ethtool_ops - optional netdev operations * @cap_link_lanes_supported: indicates if the driver supports lanes * parameter. + * @cap_rss_ctx_supported: indicates if the driver supports RSS + * contexts. * @supported_coalesce_params: supported types of interrupt coalescing. * @supported_ring_params: supported ring params. * @get_drvinfo: Report driver/device information. Modern drivers no @@ -718,15 +728,6 @@ struct ethtool_rxfh_param { * will remain unchanged. * Returns a negative error code or zero. An error code must be returned * if at least one unsupported change was requested. - * @get_rxfh_context: Get the contents of the RX flow hash indirection table, - * hash key, and/or hash function assiciated to the given rss context. - * Returns a negative error code or zero. - * @set_rxfh_context: Create, remove and configure RSS contexts. Allows setting - * the contents of the RX flow hash indirection table, hash key, and/or - * hash function associated to the given context. Arguments which are set - * to %NULL or zero will remain unchanged. - * Returns a negative error code or zero. An error code must be returned - * if at least one unsupported change was requested. * @get_channels: Get number of channels. * @set_channels: Set number of channels. Returns a negative error code or * zero. @@ -809,6 +810,7 @@ struct ethtool_rxfh_param { */ struct ethtool_ops { u32 cap_link_lanes_supported:1; + u32 cap_rss_ctx_supported:1; u32 supported_coalesce_params; u32 supported_ring_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); @@ -871,12 +873,6 @@ struct ethtool_ops { int (*get_rxfh)(struct net_device *, struct ethtool_rxfh_param *); int (*set_rxfh)(struct net_device *, struct ethtool_rxfh_param *, struct netlink_ext_ack *extack); - int (*get_rxfh_context)(struct net_device *, - struct ethtool_rxfh_param *, - u32 rss_context); - int (*set_rxfh_context)(struct net_device *, - struct ethtool_rxfh_param *, - u32 *rss_context, bool delete); void (*get_channels)(struct net_device *, struct ethtool_channels *); int (*set_channels)(struct net_device *, struct ethtool_channels *); int (*get_dump_flag)(struct net_device *, struct ethtool_dump *); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 5f49a105fc73..86e5fc64b711 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -1201,7 +1201,7 @@ static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev, if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd8[2] || rxfh.rsvd32) return -EINVAL; /* Most drivers don't handle rss_context, check it's 0 as well */ - if (rxfh.rss_context && !ops->get_rxfh_context) + if (rxfh.rss_context && !ops->cap_rss_ctx_supported) return -EOPNOTSUPP; rxfh.indir_size = rxfh_dev.indir_size; @@ -1225,11 +1225,9 @@ static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev, if (user_key_size) rxfh_dev.key = rss_config + indir_bytes; - if (rxfh.rss_context) - ret = dev->ethtool_ops->get_rxfh_context(dev, &rxfh_dev, - rxfh.rss_context); - else - ret = dev->ethtool_ops->get_rxfh(dev, &rxfh_dev); + rxfh_dev.rss_context = rxfh.rss_context; + + ret = dev->ethtool_ops->get_rxfh(dev, &rxfh_dev); if (ret) goto out; @@ -1257,7 +1255,6 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, struct netlink_ext_ack *extack = NULL; struct ethtool_rxnfc rx_rings; struct ethtool_rxfh rxfh; - bool delete = false; u32 indir_bytes = 0; u8 *rss_config; int ret; @@ -1277,7 +1274,7 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd8[2] || rxfh.rsvd32) return -EINVAL; /* Most drivers don't handle rss_context, check it's 0 as well */ - if (rxfh.rss_context && !ops->set_rxfh_context) + if (rxfh.rss_context && !ops->cap_rss_ctx_supported) return -EOPNOTSUPP; /* If either indir, hash key or function is valid, proceed further. @@ -1327,7 +1324,7 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, for (i = 0; i < dev_indir_size; i++) indir[i] = ethtool_rxfh_indir_default(i, rx_rings.data); } else { - delete = true; + rxfh_dev.rss_delete = true; } } @@ -1343,21 +1340,17 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, } rxfh_dev.hfunc = rxfh.hfunc; + rxfh_dev.rss_context = rxfh.rss_context; - if (rxfh.rss_context) - ret = ops->set_rxfh_context(dev, &rxfh_dev, - &rxfh.rss_context, delete); - else - ret = ops->set_rxfh(dev, &rxfh_dev, extack); - + ret = ops->set_rxfh(dev, &rxfh_dev, extack); if (ret) goto out; if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh, rss_context), - &rxfh.rss_context, sizeof(rxfh.rss_context))) + &rxfh_dev.rss_context, sizeof(rxfh_dev.rss_context))) ret = -EFAULT; - if (!rxfh.rss_context) { + if (!rxfh_dev.rss_context) { /* indicate whether rxfh was set to default */ if (rxfh.indir_size == 0) dev->priv_flags &= ~IFF_RXFH_CONFIGURED; diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index 56fae7d5c0f7..efc9f4409e40 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -59,7 +59,7 @@ rss_prepare_data(const struct ethnl_req_info *req_base, return -EOPNOTSUPP; /* Some drivers don't handle rss_context */ - if (request->rss_context && !ops->get_rxfh_context) + if (request->rss_context && !ops->cap_rss_ctx_supported) return -EOPNOTSUPP; ret = ethnl_ops_begin(dev); @@ -90,12 +90,9 @@ rss_prepare_data(const struct ethnl_req_info *req_base, rxfh.indir = data->indir_table; rxfh.key_size = data->hkey_size; rxfh.key = data->hkey; + rxfh.rss_context = request->rss_context; - if (request->rss_context) - ret = ops->get_rxfh_context(dev, &rxfh, request->rss_context); - else - ret = ops->get_rxfh(dev, &rxfh); - + ret = ops->get_rxfh(dev, &rxfh); if (ret) goto out_ops; -- cgit v1.2.3 From 13e59344fb9d3c9d3acd138ae320b5b67b658694 Mon Sep 17 00:00:00 2001 From: Ahmed Zaki Date: Tue, 12 Dec 2023 17:33:16 -0700 Subject: net: ethtool: add support for symmetric-xor RSS hash Symmetric RSS hash functions are beneficial in applications that monitor both Tx and Rx packets of the same flow (IDS, software firewalls, ..etc). Getting all traffic of the same flow on the same RX queue results in higher CPU cache efficiency. A NIC that supports "symmetric-xor" can achieve this RSS hash symmetry by XORing the source and destination fields and pass the values to the RSS hash algorithm. The user may request RSS hash symmetry for a specific algorithm, via: # ethtool -X eth0 hfunc symmetric-xor or turn symmetry off (asymmetric) by: # ethtool -X eth0 hfunc The specific fields for each flow type should then be specified as usual via: # ethtool -N|-U eth0 rx-flow-hash s|d|f|n Reviewed-by: Wojciech Drewek Signed-off-by: Ahmed Zaki Link: https://lore.kernel.org/r/20231213003321.605376-4-ahmed.zaki@intel.com Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/ethtool.yaml | 4 ++++ Documentation/networking/ethtool-netlink.rst | 6 +++++- Documentation/networking/scaling.rst | 15 ++++++++++++++ include/linux/ethtool.h | 6 ++++++ include/uapi/linux/ethtool.h | 13 +++++++++++- include/uapi/linux/ethtool_netlink.h | 1 + net/ethtool/ioctl.c | 30 ++++++++++++++++++++++++---- net/ethtool/rss.c | 5 +++++ 8 files changed, 74 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 5c7a65b009b4..197208f419dc 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -908,6 +908,9 @@ attribute-sets: - name: hkey type: binary + - + name: input_xfrm + type: u32 - name: plca attributes: @@ -1598,6 +1601,7 @@ operations: - hfunc - indir - hkey + - input_xfrm dump: *rss-get-op - name: plca-get-cfg diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 6a49624a9cbf..d583d9abf2f8 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -1774,12 +1774,16 @@ Kernel response contents: ``ETHTOOL_A_RSS_HFUNC`` u32 RSS hash func ``ETHTOOL_A_RSS_INDIR`` binary Indir table bytes ``ETHTOOL_A_RSS_HKEY`` binary Hash key bytes + ``ETHTOOL_A_RSS_INPUT_XFRM`` u32 RSS input data transformation ===================================== ====== ========================== ETHTOOL_A_RSS_HFUNC attribute is bitmap indicating the hash function being used. Current supported options are toeplitz, xor or crc32. -ETHTOOL_A_RSS_INDIR attribute returns RSS indrection table where each byte +ETHTOOL_A_RSS_INDIR attribute returns RSS indirection table where each byte indicates queue number. +ETHTOOL_A_RSS_INPUT_XFRM attribute is a bitmap indicating the type of +transformation applied to the input protocol fields before given to the RSS +hfunc. Current supported option is symmetric-xor. PLCA_GET_CFG ============ diff --git a/Documentation/networking/scaling.rst b/Documentation/networking/scaling.rst index 03ae19a689fc..4eb50bcb9d42 100644 --- a/Documentation/networking/scaling.rst +++ b/Documentation/networking/scaling.rst @@ -44,6 +44,21 @@ by masking out the low order seven bits of the computed hash for the packet (usually a Toeplitz hash), taking this number as a key into the indirection table and reading the corresponding value. +Some NICs support symmetric RSS hashing where, if the IP (source address, +destination address) and TCP/UDP (source port, destination port) tuples +are swapped, the computed hash is the same. This is beneficial in some +applications that monitor TCP/IP flows (IDS, firewalls, ...etc) and need +both directions of the flow to land on the same Rx queue (and CPU). The +"Symmetric-XOR" is a type of RSS algorithms that achieves this hash +symmetry by XORing the input source and destination fields of the IP +and/or L4 protocols. This, however, results in reduced input entropy and +could potentially be exploited. Specifically, the algorithm XORs the input +as follows:: + + # (SRC_IP ^ DST_IP, SRC_IP ^ DST_IP, SRC_PORT ^ DST_PORT, SRC_PORT ^ DST_PORT) + +The result is then fed to the underlying RSS algorithm. + Some advanced NICs allow steering packets to queues based on programmable filters. For example, webserver bound TCP port 80 packets can be directed to their own receive queue. Such “n-tuple” filters can diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 66fe254c3e51..cfcd952a1d4f 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -615,6 +615,8 @@ struct ethtool_mm_stats { * to allocate a new RSS context; on return this field will * contain the ID of the newly allocated context. * @rss_delete: Set to non-ZERO to remove the @rss_context context. + * @input_xfrm: Defines how the input data is transformed. Valid values are one + * of %RXH_XFRM_*. */ struct ethtool_rxfh_param { u8 hfunc; @@ -624,6 +626,7 @@ struct ethtool_rxfh_param { u8 *key; u32 rss_context; u8 rss_delete; + u8 input_xfrm; }; /** @@ -632,6 +635,8 @@ struct ethtool_rxfh_param { * parameter. * @cap_rss_ctx_supported: indicates if the driver supports RSS * contexts. + * @cap_rss_sym_xor_supported: indicates if the driver supports symmetric-xor + * RSS. * @supported_coalesce_params: supported types of interrupt coalescing. * @supported_ring_params: supported ring params. * @get_drvinfo: Report driver/device information. Modern drivers no @@ -811,6 +816,7 @@ struct ethtool_rxfh_param { struct ethtool_ops { u32 cap_link_lanes_supported:1; u32 cap_rss_ctx_supported:1; + u32 cap_rss_sym_xor_supported:1; u32 supported_coalesce_params; u32 supported_ring_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index f7fba0dc87e5..0787d561ace0 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -1266,6 +1266,8 @@ struct ethtool_rxfh_indir { * hardware hash key. * @hfunc: Defines the current RSS hash function used by HW (or to be set to). * Valid values are one of the %ETH_RSS_HASH_*. + * @input_xfrm: Defines how the input data is transformed. Valid values are one + * of %RXH_XFRM_*. * @rsvd8: Reserved for future use; see the note on reserved space. * @rsvd32: Reserved for future use; see the note on reserved space. * @rss_config: RX ring/queue index for each hash value i.e., indirection table @@ -1285,7 +1287,8 @@ struct ethtool_rxfh { __u32 indir_size; __u32 key_size; __u8 hfunc; - __u8 rsvd8[3]; + __u8 input_xfrm; + __u8 rsvd8[2]; __u32 rsvd32; __u32 rss_config[]; }; @@ -1992,6 +1995,14 @@ static inline int ethtool_validate_duplex(__u8 duplex) #define WOL_MODE_COUNT 8 +/* RSS hash function data + * XOR the corresponding source and destination fields of each specified + * protocol. Both copies of the XOR'ed fields are fed into the RSS and RXHASH + * calculation. Note that this XORing reduces the input set entropy and could + * be exploited to reduce the RSS queue spread. + */ +#define RXH_XFRM_SYM_XOR (1 << 0) + /* L2-L4 network traffic flow types */ #define TCP_V4_FLOW 0x01 /* hash or spec (tcp_ip4_spec) */ #define UDP_V4_FLOW 0x02 /* hash or spec (udp_ip4_spec) */ diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index 73e2c10dc2cc..3f89074aa06c 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -908,6 +908,7 @@ enum { ETHTOOL_A_RSS_HFUNC, /* u32 */ ETHTOOL_A_RSS_INDIR, /* binary */ ETHTOOL_A_RSS_HKEY, /* binary */ + ETHTOOL_A_RSS_INPUT_XFRM, /* u32 */ __ETHTOOL_A_RSS_CNT, ETHTOOL_A_RSS_MAX = (__ETHTOOL_A_RSS_CNT - 1), diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 86e5fc64b711..86d47425038b 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -972,18 +972,35 @@ static int ethtool_rxnfc_copy_to_user(void __user *useraddr, static noinline_for_stack int ethtool_set_rxnfc(struct net_device *dev, u32 cmd, void __user *useraddr) { + const struct ethtool_ops *ops = dev->ethtool_ops; + struct ethtool_rxfh_param rxfh = {}; struct ethtool_rxnfc info; size_t info_size = sizeof(info); int rc; - if (!dev->ethtool_ops->set_rxnfc) + if (!ops->set_rxnfc || !ops->get_rxfh) return -EOPNOTSUPP; rc = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr); if (rc) return rc; - rc = dev->ethtool_ops->set_rxnfc(dev, &info); + rc = ops->get_rxfh(dev, &rxfh); + if (rc) + return rc; + + /* Sanity check: if symmetric-xor is set, then: + * 1 - no other fields besides IP src/dst and/or L4 src/dst + * 2 - If src is set, dst must also be set + */ + if ((rxfh.input_xfrm & RXH_XFRM_SYM_XOR) && + ((info.data & ~(RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3)) || + (!!(info.data & RXH_IP_SRC) ^ !!(info.data & RXH_IP_DST)) || + (!!(info.data & RXH_L4_B_0_1) ^ !!(info.data & RXH_L4_B_2_3)))) + return -EINVAL; + + rc = ops->set_rxnfc(dev, &info); if (rc) return rc; @@ -1198,7 +1215,7 @@ static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev, user_key_size = rxfh.key_size; /* Check that reserved fields are 0 for now */ - if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd8[2] || rxfh.rsvd32) + if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd32) return -EINVAL; /* Most drivers don't handle rss_context, check it's 0 as well */ if (rxfh.rss_context && !ops->cap_rss_ctx_supported) @@ -1271,11 +1288,15 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, return -EFAULT; /* Check that reserved fields are 0 for now */ - if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd8[2] || rxfh.rsvd32) + if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd32) return -EINVAL; /* Most drivers don't handle rss_context, check it's 0 as well */ if (rxfh.rss_context && !ops->cap_rss_ctx_supported) return -EOPNOTSUPP; + /* Check input data transformation capabilities */ + if ((rxfh.input_xfrm & RXH_XFRM_SYM_XOR) && + !ops->cap_rss_sym_xor_supported) + return -EOPNOTSUPP; /* If either indir, hash key or function is valid, proceed further. * Must request at least one change: indir size, hash key or function. @@ -1341,6 +1362,7 @@ static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev, rxfh_dev.hfunc = rxfh.hfunc; rxfh_dev.rss_context = rxfh.rss_context; + rxfh_dev.input_xfrm = rxfh.input_xfrm; ret = ops->set_rxfh(dev, &rxfh_dev, extack); if (ret) diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index efc9f4409e40..71679137eff2 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -13,6 +13,7 @@ struct rss_reply_data { u32 indir_size; u32 hkey_size; u32 hfunc; + u32 input_xfrm; u32 *indir_table; u8 *hkey; }; @@ -97,6 +98,7 @@ rss_prepare_data(const struct ethnl_req_info *req_base, goto out_ops; data->hfunc = rxfh.hfunc; + data->input_xfrm = rxfh.input_xfrm; out_ops: ethnl_ops_complete(dev); return ret; @@ -110,6 +112,7 @@ rss_reply_size(const struct ethnl_req_info *req_base, int len; len = nla_total_size(sizeof(u32)) + /* _RSS_HFUNC */ + nla_total_size(sizeof(u32)) + /* _RSS_INPUT_XFRM */ nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */ nla_total_size(data->hkey_size); /* _RSS_HKEY */ @@ -124,6 +127,8 @@ rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, if ((data->hfunc && nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc)) || + (data->input_xfrm && + nla_put_u32(skb, ETHTOOL_A_RSS_INPUT_XFRM, data->input_xfrm)) || (data->indir_size && nla_put(skb, ETHTOOL_A_RSS_INDIR, sizeof(u32) * data->indir_size, data->indir_table)) || -- cgit v1.2.3 From dc6e44c9d6d68e8aa5de78d15f43f93145719b72 Mon Sep 17 00:00:00 2001 From: Qi Zhang Date: Tue, 12 Dec 2023 17:33:18 -0700 Subject: ice: refactor RSS configuration Refactor the driver to use a communication data structure for RSS config. To do so we introduce the new ice_rss_hash_cfg struct, and then pass it as an argument to several functions. Also introduce enum ice_rss_cfg_hdr_type to specify a more granular and flexible RSS configuration: ICE_RSS_OUTER_HEADERS - take outer layer as RSS input set ICE_RSS_INNER_HEADERS - take inner layer as RSS input set ICE_RSS_INNER_HEADERS_W_OUTER_IPV4 - take inner layer as RSS input set for packet with outer IPV4 ICE_RSS_INNER_HEADERS_W_OUTER_IPV6 - take inner layer as RSS input set for packet with outer IPV6 ICE_RSS_ANY_HEADERS - try with outer first then inner (same as the behaviour without this change) Finally, move the virtchnl_rss_algorithm enum to be with the other RSS related structures in the virtchnl.h file. There should be no functional change due to this patch. Reviewed-by: Wojciech Drewek Signed-off-by: Qi Zhang Co-developed-by: Jesse Brandeburg Signed-off-by: Jesse Brandeburg Co-developed-by: Ahmed Zaki Signed-off-by: Ahmed Zaki Link: https://lore.kernel.org/r/20231213003321.605376-6-ahmed.zaki@intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/ice/ice_ethtool.c | 6 +- drivers/net/ethernet/intel/ice/ice_flow.c | 232 ++++++++++++++++---------- drivers/net/ethernet/intel/ice/ice_flow.h | 33 +++- drivers/net/ethernet/intel/ice/ice_lib.c | 100 +++++------ drivers/net/ethernet/intel/ice/ice_virtchnl.c | 36 ++-- include/linux/avf/virtchnl.h | 16 +- 6 files changed, 246 insertions(+), 177 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index c77605b42833..fcd8678bea66 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -2591,6 +2591,7 @@ static int ice_set_rss_hash_opt(struct ice_vsi *vsi, struct ethtool_rxnfc *nfc) { struct ice_pf *pf = vsi->back; + struct ice_rss_hash_cfg cfg; struct device *dev; u64 hashed_flds; int status; @@ -2617,7 +2618,10 @@ ice_set_rss_hash_opt(struct ice_vsi *vsi, struct ethtool_rxnfc *nfc) return -EINVAL; } - status = ice_add_rss_cfg(&pf->hw, vsi->idx, hashed_flds, hdrs); + cfg.hash_flds = hashed_flds; + cfg.addl_hdrs = hdrs; + cfg.hdr_type = ICE_RSS_ANY_HEADERS; + status = ice_add_rss_cfg(&pf->hw, vsi->idx, &cfg); if (status) { dev_dbg(dev, "ice_add_rss_cfg failed, vsi num = %d, error = %d\n", vsi->vsi_num, status); diff --git a/drivers/net/ethernet/intel/ice/ice_flow.c b/drivers/net/ethernet/intel/ice/ice_flow.c index fb8b925aaf8b..c34be72f99b3 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.c +++ b/drivers/net/ethernet/intel/ice/ice_flow.c @@ -1855,37 +1855,49 @@ int ice_flow_rem_vsi_prof(struct ice_hw *hw, u16 vsi_handle, u64 prof_id) /** * ice_flow_set_rss_seg_info - setup packet segments for RSS * @segs: pointer to the flow field segment(s) - * @hash_fields: fields to be hashed on for the segment(s) - * @flow_hdr: protocol header fields within a packet segment + * @seg_cnt: segment count + * @cfg: configure parameters * * Helper function to extract fields from hash bitmap and use flow * header value to set flow field segment for further use in flow * profile entry or removal. */ static int -ice_flow_set_rss_seg_info(struct ice_flow_seg_info *segs, u64 hash_fields, - u32 flow_hdr) +ice_flow_set_rss_seg_info(struct ice_flow_seg_info *segs, u8 seg_cnt, + const struct ice_rss_hash_cfg *cfg) { + struct ice_flow_seg_info *seg; u64 val; - u8 i; + u16 i; + + /* set inner most segment */ + seg = &segs[seg_cnt - 1]; - for_each_set_bit(i, (unsigned long *)&hash_fields, - ICE_FLOW_FIELD_IDX_MAX) - ice_flow_set_fld(segs, (enum ice_flow_field)i, + for_each_set_bit(i, (const unsigned long *)&cfg->hash_flds, + (u16)ICE_FLOW_FIELD_IDX_MAX) + ice_flow_set_fld(seg, (enum ice_flow_field)i, ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false); - ICE_FLOW_SET_HDRS(segs, flow_hdr); + ICE_FLOW_SET_HDRS(seg, cfg->addl_hdrs); - if (segs->hdrs & ~ICE_FLOW_RSS_SEG_HDR_VAL_MASKS & + /* set outer most header */ + if (cfg->hdr_type == ICE_RSS_INNER_HEADERS_W_OUTER_IPV4) + segs[ICE_RSS_OUTER_HEADERS].hdrs |= ICE_FLOW_SEG_HDR_IPV4 | + ICE_FLOW_SEG_HDR_IPV_OTHER; + else if (cfg->hdr_type == ICE_RSS_INNER_HEADERS_W_OUTER_IPV6) + segs[ICE_RSS_OUTER_HEADERS].hdrs |= ICE_FLOW_SEG_HDR_IPV6 | + ICE_FLOW_SEG_HDR_IPV_OTHER; + + if (seg->hdrs & ~ICE_FLOW_RSS_SEG_HDR_VAL_MASKS & ~ICE_FLOW_RSS_HDRS_INNER_MASK & ~ICE_FLOW_SEG_HDR_IPV_OTHER) return -EINVAL; - val = (u64)(segs->hdrs & ICE_FLOW_RSS_SEG_HDR_L3_MASKS); + val = (u64)(seg->hdrs & ICE_FLOW_RSS_SEG_HDR_L3_MASKS); if (val && !is_power_of_2(val)) return -EIO; - val = (u64)(segs->hdrs & ICE_FLOW_RSS_SEG_HDR_L4_MASKS); + val = (u64)(seg->hdrs & ICE_FLOW_RSS_SEG_HDR_L4_MASKS); if (val && !is_power_of_2(val)) return -EIO; @@ -1955,6 +1967,39 @@ int ice_rem_vsi_rss_cfg(struct ice_hw *hw, u16 vsi_handle) return status; } +/** + * ice_get_rss_hdr_type - get a RSS profile's header type + * @prof: RSS flow profile + */ +static enum ice_rss_cfg_hdr_type +ice_get_rss_hdr_type(struct ice_flow_prof *prof) +{ + if (prof->segs_cnt == ICE_FLOW_SEG_SINGLE) { + return ICE_RSS_OUTER_HEADERS; + } else if (prof->segs_cnt == ICE_FLOW_SEG_MAX) { + const struct ice_flow_seg_info *s; + + s = &prof->segs[ICE_RSS_OUTER_HEADERS]; + if (s->hdrs == ICE_FLOW_SEG_HDR_NONE) + return ICE_RSS_INNER_HEADERS; + if (s->hdrs & ICE_FLOW_SEG_HDR_IPV4) + return ICE_RSS_INNER_HEADERS_W_OUTER_IPV4; + if (s->hdrs & ICE_FLOW_SEG_HDR_IPV6) + return ICE_RSS_INNER_HEADERS_W_OUTER_IPV6; + } + + return ICE_RSS_ANY_HEADERS; +} + +static bool +ice_rss_match_prof(struct ice_rss_cfg *r, struct ice_flow_prof *prof, + enum ice_rss_cfg_hdr_type hdr_type) +{ + return (r->hash.hdr_type == hdr_type && + r->hash.hash_flds == prof->segs[prof->segs_cnt - 1].match && + r->hash.addl_hdrs == prof->segs[prof->segs_cnt - 1].hdrs); +} + /** * ice_rem_rss_list - remove RSS configuration from list * @hw: pointer to the hardware structure @@ -1966,15 +2011,16 @@ int ice_rem_vsi_rss_cfg(struct ice_hw *hw, u16 vsi_handle) static void ice_rem_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) { + enum ice_rss_cfg_hdr_type hdr_type; struct ice_rss_cfg *r, *tmp; /* Search for RSS hash fields associated to the VSI that match the * hash configurations associated to the flow profile. If found * remove from the RSS entry list of the VSI context and delete entry. */ + hdr_type = ice_get_rss_hdr_type(prof); list_for_each_entry_safe(r, tmp, &hw->rss_list_head, l_entry) - if (r->hashed_flds == prof->segs[prof->segs_cnt - 1].match && - r->packet_hdr == prof->segs[prof->segs_cnt - 1].hdrs) { + if (ice_rss_match_prof(r, prof, hdr_type)) { clear_bit(vsi_handle, r->vsis); if (bitmap_empty(r->vsis, ICE_MAX_VSI)) { list_del(&r->l_entry); @@ -1995,11 +2041,12 @@ ice_rem_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) static int ice_add_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) { + enum ice_rss_cfg_hdr_type hdr_type; struct ice_rss_cfg *r, *rss_cfg; + hdr_type = ice_get_rss_hdr_type(prof); list_for_each_entry(r, &hw->rss_list_head, l_entry) - if (r->hashed_flds == prof->segs[prof->segs_cnt - 1].match && - r->packet_hdr == prof->segs[prof->segs_cnt - 1].hdrs) { + if (ice_rss_match_prof(r, prof, hdr_type)) { set_bit(vsi_handle, r->vsis); return 0; } @@ -2009,8 +2056,9 @@ ice_add_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) if (!rss_cfg) return -ENOMEM; - rss_cfg->hashed_flds = prof->segs[prof->segs_cnt - 1].match; - rss_cfg->packet_hdr = prof->segs[prof->segs_cnt - 1].hdrs; + rss_cfg->hash.hash_flds = prof->segs[prof->segs_cnt - 1].match; + rss_cfg->hash.addl_hdrs = prof->segs[prof->segs_cnt - 1].hdrs; + rss_cfg->hash.hdr_type = hdr_type; set_bit(vsi_handle, rss_cfg->vsis); list_add_tail(&rss_cfg->l_entry, &hw->rss_list_head); @@ -2019,54 +2067,55 @@ ice_add_rss_list(struct ice_hw *hw, u16 vsi_handle, struct ice_flow_prof *prof) } #define ICE_FLOW_PROF_HASH_S 0 -#define ICE_FLOW_PROF_HASH_M (0xFFFFFFFFULL << ICE_FLOW_PROF_HASH_S) +#define ICE_FLOW_PROF_HASH_M GENMASK_ULL(31, 0) #define ICE_FLOW_PROF_HDR_S 32 -#define ICE_FLOW_PROF_HDR_M (0x3FFFFFFFULL << ICE_FLOW_PROF_HDR_S) -#define ICE_FLOW_PROF_ENCAP_S 63 -#define ICE_FLOW_PROF_ENCAP_M (BIT_ULL(ICE_FLOW_PROF_ENCAP_S)) - -#define ICE_RSS_OUTER_HEADERS 1 -#define ICE_RSS_INNER_HEADERS 2 +#define ICE_FLOW_PROF_HDR_M GENMASK_ULL(61, 32) +#define ICE_FLOW_PROF_ENCAP_S 62 +#define ICE_FLOW_PROF_ENCAP_M GENMASK_ULL(63, 62) /* Flow profile ID format: * [0:31] - Packet match fields - * [32:62] - Protocol header - * [63] - Encapsulation flag, 0 if non-tunneled, 1 if tunneled + * [32:61] - Protocol header + * [62:63] - Encapsulation flag: + * 0 if non-tunneled + * 1 if tunneled + * 2 for tunneled with outer ipv4 + * 3 for tunneled with outer ipv6 */ -#define ICE_FLOW_GEN_PROFID(hash, hdr, segs_cnt) \ - ((u64)(((u64)(hash) & ICE_FLOW_PROF_HASH_M) | \ +#define ICE_FLOW_GEN_PROFID(hash, hdr, encap) \ + ((u64)(((u64)(hash) & ICE_FLOW_PROF_HASH_M) | \ (((u64)(hdr) << ICE_FLOW_PROF_HDR_S) & ICE_FLOW_PROF_HDR_M) | \ - ((u8)((segs_cnt) - 1) ? ICE_FLOW_PROF_ENCAP_M : 0))) + (((u64)(encap) << ICE_FLOW_PROF_ENCAP_S) & \ + ICE_FLOW_PROF_ENCAP_M))) /** * ice_add_rss_cfg_sync - add an RSS configuration * @hw: pointer to the hardware structure * @vsi_handle: software VSI handle - * @hashed_flds: hash bit fields (ICE_FLOW_HASH_*) to configure - * @addl_hdrs: protocol header fields - * @segs_cnt: packet segment count + * @cfg: configure parameters * * Assumption: lock has already been acquired for RSS list */ static int -ice_add_rss_cfg_sync(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs, u8 segs_cnt) +ice_add_rss_cfg_sync(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg) { const enum ice_block blk = ICE_BLK_RSS; struct ice_flow_prof *prof = NULL; struct ice_flow_seg_info *segs; + u8 segs_cnt; int status; - if (!segs_cnt || segs_cnt > ICE_FLOW_SEG_MAX) - return -EINVAL; + segs_cnt = (cfg->hdr_type == ICE_RSS_OUTER_HEADERS) ? + ICE_FLOW_SEG_SINGLE : + ICE_FLOW_SEG_MAX; segs = kcalloc(segs_cnt, sizeof(*segs), GFP_KERNEL); if (!segs) return -ENOMEM; /* Construct the packet segment info from the hashed fields */ - status = ice_flow_set_rss_seg_info(&segs[segs_cnt - 1], hashed_flds, - addl_hdrs); + status = ice_flow_set_rss_seg_info(segs, segs_cnt, cfg); if (status) goto exit; @@ -2120,9 +2169,9 @@ ice_add_rss_cfg_sync(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, * segment information. */ status = ice_flow_add_prof(hw, blk, ICE_FLOW_RX, - ICE_FLOW_GEN_PROFID(hashed_flds, + ICE_FLOW_GEN_PROFID(cfg->hash_flds, segs[segs_cnt - 1].hdrs, - segs_cnt), + cfg->hdr_type), segs, segs_cnt, &prof); if (status) goto exit; @@ -2147,29 +2196,37 @@ exit: * ice_add_rss_cfg - add an RSS configuration with specified hashed fields * @hw: pointer to the hardware structure * @vsi_handle: software VSI handle - * @hashed_flds: hash bit fields (ICE_FLOW_HASH_*) to configure - * @addl_hdrs: protocol header fields + * @cfg: configure parameters * * This function will generate a flow profile based on fields associated with * the input fields to hash on, the flow type and use the VSI number to add * a flow entry to the profile. */ int -ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs) +ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg) { + struct ice_rss_hash_cfg local_cfg; int status; - if (hashed_flds == ICE_HASH_INVALID || - !ice_is_vsi_valid(hw, vsi_handle)) + if (!ice_is_vsi_valid(hw, vsi_handle) || !cfg || + cfg->hdr_type > ICE_RSS_ANY_HEADERS || + cfg->hash_flds == ICE_HASH_INVALID) return -EINVAL; mutex_lock(&hw->rss_locks); - status = ice_add_rss_cfg_sync(hw, vsi_handle, hashed_flds, addl_hdrs, - ICE_RSS_OUTER_HEADERS); - if (!status) - status = ice_add_rss_cfg_sync(hw, vsi_handle, hashed_flds, - addl_hdrs, ICE_RSS_INNER_HEADERS); + local_cfg = *cfg; + if (cfg->hdr_type < ICE_RSS_ANY_HEADERS) { + status = ice_add_rss_cfg_sync(hw, vsi_handle, &local_cfg); + } else { + local_cfg.hdr_type = ICE_RSS_OUTER_HEADERS; + status = ice_add_rss_cfg_sync(hw, vsi_handle, &local_cfg); + if (!status) { + local_cfg.hdr_type = ICE_RSS_INNER_HEADERS; + status = ice_add_rss_cfg_sync(hw, vsi_handle, + &local_cfg); + } + } mutex_unlock(&hw->rss_locks); return status; @@ -2179,28 +2236,29 @@ ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, * ice_rem_rss_cfg_sync - remove an existing RSS configuration * @hw: pointer to the hardware structure * @vsi_handle: software VSI handle - * @hashed_flds: Packet hash types (ICE_FLOW_HASH_*) to remove - * @addl_hdrs: Protocol header fields within a packet segment - * @segs_cnt: packet segment count + * @cfg: configure parameters * * Assumption: lock has already been acquired for RSS list */ static int -ice_rem_rss_cfg_sync(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs, u8 segs_cnt) +ice_rem_rss_cfg_sync(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg) { const enum ice_block blk = ICE_BLK_RSS; struct ice_flow_seg_info *segs; struct ice_flow_prof *prof; + u8 segs_cnt; int status; + segs_cnt = (cfg->hdr_type == ICE_RSS_OUTER_HEADERS) ? + ICE_FLOW_SEG_SINGLE : + ICE_FLOW_SEG_MAX; segs = kcalloc(segs_cnt, sizeof(*segs), GFP_KERNEL); if (!segs) return -ENOMEM; /* Construct the packet segment info from the hashed fields */ - status = ice_flow_set_rss_seg_info(&segs[segs_cnt - 1], hashed_flds, - addl_hdrs); + status = ice_flow_set_rss_seg_info(segs, segs_cnt, cfg); if (status) goto out; @@ -2233,31 +2291,39 @@ out: * ice_rem_rss_cfg - remove an existing RSS config with matching hashed fields * @hw: pointer to the hardware structure * @vsi_handle: software VSI handle - * @hashed_flds: Packet hash types (ICE_FLOW_HASH_*) to remove - * @addl_hdrs: Protocol header fields within a packet segment + * @cfg: configure parameters * * This function will lookup the flow profile based on the input * hash field bitmap, iterate through the profile entry list of * that profile and find entry associated with input VSI to be - * removed. Calls are made to underlying flow s which will APIs + * removed. Calls are made to underlying flow apis which will in * turn build or update buffers for RSS XLT1 section. */ -int __maybe_unused -ice_rem_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs) +int +ice_rem_rss_cfg(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg) { + struct ice_rss_hash_cfg local_cfg; int status; - if (hashed_flds == ICE_HASH_INVALID || - !ice_is_vsi_valid(hw, vsi_handle)) + if (!ice_is_vsi_valid(hw, vsi_handle) || !cfg || + cfg->hdr_type > ICE_RSS_ANY_HEADERS || + cfg->hash_flds == ICE_HASH_INVALID) return -EINVAL; mutex_lock(&hw->rss_locks); - status = ice_rem_rss_cfg_sync(hw, vsi_handle, hashed_flds, addl_hdrs, - ICE_RSS_OUTER_HEADERS); - if (!status) - status = ice_rem_rss_cfg_sync(hw, vsi_handle, hashed_flds, - addl_hdrs, ICE_RSS_INNER_HEADERS); + local_cfg = *cfg; + if (cfg->hdr_type < ICE_RSS_ANY_HEADERS) { + status = ice_rem_rss_cfg_sync(hw, vsi_handle, &local_cfg); + } else { + local_cfg.hdr_type = ICE_RSS_OUTER_HEADERS; + status = ice_rem_rss_cfg_sync(hw, vsi_handle, &local_cfg); + if (!status) { + local_cfg.hdr_type = ICE_RSS_INNER_HEADERS; + status = ice_rem_rss_cfg_sync(hw, vsi_handle, + &local_cfg); + } + } mutex_unlock(&hw->rss_locks); return status; @@ -2307,6 +2373,7 @@ ice_rem_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, */ int ice_add_avf_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 avf_hash) { + struct ice_rss_hash_cfg hcfg; int status = 0; u64 hash_flds; @@ -2379,8 +2446,10 @@ int ice_add_avf_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 avf_hash) if (rss_hash == ICE_HASH_INVALID) return -EIO; - status = ice_add_rss_cfg(hw, vsi_handle, rss_hash, - ICE_FLOW_SEG_HDR_NONE); + hcfg.addl_hdrs = ICE_FLOW_SEG_HDR_NONE; + hcfg.hash_flds = rss_hash; + hcfg.hdr_type = ICE_RSS_ANY_HEADERS; + status = ice_add_rss_cfg(hw, vsi_handle, &hcfg); if (status) break; } @@ -2404,16 +2473,7 @@ int ice_replay_rss_cfg(struct ice_hw *hw, u16 vsi_handle) mutex_lock(&hw->rss_locks); list_for_each_entry(r, &hw->rss_list_head, l_entry) { if (test_bit(vsi_handle, r->vsis)) { - status = ice_add_rss_cfg_sync(hw, vsi_handle, - r->hashed_flds, - r->packet_hdr, - ICE_RSS_OUTER_HEADERS); - if (status) - break; - status = ice_add_rss_cfg_sync(hw, vsi_handle, - r->hashed_flds, - r->packet_hdr, - ICE_RSS_INNER_HEADERS); + status = ice_add_rss_cfg_sync(hw, vsi_handle, &r->hash); if (status) break; } @@ -2444,8 +2504,8 @@ u64 ice_get_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u32 hdrs) mutex_lock(&hw->rss_locks); list_for_each_entry(r, &hw->rss_list_head, l_entry) if (test_bit(vsi_handle, r->vsis) && - r->packet_hdr == hdrs) { - rss_hash = r->hashed_flds; + r->hash.addl_hdrs == hdrs) { + rss_hash = r->hash.hash_flds; break; } mutex_unlock(&hw->rss_locks); diff --git a/drivers/net/ethernet/intel/ice/ice_flow.h b/drivers/net/ethernet/intel/ice/ice_flow.h index 96923ef0a5a8..d1de84b78386 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.h +++ b/drivers/net/ethernet/intel/ice/ice_flow.h @@ -34,6 +34,8 @@ #define ICE_HASH_TCP_IPV6 (ICE_FLOW_HASH_IPV6 | ICE_FLOW_HASH_TCP_PORT) #define ICE_HASH_UDP_IPV4 (ICE_FLOW_HASH_IPV4 | ICE_FLOW_HASH_UDP_PORT) #define ICE_HASH_UDP_IPV6 (ICE_FLOW_HASH_IPV6 | ICE_FLOW_HASH_UDP_PORT) +#define ICE_HASH_SCTP_IPV4 (ICE_FLOW_HASH_IPV4 | ICE_FLOW_HASH_SCTP_PORT) +#define ICE_HASH_SCTP_IPV6 (ICE_FLOW_HASH_IPV6 | ICE_FLOW_HASH_SCTP_PORT) #define ICE_FLOW_HASH_GTP_TEID \ (BIT_ULL(ICE_FLOW_FIELD_IDX_GTPC_TEID)) @@ -279,6 +281,23 @@ enum ice_flow_avf_hdr_field { BIT_ULL(ICE_AVF_FLOW_FIELD_UNICAST_IPV6_UDP) | \ BIT_ULL(ICE_AVF_FLOW_FIELD_MULTICAST_IPV6_UDP)) +enum ice_rss_cfg_hdr_type { + ICE_RSS_OUTER_HEADERS, /* take outer headers as inputset. */ + ICE_RSS_INNER_HEADERS, /* take inner headers as inputset. */ + /* take inner headers as inputset for packet with outer ipv4. */ + ICE_RSS_INNER_HEADERS_W_OUTER_IPV4, + /* take inner headers as inputset for packet with outer ipv6. */ + ICE_RSS_INNER_HEADERS_W_OUTER_IPV6, + /* take outer headers first then inner headers as inputset */ + ICE_RSS_ANY_HEADERS +}; + +struct ice_rss_hash_cfg { + u32 addl_hdrs; /* protocol header fields */ + u64 hash_flds; /* hash bit field (ICE_FLOW_HASH_*) to configure */ + enum ice_rss_cfg_hdr_type hdr_type; /* to specify inner or outer */ +}; + enum ice_flow_dir { ICE_FLOW_RX = 0x02, }; @@ -289,6 +308,7 @@ enum ice_flow_priority { ICE_FLOW_PRIO_HIGH }; +#define ICE_FLOW_SEG_SINGLE 1 #define ICE_FLOW_SEG_MAX 2 #define ICE_FLOW_SEG_RAW_FLD_MAX 2 #define ICE_FLOW_FV_EXTRACT_SZ 2 @@ -378,8 +398,7 @@ struct ice_rss_cfg { struct list_head l_entry; /* bitmap of VSIs added to the RSS entry */ DECLARE_BITMAP(vsis, ICE_MAX_VSI); - u64 hashed_flds; - u32 packet_hdr; + struct ice_rss_hash_cfg hash; }; int @@ -403,11 +422,9 @@ void ice_rem_vsi_rss_list(struct ice_hw *hw, u16 vsi_handle); int ice_replay_rss_cfg(struct ice_hw *hw, u16 vsi_handle); int ice_add_avf_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds); int ice_rem_vsi_rss_cfg(struct ice_hw *hw, u16 vsi_handle); -int -ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs); -int -ice_rem_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u64 hashed_flds, - u32 addl_hdrs); +int ice_add_rss_cfg(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg); +int ice_rem_rss_cfg(struct ice_hw *hw, u16 vsi_handle, + const struct ice_rss_hash_cfg *cfg); u64 ice_get_rss_cfg(struct ice_hw *hw, u16 vsi_handle, u32 hdrs); #endif /* _ICE_FLOW_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index bb6151e798e4..b22f1b861831 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -1611,6 +1611,38 @@ static void ice_vsi_set_vf_rss_flow_fld(struct ice_vsi *vsi) vsi->vsi_num, status); } +static const struct ice_rss_hash_cfg default_rss_cfgs[] = { + /* configure RSS for IPv4 with input set IP src/dst */ + {ICE_FLOW_SEG_HDR_IPV4, ICE_FLOW_HASH_IPV4, ICE_RSS_ANY_HEADERS}, + /* configure RSS for IPv6 with input set IPv6 src/dst */ + {ICE_FLOW_SEG_HDR_IPV6, ICE_FLOW_HASH_IPV6, ICE_RSS_ANY_HEADERS}, + /* configure RSS for tcp4 with input set IP src/dst, TCP src/dst */ + {ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_TCP_IPV4, ICE_RSS_ANY_HEADERS}, + /* configure RSS for udp4 with input set IP src/dst, UDP src/dst */ + {ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_UDP_IPV4, ICE_RSS_ANY_HEADERS}, + /* configure RSS for sctp4 with input set IP src/dst - only support + * RSS on SCTPv4 on outer headers (non-tunneled) + */ + {ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4, + ICE_HASH_SCTP_IPV4, ICE_RSS_OUTER_HEADERS}, + /* configure RSS for tcp6 with input set IPv6 src/dst, TCP src/dst */ + {ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_TCP_IPV6, ICE_RSS_ANY_HEADERS}, + /* configure RSS for udp6 with input set IPv6 src/dst, UDP src/dst */ + {ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_UDP_IPV6, ICE_RSS_ANY_HEADERS}, + /* configure RSS for sctp6 with input set IPv6 src/dst - only support + * RSS on SCTPv6 on outer headers (non-tunneled) + */ + {ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6, + ICE_HASH_SCTP_IPV6, ICE_RSS_OUTER_HEADERS}, + /* configure RSS for IPSEC ESP SPI with input set MAC_IPV4_SPI */ + {ICE_FLOW_SEG_HDR_ESP, + ICE_FLOW_HASH_ESP_SPI, ICE_RSS_OUTER_HEADERS}, +}; + /** * ice_vsi_set_rss_flow_fld - Sets RSS input set for different flows * @vsi: VSI to be configured @@ -1629,6 +1661,7 @@ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi) struct ice_hw *hw = &pf->hw; struct device *dev; int status; + u32 i; dev = ice_pf_to_dev(pf); if (ice_is_safe_mode(pf)) { @@ -1636,67 +1669,14 @@ static void ice_vsi_set_rss_flow_fld(struct ice_vsi *vsi) vsi_num); return; } - /* configure RSS for IPv4 with input set IP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, - ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv4 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for IPv6 with input set IPv6 src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, - ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for ipv6 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for tcp4 with input set IP src/dst, TCP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV4, - ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp4 flow, vsi = %d, error = %d\n", - vsi_num, status); + for (i = 0; i < ARRAY_SIZE(default_rss_cfgs); i++) { + const struct ice_rss_hash_cfg *cfg = &default_rss_cfgs[i]; - /* configure RSS for udp4 with input set IP src/dst, UDP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV4, - ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp4 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for sctp4 with input set IP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV4, - ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp4 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for tcp6 with input set IPv6 src/dst, TCP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_TCP_IPV6, - ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for tcp6 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for udp6 with input set IPv6 src/dst, UDP src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_HASH_UDP_IPV6, - ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for udp6 flow, vsi = %d, error = %d\n", - vsi_num, status); - - /* configure RSS for sctp6 with input set IPv6 src/dst */ - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_IPV6, - ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for sctp6 flow, vsi = %d, error = %d\n", - vsi_num, status); - - status = ice_add_rss_cfg(hw, vsi_handle, ICE_FLOW_HASH_ESP_SPI, - ICE_FLOW_SEG_HDR_ESP); - if (status) - dev_dbg(dev, "ice_add_rss_cfg failed for esp/spi flow, vsi = %d, error = %d\n", - vsi_num, status); + status = ice_add_rss_cfg(hw, vsi_handle, cfg); + if (status) + dev_dbg(dev, "ice_add_rss_cfg failed, addl_hdrs = %x, hash_flds = %llx, hdr_type = %d\n", + cfg->addl_hdrs, cfg->hash_flds, cfg->hdr_type); + } } /** diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index 8872f7a4f432..0e7b5984a7c7 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -689,9 +689,7 @@ out: * a specific virtchnl RSS cfg * @hw: pointer to the hardware * @rss_cfg: pointer to the virtchnl RSS cfg - * @addl_hdrs: pointer to the protocol header fields (ICE_FLOW_SEG_HDR_*) - * to configure - * @hash_flds: pointer to the hash bit fields (ICE_FLOW_HASH_*) to configure + * @hash_cfg: pointer to the HW hash configuration * * Return true if all the protocol header and hash fields in the RSS cfg could * be parsed, else return false @@ -699,13 +697,18 @@ out: * This function parses the virtchnl RSS cfg to be the intended * hash fields and the intended header for RSS configuration */ -static bool -ice_vc_parse_rss_cfg(struct ice_hw *hw, struct virtchnl_rss_cfg *rss_cfg, - u32 *addl_hdrs, u64 *hash_flds) +static bool ice_vc_parse_rss_cfg(struct ice_hw *hw, + struct virtchnl_rss_cfg *rss_cfg, + struct ice_rss_hash_cfg *hash_cfg) { const struct ice_vc_hash_field_match_type *hf_list; const struct ice_vc_hdr_match_type *hdr_list; int i, hf_list_len, hdr_list_len; + u32 *addl_hdrs = &hash_cfg->addl_hdrs; + u64 *hash_flds = &hash_cfg->hash_flds; + + /* set outer layer RSS as default */ + hash_cfg->hdr_type = ICE_RSS_OUTER_HEADERS; hf_list = ice_vc_hash_field_list; hf_list_len = ARRAY_SIZE(ice_vc_hash_field_list); @@ -856,18 +859,24 @@ static int ice_vc_handle_rss_cfg(struct ice_vf *vf, u8 *msg, bool add) kfree(ctx); } else { - u32 addl_hdrs = ICE_FLOW_SEG_HDR_NONE; - u64 hash_flds = ICE_HASH_INVALID; + struct ice_rss_hash_cfg cfg; + + /* Only check for none raw pattern case */ + if (!ice_vc_validate_pattern(vf, &rss_cfg->proto_hdrs)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + cfg.addl_hdrs = ICE_FLOW_SEG_HDR_NONE; + cfg.hash_flds = ICE_HASH_INVALID; + cfg.hdr_type = ICE_RSS_ANY_HEADERS; - if (!ice_vc_parse_rss_cfg(hw, rss_cfg, &addl_hdrs, - &hash_flds)) { + if (!ice_vc_parse_rss_cfg(hw, rss_cfg, &cfg)) { v_ret = VIRTCHNL_STATUS_ERR_PARAM; goto error_param; } if (add) { - if (ice_add_rss_cfg(hw, vsi->idx, hash_flds, - addl_hdrs)) { + if (ice_add_rss_cfg(hw, vsi->idx, &cfg)) { v_ret = VIRTCHNL_STATUS_ERR_PARAM; dev_err(dev, "ice_add_rss_cfg failed for vsi = %d, v_ret = %d\n", vsi->vsi_num, v_ret); @@ -875,8 +884,7 @@ static int ice_vc_handle_rss_cfg(struct ice_vf *vf, u8 *msg, bool add) } else { int status; - status = ice_rem_rss_cfg(hw, vsi->idx, hash_flds, - addl_hdrs); + status = ice_rem_rss_cfg(hw, vsi->idx, &cfg); /* We just ignore -ENOENT, because if two configurations * share the same profile remove one of them actually * removes both, since the profile is deleted. diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h index 6b3acf15be5c..b0e060cc79ac 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/avf/virtchnl.h @@ -911,6 +911,14 @@ struct virtchnl_rss_hena { VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hena); +/* Type of RSS algorithm */ +enum virtchnl_rss_algorithm { + VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC = 0, + VIRTCHNL_RSS_ALG_R_ASYMMETRIC = 1, + VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC = 2, + VIRTCHNL_RSS_ALG_XOR_SYMMETRIC = 3, +}; + /* VIRTCHNL_OP_ENABLE_CHANNELS * VIRTCHNL_OP_DISABLE_CHANNELS * VF sends these messages to enable or disable channels based on @@ -1095,14 +1103,6 @@ enum virtchnl_vfr_states { VIRTCHNL_VFR_VFACTIVE, }; -/* Type of RSS algorithm */ -enum virtchnl_rss_algorithm { - VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC = 0, - VIRTCHNL_RSS_ALG_R_ASYMMETRIC = 1, - VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC = 2, - VIRTCHNL_RSS_ALG_XOR_SYMMETRIC = 3, -}; - #define VIRTCHNL_MAX_NUM_PROTO_HDRS 32 #define PROTO_HDR_SHIFT 5 #define PROTO_HDR_FIELD_START(proto_hdr_type) ((proto_hdr_type) << PROTO_HDR_SHIFT) -- cgit v1.2.3 From 4a3de3fb0eb6897488dd510006abd9673f1fb34c Mon Sep 17 00:00:00 2001 From: Ahmed Zaki Date: Tue, 12 Dec 2023 17:33:21 -0700 Subject: iavf: enable symmetric-xor RSS for Toeplitz hash function Allow the user to set the symmetric Toeplitz hash function via: # ethtool -X eth0 hfunc toeplitz symmetric-xor The driver will reject any new RSS configuration if a field other than (IP src/dst and L4 src/dst ports) is requested for hashing. The symmetric RSS will not be supported on PFs not advertising the ADV RSS Offload flag (ADV_RSS_SUPPORT()), for example the E700 series (i40e). Reviewed-by: Madhu Chittim Signed-off-by: Ahmed Zaki Link: https://lore.kernel.org/r/20231213003321.605376-9-ahmed.zaki@intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/intel/iavf/iavf.h | 5 ++- drivers/net/ethernet/intel/iavf/iavf_adv_rss.c | 8 +++- drivers/net/ethernet/intel/iavf/iavf_adv_rss.h | 3 +- drivers/net/ethernet/intel/iavf/iavf_ethtool.c | 32 ++++++++++++-- drivers/net/ethernet/intel/iavf/iavf_main.c | 4 ++ drivers/net/ethernet/intel/iavf/iavf_virtchnl.c | 41 ++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_virtchnl.c | 50 ++++++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_virtchnl.h | 1 + .../ethernet/intel/ice/ice_virtchnl_allowlist.c | 1 + include/linux/avf/virtchnl.h | 19 ++++++++ 10 files changed, 156 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index e7ab89dc883a..f83fbcc72075 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -312,7 +312,8 @@ struct iavf_adapter { #define IAVF_FLAG_AQ_SET_HENA BIT_ULL(12) #define IAVF_FLAG_AQ_SET_RSS_KEY BIT_ULL(13) #define IAVF_FLAG_AQ_SET_RSS_LUT BIT_ULL(14) -#define IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE BIT_ULL(15) +#define IAVF_FLAG_AQ_SET_RSS_HFUNC BIT_ULL(15) +#define IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE BIT_ULL(16) #define IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING BIT_ULL(19) #define IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING BIT_ULL(20) #define IAVF_FLAG_AQ_ENABLE_CHANNELS BIT_ULL(21) @@ -414,6 +415,7 @@ struct iavf_adapter { struct iavf_vsi vsi; u32 aq_wait_count; /* RSS stuff */ + enum virtchnl_rss_algorithm hfunc; u64 hena; u16 rss_key_size; u16 rss_lut_size; @@ -539,6 +541,7 @@ void iavf_get_hena(struct iavf_adapter *adapter); void iavf_set_hena(struct iavf_adapter *adapter); void iavf_set_rss_key(struct iavf_adapter *adapter); void iavf_set_rss_lut(struct iavf_adapter *adapter); +void iavf_set_rss_hfunc(struct iavf_adapter *adapter); void iavf_enable_vlan_stripping(struct iavf_adapter *adapter); void iavf_disable_vlan_stripping(struct iavf_adapter *adapter); void iavf_virtchnl_completion(struct iavf_adapter *adapter, diff --git a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c index 6edbf134b73f..a9e1da35e248 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c +++ b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.c @@ -95,17 +95,21 @@ iavf_fill_adv_rss_sctp_hdr(struct virtchnl_proto_hdr *hdr, u64 hash_flds) * @rss_cfg: the virtchnl message to be filled with RSS configuration setting * @packet_hdrs: the RSS configuration protocol header types * @hash_flds: the RSS configuration protocol hash fields + * @symm: if true, symmetric hash is required * * Returns 0 if the RSS configuration virtchnl message is filled successfully */ int iavf_fill_adv_rss_cfg_msg(struct virtchnl_rss_cfg *rss_cfg, - u32 packet_hdrs, u64 hash_flds) + u32 packet_hdrs, u64 hash_flds, bool symm) { struct virtchnl_proto_hdrs *proto_hdrs = &rss_cfg->proto_hdrs; struct virtchnl_proto_hdr *hdr; - rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; + if (symm) + rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC; + else + rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; proto_hdrs->tunnel_level = 0; /* always outer layer */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h index 4d3be11af7aa..e31eb2afebea 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h +++ b/drivers/net/ethernet/intel/iavf/iavf_adv_rss.h @@ -80,13 +80,14 @@ struct iavf_adv_rss { u32 packet_hdrs; u64 hash_flds; + bool symm; struct virtchnl_rss_cfg cfg_msg; }; int iavf_fill_adv_rss_cfg_msg(struct virtchnl_rss_cfg *rss_cfg, - u32 packet_hdrs, u64 hash_flds); + u32 packet_hdrs, u64 hash_flds, bool symm); struct iavf_adv_rss * iavf_find_adv_rss_cfg_by_hdrs(struct iavf_adapter *adapter, u32 packet_hdrs); void diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index c376a489e4c2..f147743792fb 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -1529,11 +1529,12 @@ static u32 iavf_adv_rss_parse_hdrs(struct ethtool_rxnfc *cmd) /** * iavf_adv_rss_parse_hash_flds - parses hash fields from RSS hash input * @cmd: ethtool rxnfc command + * @symm: true if Symmetric Topelitz is set * * This function parses the rxnfc command and returns intended hash fields for * RSS configuration */ -static u64 iavf_adv_rss_parse_hash_flds(struct ethtool_rxnfc *cmd) +static u64 iavf_adv_rss_parse_hash_flds(struct ethtool_rxnfc *cmd, bool symm) { u64 hfld = IAVF_ADV_RSS_HASH_INVALID; @@ -1605,17 +1606,20 @@ iavf_set_adv_rss_hash_opt(struct iavf_adapter *adapter, struct iavf_adv_rss *rss_old, *rss_new; bool rss_new_add = false; int count = 50, err = 0; + bool symm = false; u64 hash_flds; u32 hdrs; if (!ADV_RSS_SUPPORT(adapter)) return -EOPNOTSUPP; + symm = !!(adapter->hfunc == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC); + hdrs = iavf_adv_rss_parse_hdrs(cmd); if (hdrs == IAVF_ADV_RSS_FLOW_SEG_HDR_NONE) return -EINVAL; - hash_flds = iavf_adv_rss_parse_hash_flds(cmd); + hash_flds = iavf_adv_rss_parse_hash_flds(cmd, symm); if (hash_flds == IAVF_ADV_RSS_HASH_INVALID) return -EINVAL; @@ -1623,7 +1627,8 @@ iavf_set_adv_rss_hash_opt(struct iavf_adapter *adapter, if (!rss_new) return -ENOMEM; - if (iavf_fill_adv_rss_cfg_msg(&rss_new->cfg_msg, hdrs, hash_flds)) { + if (iavf_fill_adv_rss_cfg_msg(&rss_new->cfg_msg, hdrs, hash_flds, + symm)) { kfree(rss_new); return -EINVAL; } @@ -1642,9 +1647,11 @@ iavf_set_adv_rss_hash_opt(struct iavf_adapter *adapter, if (rss_old) { if (rss_old->state != IAVF_ADV_RSS_ACTIVE) { err = -EBUSY; - } else if (rss_old->hash_flds != hash_flds) { + } else if (rss_old->hash_flds != hash_flds || + rss_old->symm != symm) { rss_old->state = IAVF_ADV_RSS_ADD_REQUEST; rss_old->hash_flds = hash_flds; + rss_old->symm = symm; memcpy(&rss_old->cfg_msg, &rss_new->cfg_msg, sizeof(rss_new->cfg_msg)); } else { @@ -1655,6 +1662,7 @@ iavf_set_adv_rss_hash_opt(struct iavf_adapter *adapter, rss_new->state = IAVF_ADV_RSS_ADD_REQUEST; rss_new->packet_hdrs = hdrs; rss_new->hash_flds = hash_flds; + rss_new->symm = symm; list_add_tail(&rss_new->list, &adapter->adv_rss_list_head); } spin_unlock_bh(&adapter->adv_rss_lock); @@ -1905,6 +1913,9 @@ static int iavf_get_rxfh(struct net_device *netdev, u16 i; rxfh->hfunc = ETH_RSS_HASH_TOP; + if (adapter->hfunc == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) + rxfh->input_xfrm |= RXH_XFRM_SYM_XOR; + if (rxfh->key) memcpy(rxfh->key, adapter->rss_key, adapter->rss_key_size); @@ -1937,6 +1948,18 @@ static int iavf_set_rxfh(struct net_device *netdev, rxfh->hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; + if ((rxfh->input_xfrm & RXH_XFRM_SYM_XOR) && + adapter->hfunc != VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) { + if (!ADV_RSS_SUPPORT(adapter)) + return -EOPNOTSUPP; + adapter->hfunc = VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC; + adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_HFUNC; + } else if (!(rxfh->input_xfrm & RXH_XFRM_SYM_XOR) && + adapter->hfunc != VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC) { + adapter->hfunc = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; + adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_HFUNC; + } + if (!rxfh->key && !rxfh->indir) return 0; @@ -1955,6 +1978,7 @@ static int iavf_set_rxfh(struct net_device *netdev, static const struct ethtool_ops iavf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE, + .cap_rss_sym_xor_supported = true, .get_drvinfo = iavf_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = iavf_get_ringparam, diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 06a87030c163..0b3b33acf1bd 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -2166,6 +2166,10 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter) iavf_set_rss_lut(adapter); return 0; } + if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_HFUNC) { + iavf_set_rss_hfunc(adapter); + return 0; + } if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) { iavf_set_promiscuous(adapter); diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 64c4443dbef9..64a351e70a56 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -1141,6 +1141,34 @@ void iavf_set_rss_lut(struct iavf_adapter *adapter) kfree(vrl); } +/** + * iavf_set_rss_hfunc + * @adapter: adapter structure + * + * Request the PF to set our RSS Hash function + **/ +void iavf_set_rss_hfunc(struct iavf_adapter *adapter) +{ + struct virtchnl_rss_hfunc *vrh; + int len = sizeof(*vrh); + + if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { + /* bail because we already have a command pending */ + dev_err(&adapter->pdev->dev, "Cannot set RSS Hash function, command %d pending\n", + adapter->current_op); + return; + } + vrh = kzalloc(len, GFP_KERNEL); + if (!vrh) + return; + vrh->vsi_id = adapter->vsi.id; + vrh->rss_algorithm = adapter->hfunc; + adapter->current_op = VIRTCHNL_OP_CONFIG_RSS_HFUNC; + adapter->aq_required &= ~IAVF_FLAG_AQ_SET_RSS_HFUNC; + iavf_send_pf_msg(adapter, VIRTCHNL_OP_CONFIG_RSS_HFUNC, (u8 *)vrh, len); + kfree(vrh); +} + /** * iavf_enable_vlan_stripping * @adapter: adapter structure @@ -2142,6 +2170,19 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, dev_warn(&adapter->pdev->dev, "Failed to add VLAN filter, error %s\n", iavf_stat_str(&adapter->hw, v_retval)); break; + case VIRTCHNL_OP_CONFIG_RSS_HFUNC: + dev_warn(&adapter->pdev->dev, "Failed to configure hash function, error %s\n", + iavf_stat_str(&adapter->hw, v_retval)); + + if (adapter->hfunc == + VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) + adapter->hfunc = + VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC; + else + adapter->hfunc = + VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC; + + break; default: dev_err(&adapter->pdev->dev, "PF returned error %d (%s) to our request %d\n", v_retval, iavf_stat_str(&adapter->hw, v_retval), diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index 482f8e8b1951..d4ad0739b57b 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -999,6 +999,51 @@ error_param: NULL, 0); } +/** + * ice_vc_config_rss_hfunc + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * Configure the VF's RSS Hash function + */ +static int ice_vc_config_rss_hfunc(struct ice_vf *vf, u8 *msg) +{ + struct virtchnl_rss_hfunc *vrh = (struct virtchnl_rss_hfunc *)msg; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + u8 hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; + struct ice_vsi *vsi; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vrh->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (vrh->rss_algorithm == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) + hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ; + + if (ice_set_rss_hfunc(vsi, hfunc)) + v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; +error_param: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_HFUNC, v_ret, + NULL, 0); +} + /** * ice_vc_cfg_promiscuous_mode_msg * @vf: pointer to the VF info @@ -3766,6 +3811,7 @@ static const struct ice_virtchnl_ops ice_virtchnl_dflt_ops = { .cfg_irq_map_msg = ice_vc_cfg_irq_map_msg, .config_rss_key = ice_vc_config_rss_key, .config_rss_lut = ice_vc_config_rss_lut, + .config_rss_hfunc = ice_vc_config_rss_hfunc, .get_stats_msg = ice_vc_get_stats_msg, .cfg_promiscuous_mode_msg = ice_vc_cfg_promiscuous_mode_msg, .add_vlan_msg = ice_vc_add_vlan_msg, @@ -3895,6 +3941,7 @@ static const struct ice_virtchnl_ops ice_virtchnl_repr_ops = { .cfg_irq_map_msg = ice_vc_cfg_irq_map_msg, .config_rss_key = ice_vc_config_rss_key, .config_rss_lut = ice_vc_config_rss_lut, + .config_rss_hfunc = ice_vc_config_rss_hfunc, .get_stats_msg = ice_vc_get_stats_msg, .cfg_promiscuous_mode_msg = ice_vc_repr_cfg_promiscuous_mode, .add_vlan_msg = ice_vc_add_vlan_msg, @@ -4077,6 +4124,9 @@ error_handler: case VIRTCHNL_OP_CONFIG_RSS_LUT: err = ops->config_rss_lut(vf, msg); break; + case VIRTCHNL_OP_CONFIG_RSS_HFUNC: + err = ops->config_rss_hfunc(vf, msg); + break; case VIRTCHNL_OP_GET_STATS: err = ops->get_stats_msg(vf, msg); break; diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.h b/drivers/net/ethernet/intel/ice/ice_virtchnl.h index cd747718de73..60dfbe05980a 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.h +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.h @@ -32,6 +32,7 @@ struct ice_virtchnl_ops { int (*cfg_irq_map_msg)(struct ice_vf *vf, u8 *msg); int (*config_rss_key)(struct ice_vf *vf, u8 *msg); int (*config_rss_lut)(struct ice_vf *vf, u8 *msg); + int (*config_rss_hfunc)(struct ice_vf *vf, u8 *msg); int (*get_stats_msg)(struct ice_vf *vf, u8 *msg); int (*cfg_promiscuous_mode_msg)(struct ice_vf *vf, u8 *msg); int (*add_vlan_msg)(struct ice_vf *vf, u8 *msg); diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c index 7d547fa616fa..5e19d48a05b4 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c @@ -68,6 +68,7 @@ static const u32 vlan_v2_allowlist_opcodes[] = { static const u32 rss_pf_allowlist_opcodes[] = { VIRTCHNL_OP_CONFIG_RSS_KEY, VIRTCHNL_OP_CONFIG_RSS_LUT, VIRTCHNL_OP_GET_RSS_HENA_CAPS, VIRTCHNL_OP_SET_RSS_HENA, + VIRTCHNL_OP_CONFIG_RSS_HFUNC, }; /* VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC */ diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h index b0e060cc79ac..a44d9dc7e3eb 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/avf/virtchnl.h @@ -118,6 +118,7 @@ enum virtchnl_ops { VIRTCHNL_OP_GET_STATS = 15, VIRTCHNL_OP_RSVD = 16, VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ + VIRTCHNL_OP_CONFIG_RSS_HFUNC = 18, /* opcode 19 is reserved */ VIRTCHNL_OP_IWARP = 20, /* advanced opcode */ VIRTCHNL_OP_RDMA = VIRTCHNL_OP_IWARP, @@ -919,6 +920,21 @@ enum virtchnl_rss_algorithm { VIRTCHNL_RSS_ALG_XOR_SYMMETRIC = 3, }; +/* VIRTCHNL_OP_CONFIG_RSS_HFUNC + * VF sends this message to configure the RSS hash function. Only supported + * if both PF and VF drivers set the VIRTCHNL_VF_OFFLOAD_RSS_PF bit during + * configuration negotiation. + * The hash function is initialized to VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC + * by the PF. + */ +struct virtchnl_rss_hfunc { + u16 vsi_id; + u16 rss_algorithm; /* enum virtchnl_rss_algorithm */ + u32 reserved; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_rss_hfunc); + /* VIRTCHNL_OP_ENABLE_CHANNELS * VIRTCHNL_OP_DISABLE_CHANNELS * VF sends these messages to enable or disable channels based on @@ -1542,6 +1558,9 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, vrl->lut_entries); } break; + case VIRTCHNL_OP_CONFIG_RSS_HFUNC: + valid_len = sizeof(struct virtchnl_rss_hfunc); + break; case VIRTCHNL_OP_GET_RSS_HENA_CAPS: break; case VIRTCHNL_OP_SET_RSS_HENA: -- cgit v1.2.3 From bf873a800ac3234eba991603a450eaa517d27022 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 12 Dec 2023 20:35:11 -0800 Subject: net: skbuff: fix spelling errors Correct spelling as reported by codespell. Signed-off-by: Randy Dunlap Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20231213043511.10357-1-rdunlap@infradead.org Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b370eb8d70f7..7ce38874dbd1 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1069,7 +1069,7 @@ struct sk_buff { refcount_t users; #ifdef CONFIG_SKB_EXTENSIONS - /* only useable after checking ->active_extensions != 0 */ + /* only usable after checking ->active_extensions != 0 */ struct skb_ext *extensions; #endif }; @@ -3311,7 +3311,7 @@ static inline struct page *__dev_alloc_pages(gfp_t gfp_mask, unsigned int order) { /* This piece of code contains several assumptions. - * 1. This is for device Rx, therefor a cold page is preferred. + * 1. This is for device Rx, therefore a cold page is preferred. * 2. The expectation is the user wants a compound page. * 3. If requesting a order 0 page it will not be compound * due to the check to see if order has a value in prep_new_page @@ -4247,7 +4247,7 @@ static inline bool __skb_metadata_differs(const struct sk_buff *skb_a, { const void *a = skb_metadata_end(skb_a); const void *b = skb_metadata_end(skb_b); - /* Using more efficient varaiant than plain call to memcmp(). */ + /* Using more efficient variant than plain call to memcmp(). */ #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 u64 diffs = 0; -- cgit v1.2.3 From 0fe1798968115488c0c02f4633032a015b1faf97 Mon Sep 17 00:00:00 2001 From: Arseniy Krasnov Date: Thu, 14 Dec 2023 15:52:29 +0300 Subject: virtio/vsock: send credit update during setting SO_RCVLOWAT Send credit update message when SO_RCVLOWAT is updated and it is bigger than number of bytes in rx queue. It is needed, because 'poll()' will wait until number of bytes in rx queue will be not smaller than O_RCVLOWAT, so kick sender to send more data. Otherwise mutual hungup for tx/rx is possible: sender waits for free space and receiver is waiting data in 'poll()'. Rename 'set_rcvlowat' callback to 'notify_set_rcvlowat' and set 'sk->sk_rcvlowat' only in one place (i.e. 'vsock_set_rcvlowat'), so the transport doesn't need to do it. Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages") Signed-off-by: Arseniy Krasnov Reviewed-by: Stefano Garzarella Acked-by: Michael S. Tsirkin Signed-off-by: David S. Miller --- drivers/vhost/vsock.c | 1 + include/linux/virtio_vsock.h | 1 + include/net/af_vsock.h | 2 +- net/vmw_vsock/af_vsock.c | 9 +++++++-- net/vmw_vsock/hyperv_transport.c | 4 ++-- net/vmw_vsock/virtio_transport.c | 1 + net/vmw_vsock/virtio_transport_common.c | 30 ++++++++++++++++++++++++++++++ net/vmw_vsock/vsock_loopback.c | 1 + 8 files changed, 44 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index f75731396b7e..ec20ecff85c7 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -449,6 +449,7 @@ static struct virtio_transport vhost_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h index ebb3ce63d64d..c82089dee0c8 100644 --- a/include/linux/virtio_vsock.h +++ b/include/linux/virtio_vsock.h @@ -256,4 +256,5 @@ void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit); void virtio_transport_deliver_tap_pkt(struct sk_buff *skb); int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list); int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t read_actor); +int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val); #endif /* _LINUX_VIRTIO_VSOCK_H */ diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index e302c0e804d0..535701efc1e5 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -137,7 +137,6 @@ struct vsock_transport { u64 (*stream_rcvhiwat)(struct vsock_sock *); bool (*stream_is_active)(struct vsock_sock *); bool (*stream_allow)(u32 cid, u32 port); - int (*set_rcvlowat)(struct vsock_sock *vsk, int val); /* SEQ_PACKET. */ ssize_t (*seqpacket_dequeue)(struct vsock_sock *vsk, struct msghdr *msg, @@ -168,6 +167,7 @@ struct vsock_transport { struct vsock_transport_send_notify_data *); /* sk_lock held by the caller */ void (*notify_buffer_size)(struct vsock_sock *, u64 *); + int (*notify_set_rcvlowat)(struct vsock_sock *vsk, int val); /* Shutdown. */ int (*shutdown)(struct vsock_sock *, int); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 816725af281f..54ba7316f808 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -2264,8 +2264,13 @@ static int vsock_set_rcvlowat(struct sock *sk, int val) transport = vsk->transport; - if (transport && transport->set_rcvlowat) - return transport->set_rcvlowat(vsk, val); + if (transport && transport->notify_set_rcvlowat) { + int err; + + err = transport->notify_set_rcvlowat(vsk, val); + if (err) + return err; + } WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); return 0; diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 7cb1a9d2cdb4..e2157e387217 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -816,7 +816,7 @@ int hvs_notify_send_post_enqueue(struct vsock_sock *vsk, ssize_t written, } static -int hvs_set_rcvlowat(struct vsock_sock *vsk, int val) +int hvs_notify_set_rcvlowat(struct vsock_sock *vsk, int val) { return -EOPNOTSUPP; } @@ -856,7 +856,7 @@ static struct vsock_transport hvs_transport = { .notify_send_pre_enqueue = hvs_notify_send_pre_enqueue, .notify_send_post_enqueue = hvs_notify_send_post_enqueue, - .set_rcvlowat = hvs_set_rcvlowat + .notify_set_rcvlowat = hvs_notify_set_rcvlowat }; static bool hvs_check_transport(struct vsock_sock *vsk) diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index af5bab1acee1..f495b9e5186b 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -537,6 +537,7 @@ static struct virtio_transport virtio_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index b35306dfcebe..16ff976a86e3 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -1690,6 +1690,36 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto } EXPORT_SYMBOL_GPL(virtio_transport_read_skb); +int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val) +{ + struct virtio_vsock_sock *vvs = vsk->trans; + bool send_update; + + spin_lock_bh(&vvs->rx_lock); + + /* If number of available bytes is less than new SO_RCVLOWAT value, + * kick sender to send more data, because sender may sleep in its + * 'send()' syscall waiting for enough space at our side. Also + * don't send credit update when peer already knows actual value - + * such transmission will be useless. + */ + send_update = (vvs->rx_bytes < val) && + (vvs->fwd_cnt != vvs->last_fwd_cnt); + + spin_unlock_bh(&vvs->rx_lock); + + if (send_update) { + int err; + + err = virtio_transport_send_credit_update(vsk); + if (err < 0) + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_transport_notify_set_rcvlowat); + MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Asias He"); MODULE_DESCRIPTION("common code for virtio vsock"); diff --git a/net/vmw_vsock/vsock_loopback.c b/net/vmw_vsock/vsock_loopback.c index 048640167411..6dea6119f5b2 100644 --- a/net/vmw_vsock/vsock_loopback.c +++ b/net/vmw_vsock/vsock_loopback.c @@ -96,6 +96,7 @@ static struct virtio_transport loopback_transport = { .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, .notify_buffer_size = virtio_transport_notify_buffer_size, + .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat, .read_skb = virtio_transport_read_skb, }, -- cgit v1.2.3 From 4382159696c9af67ee047ed55f2dbf05480f52f6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 15 Dec 2023 10:12:17 +0100 Subject: cfi: Flip headers Normal include order is that linux/foo.h should include asm/foo.h, CFI has it the wrong way around. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Sami Tolvanen Link: https://lore.kernel.org/r/20231215092707.231038174@infradead.org Signed-off-by: Alexei Starovoitov --- arch/riscv/include/asm/cfi.h | 3 ++- arch/riscv/kernel/cfi.c | 2 +- arch/x86/include/asm/cfi.h | 3 ++- arch/x86/kernel/cfi.c | 4 ++-- include/asm-generic/Kbuild | 1 + include/asm-generic/cfi.h | 5 +++++ include/linux/cfi.h | 1 + 7 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 include/asm-generic/cfi.h (limited to 'include/linux') diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h index 56bf9d69d5e3..8f7a62257044 100644 --- a/arch/riscv/include/asm/cfi.h +++ b/arch/riscv/include/asm/cfi.h @@ -7,8 +7,9 @@ * * Copyright (C) 2023 Google LLC */ +#include -#include +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 820158d7a291..6ec9dbd7292e 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -4,7 +4,7 @@ * * Copyright (C) 2023 Google LLC */ -#include +#include #include /* diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 58dacd90daef..2a494643089d 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -7,8 +7,9 @@ * * Copyright (C) 2022 Google LLC */ +#include -#include +struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c index 8674a5c0c031..e6bf78fac146 100644 --- a/arch/x86/kernel/cfi.c +++ b/arch/x86/kernel/cfi.c @@ -4,10 +4,10 @@ * * Copyright (C) 2022 Google LLC */ -#include +#include +#include #include #include -#include /* * Returns the target address and the expected type when regs->ip points diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index def242528b1d..d436bee4d129 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -11,6 +11,7 @@ mandatory-y += bitops.h mandatory-y += bug.h mandatory-y += bugs.h mandatory-y += cacheflush.h +mandatory-y += cfi.h mandatory-y += checksum.h mandatory-y += compat.h mandatory-y += current.h diff --git a/include/asm-generic/cfi.h b/include/asm-generic/cfi.h new file mode 100644 index 000000000000..41fac3537bf9 --- /dev/null +++ b/include/asm-generic/cfi.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_CFI_H +#define __ASM_GENERIC_CFI_H + +#endif /* __ASM_GENERIC_CFI_H */ diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 3552ec82b725..2309d74e77e6 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -9,6 +9,7 @@ #include #include +#include #ifdef CONFIG_CFI_CLANG enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, -- cgit v1.2.3 From 4f9087f16651aca4a5f32da840a53f6660f0579a Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 15 Dec 2023 10:12:18 +0100 Subject: x86/cfi,bpf: Fix BPF JIT call The current BPF call convention is __nocfi, except when it calls !JIT things, then it calls regular C functions. It so happens that with FineIBT the __nocfi and C calling conventions are incompatible. Specifically __nocfi will call at func+0, while FineIBT will have endbr-poison there, which is not a valid indirect target. Causing #CP. Notably this only triggers on IBT enabled hardware, which is probably why this hasn't been reported (also, most people will have JIT on anyway). Implement proper CFI prologues for the BPF JIT codegen and drop __nocfi for x86. Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20231215092707.345270396@infradead.org Signed-off-by: Alexei Starovoitov --- arch/x86/include/asm/cfi.h | 110 ++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/alternative.c | 47 +++++++++++++++--- arch/x86/net/bpf_jit_comp.c | 82 +++++++++++++++++++++++++++++-- include/linux/bpf.h | 12 ++++- include/linux/cfi.h | 7 +++ kernel/bpf/core.c | 25 ++++++++++ 6 files changed, 269 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 2a494643089d..7a7b0b823a98 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -9,15 +9,125 @@ */ #include +/* + * An overview of the various calling conventions... + * + * Traditional: + * + * foo: + * ... code here ... + * ret + * + * direct caller: + * call foo + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * IBT: + * + * foo: + * endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * call *%r11 + * + * + * kCFI: + * + * __cfi_foo: + * movl $0x12345678, %eax + * # 11 nops when CONFIG_CALL_PADDING + * foo: + * endbr64 # when IBT + * ... code here ... + * ret + * + * direct call: + * call foo # / call foo+4 when IBT + * + * indirect call: + * lea foo(%rip), %r11 + * ... + * movl $(-0x12345678), %r10d + * addl -4(%r11), %r10d # -15 when CONFIG_CALL_PADDING + * jz 1f + * ud2 + * 1:call *%r11 + * + * + * FineIBT (builds as kCFI + CALL_PADDING + IBT + RETPOLINE and runtime patches into): + * + * __cfi_foo: + * endbr64 + * subl 0x12345678, %r10d + * jz foo + * ud2 + * nop + * foo: + * osp nop3 # was endbr64 + * ... code here ... + * ret + * + * direct caller: + * call foo / call foo+4 + * + * indirect caller: + * lea foo(%rip), %r11 + * ... + * movl $0x12345678, %r10d + * subl $16, %r11 + * nop4 + * call *%r11 + * + */ +enum cfi_mode { + CFI_DEFAULT, /* FineIBT if hardware has IBT, otherwise kCFI */ + CFI_OFF, /* Taditional / IBT depending on .config */ + CFI_KCFI, /* Optionally CALL_PADDING, IBT, RETPOLINE */ + CFI_FINEIBT, /* see arch/x86/kernel/alternative.c */ +}; + +extern enum cfi_mode cfi_mode; + struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#define __bpfcall +extern u32 cfi_bpf_hash; + +static inline int cfi_get_offset(void) +{ + switch (cfi_mode) { + case CFI_FINEIBT: + return 16; + case CFI_KCFI: + if (IS_ENABLED(CONFIG_CALL_PADDING)) + return 16; + return 5; + default: + return 0; + } +} +#define cfi_get_offset cfi_get_offset + #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { return BUG_TRAP_TYPE_NONE; } +#define cfi_bpf_hash 0U #endif /* CONFIG_CFI_CLANG */ #endif /* _ASM_X86_CFI_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 73be3931e4f0..d808d3aaec7e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -30,6 +30,7 @@ #include #include #include +#include int __read_mostly alternatives_patched; @@ -832,15 +833,43 @@ void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { } #endif /* CONFIG_X86_KERNEL_IBT */ #ifdef CONFIG_FINEIBT +#define __CFI_DEFAULT CFI_DEFAULT +#elif defined(CONFIG_CFI_CLANG) +#define __CFI_DEFAULT CFI_KCFI +#else +#define __CFI_DEFAULT CFI_OFF +#endif -enum cfi_mode { - CFI_DEFAULT, - CFI_OFF, - CFI_KCFI, - CFI_FINEIBT, -}; +enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT; + +#ifdef CONFIG_CFI_CLANG +struct bpf_insn; + +/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */ +extern unsigned int __bpf_prog_runX(const void *ctx, + const struct bpf_insn *insn); + +/* + * Force a reference to the external symbol so the compiler generates + * __kcfi_typid. + */ +__ADDRESSABLE(__bpf_prog_runX); + +/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ +asm ( +" .pushsection .data..ro_after_init,\"aw\",@progbits \n" +" .type cfi_bpf_hash,@object \n" +" .globl cfi_bpf_hash \n" +" .p2align 2, 0x0 \n" +"cfi_bpf_hash: \n" +" .long __kcfi_typeid___bpf_prog_runX \n" +" .size cfi_bpf_hash, 4 \n" +" .popsection \n" +); +#endif + +#ifdef CONFIG_FINEIBT -static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT; static bool cfi_rand __ro_after_init = true; static u32 cfi_seed __ro_after_init; @@ -1149,8 +1178,10 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, goto err; if (cfi_rand) { - if (builtin) + if (builtin) { cfi_seed = get_random_u32(); + cfi_bpf_hash = cfi_rehash(cfi_bpf_hash); + } ret = cfi_rand_preamble(start_cfi, end_cfi); if (ret) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index af4a5de7d93a..5d5b967b111d 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -17,6 +17,7 @@ #include #include #include +#include static bool all_callee_regs_used[4] = {true, true, true, true}; @@ -51,9 +52,11 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) #ifdef CONFIG_X86_KERNEL_IBT -#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR() EMIT(gen_endbr(), 4) +#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4) #else #define EMIT_ENDBR() +#define EMIT_ENDBR_POISON() #endif static bool is_imm8(int value) @@ -304,6 +307,69 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) *pprog = prog; } +/* + * Emit the various CFI preambles, see asm/cfi.h and the comments about FineIBT + * in arch/x86/kernel/alternative.c + */ + +static void emit_fineibt(u8 **pprog) +{ + u8 *prog = *pprog; + + EMIT_ENDBR(); + EMIT3_off32(0x41, 0x81, 0xea, cfi_bpf_hash); /* subl $hash, %r10d */ + EMIT2(0x74, 0x07); /* jz.d8 +7 */ + EMIT2(0x0f, 0x0b); /* ud2 */ + EMIT1(0x90); /* nop */ + EMIT_ENDBR_POISON(); + + *pprog = prog; +} + +static void emit_kcfi(u8 **pprog) +{ + u8 *prog = *pprog; + + EMIT1_off32(0xb8, cfi_bpf_hash); /* movl $hash, %eax */ +#ifdef CONFIG_CALL_PADDING + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); + EMIT1(0x90); +#endif + EMIT_ENDBR(); + + *pprog = prog; +} + +static void emit_cfi(u8 **pprog) +{ + u8 *prog = *pprog; + + switch (cfi_mode) { + case CFI_FINEIBT: + emit_fineibt(&prog); + break; + + case CFI_KCFI: + emit_kcfi(&prog); + break; + + default: + EMIT_ENDBR(); + break; + } + + *pprog = prog; +} + /* * Emit x86-64 prologue code for BPF program. * bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes @@ -315,10 +381,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, { u8 *prog = *pprog; + emit_cfi(&prog); /* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ - EMIT_ENDBR(); memcpy(prog, x86_nops[5], X86_PATCH_SIZE); prog += X86_PATCH_SIZE; if (!ebpf_from_cbpf) { @@ -3013,9 +3079,16 @@ out_image: jit_data->header = header; jit_data->rw_header = rw_header; } - prog->bpf_func = (void *)image; + /* + * ctx.prog_offset is used when CFI preambles put code *before* + * the function. See emit_cfi(). For FineIBT specifically this code + * can also be executed and bpf_prog_kallsyms_add() will + * generate an additional symbol to cover this, hence also + * decrement proglen. + */ + prog->bpf_func = (void *)image + cfi_get_offset(); prog->jited = 1; - prog->jited_len = proglen; + prog->jited_len = proglen - cfi_get_offset(); } else { prog = orig_prog; } @@ -3070,6 +3143,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->addrs); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c87c608a3689..9d84c376851a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -29,6 +29,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -1211,7 +1212,11 @@ struct bpf_dispatcher { #endif }; -static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( +#ifndef __bpfcall +#define __bpfcall __nocfi +#endif + +static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func( const void *ctx, const struct bpf_insn *insnsi, bpf_func_t bpf_func) @@ -1303,7 +1308,7 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func #define DEFINE_BPF_DISPATCHER(name) \ __BPF_DISPATCHER_SC(name); \ - noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ + noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func) \ @@ -1453,6 +1458,9 @@ struct bpf_prog_aux { struct bpf_kfunc_desc_tab *kfunc_tab; struct bpf_kfunc_btf_tab *kfunc_btf_tab; u32 size_poke_tab; +#ifdef CONFIG_FINEIBT + struct bpf_ksym ksym_prefix; +#endif struct bpf_ksym ksym; const struct bpf_prog_ops *ops; struct bpf_map **used_maps; diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 2309d74e77e6..1ed2d96c0cfc 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -11,6 +11,13 @@ #include #include +#ifndef cfi_get_offset +static inline int cfi_get_offset(void) +{ + return 0; +} +#endif + #ifdef CONFIG_CFI_CLANG enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, unsigned long *target, u32 type); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c34513d645c4..5aa6863ac33b 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -121,6 +121,9 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag #endif INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); +#ifdef CONFIG_FINEIBT + INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode); +#endif mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->dst_mutex); @@ -683,6 +686,23 @@ void bpf_prog_kallsyms_add(struct bpf_prog *fp) fp->aux->ksym.prog = true; bpf_ksym_add(&fp->aux->ksym); + +#ifdef CONFIG_FINEIBT + /* + * When FineIBT, code in the __cfi_foo() symbols can get executed + * and hence unwinder needs help. + */ + if (cfi_mode != CFI_FINEIBT) + return; + + snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN, + "__cfi_%s", fp->aux->ksym.name); + + fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16; + fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func; + + bpf_ksym_add(&fp->aux->ksym_prefix); +#endif } void bpf_prog_kallsyms_del(struct bpf_prog *fp) @@ -691,6 +711,11 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp) return; bpf_ksym_del(&fp->aux->ksym); +#ifdef CONFIG_FINEIBT + if (cfi_mode != CFI_FINEIBT) + return; + bpf_ksym_del(&fp->aux->ksym_prefix); +#endif } static struct bpf_ksym *bpf_ksym_find(unsigned long addr) -- cgit v1.2.3 From 2cd3e3772e41377f32d6eea643e0590774e9187c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 15 Dec 2023 10:12:20 +0100 Subject: x86/cfi,bpf: Fix bpf_struct_ops CFI BPF struct_ops uses __arch_prepare_bpf_trampoline() to write trampolines for indirect function calls. These tramplines much have matching CFI. In order to obtain the correct CFI hash for the various methods, add a matching structure that contains stub functions, the compiler will generate correct CFI which we can pilfer for the trampolines. Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20231215092707.566977112@infradead.org Signed-off-by: Alexei Starovoitov --- arch/x86/include/asm/cfi.h | 6 ++++ arch/x86/kernel/alternative.c | 22 ++++++++++++++ arch/x86/net/bpf_jit_comp.c | 66 ++++++++++++++++++++++++++-------------- include/linux/bpf.h | 13 ++++++++ kernel/bpf/bpf_struct_ops.c | 16 +++++----- net/bpf/bpf_dummy_struct_ops.c | 31 ++++++++++++++++++- net/ipv4/bpf_tcp_ca.c | 69 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 191 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 8779abd217b7..1a50b2cd4713 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -123,6 +123,8 @@ static inline int cfi_get_offset(void) } #define cfi_get_offset cfi_get_offset +extern u32 cfi_get_func_hash(void *func); + #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { @@ -130,6 +132,10 @@ static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) } #define cfi_bpf_hash 0U #define cfi_bpf_subprog_hash 0U +static inline u32 cfi_get_func_hash(void *func) +{ + return 0; +} #endif /* CONFIG_CFI_CLANG */ #endif /* _ASM_X86_CFI_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index cb393efd5ccd..49c2a62ba5e4 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -883,6 +883,28 @@ asm ( " .size cfi_bpf_subprog_hash, 4 \n" " .popsection \n" ); + +u32 cfi_get_func_hash(void *func) +{ + u32 hash; + + func -= cfi_get_offset(); + switch (cfi_mode) { + case CFI_FINEIBT: + func += 7; + break; + case CFI_KCFI: + func += 1; + break; + default: + return 0; + } + + if (get_kernel_nofault(hash, func)) + return 0; + + return hash; +} #endif #ifdef CONFIG_FINEIBT diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 43b8c08fdf8d..c89a4abdd726 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -312,9 +312,8 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) * in arch/x86/kernel/alternative.c */ -static void emit_fineibt(u8 **pprog, bool is_subprog) +static void emit_fineibt(u8 **pprog, u32 hash) { - u32 hash = is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash; u8 *prog = *pprog; EMIT_ENDBR(); @@ -327,9 +326,8 @@ static void emit_fineibt(u8 **pprog, bool is_subprog) *pprog = prog; } -static void emit_kcfi(u8 **pprog, bool is_subprog) +static void emit_kcfi(u8 **pprog, u32 hash) { - u32 hash = is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash; u8 *prog = *pprog; EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ @@ -351,17 +349,17 @@ static void emit_kcfi(u8 **pprog, bool is_subprog) *pprog = prog; } -static void emit_cfi(u8 **pprog, bool is_subprog) +static void emit_cfi(u8 **pprog, u32 hash) { u8 *prog = *pprog; switch (cfi_mode) { case CFI_FINEIBT: - emit_fineibt(&prog, is_subprog); + emit_fineibt(&prog, hash); break; case CFI_KCFI: - emit_kcfi(&prog, is_subprog); + emit_kcfi(&prog, hash); break; default: @@ -383,7 +381,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, { u8 *prog = *pprog; - emit_cfi(&prog, is_subprog); + emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash); /* BPF trampoline can be made to work without these nops, * but let's waste 5 bytes for now and optimize later */ @@ -2510,10 +2508,19 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im u8 *prog; bool save_ret; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is + * explicitly incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG + * because @func_addr. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) + for (i = 0; i < m->nr_args; i++) { if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) nr_regs += (m->arg_size[i] + 7) / 8 - 1; + } /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. @@ -2596,20 +2603,27 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im prog = rw_image; - EMIT_ENDBR(); - /* - * This is the direct-call trampoline, as such it needs accounting - * for the __fentry__ call. - */ - x86_call_depth_emit_accounting(&prog, NULL); + if (flags & BPF_TRAMP_F_INDIRECT) { + /* + * Indirect call for bpf_struct_ops + */ + emit_cfi(&prog, cfi_get_func_hash(func_addr)); + } else { + /* + * Direct-call fentry stub, as such it needs accounting for the + * __fentry__ call. + */ + x86_call_depth_emit_accounting(&prog, NULL); + } EMIT1(0x55); /* push rbp */ EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ - if (!is_imm8(stack_size)) + if (!is_imm8(stack_size)) { /* sub rsp, stack_size */ EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - else + } else { /* sub rsp, stack_size */ EMIT4(0x48, 0x83, 0xEC, stack_size); + } if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) EMIT1(0x50); /* push rax */ /* mov QWORD PTR [rbp - rbx_off], rbx */ @@ -2643,10 +2657,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } } - if (fentry->nr_links) + if (fentry->nr_links) { if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image)) return -EINVAL; + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *), @@ -2665,11 +2680,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im restore_regs(m, &prog, regs_off); save_args(m, &prog, arg_stack_off, true); - if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } if (flags & BPF_TRAMP_F_ORIG_STACK) { emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); @@ -2698,17 +2714,19 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Update the branches saved in invoke_bpf_mod_ret with the * aligned address of do_fexit. */ - for (i = 0; i < fmod_ret->nr_links; i++) + for (i = 0; i < fmod_ret->nr_links; i++) { emit_cond_near_jump(&branches[i], image + (prog - (u8 *)rw_image), image + (branches[i] - (u8 *)rw_image), X86_JNE); + } } - if (fexit->nr_links) + if (fexit->nr_links) { if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false, image, rw_image)) { ret = -EINVAL; goto cleanup; } + } if (flags & BPF_TRAMP_F_RESTORE_REGS) restore_regs(m, &prog, regs_off); @@ -2725,11 +2743,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im ret = -EINVAL; goto cleanup; } - } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before running the original function, restore the * tail_call_cnt from stack to rax. */ RESTORE_TAIL_CALL_CNT(stack_size); + } /* restore return value of orig_call or fentry prog back into RAX */ if (save_ret) @@ -2737,9 +2756,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); EMIT1(0xC9); /* leave */ - if (flags & BPF_TRAMP_F_SKIP_FRAME) + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ + } emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ if (WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 9d84c376851a..db46b3359bf5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1060,6 +1060,17 @@ struct btf_func_model { */ #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) +/* + * Indicate the trampoline should be suitable to receive indirect calls; + * without this indirectly calling the generated code can result in #UD/#CP, + * depending on the CFI options. + * + * Used by bpf_struct_ops. + * + * Incompatible with FENTRY usage, overloads @func_addr argument. + */ +#define BPF_TRAMP_F_INDIRECT BIT(8) + /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ @@ -1697,6 +1708,7 @@ struct bpf_struct_ops { struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; u32 type_id; u32 value_id; + void *cfi_stubs; }; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) @@ -1710,6 +1722,7 @@ int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, + void *stub_func, void *image, void *image_end); static inline bool bpf_try_module_get(const void *data, struct module *owner) { diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 4d53c53fc5aa..02068bd0e4d9 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -352,17 +352,16 @@ const struct bpf_link_ops bpf_struct_ops_link_lops = { int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, - void *image, void *image_end) + void *stub_func, void *image, void *image_end) { - u32 flags; + u32 flags = BPF_TRAMP_F_INDIRECT; int size; tlinks[BPF_TRAMP_FENTRY].links[0] = link; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; - /* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops, - * and it must be used alone. - */ - flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0; + + if (model->ret_size > 0) + flags |= BPF_TRAMP_F_RET_FENTRY_RET; size = arch_bpf_trampoline_size(model, flags, tlinks, NULL); if (size < 0) @@ -370,7 +369,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, if (size > (unsigned long)image_end - (unsigned long)image) return -E2BIG; return arch_prepare_bpf_trampoline(NULL, image, image_end, - model, flags, tlinks, NULL); + model, flags, tlinks, stub_func); } static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, @@ -504,11 +503,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[i], + *(void **)(st_ops->cfi_stubs + moff), image, image_end); if (err < 0) goto reset_unlock; - *(void **)(kdata + moff) = image; + *(void **)(kdata + moff) = image + cfi_get_offset(); image += err; /* put prog_id to udata */ diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c index 2748f9d77b18..8906f7bdf4a9 100644 --- a/net/bpf/bpf_dummy_struct_ops.c +++ b/net/bpf/bpf_dummy_struct_ops.c @@ -12,6 +12,11 @@ extern struct bpf_struct_ops bpf_bpf_dummy_ops; /* A common type for test_N with return value in bpf_dummy_ops */ typedef int (*dummy_ops_test_ret_fn)(struct bpf_dummy_ops_state *state, ...); +static int dummy_ops_test_ret_function(struct bpf_dummy_ops_state *state, ...) +{ + return 0; +} + struct bpf_dummy_ops_test_args { u64 args[MAX_BPF_FUNC_ARGS]; struct bpf_dummy_ops_state state; @@ -62,7 +67,7 @@ static int dummy_ops_copy_args(struct bpf_dummy_ops_test_args *args) static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args) { - dummy_ops_test_ret_fn test = (void *)image; + dummy_ops_test_ret_fn test = (void *)image + cfi_get_offset(); struct bpf_dummy_ops_state *state = NULL; /* state needs to be NULL if args[0] is 0 */ @@ -119,6 +124,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, op_idx = prog->expected_attach_type; err = bpf_struct_ops_prepare_trampoline(tlinks, link, &st_ops->func_models[op_idx], + &dummy_ops_test_ret_function, image, image + PAGE_SIZE); if (err < 0) goto out; @@ -219,6 +225,28 @@ static void bpf_dummy_unreg(void *kdata) { } +static int bpf_dummy_test_1(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static int bpf_dummy_test_2(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, + char a3, unsigned long a4) +{ + return 0; +} + +static int bpf_dummy_test_sleepable(struct bpf_dummy_ops_state *cb) +{ + return 0; +} + +static struct bpf_dummy_ops __bpf_bpf_dummy_ops = { + .test_1 = bpf_dummy_test_1, + .test_2 = bpf_dummy_test_2, + .test_sleepable = bpf_dummy_test_sleepable, +}; + struct bpf_struct_ops bpf_bpf_dummy_ops = { .verifier_ops = &bpf_dummy_verifier_ops, .init = bpf_dummy_init, @@ -227,4 +255,5 @@ struct bpf_struct_ops bpf_bpf_dummy_ops = { .reg = bpf_dummy_reg, .unreg = bpf_dummy_unreg, .name = "bpf_dummy_ops", + .cfi_stubs = &__bpf_bpf_dummy_ops, }; diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index c7bbd8f3c708..634cfafa583d 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -271,6 +271,74 @@ static int bpf_tcp_ca_validate(void *kdata) return tcp_validate_congestion_control(kdata); } +static u32 bpf_tcp_ca_ssthresh(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_avoid(struct sock *sk, u32 ack, u32 acked) +{ +} + +static void bpf_tcp_ca_set_state(struct sock *sk, u8 new_state) +{ +} + +static void bpf_tcp_ca_cwnd_event(struct sock *sk, enum tcp_ca_event ev) +{ +} + +static void bpf_tcp_ca_in_ack_event(struct sock *sk, u32 flags) +{ +} + +static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *sample) +{ +} + +static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +{ + return 0; +} + +static void bpf_tcp_ca_cong_control(struct sock *sk, const struct rate_sample *rs) +{ +} + +static u32 bpf_tcp_ca_undo_cwnd(struct sock *sk) +{ + return 0; +} + +static u32 bpf_tcp_ca_sndbuf_expand(struct sock *sk) +{ + return 0; +} + +static void __bpf_tcp_ca_init(struct sock *sk) +{ +} + +static void __bpf_tcp_ca_release(struct sock *sk) +{ +} + +static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { + .ssthresh = bpf_tcp_ca_ssthresh, + .cong_avoid = bpf_tcp_ca_cong_avoid, + .set_state = bpf_tcp_ca_set_state, + .cwnd_event = bpf_tcp_ca_cwnd_event, + .in_ack_event = bpf_tcp_ca_in_ack_event, + .pkts_acked = bpf_tcp_ca_pkts_acked, + .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .cong_control = bpf_tcp_ca_cong_control, + .undo_cwnd = bpf_tcp_ca_undo_cwnd, + .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, + + .init = __bpf_tcp_ca_init, + .release = __bpf_tcp_ca_release, +}; + struct bpf_struct_ops bpf_tcp_congestion_ops = { .verifier_ops = &bpf_tcp_ca_verifier_ops, .reg = bpf_tcp_ca_reg, @@ -281,6 +349,7 @@ struct bpf_struct_ops bpf_tcp_congestion_ops = { .init = bpf_tcp_ca_init, .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", + .cfi_stubs = &__bpf_ops_tcp_congestion_ops, }; static int __init bpf_tcp_ca_kfunc_init(void) -- cgit v1.2.3 From e9d13b9d2f99ccf7afeab490d97eaa5ac9846598 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 15 Dec 2023 10:12:21 +0100 Subject: cfi: Add CFI_NOSEAL() Add a CFI_NOSEAL() helper to mark functions that need to retain their CFI information, despite not otherwise leaking their address. Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20231215092707.669401084@infradead.org Signed-off-by: Alexei Starovoitov --- arch/x86/include/asm/cfi.h | 5 +++++ include/linux/cfi.h | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index 1a50b2cd4713..7cd752557905 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -8,6 +8,7 @@ * Copyright (C) 2022 Google LLC */ #include +#include /* * An overview of the various calling conventions... @@ -138,4 +139,8 @@ static inline u32 cfi_get_func_hash(void *func) } #endif /* CONFIG_CFI_CLANG */ +#if HAS_KERNEL_IBT == 1 +#define CFI_NOSEAL(x) asm(IBT_NOSEAL(__stringify(x))) +#endif + #endif /* _ASM_X86_CFI_H */ diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 1ed2d96c0cfc..f0df518e11dd 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -46,4 +46,8 @@ static inline void module_cfi_finalize(const Elf_Ehdr *hdr, #endif /* CONFIG_ARCH_USES_CFI_TRAPS */ #endif /* CONFIG_MODULES */ +#ifndef CFI_NOSEAL +#define CFI_NOSEAL(x) +#endif + #endif /* _LINUX_CFI_H */ -- cgit v1.2.3 From 852486b35f344887786d63250946dd921a05d7e8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 15 Dec 2023 10:12:23 +0100 Subject: x86/cfi,bpf: Fix bpf_exception_cb() signature As per the earlier patches, BPF sub-programs have bpf_callback_t signature and CFI expects callers to have matching signature. This is violated by bpf_prog_aux::bpf_exception_cb(). [peterz: Changelog] Reported-by: Peter Zijlstra Signed-off-by: Alexei Starovoitov Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/CAADnVQ+Z7UcXXBBhMubhcMM=R-dExk-uHtfOLtoLxQ1XxEpqEA@mail.gmail.com Link: https://lore.kernel.org/r/20231215092707.910319166@infradead.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 +- kernel/bpf/helpers.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index db46b3359bf5..5e694934cf37 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1484,7 +1484,7 @@ struct bpf_prog_aux { int cgroup_atype; /* enum cgroup_bpf_attach_type */ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; char name[BPF_OBJ_NAME_LEN]; - unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp); + u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); #ifdef CONFIG_SECURITY void *security; #endif diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index e0c0e3676df8..07fd4b5704f3 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2537,7 +2537,7 @@ __bpf_kfunc void bpf_throw(u64 cookie) * which skips compiler generated instrumentation to do the same. */ kasan_unpoison_task_stack_below((void *)(long)ctx.sp); - ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp); + ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } -- cgit v1.2.3 From ebb30ccbbdbd6fae5177b676da4f4ac92bb4f635 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 15 Dec 2023 14:15:31 +0100 Subject: net: phy: make addr type u8 in phy_package_shared struct Switch addr type in phy_package_shared struct to u8. The value is already checked to be non negative and to be less than PHY_MAX_ADDR, hence u8 is better suited than using int. Signed-off-by: Christian Marangi Reviewed-by: Russell King (Oracle) Signed-off-by: David S. Miller --- include/linux/phy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index dbb5e13e3e1b..4b13cc85c4f5 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -338,7 +338,7 @@ struct mdio_bus_stats { * phy_package_leave(). */ struct phy_package_shared { - int addr; + u8 addr; refcount_t refcnt; unsigned long flags; size_t priv_size; -- cgit v1.2.3 From 9eea577eb1155fe4a183bc5e7bf269b0b2e7a6ba Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 15 Dec 2023 14:15:32 +0100 Subject: net: phy: extend PHY package API to support multiple global address Current API for PHY package are limited to single address to configure global settings for the PHY package. It was found that some PHY package (for example the qca807x, a PHY package that is shipped with a bundle of 5 PHY) requires multiple PHY address to configure global settings. An example scenario is a PHY that have a dedicated PHY for PSGMII/serdes calibrarion and have a specific PHY in the package where the global PHY mode is set and affects every other PHY in the package. Change the API in the following way: - Change phy_package_join() to take the base addr of the PHY package instead of the global PHY addr. - Make __/phy_package_write/read() require an additional arg that select what global PHY address to use by passing the offset from the base addr passed on phy_package_join(). Each user of this API is updated to follow this new implementation following a pattern where an enum is defined to declare the offset of the addr. We also drop the check if shared is defined as any user of the phy_package_read/write is expected to use phy_package_join first. Misuse of this will correctly trigger a kernel panic for NULL pointer exception. Signed-off-by: Christian Marangi Signed-off-by: David S. Miller --- drivers/net/phy/bcm54140.c | 16 +++++++--- drivers/net/phy/mscc/mscc.h | 5 ++++ drivers/net/phy/mscc/mscc_main.c | 4 +-- drivers/net/phy/phy_device.c | 35 ++++++++++++---------- include/linux/phy.h | 64 ++++++++++++++++++++++++++-------------- 5 files changed, 80 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/bcm54140.c b/drivers/net/phy/bcm54140.c index d43076592f81..2eea3d09b1e6 100644 --- a/drivers/net/phy/bcm54140.c +++ b/drivers/net/phy/bcm54140.c @@ -128,6 +128,10 @@ #define BCM54140_DEFAULT_DOWNSHIFT 5 #define BCM54140_MAX_DOWNSHIFT 9 +enum bcm54140_global_phy { + BCM54140_BASE_ADDR = 0, +}; + struct bcm54140_priv { int port; int base_addr; @@ -429,11 +433,13 @@ static int bcm54140_base_read_rdb(struct phy_device *phydev, u16 rdb) int ret; phy_lock_mdio_bus(phydev); - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_ADDR, rdb); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_ADDR, rdb); if (ret < 0) goto out; - ret = __phy_package_read(phydev, MII_BCM54XX_RDB_DATA); + ret = __phy_package_read(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_DATA); out: phy_unlock_mdio_bus(phydev); @@ -446,11 +452,13 @@ static int bcm54140_base_write_rdb(struct phy_device *phydev, int ret; phy_lock_mdio_bus(phydev); - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_ADDR, rdb); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_ADDR, rdb); if (ret < 0) goto out; - ret = __phy_package_write(phydev, MII_BCM54XX_RDB_DATA, val); + ret = __phy_package_write(phydev, BCM54140_BASE_ADDR, + MII_BCM54XX_RDB_DATA, val); out: phy_unlock_mdio_bus(phydev); diff --git a/drivers/net/phy/mscc/mscc.h b/drivers/net/phy/mscc/mscc.h index 7a962050a4d4..6a3d8a754eb8 100644 --- a/drivers/net/phy/mscc/mscc.h +++ b/drivers/net/phy/mscc/mscc.h @@ -416,6 +416,11 @@ struct vsc8531_private { * gpio_lock: used for PHC operations. Common for all PHYs as the load/save GPIO * is shared. */ + +enum vsc85xx_global_phy { + VSC88XX_BASE_ADDR = 0, +}; + struct vsc85xx_shared_private { struct mutex gpio_lock; }; diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c index 4171f01d34e5..6f74ce0ab1aa 100644 --- a/drivers/net/phy/mscc/mscc_main.c +++ b/drivers/net/phy/mscc/mscc_main.c @@ -711,7 +711,7 @@ int phy_base_write(struct phy_device *phydev, u32 regnum, u16 val) dump_stack(); } - return __phy_package_write(phydev, regnum, val); + return __phy_package_write(phydev, VSC88XX_BASE_ADDR, regnum, val); } /* phydev->bus->mdio_lock should be locked when using this function */ @@ -722,7 +722,7 @@ int phy_base_read(struct phy_device *phydev, u32 regnum) dump_stack(); } - return __phy_package_read(phydev, regnum); + return __phy_package_read(phydev, VSC88XX_BASE_ADDR, regnum); } u32 vsc85xx_csr_read(struct phy_device *phydev, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index d8e9335d415c..0c52a9eff188 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1651,20 +1651,22 @@ EXPORT_SYMBOL_GPL(phy_driver_is_genphy_10g); /** * phy_package_join - join a common PHY group * @phydev: target phy_device struct - * @addr: cookie and PHY address for global register access + * @base_addr: cookie and base PHY address of PHY package for offset + * calculation of global register access * @priv_size: if non-zero allocate this amount of bytes for private data * * This joins a PHY group and provides a shared storage for all phydevs in * this group. This is intended to be used for packages which contain * more than one PHY, for example a quad PHY transceiver. * - * The addr parameter serves as a cookie which has to have the same value - * for all members of one group and as a PHY address to access generic - * registers of a PHY package. Usually, one of the PHY addresses of the - * different PHYs in the package provides access to these global registers. + * The base_addr parameter serves as cookie which has to have the same values + * for all members of one group and as the base PHY address of the PHY package + * for offset calculation to access generic registers of a PHY package. + * Usually, one of the PHY addresses of the different PHYs in the package + * provides access to these global registers. * The address which is given here, will be used in the phy_package_read() - * and phy_package_write() convenience functions. If your PHY doesn't have - * global registers you can just pick any of the PHY addresses. + * and phy_package_write() convenience functions as base and added to the + * passed offset in those functions. * * This will set the shared pointer of the phydev to the shared storage. * If this is the first call for a this cookie the shared storage will be @@ -1674,17 +1676,17 @@ EXPORT_SYMBOL_GPL(phy_driver_is_genphy_10g); * Returns < 1 on error, 0 on success. Esp. calling phy_package_join() * with the same cookie but a different priv_size is an error. */ -int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size) +int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size) { struct mii_bus *bus = phydev->mdio.bus; struct phy_package_shared *shared; int ret; - if (addr < 0 || addr >= PHY_MAX_ADDR) + if (base_addr < 0 || base_addr >= PHY_MAX_ADDR) return -EINVAL; mutex_lock(&bus->shared_lock); - shared = bus->shared[addr]; + shared = bus->shared[base_addr]; if (!shared) { ret = -ENOMEM; shared = kzalloc(sizeof(*shared), GFP_KERNEL); @@ -1696,9 +1698,9 @@ int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size) goto err_free; shared->priv_size = priv_size; } - shared->addr = addr; + shared->base_addr = base_addr; refcount_set(&shared->refcnt, 1); - bus->shared[addr] = shared; + bus->shared[base_addr] = shared; } else { ret = -EINVAL; if (priv_size && priv_size != shared->priv_size) @@ -1736,7 +1738,7 @@ void phy_package_leave(struct phy_device *phydev) return; if (refcount_dec_and_mutex_lock(&shared->refcnt, &bus->shared_lock)) { - bus->shared[shared->addr] = NULL; + bus->shared[shared->base_addr] = NULL; mutex_unlock(&bus->shared_lock); kfree(shared->priv); kfree(shared); @@ -1755,7 +1757,8 @@ static void devm_phy_package_leave(struct device *dev, void *res) * devm_phy_package_join - resource managed phy_package_join() * @dev: device that is registering this PHY package * @phydev: target phy_device struct - * @addr: cookie and PHY address for global register access + * @base_addr: cookie and base PHY address of PHY package for offset + * calculation of global register access * @priv_size: if non-zero allocate this amount of bytes for private data * * Managed phy_package_join(). Shared storage fetched by this function, @@ -1763,7 +1766,7 @@ static void devm_phy_package_leave(struct device *dev, void *res) * phy_package_join() for more information. */ int devm_phy_package_join(struct device *dev, struct phy_device *phydev, - int addr, size_t priv_size) + int base_addr, size_t priv_size) { struct phy_device **ptr; int ret; @@ -1773,7 +1776,7 @@ int devm_phy_package_join(struct device *dev, struct phy_device *phydev, if (!ptr) return -ENOMEM; - ret = phy_package_join(phydev, addr, priv_size); + ret = phy_package_join(phydev, base_addr, priv_size); if (!ret) { *ptr = phydev; diff --git a/include/linux/phy.h b/include/linux/phy.h index 4b13cc85c4f5..d653f660c39d 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -327,7 +327,8 @@ struct mdio_bus_stats { /** * struct phy_package_shared - Shared information in PHY packages - * @addr: Common PHY address used to combine PHYs in one package + * @base_addr: Base PHY address of PHY package used to combine PHYs + * in one package and for offset calculation of phy_package_read/write * @refcnt: Number of PHYs connected to this shared data * @flags: Initialization of PHY package * @priv_size: Size of the shared private data @priv @@ -338,7 +339,7 @@ struct mdio_bus_stats { * phy_package_leave(). */ struct phy_package_shared { - u8 addr; + u8 base_addr; refcount_t refcnt; unsigned long flags; size_t priv_size; @@ -1976,10 +1977,10 @@ int phy_ethtool_get_link_ksettings(struct net_device *ndev, int phy_ethtool_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *cmd); int phy_ethtool_nway_reset(struct net_device *ndev); -int phy_package_join(struct phy_device *phydev, int addr, size_t priv_size); +int phy_package_join(struct phy_device *phydev, int base_addr, size_t priv_size); void phy_package_leave(struct phy_device *phydev); int devm_phy_package_join(struct device *dev, struct phy_device *phydev, - int addr, size_t priv_size); + int base_addr, size_t priv_size); int __init mdio_bus_init(void); void mdio_bus_exit(void); @@ -2002,46 +2003,65 @@ int __phy_hwtstamp_set(struct phy_device *phydev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack); -static inline int phy_package_read(struct phy_device *phydev, u32 regnum) +static inline int phy_package_address(struct phy_device *phydev, + unsigned int addr_offset) { struct phy_package_shared *shared = phydev->shared; + u8 base_addr = shared->base_addr; - if (!shared) + if (addr_offset >= PHY_MAX_ADDR - base_addr) return -EIO; - return mdiobus_read(phydev->mdio.bus, shared->addr, regnum); + /* we know that addr will be in the range 0..31 and thus the + * implicit cast to a signed int is not a problem. + */ + return base_addr + addr_offset; } -static inline int __phy_package_read(struct phy_device *phydev, u32 regnum) +static inline int phy_package_read(struct phy_device *phydev, + unsigned int addr_offset, u32 regnum) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; + + return mdiobus_read(phydev->mdio.bus, addr, regnum); +} + +static inline int __phy_package_read(struct phy_device *phydev, + unsigned int addr_offset, u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; - return __mdiobus_read(phydev->mdio.bus, shared->addr, regnum); + return __mdiobus_read(phydev->mdio.bus, addr, regnum); } static inline int phy_package_write(struct phy_device *phydev, - u32 regnum, u16 val) + unsigned int addr_offset, u32 regnum, + u16 val) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; - return mdiobus_write(phydev->mdio.bus, shared->addr, regnum, val); + return mdiobus_write(phydev->mdio.bus, addr, regnum, val); } static inline int __phy_package_write(struct phy_device *phydev, - u32 regnum, u16 val) + unsigned int addr_offset, u32 regnum, + u16 val) { - struct phy_package_shared *shared = phydev->shared; + int addr = phy_package_address(phydev, addr_offset); - if (!shared) - return -EIO; + if (addr < 0) + return addr; - return __mdiobus_write(phydev->mdio.bus, shared->addr, regnum, val); + return __mdiobus_write(phydev->mdio.bus, addr, regnum, val); } static inline bool __phy_package_set_once(struct phy_device *phydev, -- cgit v1.2.3 From d63710fc0f1a501fd75a7025e3070a96ffa1645f Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 15 Dec 2023 14:15:34 +0100 Subject: net: phy: add support for PHY package MMD read/write Some PHY in PHY package may require to read/write MMD regs to correctly configure the PHY package. Add support for these additional required function in both lock and no lock variant. It's assumed that the entire PHY package is either C22 or C45. We use C22 or C45 way of writing/reading to mmd regs based on the passed phydev whether it's C22 or C45. Signed-off-by: Christian Marangi Signed-off-by: David S. Miller --- drivers/net/phy/phy-core.c | 140 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/phy.h | 16 ++++++ 2 files changed, 156 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c index b729ac8b2640..15f349e5995a 100644 --- a/drivers/net/phy/phy-core.c +++ b/drivers/net/phy/phy-core.c @@ -650,6 +650,146 @@ int phy_write_mmd(struct phy_device *phydev, int devad, u32 regnum, u16 val) } EXPORT_SYMBOL(phy_write_mmd); +/** + * __phy_package_read_mmd - read MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to read from + * @regnum: The register on the MMD to read + * + * Convenience helper for reading a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for __phy_read(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int __phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + return mmd_phy_read(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum); +} +EXPORT_SYMBOL(__phy_package_read_mmd); + +/** + * phy_package_read_mmd - read MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to read from + * @regnum: The register on the MMD to read + * + * Convenience helper for reading a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for phy_read(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum) +{ + int addr = phy_package_address(phydev, addr_offset); + int val; + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + phy_lock_mdio_bus(phydev); + val = mmd_phy_read(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum); + phy_unlock_mdio_bus(phydev); + + return val; +} +EXPORT_SYMBOL(phy_package_read_mmd); + +/** + * __phy_package_write_mmd - write MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to write to + * @regnum: The register on the MMD to write + * @val: value to write to @regnum + * + * Convenience helper for writing a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for __phy_write(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int __phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val) +{ + int addr = phy_package_address(phydev, addr_offset); + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + return mmd_phy_write(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum, val); +} +EXPORT_SYMBOL(__phy_package_write_mmd); + +/** + * phy_package_write_mmd - write MMD reg relative to PHY package base addr + * @phydev: The phy_device struct + * @addr_offset: The offset to be added to PHY package base_addr + * @devad: The MMD to write to + * @regnum: The register on the MMD to write + * @val: value to write to @regnum + * + * Convenience helper for writing a register of an MMD on a given PHY + * using the PHY package base address. The base address is added to + * the addr_offset value. + * + * Same calling rules as for phy_write(); + * + * NOTE: It's assumed that the entire PHY package is either C22 or C45. + */ +int phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val) +{ + int addr = phy_package_address(phydev, addr_offset); + int ret; + + if (addr < 0) + return addr; + + if (regnum > (u16)~0 || devad > 32) + return -EINVAL; + + phy_lock_mdio_bus(phydev); + ret = mmd_phy_write(phydev->mdio.bus, addr, phydev->is_c45, devad, + regnum, val); + phy_unlock_mdio_bus(phydev); + + return ret; +} +EXPORT_SYMBOL(phy_package_write_mmd); + /** * phy_modify_changed - Function for modifying a PHY register * @phydev: the phy_device struct diff --git a/include/linux/phy.h b/include/linux/phy.h index d653f660c39d..e9e85d347587 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -2064,6 +2064,22 @@ static inline int __phy_package_write(struct phy_device *phydev, return __mdiobus_write(phydev->mdio.bus, addr, regnum, val); } +int __phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum); + +int phy_package_read_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum); + +int __phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val); + +int phy_package_write_mmd(struct phy_device *phydev, + unsigned int addr_offset, int devad, + u32 regnum, u16 val); + static inline bool __phy_package_set_once(struct phy_device *phydev, unsigned int b) { -- cgit v1.2.3 From 32da0f00ddcb101730cf242289b2b10ede0e1156 Mon Sep 17 00:00:00 2001 From: Jamal Hadi Salim Date: Fri, 15 Dec 2023 14:57:10 -0300 Subject: net: rtnl: introduce rcu_replace_pointer_rtnl Introduce the rcu_replace_pointer_rtnl helper to lockdep check rtnl lock rcu replacements, alongside the already existing helpers. This is a quality of life helper so instead of using: rcu_replace_pointer(rp, p, lockdep_rtnl_is_held()) .. or the open coded.. rtnl_dereference() / rcu_assign_pointer() .. or the lazy check version .. rcu_replace_pointer(rp, p, 1) Use: rcu_replace_pointer_rtnl(rp, p) Signed-off-by: Jamal Hadi Salim Signed-off-by: Victor Nogueira Signed-off-by: Pedro Tammela Reviewed-by: Ido Schimmel Reviewed-by: Nikolay Aleksandrov Signed-off-by: David S. Miller --- include/linux/rtnetlink.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 6a8543b34e2c..410529fca18b 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -79,6 +79,18 @@ static inline bool lockdep_rtnl_is_held(void) #define rtnl_dereference(p) \ rcu_dereference_protected(p, lockdep_rtnl_is_held()) +/** + * rcu_replace_pointer_rtnl - replace an RCU pointer under rtnl_lock, returning + * its old value + * @rp: RCU pointer, whose value is returned + * @p: regular pointer + * + * Perform a replacement under rtnl_lock, where @rp is an RCU-annotated + * pointer. The old value of @rp is returned, and @rp is set to @p + */ +#define rcu_replace_pointer_rtnl(rp, p) \ + rcu_replace_pointer(rp, p, lockdep_rtnl_is_held()) + static inline struct netdev_queue *dev_ingress_queue(struct net_device *dev) { return rtnl_dereference(dev->ingress_queue); -- cgit v1.2.3 From 3314f2097dee43defc20554f961a8b17f4787e2d Mon Sep 17 00:00:00 2001 From: Jesse Brandeburg Date: Tue, 5 Dec 2023 17:01:01 -0800 Subject: intel: add bit macro includes where needed This series is introducing the use of FIELD_GET and FIELD_PREP which requires bitfield.h to be included. Fix all the includes in this one change, and rearrange includes into alphabetical order to ease readability and future maintenance. virtchnl.h and it's usage was modified to have it's own includes as it should. This required including bits.h for virtchnl.h. Reviewed-by: Marcin Szycik Signed-off-by: Jesse Brandeburg Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/e1000/e1000_hw.c | 1 + drivers/net/ethernet/intel/fm10k/fm10k_pf.c | 1 + drivers/net/ethernet/intel/fm10k/fm10k_vf.c | 1 + drivers/net/ethernet/intel/i40e/i40e_common.c | 1 + drivers/net/ethernet/intel/i40e/i40e_dcb.c | 2 ++ drivers/net/ethernet/intel/i40e/i40e_nvm.c | 1 + drivers/net/ethernet/intel/iavf/iavf_common.c | 3 ++- drivers/net/ethernet/intel/iavf/iavf_ethtool.c | 5 +++-- drivers/net/ethernet/intel/iavf/iavf_fdir.c | 1 + drivers/net/ethernet/intel/iavf/iavf_txrx.c | 1 + drivers/net/ethernet/intel/igb/e1000_i210.c | 4 ++-- drivers/net/ethernet/intel/igb/e1000_nvm.c | 4 ++-- drivers/net/ethernet/intel/igb/e1000_phy.c | 4 ++-- drivers/net/ethernet/intel/igbvf/netdev.c | 28 +++++++++++++------------- drivers/net/ethernet/intel/igc/igc_i225.c | 1 + drivers/net/ethernet/intel/igc/igc_phy.c | 1 + include/linux/avf/virtchnl.h | 1 + 17 files changed, 37 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.c b/drivers/net/ethernet/intel/e1000/e1000_hw.c index 4542e2bc28e8..4576511c99f5 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_hw.c +++ b/drivers/net/ethernet/intel/e1000/e1000_hw.c @@ -5,6 +5,7 @@ * Shared functions for accessing and configuring the MAC */ +#include #include "e1000.h" static s32 e1000_check_downshift(struct e1000_hw *hw); diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c index af1b0cde3670..ae700a1807c6 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2019 Intel Corporation. */ +#include #include "fm10k_pf.h" #include "fm10k_vf.h" diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c index dc8ccd378ec9..c50928ec14ff 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2019 Intel Corporation. */ +#include #include "fm10k_vf.h" /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index bd52b73cf61f..522cf2e5f365 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -2,6 +2,7 @@ /* Copyright(c) 2013 - 2021 Intel Corporation. */ #include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb.c b/drivers/net/ethernet/intel/i40e/i40e_dcb.c index 498728e16a37..0d334036ab8b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_dcb.c +++ b/drivers/net/ethernet/intel/i40e/i40e_dcb.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2021 Intel Corporation. */ +#include +#include "i40e_adminq.h" #include "i40e_alloc.h" #include "i40e_dcb.h" #include "i40e_prototype.h" diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c index 62eb34871094..157eacfdc918 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c +++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ +#include #include #include "i40e_alloc.h" #include "i40e_prototype.h" diff --git a/drivers/net/ethernet/intel/iavf/iavf_common.c b/drivers/net/ethernet/intel/iavf/iavf_common.c index 89d2bce529ae..af5cc69f26e3 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_common.c +++ b/drivers/net/ethernet/intel/iavf/iavf_common.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ +#include +#include #include "iavf_type.h" #include "iavf_adminq.h" #include "iavf_prototype.h" -#include /** * iavf_aq_str - convert AQ err code to a string diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index cdb4849f5db4..bffb45802dab 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ +#include +#include + /* ethtool support for iavf */ #include "iavf.h" -#include - /* ethtool statistics helpers */ /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.c b/drivers/net/ethernet/intel/iavf/iavf_fdir.c index 03e774bd2a5b..65ddcd81c993 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_fdir.c +++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.c @@ -3,6 +3,7 @@ /* flow director ethtool support for iavf */ +#include #include "iavf.h" #define GTPU_PORT 2152 diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index d64c4997136b..fb7edba9c2f8 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ +#include #include #include "iavf.h" diff --git a/drivers/net/ethernet/intel/igb/e1000_i210.c b/drivers/net/ethernet/intel/igb/e1000_i210.c index b9b9d35494d2..53b396fd194a 100644 --- a/drivers/net/ethernet/intel/igb/e1000_i210.c +++ b/drivers/net/ethernet/intel/igb/e1000_i210.c @@ -5,9 +5,9 @@ * e1000_i211 */ -#include +#include #include - +#include #include "e1000_hw.h" #include "e1000_i210.h" diff --git a/drivers/net/ethernet/intel/igb/e1000_nvm.c b/drivers/net/ethernet/intel/igb/e1000_nvm.c index fa136e6e9328..0da57e89593a 100644 --- a/drivers/net/ethernet/intel/igb/e1000_nvm.c +++ b/drivers/net/ethernet/intel/igb/e1000_nvm.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2007 - 2018 Intel Corporation. */ -#include +#include #include - +#include #include "e1000_mac.h" #include "e1000_nvm.h" diff --git a/drivers/net/ethernet/intel/igb/e1000_phy.c b/drivers/net/ethernet/intel/igb/e1000_phy.c index a018000f7db9..3c1b562a3271 100644 --- a/drivers/net/ethernet/intel/igb/e1000_phy.c +++ b/drivers/net/ethernet/intel/igb/e1000_phy.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2007 - 2018 Intel Corporation. */ -#include +#include #include - +#include #include "e1000_mac.h" #include "e1000_phy.h" diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c index fd712585af27..e6c1fbee049e 100644 --- a/drivers/net/ethernet/intel/igbvf/netdev.c +++ b/drivers/net/ethernet/intel/igbvf/netdev.c @@ -3,25 +3,25 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include - +#include +#include +#include +#include +#include +#include #include "igbvf.h" char igbvf_driver_name[] = "igbvf"; diff --git a/drivers/net/ethernet/intel/igc/igc_i225.c b/drivers/net/ethernet/intel/igc/igc_i225.c index 17546a035ab1..d2562c8e8015 100644 --- a/drivers/net/ethernet/intel/igc/igc_i225.c +++ b/drivers/net/ethernet/intel/igc/igc_i225.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018 Intel Corporation */ +#include #include #include "igc_hw.h" diff --git a/drivers/net/ethernet/intel/igc/igc_phy.c b/drivers/net/ethernet/intel/igc/igc_phy.c index 53b77c969c85..d0d9e7170154 100644 --- a/drivers/net/ethernet/intel/igc/igc_phy.c +++ b/drivers/net/ethernet/intel/igc/igc_phy.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018 Intel Corporation */ +#include #include "igc_phy.h" /** diff --git a/include/linux/avf/virtchnl.h b/include/linux/avf/virtchnl.h index a44d9dc7e3eb..8e177b67e82f 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/avf/virtchnl.h @@ -5,6 +5,7 @@ #define _VIRTCHNL_H_ #include +#include #include #include -- cgit v1.2.3 From 403863e985e8eba608d53b2907caaf37b6176290 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Sat, 16 Dec 2023 13:29:58 +0100 Subject: netlink: introduce typedef for filter function Make the code using filter function a bit nicer by consolidating the filter function arguments using typedef. Suggested-by: Andy Shevchenko Signed-off-by: Jiri Pirko Signed-off-by: Paolo Abeni --- drivers/connector/connector.c | 5 ++--- include/linux/connector.h | 3 +-- include/linux/netlink.h | 6 ++++-- net/netlink/af_netlink.c | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/connector/connector.c b/drivers/connector/connector.c index 7f7b94f616a6..4028e8eeba82 100644 --- a/drivers/connector/connector.c +++ b/drivers/connector/connector.c @@ -59,9 +59,8 @@ static int cn_already_initialized; * both, or if both are zero then the group is looked up and sent there. */ int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 __group, - gfp_t gfp_mask, - int (*filter)(struct sock *dsk, struct sk_buff *skb, void *data), - void *filter_data) + gfp_t gfp_mask, netlink_filter_fn filter, + void *filter_data) { struct cn_callback_entry *__cbq; unsigned int size; diff --git a/include/linux/connector.h b/include/linux/connector.h index cec2d99ae902..70bc1160f3d8 100644 --- a/include/linux/connector.h +++ b/include/linux/connector.h @@ -100,8 +100,7 @@ void cn_del_callback(const struct cb_id *id); */ int cn_netlink_send_mult(struct cn_msg *msg, u16 len, u32 portid, u32 group, gfp_t gfp_mask, - int (*filter)(struct sock *dsk, struct sk_buff *skb, - void *data), + netlink_filter_fn filter, void *filter_data); /** diff --git a/include/linux/netlink.h b/include/linux/netlink.h index abe91ed6b9aa..1a4445bf2ab9 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -228,10 +228,12 @@ bool netlink_strict_get_check(struct sk_buff *skb); int netlink_unicast(struct sock *ssk, struct sk_buff *skb, __u32 portid, int nonblock); int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, __u32 portid, __u32 group, gfp_t allocation); + +typedef int (*netlink_filter_fn)(struct sock *dsk, struct sk_buff *skb, void *data); + int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb, __u32 portid, __u32 group, gfp_t allocation, - int (*filter)(struct sock *dsk, - struct sk_buff *skb, void *data), + netlink_filter_fn filter, void *filter_data); int netlink_set_err(struct sock *ssk, __u32 portid, __u32 group, int code); int netlink_register_notifier(struct notifier_block *nb); diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 177126fb0484..4ed8ffd58ff3 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -1519,8 +1519,7 @@ out: int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb, u32 portid, u32 group, gfp_t allocation, - int (*filter)(struct sock *dsk, - struct sk_buff *skb, void *data), + netlink_filter_fn filter, void *filter_data) { struct net *net = sock_net(ssk); -- cgit v1.2.3 From d17aff807f845cf93926c28705216639c7279110 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Tue, 19 Dec 2023 07:37:35 -0800 Subject: Revert BPF token-related functionality This patch includes the following revert (one conflicting BPF FS patch and three token patch sets, represented by merge commits): - revert 0f5d5454c723 "Merge branch 'bpf-fs-mount-options-parsing-follow-ups'"; - revert 750e785796bb "bpf: Support uid and gid when mounting bpffs"; - revert 733763285acf "Merge branch 'bpf-token-support-in-libbpf-s-bpf-object'"; - revert c35919dcce28 "Merge branch 'bpf-token-and-bpf-fs-based-delegation'". Link: https://lore.kernel.org/bpf/CAHk-=wg7JuFYwGy=GOMbRCtOL+jwSQsdUaBsRWkDVYbxipbM5A@mail.gmail.com Signed-off-by: Andrii Nakryiko --- drivers/media/rc/bpf-lirc.c | 2 +- include/linux/bpf.h | 85 +- include/linux/filter.h | 2 +- include/linux/lsm_hook_defs.h | 15 +- include/linux/security.h | 43 +- include/uapi/linux/bpf.h | 42 - kernel/bpf/Makefile | 2 +- kernel/bpf/arraymap.c | 2 +- kernel/bpf/bpf_lsm.c | 15 +- kernel/bpf/cgroup.c | 6 +- kernel/bpf/core.c | 3 +- kernel/bpf/helpers.c | 6 +- kernel/bpf/inode.c | 326 +------ kernel/bpf/syscall.c | 215 ++-- kernel/bpf/token.c | 271 ----- kernel/bpf/verifier.c | 13 +- kernel/trace/bpf_trace.c | 2 +- net/core/filter.c | 36 +- net/ipv4/bpf_tcp_ca.c | 2 +- net/netfilter/nf_bpf_link.c | 2 +- security/security.c | 101 +- security/selinux/hooks.c | 47 +- tools/include/uapi/linux/bpf.h | 42 - tools/lib/bpf/Build | 2 +- tools/lib/bpf/bpf.c | 37 +- tools/lib/bpf/bpf.h | 35 +- tools/lib/bpf/btf.c | 7 +- tools/lib/bpf/elf.c | 2 + tools/lib/bpf/features.c | 478 --------- tools/lib/bpf/libbpf.c | 573 ++++++++--- tools/lib/bpf/libbpf.h | 37 +- tools/lib/bpf/libbpf.map | 1 - tools/lib/bpf/libbpf_internal.h | 36 +- tools/lib/bpf/libbpf_probes.c | 8 +- tools/lib/bpf/str_error.h | 3 - .../selftests/bpf/prog_tests/libbpf_probes.c | 4 - .../testing/selftests/bpf/prog_tests/libbpf_str.c | 6 - tools/testing/selftests/bpf/prog_tests/token.c | 1031 -------------------- tools/testing/selftests/bpf/progs/priv_map.c | 13 - tools/testing/selftests/bpf/progs/priv_prog.c | 13 - 40 files changed, 641 insertions(+), 2925 deletions(-) delete mode 100644 kernel/bpf/token.c delete mode 100644 tools/lib/bpf/features.c delete mode 100644 tools/testing/selftests/bpf/prog_tests/token.c delete mode 100644 tools/testing/selftests/bpf/progs/priv_map.c delete mode 100644 tools/testing/selftests/bpf/progs/priv_prog.c (limited to 'include/linux') diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index 6d07693c6b9f..fe17c7f98e81 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -110,7 +110,7 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_get_prandom_u32: return &bpf_get_prandom_u32_proto; case BPF_FUNC_trace_printk: - if (bpf_token_capable(prog->aux->token, CAP_PERFMON)) + if (perfmon_capable()) return bpf_get_trace_printk_proto(); fallthrough; default: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 2f54cc0436c4..7a8d4c81a39a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -52,10 +52,6 @@ struct module; struct bpf_func_state; struct ftrace_ops; struct cgroup; -struct bpf_token; -struct user_namespace; -struct super_block; -struct inode; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -1488,7 +1484,6 @@ struct bpf_prog_aux { #ifdef CONFIG_SECURITY void *security; #endif - struct bpf_token *token; struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; @@ -1613,31 +1608,6 @@ struct bpf_link_primer { u32 id; }; -struct bpf_mount_opts { - kuid_t uid; - kgid_t gid; - umode_t mode; - - /* BPF token-related delegation options */ - u64 delegate_cmds; - u64 delegate_maps; - u64 delegate_progs; - u64 delegate_attachs; -}; - -struct bpf_token { - struct work_struct work; - atomic64_t refcnt; - struct user_namespace *userns; - u64 allowed_cmds; - u64 allowed_maps; - u64 allowed_progs; - u64 allowed_attachs; -#ifdef CONFIG_SECURITY - void *security; -#endif -}; - struct bpf_struct_ops_value; struct btf_member; @@ -2097,7 +2067,6 @@ static inline void bpf_enable_instrumentation(void) migrate_enable(); } -extern const struct super_operations bpf_super_ops; extern const struct file_operations bpf_map_fops; extern const struct file_operations bpf_prog_fops; extern const struct file_operations bpf_iter_fops; @@ -2232,26 +2201,24 @@ static inline void bpf_map_dec_elem_count(struct bpf_map *map) extern int sysctl_unprivileged_bpf_disabled; -bool bpf_token_capable(const struct bpf_token *token, int cap); - -static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token) +static inline bool bpf_allow_ptr_leaks(void) { - return bpf_token_capable(token, CAP_PERFMON); + return perfmon_capable(); } -static inline bool bpf_allow_uninit_stack(const struct bpf_token *token) +static inline bool bpf_allow_uninit_stack(void) { - return bpf_token_capable(token, CAP_PERFMON); + return perfmon_capable(); } -static inline bool bpf_bypass_spec_v1(const struct bpf_token *token) +static inline bool bpf_bypass_spec_v1(void) { - return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); + return cpu_mitigations_off() || perfmon_capable(); } -static inline bool bpf_bypass_spec_v4(const struct bpf_token *token) +static inline bool bpf_bypass_spec_v4(void) { - return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); + return cpu_mitigations_off() || perfmon_capable(); } int bpf_map_new_fd(struct bpf_map *map, int flags); @@ -2268,21 +2235,8 @@ int bpf_link_new_fd(struct bpf_link *link); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); -void bpf_token_inc(struct bpf_token *token); -void bpf_token_put(struct bpf_token *token); -int bpf_token_create(union bpf_attr *attr); -struct bpf_token *bpf_token_get_from_fd(u32 ufd); - -bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); -bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); -bool bpf_token_allow_prog_type(const struct bpf_token *token, - enum bpf_prog_type prog_type, - enum bpf_attach_type attach_type); - int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); -struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, - umode_t mode); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ @@ -2526,8 +2480,7 @@ const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); -const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, - const struct bpf_prog *prog); +const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); @@ -2646,24 +2599,6 @@ static inline int bpf_obj_get_user(const char __user *pathname, int flags) return -EOPNOTSUPP; } -static inline bool bpf_token_capable(const struct bpf_token *token, int cap) -{ - return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); -} - -static inline void bpf_token_inc(struct bpf_token *token) -{ -} - -static inline void bpf_token_put(struct bpf_token *token) -{ -} - -static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd) -{ - return ERR_PTR(-EOPNOTSUPP); -} - static inline void __dev_flush(void) { } @@ -2787,7 +2722,7 @@ static inline int btf_struct_access(struct bpf_verifier_log *log, } static inline const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +bpf_base_func_proto(enum bpf_func_id func_id) { return NULL; } diff --git a/include/linux/filter.h b/include/linux/filter.h index 12d907f17d36..68fb6c8142fe 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1139,7 +1139,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) return false; if (!bpf_jit_harden) return false; - if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF)) + if (bpf_jit_harden == 1 && bpf_capable()) return false; return true; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 3fdd00b452ac..ff217a5ce552 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -398,17 +398,10 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size) LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode) LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog) -LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr, - struct bpf_token *token) -LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map) -LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr, - struct bpf_token *token) -LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog) -LSM_HOOK(int, 0, bpf_token_create, struct bpf_token *token, union bpf_attr *attr, - struct path *path) -LSM_HOOK(void, LSM_RET_VOID, bpf_token_free, struct bpf_token *token) -LSM_HOOK(int, 0, bpf_token_cmd, const struct bpf_token *token, enum bpf_cmd cmd) -LSM_HOOK(int, 0, bpf_token_capable, const struct bpf_token *token, int cap) +LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map) +LSM_HOOK(void, LSM_RET_VOID, bpf_map_free_security, struct bpf_map *map) +LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux) +LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free_security, struct bpf_prog_aux *aux) #endif /* CONFIG_BPF_SYSCALL */ LSM_HOOK(int, 0, locked_down, enum lockdown_reason what) diff --git a/include/linux/security.h b/include/linux/security.h index 00809d2d5c38..1d1df326c881 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -32,7 +32,6 @@ #include #include #include -#include struct linux_binprm; struct cred; @@ -2021,22 +2020,15 @@ static inline void securityfs_remove(struct dentry *dentry) union bpf_attr; struct bpf_map; struct bpf_prog; -struct bpf_token; +struct bpf_prog_aux; #ifdef CONFIG_SECURITY extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size); extern int security_bpf_map(struct bpf_map *map, fmode_t fmode); extern int security_bpf_prog(struct bpf_prog *prog); -extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, - struct bpf_token *token); +extern int security_bpf_map_alloc(struct bpf_map *map); extern void security_bpf_map_free(struct bpf_map *map); -extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, - struct bpf_token *token); -extern void security_bpf_prog_free(struct bpf_prog *prog); -extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, - struct path *path); -extern void security_bpf_token_free(struct bpf_token *token); -extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd); -extern int security_bpf_token_capable(const struct bpf_token *token, int cap); +extern int security_bpf_prog_alloc(struct bpf_prog_aux *aux); +extern void security_bpf_prog_free(struct bpf_prog_aux *aux); #else static inline int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) @@ -2054,8 +2046,7 @@ static inline int security_bpf_prog(struct bpf_prog *prog) return 0; } -static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, - struct bpf_token *token) +static inline int security_bpf_map_alloc(struct bpf_map *map) { return 0; } @@ -2063,33 +2054,13 @@ static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *a static inline void security_bpf_map_free(struct bpf_map *map) { } -static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, - struct bpf_token *token) +static inline int security_bpf_prog_alloc(struct bpf_prog_aux *aux) { return 0; } -static inline void security_bpf_prog_free(struct bpf_prog *prog) +static inline void security_bpf_prog_free(struct bpf_prog_aux *aux) { } - -static inline int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, - struct path *path) -{ - return 0; -} - -static inline void security_bpf_token_free(struct bpf_token *token) -{ } - -static inline int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) -{ - return 0; -} - -static inline int security_bpf_token_capable(const struct bpf_token *token, int cap) -{ - return 0; -} #endif /* CONFIG_SECURITY */ #endif /* CONFIG_BPF_SYSCALL */ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 42f4d3090efe..754e68ca8744 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -847,36 +847,6 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * - * BPF_TOKEN_CREATE - * Description - * Create BPF token with embedded information about what - * BPF-related functionality it allows: - * - a set of allowed bpf() syscall commands; - * - a set of allowed BPF map types to be created with - * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; - * - a set of allowed BPF program types and BPF program attach - * types to be loaded with BPF_PROG_LOAD command, if - * BPF_PROG_LOAD itself is allowed. - * - * BPF token is created (derived) from an instance of BPF FS, - * assuming it has necessary delegation mount options specified. - * This BPF token can be passed as an extra parameter to various - * bpf() syscall commands to grant BPF subsystem functionality to - * unprivileged processes. - * - * When created, BPF token is "associated" with the owning - * user namespace of BPF FS instance (super block) that it was - * derived from, and subsequent BPF operations performed with - * BPF token would be performing capabilities checks (i.e., - * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within - * that user namespace. Without BPF token, such capabilities - * have to be granted in init user namespace, making bpf() - * syscall incompatible with user namespace, for the most part. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -931,8 +901,6 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, - BPF_TOKEN_CREATE, - __MAX_BPF_CMD, }; enum bpf_map_type { @@ -983,7 +951,6 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, - __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1028,7 +995,6 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, - __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1437,7 +1403,6 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; - __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ @@ -1507,7 +1472,6 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; - __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -1620,7 +1584,6 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 btf_log_true_size; - __u32 btf_token_fd; }; struct { @@ -1751,11 +1714,6 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; - struct { /* struct used by BPF_TOKEN_CREATE command */ - __u32 flags; - __u32 bpffs_fd; - } token_create; - } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4ce95acfcaa7..f526b7573e97 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 13358675ff2e..0bdbbbeab155 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -82,7 +82,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; int numa_node = bpf_map_attr_numa_node(attr); u32 elem_size, index_mask, max_entries; - bool bypass_spec_v1 = bpf_bypass_spec_v1(NULL); + bool bypass_spec_v1 = bpf_bypass_spec_v1(); u64 array_size, mask64; struct bpf_array *array; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 63b4dc495125..e8e910395bf6 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -260,15 +260,9 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) BTF_SET_START(sleepable_lsm_hooks) BTF_ID(func, bpf_lsm_bpf) BTF_ID(func, bpf_lsm_bpf_map) -BTF_ID(func, bpf_lsm_bpf_map_create) -BTF_ID(func, bpf_lsm_bpf_map_free) +BTF_ID(func, bpf_lsm_bpf_map_alloc_security) +BTF_ID(func, bpf_lsm_bpf_map_free_security) BTF_ID(func, bpf_lsm_bpf_prog) -BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) -BTF_ID(func, bpf_lsm_bpf_token_create) -BTF_ID(func, bpf_lsm_bpf_token_free) -BTF_ID(func, bpf_lsm_bpf_token_cmd) -BTF_ID(func, bpf_lsm_bpf_token_capable) BTF_ID(func, bpf_lsm_bprm_check_security) BTF_ID(func, bpf_lsm_bprm_committed_creds) BTF_ID(func, bpf_lsm_bprm_committing_creds) @@ -363,8 +357,9 @@ BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) BTF_SET_START(untrusted_lsm_hooks) -BTF_ID(func, bpf_lsm_bpf_map_free) -BTF_ID(func, bpf_lsm_bpf_prog_free) +BTF_ID(func, bpf_lsm_bpf_map_free_security) +BTF_ID(func, bpf_lsm_bpf_prog_alloc_security) +BTF_ID(func, bpf_lsm_bpf_prog_free_security) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_ID(func, bpf_lsm_file_free_security) #ifdef CONFIG_SECURITY_NETWORK diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 98e0e3835b28..491d20038cbe 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1630,7 +1630,7 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -2191,7 +2191,7 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -2348,7 +2348,7 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 14ace23d517b..ea6843be2616 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -682,7 +682,7 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) void bpf_prog_kallsyms_add(struct bpf_prog *fp) { if (!bpf_prog_kallsyms_candidate(fp) || - !bpf_token_capable(fp->aux->token, CAP_BPF)) + !bpf_capable()) return; bpf_prog_ksym_set_addr(fp); @@ -2779,7 +2779,6 @@ void bpf_prog_free(struct bpf_prog *fp) if (aux->dst_prog) bpf_prog_put(aux->dst_prog); - bpf_token_put(aux->token); INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 07fd4b5704f3..be72824f32b2 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1679,7 +1679,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto __weak; const struct bpf_func_proto bpf_task_pt_regs_proto __weak; const struct bpf_func_proto * -bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +bpf_base_func_proto(enum bpf_func_id func_id) { switch (func_id) { case BPF_FUNC_map_lookup_elem: @@ -1730,7 +1730,7 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) break; } - if (!bpf_token_capable(prog->aux->token, CAP_BPF)) + if (!bpf_capable()) return NULL; switch (func_id) { @@ -1788,7 +1788,7 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) break; } - if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) + if (!perfmon_capable()) return NULL; switch (func_id) { diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c index 4383b3d13a55..1aafb2ff2e95 100644 --- a/kernel/bpf/inode.c +++ b/kernel/bpf/inode.c @@ -20,7 +20,6 @@ #include #include #include -#include #include "preload/bpf_preload.h" enum bpf_type { @@ -99,9 +98,9 @@ static const struct inode_operations bpf_prog_iops = { }; static const struct inode_operations bpf_map_iops = { }; static const struct inode_operations bpf_link_iops = { }; -struct inode *bpf_get_inode(struct super_block *sb, - const struct inode *dir, - umode_t mode) +static struct inode *bpf_get_inode(struct super_block *sb, + const struct inode *dir, + umode_t mode) { struct inode *inode; @@ -595,183 +594,15 @@ struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type typ } EXPORT_SYMBOL(bpf_prog_get_type_path); -struct bpffs_btf_enums { - const struct btf *btf; - const struct btf_type *cmd_t; - const struct btf_type *map_t; - const struct btf_type *prog_t; - const struct btf_type *attach_t; -}; - -static int find_bpffs_btf_enums(struct bpffs_btf_enums *info) -{ - const struct btf *btf; - const struct btf_type *t; - const char *name; - int i, n; - - memset(info, 0, sizeof(*info)); - - btf = bpf_get_btf_vmlinux(); - if (IS_ERR(btf)) - return PTR_ERR(btf); - if (!btf) - return -ENOENT; - - info->btf = btf; - - for (i = 1, n = btf_nr_types(btf); i < n; i++) { - t = btf_type_by_id(btf, i); - if (!btf_type_is_enum(t)) - continue; - - name = btf_name_by_offset(btf, t->name_off); - if (!name) - continue; - - if (strcmp(name, "bpf_cmd") == 0) - info->cmd_t = t; - else if (strcmp(name, "bpf_map_type") == 0) - info->map_t = t; - else if (strcmp(name, "bpf_prog_type") == 0) - info->prog_t = t; - else if (strcmp(name, "bpf_attach_type") == 0) - info->attach_t = t; - else - continue; - - if (info->cmd_t && info->map_t && info->prog_t && info->attach_t) - return 0; - } - - return -ESRCH; -} - -static bool find_btf_enum_const(const struct btf *btf, const struct btf_type *enum_t, - const char *prefix, const char *str, int *value) -{ - const struct btf_enum *e; - const char *name; - int i, n, pfx_len = strlen(prefix); - - *value = 0; - - if (!btf || !enum_t) - return false; - - for (i = 0, n = btf_vlen(enum_t); i < n; i++) { - e = &btf_enum(enum_t)[i]; - - name = btf_name_by_offset(btf, e->name_off); - if (!name || strncasecmp(name, prefix, pfx_len) != 0) - continue; - - /* match symbolic name case insensitive and ignoring prefix */ - if (strcasecmp(name + pfx_len, str) == 0) { - *value = e->val; - return true; - } - } - - return false; -} - -static void seq_print_delegate_opts(struct seq_file *m, - const char *opt_name, - const struct btf *btf, - const struct btf_type *enum_t, - const char *prefix, - u64 delegate_msk, u64 any_msk) -{ - const struct btf_enum *e; - bool first = true; - const char *name; - u64 msk; - int i, n, pfx_len = strlen(prefix); - - delegate_msk &= any_msk; /* clear unknown bits */ - - if (delegate_msk == 0) - return; - - seq_printf(m, ",%s", opt_name); - if (delegate_msk == any_msk) { - seq_printf(m, "=any"); - return; - } - - if (btf && enum_t) { - for (i = 0, n = btf_vlen(enum_t); i < n; i++) { - e = &btf_enum(enum_t)[i]; - name = btf_name_by_offset(btf, e->name_off); - if (!name || strncasecmp(name, prefix, pfx_len) != 0) - continue; - msk = 1ULL << e->val; - if (delegate_msk & msk) { - /* emit lower-case name without prefix */ - seq_printf(m, "%c", first ? '=' : ':'); - name += pfx_len; - while (*name) { - seq_printf(m, "%c", tolower(*name)); - name++; - } - - delegate_msk &= ~msk; - first = false; - } - } - } - if (delegate_msk) - seq_printf(m, "%c0x%llx", first ? '=' : ':', delegate_msk); -} - /* * Display the mount options in /proc/mounts. */ static int bpf_show_options(struct seq_file *m, struct dentry *root) { - struct bpf_mount_opts *opts = root->d_sb->s_fs_info; - struct inode *inode = d_inode(root); - umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; - u64 mask; - - if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) - seq_printf(m, ",uid=%u", - from_kuid_munged(&init_user_ns, inode->i_uid)); - if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) - seq_printf(m, ",gid=%u", - from_kgid_munged(&init_user_ns, inode->i_gid)); + umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; + if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); - - if (opts->delegate_cmds || opts->delegate_maps || - opts->delegate_progs || opts->delegate_attachs) { - struct bpffs_btf_enums info; - - /* ignore errors, fallback to hex */ - (void)find_bpffs_btf_enums(&info); - - mask = (1ULL << __MAX_BPF_CMD) - 1; - seq_print_delegate_opts(m, "delegate_cmds", - info.btf, info.cmd_t, "BPF_", - opts->delegate_cmds, mask); - - mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; - seq_print_delegate_opts(m, "delegate_maps", - info.btf, info.map_t, "BPF_MAP_TYPE_", - opts->delegate_maps, mask); - - mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; - seq_print_delegate_opts(m, "delegate_progs", - info.btf, info.prog_t, "BPF_PROG_TYPE_", - opts->delegate_progs, mask); - - mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; - seq_print_delegate_opts(m, "delegate_attachs", - info.btf, info.attach_t, "BPF_", - opts->delegate_attachs, mask); - } - return 0; } @@ -786,7 +617,7 @@ static void bpf_free_inode(struct inode *inode) free_inode_nonrcu(inode); } -const struct super_operations bpf_super_ops = { +static const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, @@ -794,33 +625,23 @@ const struct super_operations bpf_super_ops = { }; enum { - OPT_UID, - OPT_GID, OPT_MODE, - OPT_DELEGATE_CMDS, - OPT_DELEGATE_MAPS, - OPT_DELEGATE_PROGS, - OPT_DELEGATE_ATTACHS, }; static const struct fs_parameter_spec bpf_fs_parameters[] = { - fsparam_u32 ("uid", OPT_UID), - fsparam_u32 ("gid", OPT_GID), fsparam_u32oct ("mode", OPT_MODE), - fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), - fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), - fsparam_string ("delegate_progs", OPT_DELEGATE_PROGS), - fsparam_string ("delegate_attachs", OPT_DELEGATE_ATTACHS), {} }; +struct bpf_mount_opts { + umode_t mode; +}; + static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { - struct bpf_mount_opts *opts = fc->s_fs_info; + struct bpf_mount_opts *opts = fc->fs_private; struct fs_parse_result result; - kuid_t uid; - kgid_t gid; - int opt, err; + int opt; opt = fs_parse(fc, bpf_fs_parameters, param, &result); if (opt < 0) { @@ -841,104 +662,12 @@ static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) } switch (opt) { - case OPT_UID: - uid = make_kuid(current_user_ns(), result.uint_32); - if (!uid_valid(uid)) - goto bad_value; - - /* - * The requested uid must be representable in the - * filesystem's idmapping. - */ - if (!kuid_has_mapping(fc->user_ns, uid)) - goto bad_value; - - opts->uid = uid; - break; - case OPT_GID: - gid = make_kgid(current_user_ns(), result.uint_32); - if (!gid_valid(gid)) - goto bad_value; - - /* - * The requested gid must be representable in the - * filesystem's idmapping. - */ - if (!kgid_has_mapping(fc->user_ns, gid)) - goto bad_value; - - opts->gid = gid; - break; case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; - case OPT_DELEGATE_CMDS: - case OPT_DELEGATE_MAPS: - case OPT_DELEGATE_PROGS: - case OPT_DELEGATE_ATTACHS: { - struct bpffs_btf_enums info; - const struct btf_type *enum_t; - const char *enum_pfx; - u64 *delegate_msk, msk = 0; - char *p; - int val; - - /* ignore errors, fallback to hex */ - (void)find_bpffs_btf_enums(&info); - - switch (opt) { - case OPT_DELEGATE_CMDS: - delegate_msk = &opts->delegate_cmds; - enum_t = info.cmd_t; - enum_pfx = "BPF_"; - break; - case OPT_DELEGATE_MAPS: - delegate_msk = &opts->delegate_maps; - enum_t = info.map_t; - enum_pfx = "BPF_MAP_TYPE_"; - break; - case OPT_DELEGATE_PROGS: - delegate_msk = &opts->delegate_progs; - enum_t = info.prog_t; - enum_pfx = "BPF_PROG_TYPE_"; - break; - case OPT_DELEGATE_ATTACHS: - delegate_msk = &opts->delegate_attachs; - enum_t = info.attach_t; - enum_pfx = "BPF_"; - break; - default: - return -EINVAL; - } - - while ((p = strsep(¶m->string, ":"))) { - if (strcmp(p, "any") == 0) { - msk |= ~0ULL; - } else if (find_btf_enum_const(info.btf, enum_t, enum_pfx, p, &val)) { - msk |= 1ULL << val; - } else { - err = kstrtou64(p, 0, &msk); - if (err) - return err; - } - } - - /* Setting delegation mount options requires privileges */ - if (msk && !capable(CAP_SYS_ADMIN)) - return -EPERM; - - *delegate_msk |= msk; - break; - } - default: - /* ignore unknown mount options */ - break; } return 0; - -bad_value: - return invalfc(fc, "Bad value for '%s'", param->key); } struct bpf_preload_ops *bpf_preload_ops; @@ -1010,14 +739,10 @@ out: static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { static const struct tree_descr bpf_rfiles[] = { { "" } }; - struct bpf_mount_opts *opts = sb->s_fs_info; + struct bpf_mount_opts *opts = fc->fs_private; struct inode *inode; int ret; - /* Mounting an instance of BPF FS requires privileges */ - if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) - return -EPERM; - ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); if (ret) return ret; @@ -1025,8 +750,6 @@ static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &bpf_super_ops; inode = sb->s_root->d_inode; - inode->i_uid = opts->uid; - inode->i_gid = opts->gid; inode->i_op = &bpf_dir_iops; inode->i_mode &= ~S_IALLUGO; populate_bpffs(sb->s_root); @@ -1041,7 +764,7 @@ static int bpf_get_tree(struct fs_context *fc) static void bpf_free_fc(struct fs_context *fc) { - kfree(fc->s_fs_info); + kfree(fc->fs_private); } static const struct fs_context_operations bpf_context_ops = { @@ -1062,35 +785,18 @@ static int bpf_init_fs_context(struct fs_context *fc) return -ENOMEM; opts->mode = S_IRWXUGO; - opts->uid = current_fsuid(); - opts->gid = current_fsgid(); - - /* start out with no BPF token delegation enabled */ - opts->delegate_cmds = 0; - opts->delegate_maps = 0; - opts->delegate_progs = 0; - opts->delegate_attachs = 0; - fc->s_fs_info = opts; + fc->fs_private = opts; fc->ops = &bpf_context_ops; return 0; } -static void bpf_kill_super(struct super_block *sb) -{ - struct bpf_mount_opts *opts = sb->s_fs_info; - - kill_litter_super(sb); - kfree(opts); -} - static struct file_system_type bpf_fs_type = { .owner = THIS_MODULE, .name = "bpf", .init_fs_context = bpf_init_fs_context, .parameters = bpf_fs_parameters, - .kill_sb = bpf_kill_super, - .fs_flags = FS_USERNS_MOUNT, + .kill_sb = kill_litter_super, }; static int __init bpf_init(void) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8faa1a20edf8..1bf9805ee185 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1011,8 +1011,8 @@ int map_check_no_btf(const struct bpf_map *map, return -ENOTSUPP; } -static int map_check_btf(struct bpf_map *map, struct bpf_token *token, - const struct btf *btf, u32 btf_key_id, u32 btf_value_id) +static int map_check_btf(struct bpf_map *map, const struct btf *btf, + u32 btf_key_id, u32 btf_value_id) { const struct btf_type *key_type, *value_type; u32 key_size, value_size; @@ -1040,7 +1040,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (!IS_ERR_OR_NULL(map->record)) { int i; - if (!bpf_token_capable(token, CAP_BPF)) { + if (!bpf_capable()) { ret = -EPERM; goto free_map_tab; } @@ -1123,17 +1123,11 @@ free_map_tab: return ret; } -static bool bpf_net_capable(void) -{ - return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); -} - -#define BPF_MAP_CREATE_LAST_FIELD map_token_fd +#define BPF_MAP_CREATE_LAST_FIELD map_extra /* called via syscall */ static int map_create(union bpf_attr *attr) { const struct bpf_map_ops *ops; - struct bpf_token *token = NULL; int numa_node = bpf_map_attr_numa_node(attr); u32 map_type = attr->map_type; struct bpf_map *map; @@ -1184,32 +1178,14 @@ static int map_create(union bpf_attr *attr) if (!ops->map_mem_usage) return -EINVAL; - if (attr->map_token_fd) { - token = bpf_token_get_from_fd(attr->map_token_fd); - if (IS_ERR(token)) - return PTR_ERR(token); - - /* if current token doesn't grant map creation permissions, - * then we can't use this token, so ignore it and rely on - * system-wide capabilities checks - */ - if (!bpf_token_allow_cmd(token, BPF_MAP_CREATE) || - !bpf_token_allow_map_type(token, attr->map_type)) { - bpf_token_put(token); - token = NULL; - } - } - - err = -EPERM; - /* Intent here is for unprivileged_bpf_disabled to block BPF map * creation for unprivileged users; other actions depend * on fd availability and access to bpffs, so are dependent on * object creation success. Even with unprivileged BPF disabled, * capability checks are still carried out. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_token_capable(token, CAP_BPF)) - goto put_token; + if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) + return -EPERM; /* check privileged map type permissions */ switch (map_type) { @@ -1242,27 +1218,25 @@ static int map_create(union bpf_attr *attr) case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_STRUCT_OPS: case BPF_MAP_TYPE_CPUMAP: - if (!bpf_token_capable(token, CAP_BPF)) - goto put_token; + if (!bpf_capable()) + return -EPERM; break; case BPF_MAP_TYPE_SOCKMAP: case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_DEVMAP: case BPF_MAP_TYPE_DEVMAP_HASH: case BPF_MAP_TYPE_XSKMAP: - if (!bpf_token_capable(token, CAP_NET_ADMIN)) - goto put_token; + if (!capable(CAP_NET_ADMIN)) + return -EPERM; break; default: WARN(1, "unsupported map type %d", map_type); - goto put_token; + return -EPERM; } map = ops->map_alloc(attr); - if (IS_ERR(map)) { - err = PTR_ERR(map); - goto put_token; - } + if (IS_ERR(map)) + return PTR_ERR(map); map->ops = ops; map->map_type = map_type; @@ -1299,7 +1273,7 @@ static int map_create(union bpf_attr *attr) map->btf = btf; if (attr->btf_value_type_id) { - err = map_check_btf(map, token, btf, attr->btf_key_type_id, + err = map_check_btf(map, btf, attr->btf_key_type_id, attr->btf_value_type_id); if (err) goto free_map; @@ -1311,16 +1285,15 @@ static int map_create(union bpf_attr *attr) attr->btf_vmlinux_value_type_id; } - err = security_bpf_map_create(map, attr, token); + err = security_bpf_map_alloc(map); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) goto free_map_sec; bpf_map_save_memcg(map); - bpf_token_put(token); err = bpf_map_new_fd(map, f_flags); if (err < 0) { @@ -1341,8 +1314,6 @@ free_map_sec: free_map: btf_put(map->btf); map->ops->map_free(map); -put_token: - bpf_token_put(token); return err; } @@ -2173,7 +2144,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); - security_bpf_prog_free(aux->prog); + security_bpf_prog_free(aux); bpf_prog_free(aux->prog); } @@ -2619,15 +2590,13 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD prog_token_fd +#define BPF_PROG_LOAD_LAST_FIELD log_true_size static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) { enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; struct btf *attach_btf = NULL; - struct bpf_token *token = NULL; - bool bpf_cap; int err; char license[128]; @@ -2644,31 +2613,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) BPF_F_TEST_REG_INVARIANTS)) return -EINVAL; - bpf_prog_load_fixup_attach_type(attr); - - if (attr->prog_token_fd) { - token = bpf_token_get_from_fd(attr->prog_token_fd); - if (IS_ERR(token)) - return PTR_ERR(token); - /* if current token doesn't grant prog loading permissions, - * then we can't use this token, so ignore it and rely on - * system-wide capabilities checks - */ - if (!bpf_token_allow_cmd(token, BPF_PROG_LOAD) || - !bpf_token_allow_prog_type(token, attr->prog_type, - attr->expected_attach_type)) { - bpf_token_put(token); - token = NULL; - } - } - - bpf_cap = bpf_token_capable(token, CAP_BPF); - err = -EPERM; - if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && - !bpf_cap) - goto put_token; + !bpf_capable()) + return -EPERM; /* Intent here is for unprivileged_bpf_disabled to block BPF program * creation for unprivileged users; other actions depend @@ -2677,23 +2625,21 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * capability checks are still carried out for these * and other operations. */ - if (sysctl_unprivileged_bpf_disabled && !bpf_cap) - goto put_token; + if (sysctl_unprivileged_bpf_disabled && !bpf_capable()) + return -EPERM; if (attr->insn_cnt == 0 || - attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { - err = -E2BIG; - goto put_token; - } + attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) + return -E2BIG; if (type != BPF_PROG_TYPE_SOCKET_FILTER && type != BPF_PROG_TYPE_CGROUP_SKB && - !bpf_cap) - goto put_token; + !bpf_capable()) + return -EPERM; - if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) - goto put_token; - if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) - goto put_token; + if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (is_perfmon_prog_type(type) && !perfmon_capable()) + return -EPERM; /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is @@ -2703,33 +2649,27 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (IS_ERR(dst_prog)) { dst_prog = NULL; attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); - if (IS_ERR(attach_btf)) { - err = -EINVAL; - goto put_token; - } + if (IS_ERR(attach_btf)) + return -EINVAL; if (!btf_is_kernel(attach_btf)) { /* attaching through specifying bpf_prog's BTF * objects directly might be supported eventually */ btf_put(attach_btf); - err = -ENOTSUPP; - goto put_token; + return -ENOTSUPP; } } } else if (attr->attach_btf_id) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(attach_btf)) { - err = PTR_ERR(attach_btf); - goto put_token; - } - if (!attach_btf) { - err = -EINVAL; - goto put_token; - } + if (IS_ERR(attach_btf)) + return PTR_ERR(attach_btf); + if (!attach_btf) + return -EINVAL; btf_get(attach_btf); } + bpf_prog_load_fixup_attach_type(attr); if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, dst_prog)) { @@ -2737,8 +2677,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - err = -EINVAL; - goto put_token; + return -EINVAL; } /* plain bpf_prog allocation */ @@ -2748,8 +2687,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - err = -EINVAL; - goto put_token; + return -ENOMEM; } prog->expected_attach_type = attr->expected_attach_type; @@ -2760,9 +2698,9 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; - /* move token into prog->aux, reuse taken refcnt */ - prog->aux->token = token; - token = NULL; + err = security_bpf_prog_alloc(prog->aux); + if (err) + goto free_prog; prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; @@ -2771,12 +2709,12 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), bpf_prog_insn_size(prog)) != 0) - goto free_prog; + goto free_prog_sec; /* copy eBPF program license from user space */ if (strncpy_from_bpfptr(license, make_bpfptr(attr->license, uattr.is_kernel), sizeof(license) - 1) < 0) - goto free_prog; + goto free_prog_sec; license[sizeof(license) - 1] = 0; /* eBPF programs must be GPL compatible to use GPL-ed functions */ @@ -2790,29 +2728,25 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (bpf_prog_is_dev_bound(prog->aux)) { err = bpf_prog_dev_bound_init(prog, attr); if (err) - goto free_prog; + goto free_prog_sec; } if (type == BPF_PROG_TYPE_EXT && dst_prog && bpf_prog_is_dev_bound(dst_prog->aux)) { err = bpf_prog_dev_bound_inherit(prog, dst_prog); if (err) - goto free_prog; + goto free_prog_sec; } /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) - goto free_prog; + goto free_prog_sec; prog->aux->load_time = ktime_get_boottime_ns(); err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, sizeof(attr->prog_name)); if (err < 0) - goto free_prog; - - err = security_bpf_prog_load(prog, attr, token); - if (err) goto free_prog_sec; /* run eBPF verifier */ @@ -2858,16 +2792,13 @@ free_used_maps: */ __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); return err; - free_prog_sec: - security_bpf_prog_free(prog); -free_prog: free_uid(prog->aux->user); + security_bpf_prog_free(prog->aux); +free_prog: if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); -put_token: - bpf_token_put(token); return err; } @@ -3857,7 +3788,7 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, case BPF_PROG_TYPE_SK_LOOKUP: return attach_type == prog->expected_attach_type ? 0 : -EINVAL; case BPF_PROG_TYPE_CGROUP_SKB: - if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) + if (!capable(CAP_NET_ADMIN)) /* cg-skb progs can be loaded by unpriv user. * check permissions at attach time. */ @@ -4060,7 +3991,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) static int bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) { - if (!bpf_net_capable()) + if (!capable(CAP_NET_ADMIN)) return -EPERM; if (CHECK_ATTR(BPF_PROG_QUERY)) return -EINVAL; @@ -4828,31 +4759,15 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, return err; } -#define BPF_BTF_LOAD_LAST_FIELD btf_token_fd +#define BPF_BTF_LOAD_LAST_FIELD btf_log_true_size static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) { - struct bpf_token *token = NULL; - if (CHECK_ATTR(BPF_BTF_LOAD)) return -EINVAL; - if (attr->btf_token_fd) { - token = bpf_token_get_from_fd(attr->btf_token_fd); - if (IS_ERR(token)) - return PTR_ERR(token); - if (!bpf_token_allow_cmd(token, BPF_BTF_LOAD)) { - bpf_token_put(token); - token = NULL; - } - } - - if (!bpf_token_capable(token, CAP_BPF)) { - bpf_token_put(token); + if (!bpf_capable()) return -EPERM; - } - - bpf_token_put(token); return btf_new_fd(attr, uattr, uattr_size); } @@ -5470,20 +5385,6 @@ out_prog_put: return ret; } -#define BPF_TOKEN_CREATE_LAST_FIELD token_create.bpffs_fd - -static int token_create(union bpf_attr *attr) -{ - if (CHECK_ATTR(BPF_TOKEN_CREATE)) - return -EINVAL; - - /* no flags are supported yet */ - if (attr->token_create.flags) - return -EINVAL; - - return bpf_token_create(attr); -} - static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -5617,9 +5518,6 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_BIND_MAP: err = bpf_prog_bind_map(&attr); break; - case BPF_TOKEN_CREATE: - err = token_create(&attr); - break; default: err = -EINVAL; break; @@ -5726,7 +5624,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { const struct bpf_func_proto * __weak tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } BPF_CALL_1(bpf_sys_close, u32, fd) @@ -5776,8 +5674,7 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_sys_bpf: - return !bpf_token_capable(prog->aux->token, CAP_PERFMON) - ? NULL : &bpf_sys_bpf_proto; + return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; case BPF_FUNC_sys_close: diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c deleted file mode 100644 index a86fccd57e2d..000000000000 --- a/kernel/bpf/token.c +++ /dev/null @@ -1,271 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -bool bpf_token_capable(const struct bpf_token *token, int cap) -{ - /* BPF token allows ns_capable() level of capabilities, but only if - * token's userns is *exactly* the same as current user's userns - */ - if (token && current_user_ns() == token->userns) { - if (ns_capable(token->userns, cap) || - (cap != CAP_SYS_ADMIN && ns_capable(token->userns, CAP_SYS_ADMIN))) - return security_bpf_token_capable(token, cap) == 0; - } - /* otherwise fallback to capable() checks */ - return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); -} - -void bpf_token_inc(struct bpf_token *token) -{ - atomic64_inc(&token->refcnt); -} - -static void bpf_token_free(struct bpf_token *token) -{ - security_bpf_token_free(token); - put_user_ns(token->userns); - kvfree(token); -} - -static void bpf_token_put_deferred(struct work_struct *work) -{ - struct bpf_token *token = container_of(work, struct bpf_token, work); - - bpf_token_free(token); -} - -void bpf_token_put(struct bpf_token *token) -{ - if (!token) - return; - - if (!atomic64_dec_and_test(&token->refcnt)) - return; - - INIT_WORK(&token->work, bpf_token_put_deferred); - schedule_work(&token->work); -} - -static int bpf_token_release(struct inode *inode, struct file *filp) -{ - struct bpf_token *token = filp->private_data; - - bpf_token_put(token); - return 0; -} - -static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp) -{ - struct bpf_token *token = filp->private_data; - u64 mask; - - BUILD_BUG_ON(__MAX_BPF_CMD >= 64); - mask = (1ULL << __MAX_BPF_CMD) - 1; - if ((token->allowed_cmds & mask) == mask) - seq_printf(m, "allowed_cmds:\tany\n"); - else - seq_printf(m, "allowed_cmds:\t0x%llx\n", token->allowed_cmds); - - BUILD_BUG_ON(__MAX_BPF_MAP_TYPE >= 64); - mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; - if ((token->allowed_maps & mask) == mask) - seq_printf(m, "allowed_maps:\tany\n"); - else - seq_printf(m, "allowed_maps:\t0x%llx\n", token->allowed_maps); - - BUILD_BUG_ON(__MAX_BPF_PROG_TYPE >= 64); - mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; - if ((token->allowed_progs & mask) == mask) - seq_printf(m, "allowed_progs:\tany\n"); - else - seq_printf(m, "allowed_progs:\t0x%llx\n", token->allowed_progs); - - BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE >= 64); - mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; - if ((token->allowed_attachs & mask) == mask) - seq_printf(m, "allowed_attachs:\tany\n"); - else - seq_printf(m, "allowed_attachs:\t0x%llx\n", token->allowed_attachs); -} - -#define BPF_TOKEN_INODE_NAME "bpf-token" - -static const struct inode_operations bpf_token_iops = { }; - -static const struct file_operations bpf_token_fops = { - .release = bpf_token_release, - .show_fdinfo = bpf_token_show_fdinfo, -}; - -int bpf_token_create(union bpf_attr *attr) -{ - struct bpf_mount_opts *mnt_opts; - struct bpf_token *token = NULL; - struct user_namespace *userns; - struct inode *inode; - struct file *file; - struct path path; - struct fd f; - umode_t mode; - int err, fd; - - f = fdget(attr->token_create.bpffs_fd); - if (!f.file) - return -EBADF; - - path = f.file->f_path; - path_get(&path); - fdput(f); - - if (path.dentry != path.mnt->mnt_sb->s_root) { - err = -EINVAL; - goto out_path; - } - if (path.mnt->mnt_sb->s_op != &bpf_super_ops) { - err = -EINVAL; - goto out_path; - } - err = path_permission(&path, MAY_ACCESS); - if (err) - goto out_path; - - userns = path.dentry->d_sb->s_user_ns; - /* - * Enforce that creators of BPF tokens are in the same user - * namespace as the BPF FS instance. This makes reasoning about - * permissions a lot easier and we can always relax this later. - */ - if (current_user_ns() != userns) { - err = -EPERM; - goto out_path; - } - if (!ns_capable(userns, CAP_BPF)) { - err = -EPERM; - goto out_path; - } - - mnt_opts = path.dentry->d_sb->s_fs_info; - if (mnt_opts->delegate_cmds == 0 && - mnt_opts->delegate_maps == 0 && - mnt_opts->delegate_progs == 0 && - mnt_opts->delegate_attachs == 0) { - err = -ENOENT; /* no BPF token delegation is set up */ - goto out_path; - } - - mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask()); - inode = bpf_get_inode(path.mnt->mnt_sb, NULL, mode); - if (IS_ERR(inode)) { - err = PTR_ERR(inode); - goto out_path; - } - - inode->i_op = &bpf_token_iops; - inode->i_fop = &bpf_token_fops; - clear_nlink(inode); /* make sure it is unlinked */ - - file = alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops); - if (IS_ERR(file)) { - iput(inode); - err = PTR_ERR(file); - goto out_path; - } - - token = kvzalloc(sizeof(*token), GFP_USER); - if (!token) { - err = -ENOMEM; - goto out_file; - } - - atomic64_set(&token->refcnt, 1); - - /* remember bpffs owning userns for future ns_capable() checks */ - token->userns = get_user_ns(userns); - - token->allowed_cmds = mnt_opts->delegate_cmds; - token->allowed_maps = mnt_opts->delegate_maps; - token->allowed_progs = mnt_opts->delegate_progs; - token->allowed_attachs = mnt_opts->delegate_attachs; - - err = security_bpf_token_create(token, attr, &path); - if (err) - goto out_token; - - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - err = fd; - goto out_token; - } - - file->private_data = token; - fd_install(fd, file); - - path_put(&path); - return fd; - -out_token: - bpf_token_free(token); -out_file: - fput(file); -out_path: - path_put(&path); - return err; -} - -struct bpf_token *bpf_token_get_from_fd(u32 ufd) -{ - struct fd f = fdget(ufd); - struct bpf_token *token; - - if (!f.file) - return ERR_PTR(-EBADF); - if (f.file->f_op != &bpf_token_fops) { - fdput(f); - return ERR_PTR(-EINVAL); - } - - token = f.file->private_data; - bpf_token_inc(token); - fdput(f); - - return token; -} - -bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd) -{ - /* BPF token can be used only within exactly the same userns in which - * it was created - */ - if (!token || current_user_ns() != token->userns) - return false; - if (!(token->allowed_cmds & (1ULL << cmd))) - return false; - return security_bpf_token_cmd(token, cmd) == 0; -} - -bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type) -{ - if (!token || type >= __MAX_BPF_MAP_TYPE) - return false; - - return token->allowed_maps & (1ULL << type); -} - -bool bpf_token_allow_prog_type(const struct bpf_token *token, - enum bpf_prog_type prog_type, - enum bpf_attach_type attach_type) -{ - if (!token || prog_type >= __MAX_BPF_PROG_TYPE || attach_type >= __MAX_BPF_ATTACH_TYPE) - return false; - - return (token->allowed_progs & (1ULL << prog_type)) && - (token->allowed_attachs & (1ULL << attach_type)); -} diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9456ee0ad129..4ceec8c2a484 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20594,12 +20594,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - - env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); - env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); - env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); - env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); - env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); + is_priv = bpf_capable(); bpf_get_btf_vmlinux(); @@ -20631,6 +20626,12 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) env->strict_alignment = false; + env->allow_ptr_leaks = bpf_allow_ptr_leaks(); + env->allow_uninit_stack = bpf_allow_uninit_stack(); + env->bypass_spec_v1 = bpf_bypass_spec_v1(); + env->bypass_spec_v4 = bpf_bypass_spec_v4(); + env->bpf_capable = bpf_capable(); + if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 492d60e9c480..7ac6c52b25eb 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1629,7 +1629,7 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } diff --git a/net/core/filter.c b/net/core/filter.c index 3cc52b82bab8..24061f29c9dd 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -87,7 +87,7 @@ #include "dev.h" static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); +bpf_sk_base_func_proto(enum bpf_func_id func_id); int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) { @@ -7862,7 +7862,7 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -7955,7 +7955,7 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return NULL; } default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -7974,7 +7974,7 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_perf_event_output: return &bpf_skb_event_output_proto; default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8161,7 +8161,7 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8220,7 +8220,7 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) #endif #endif default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } #if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) @@ -8281,7 +8281,7 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_tcp_sock_proto; #endif /* CONFIG_INET */ default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8323,7 +8323,7 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_cgroup_classid_curr_proto; #endif default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8367,7 +8367,7 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_skc_lookup_tcp_proto; #endif default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8378,7 +8378,7 @@ flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_load_bytes: return &bpf_flow_dissector_load_bytes_proto; default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8405,7 +8405,7 @@ lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_skb_under_cgroup: return &bpf_skb_under_cgroup_proto; default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -8580,7 +8580,7 @@ static bool cg_skb_is_valid_access(int off, int size, return false; case bpf_ctx_range(struct __sk_buff, data): case bpf_ctx_range(struct __sk_buff, data_end): - if (!bpf_token_capable(prog->aux->token, CAP_BPF)) + if (!bpf_capable()) return false; break; } @@ -8592,7 +8592,7 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): break; case bpf_ctx_range(struct __sk_buff, tstamp): - if (!bpf_token_capable(prog->aux->token, CAP_BPF)) + if (!bpf_capable()) return false; break; default: @@ -11236,7 +11236,7 @@ sk_reuseport_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -11418,7 +11418,7 @@ sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_sk_release: return &bpf_sk_release_proto; default: - return bpf_sk_base_func_proto(func_id, prog); + return bpf_sk_base_func_proto(func_id); } } @@ -11752,7 +11752,7 @@ const struct bpf_func_proto bpf_sock_from_file_proto = { }; static const struct bpf_func_proto * -bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +bpf_sk_base_func_proto(enum bpf_func_id func_id) { const struct bpf_func_proto *func; @@ -11781,10 +11781,10 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } - if (!bpf_token_capable(prog->aux->token, CAP_PERFMON)) + if (!perfmon_capable()) return NULL; return func; diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 634cfafa583d..ae8b15e6896f 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -191,7 +191,7 @@ bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; default: - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } diff --git a/net/netfilter/nf_bpf_link.c b/net/netfilter/nf_bpf_link.c index 5257d5e7eb09..0e4beae421f8 100644 --- a/net/netfilter/nf_bpf_link.c +++ b/net/netfilter/nf_bpf_link.c @@ -314,7 +314,7 @@ static bool nf_is_valid_access(int off, int size, enum bpf_access_type type, static const struct bpf_func_proto * bpf_nf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - return bpf_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } const struct bpf_verifier_ops netfilter_verifier_ops = { diff --git a/security/security.c b/security/security.c index 088a79c35c26..dcb3e7014f9b 100644 --- a/security/security.c +++ b/security/security.c @@ -5167,87 +5167,29 @@ int security_bpf_prog(struct bpf_prog *prog) } /** - * security_bpf_map_create() - Check if BPF map creation is allowed - * @map: BPF map object - * @attr: BPF syscall attributes used to create BPF map - * @token: BPF token used to grant user access - * - * Do a check when the kernel creates a new BPF map. This is also the - * point where LSM blob is allocated for LSMs that need them. - * - * Return: Returns 0 on success, error on failure. - */ -int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, - struct bpf_token *token) -{ - return call_int_hook(bpf_map_create, 0, map, attr, token); -} - -/** - * security_bpf_prog_load() - Check if loading of BPF program is allowed - * @prog: BPF program object - * @attr: BPF syscall attributes used to create BPF program - * @token: BPF token used to grant user access to BPF subsystem - * - * Perform an access control check when the kernel loads a BPF program and - * allocates associated BPF program object. This hook is also responsible for - * allocating any required LSM state for the BPF program. - * - * Return: Returns 0 on success, error on failure. - */ -int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, - struct bpf_token *token) -{ - return call_int_hook(bpf_prog_load, 0, prog, attr, token); -} - -/** - * security_bpf_token_create() - Check if creating of BPF token is allowed - * @token: BPF token object - * @attr: BPF syscall attributes used to create BPF token - * @path: path pointing to BPF FS mount point from which BPF token is created - * - * Do a check when the kernel instantiates a new BPF token object from BPF FS - * instance. This is also the point where LSM blob can be allocated for LSMs. - * - * Return: Returns 0 on success, error on failure. - */ -int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, - struct path *path) -{ - return call_int_hook(bpf_token_create, 0, token, attr, path); -} - -/** - * security_bpf_token_cmd() - Check if BPF token is allowed to delegate - * requested BPF syscall command - * @token: BPF token object - * @cmd: BPF syscall command requested to be delegated by BPF token + * security_bpf_map_alloc() - Allocate a bpf map LSM blob + * @map: bpf map * - * Do a check when the kernel decides whether provided BPF token should allow - * delegation of requested BPF syscall command. + * Initialize the security field inside bpf map. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) +int security_bpf_map_alloc(struct bpf_map *map) { - return call_int_hook(bpf_token_cmd, 0, token, cmd); + return call_int_hook(bpf_map_alloc_security, 0, map); } /** - * security_bpf_token_capable() - Check if BPF token is allowed to delegate - * requested BPF-related capability - * @token: BPF token object - * @cap: capabilities requested to be delegated by BPF token + * security_bpf_prog_alloc() - Allocate a bpf program LSM blob + * @aux: bpf program aux info struct * - * Do a check when the kernel decides whether provided BPF token should allow - * delegation of requested BPF-related capabilities. + * Initialize the security field inside bpf program. * * Return: Returns 0 on success, error on failure. */ -int security_bpf_token_capable(const struct bpf_token *token, int cap) +int security_bpf_prog_alloc(struct bpf_prog_aux *aux) { - return call_int_hook(bpf_token_capable, 0, token, cap); + return call_int_hook(bpf_prog_alloc_security, 0, aux); } /** @@ -5258,29 +5200,18 @@ int security_bpf_token_capable(const struct bpf_token *token, int cap) */ void security_bpf_map_free(struct bpf_map *map) { - call_void_hook(bpf_map_free, map); -} - -/** - * security_bpf_prog_free() - Free a BPF program's LSM blob - * @prog: BPF program struct - * - * Clean up the security information stored inside BPF program. - */ -void security_bpf_prog_free(struct bpf_prog *prog) -{ - call_void_hook(bpf_prog_free, prog); + call_void_hook(bpf_map_free_security, map); } /** - * security_bpf_token_free() - Free a BPF token's LSM blob - * @token: BPF token struct + * security_bpf_prog_free() - Free a bpf program's LSM blob + * @aux: bpf program aux info struct * - * Clean up the security information stored inside BPF token. + * Clean up the security information stored inside bpf prog. */ -void security_bpf_token_free(struct bpf_token *token) +void security_bpf_prog_free(struct bpf_prog_aux *aux) { - call_void_hook(bpf_token_free, token); + call_void_hook(bpf_prog_free_security, aux); } #endif /* CONFIG_BPF_SYSCALL */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 1501e95366a1..feda711c6b7b 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6783,8 +6783,7 @@ static int selinux_bpf_prog(struct bpf_prog *prog) BPF__PROG_RUN, NULL); } -static int selinux_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, - struct bpf_token *token) +static int selinux_bpf_map_alloc(struct bpf_map *map) { struct bpf_security_struct *bpfsec; @@ -6806,8 +6805,7 @@ static void selinux_bpf_map_free(struct bpf_map *map) kfree(bpfsec); } -static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, - struct bpf_token *token) +static int selinux_bpf_prog_alloc(struct bpf_prog_aux *aux) { struct bpf_security_struct *bpfsec; @@ -6816,39 +6814,16 @@ static int selinux_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, return -ENOMEM; bpfsec->sid = current_sid(); - prog->aux->security = bpfsec; + aux->security = bpfsec; return 0; } -static void selinux_bpf_prog_free(struct bpf_prog *prog) +static void selinux_bpf_prog_free(struct bpf_prog_aux *aux) { - struct bpf_security_struct *bpfsec = prog->aux->security; + struct bpf_security_struct *bpfsec = aux->security; - prog->aux->security = NULL; - kfree(bpfsec); -} - -static int selinux_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, - struct path *path) -{ - struct bpf_security_struct *bpfsec; - - bpfsec = kzalloc(sizeof(*bpfsec), GFP_KERNEL); - if (!bpfsec) - return -ENOMEM; - - bpfsec->sid = current_sid(); - token->security = bpfsec; - - return 0; -} - -static void selinux_bpf_token_free(struct bpf_token *token) -{ - struct bpf_security_struct *bpfsec = token->security; - - token->security = NULL; + aux->security = NULL; kfree(bpfsec); } #endif @@ -7204,9 +7179,8 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(bpf, selinux_bpf), LSM_HOOK_INIT(bpf_map, selinux_bpf_map), LSM_HOOK_INIT(bpf_prog, selinux_bpf_prog), - LSM_HOOK_INIT(bpf_map_free, selinux_bpf_map_free), - LSM_HOOK_INIT(bpf_prog_free, selinux_bpf_prog_free), - LSM_HOOK_INIT(bpf_token_free, selinux_bpf_token_free), + LSM_HOOK_INIT(bpf_map_free_security, selinux_bpf_map_free), + LSM_HOOK_INIT(bpf_prog_free_security, selinux_bpf_prog_free), #endif #ifdef CONFIG_PERF_EVENTS @@ -7263,9 +7237,8 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(audit_rule_init, selinux_audit_rule_init), #endif #ifdef CONFIG_BPF_SYSCALL - LSM_HOOK_INIT(bpf_map_create, selinux_bpf_map_create), - LSM_HOOK_INIT(bpf_prog_load, selinux_bpf_prog_load), - LSM_HOOK_INIT(bpf_token_create, selinux_bpf_token_create), + LSM_HOOK_INIT(bpf_map_alloc_security, selinux_bpf_map_alloc), + LSM_HOOK_INIT(bpf_prog_alloc_security, selinux_bpf_prog_alloc), #endif #ifdef CONFIG_PERF_EVENTS LSM_HOOK_INIT(perf_event_alloc, selinux_perf_event_alloc), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index e0545201b55f..7f24d898efbb 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -847,36 +847,6 @@ union bpf_iter_link_info { * Returns zero on success. On error, -1 is returned and *errno* * is set appropriately. * - * BPF_TOKEN_CREATE - * Description - * Create BPF token with embedded information about what - * BPF-related functionality it allows: - * - a set of allowed bpf() syscall commands; - * - a set of allowed BPF map types to be created with - * BPF_MAP_CREATE command, if BPF_MAP_CREATE itself is allowed; - * - a set of allowed BPF program types and BPF program attach - * types to be loaded with BPF_PROG_LOAD command, if - * BPF_PROG_LOAD itself is allowed. - * - * BPF token is created (derived) from an instance of BPF FS, - * assuming it has necessary delegation mount options specified. - * This BPF token can be passed as an extra parameter to various - * bpf() syscall commands to grant BPF subsystem functionality to - * unprivileged processes. - * - * When created, BPF token is "associated" with the owning - * user namespace of BPF FS instance (super block) that it was - * derived from, and subsequent BPF operations performed with - * BPF token would be performing capabilities checks (i.e., - * CAP_BPF, CAP_PERFMON, CAP_NET_ADMIN, CAP_SYS_ADMIN) within - * that user namespace. Without BPF token, such capabilities - * have to be granted in init user namespace, making bpf() - * syscall incompatible with user namespace, for the most part. - * - * Return - * A new file descriptor (a nonnegative integer), or -1 if an - * error occurred (in which case, *errno* is set appropriately). - * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -931,8 +901,6 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, - BPF_TOKEN_CREATE, - __MAX_BPF_CMD, }; enum bpf_map_type { @@ -983,7 +951,6 @@ enum bpf_map_type { BPF_MAP_TYPE_BLOOM_FILTER, BPF_MAP_TYPE_USER_RINGBUF, BPF_MAP_TYPE_CGRP_STORAGE, - __MAX_BPF_MAP_TYPE }; /* Note that tracing related programs such as @@ -1028,7 +995,6 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, - __MAX_BPF_PROG_TYPE }; enum bpf_attach_type { @@ -1437,7 +1403,6 @@ union bpf_attr { * to using 5 hash functions). */ __u64 map_extra; - __u32 map_token_fd; }; struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ @@ -1507,7 +1472,6 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 log_true_size; - __u32 prog_token_fd; }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -1620,7 +1584,6 @@ union bpf_attr { * truncated), or smaller (if log buffer wasn't filled completely). */ __u32 btf_log_true_size; - __u32 btf_token_fd; }; struct { @@ -1751,11 +1714,6 @@ union bpf_attr { __u32 flags; /* extra flags */ } prog_bind_map; - struct { /* struct used by BPF_TOKEN_CREATE command */ - __u32 flags; - __u32 bpffs_fd; - } token_create; - } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index b6619199a706..2d0c282c8588 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1,4 +1,4 @@ libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \ netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \ btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \ - usdt.o zip.o elf.o features.o + usdt.o zip.o elf.o diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 0ad8e532b3cf..9dc9625651dc 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -103,7 +103,7 @@ int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") */ -int probe_memcg_account(int token_fd) +int probe_memcg_account(void) { const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); struct bpf_insn insns[] = { @@ -120,7 +120,6 @@ int probe_memcg_account(int token_fd) attr.insns = ptr_to_u64(insns); attr.insn_cnt = insn_cnt; attr.license = ptr_to_u64("GPL"); - attr.prog_token_fd = token_fd; prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); if (prog_fd >= 0) { @@ -147,7 +146,7 @@ int bump_rlimit_memlock(void) struct rlimit rlim; /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ - if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT)) + if (memlock_bumped || kernel_supports(NULL, FEAT_MEMCG_ACCOUNT)) return 0; memlock_bumped = true; @@ -170,7 +169,7 @@ int bpf_map_create(enum bpf_map_type map_type, __u32 max_entries, const struct bpf_map_create_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, map_token_fd); + const size_t attr_sz = offsetofend(union bpf_attr, map_extra); union bpf_attr attr; int fd; @@ -182,7 +181,7 @@ int bpf_map_create(enum bpf_map_type map_type, return libbpf_err(-EINVAL); attr.map_type = map_type; - if (map_name && feat_supported(NULL, FEAT_PROG_NAME)) + if (map_name && kernel_supports(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; @@ -199,8 +198,6 @@ int bpf_map_create(enum bpf_map_type map_type, attr.numa_node = OPTS_GET(opts, numa_node, 0); attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0); - attr.map_token_fd = OPTS_GET(opts, token_fd, 0); - fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); return libbpf_err_errno(fd); } @@ -235,7 +232,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); + const size_t attr_sz = offsetofend(union bpf_attr, log_true_size); void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; @@ -264,9 +261,8 @@ int bpf_prog_load(enum bpf_prog_type prog_type, attr.prog_flags = OPTS_GET(opts, prog_flags, 0); attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); attr.kern_version = OPTS_GET(opts, kern_version, 0); - attr.prog_token_fd = OPTS_GET(opts, token_fd, 0); - if (prog_name && feat_supported(NULL, FEAT_PROG_NAME)) + if (prog_name && kernel_supports(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); @@ -1186,7 +1182,7 @@ int bpf_raw_tracepoint_open(const char *name, int prog_fd) int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts) { - const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd); + const size_t attr_sz = offsetofend(union bpf_attr, btf_log_true_size); union bpf_attr attr; char *log_buf; size_t log_size; @@ -1211,8 +1207,6 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts attr.btf = ptr_to_u64(btf_data); attr.btf_size = btf_size; - attr.btf_token_fd = OPTS_GET(opts, token_fd, 0); - /* log_level == 0 and log_buf != NULL means "try loading without * log_buf, but retry with log_buf and log_level=1 on error", which is * consistent across low-level and high-level BTF and program loading @@ -1293,20 +1287,3 @@ int bpf_prog_bind_map(int prog_fd, int map_fd, ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz); return libbpf_err_errno(ret); } - -int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts) -{ - const size_t attr_sz = offsetofend(union bpf_attr, token_create); - union bpf_attr attr; - int fd; - - if (!OPTS_VALID(opts, bpf_token_create_opts)) - return libbpf_err(-EINVAL); - - memset(&attr, 0, attr_sz); - attr.token_create.bpffs_fd = bpffs_fd; - attr.token_create.flags = OPTS_GET(opts, flags, 0); - - fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz); - return libbpf_err_errno(fd); -} diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 991b86bfe7e4..d0f53772bdc0 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -51,11 +51,8 @@ struct bpf_map_create_opts { __u32 numa_node; __u32 map_ifindex; - - __u32 token_fd; - size_t :0; }; -#define bpf_map_create_opts__last_field token_fd +#define bpf_map_create_opts__last_field map_ifindex LIBBPF_API int bpf_map_create(enum bpf_map_type map_type, const char *map_name, @@ -105,10 +102,9 @@ struct bpf_prog_load_opts { * If kernel doesn't support this feature, log_size is left unchanged. */ __u32 log_true_size; - __u32 token_fd; size_t :0; }; -#define bpf_prog_load_opts__last_field token_fd +#define bpf_prog_load_opts__last_field log_true_size LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name, const char *license, @@ -134,10 +130,9 @@ struct bpf_btf_load_opts { * If kernel doesn't support this feature, log_size is left unchanged. */ __u32 log_true_size; - __u32 token_fd; size_t :0; }; -#define bpf_btf_load_opts__last_field token_fd +#define bpf_btf_load_opts__last_field log_true_size LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts); @@ -645,30 +640,6 @@ struct bpf_test_run_opts { LIBBPF_API int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts); -struct bpf_token_create_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - size_t :0; -}; -#define bpf_token_create_opts__last_field flags - -/** - * @brief **bpf_token_create()** creates a new instance of BPF token derived - * from specified BPF FS mount point. - * - * BPF token created with this API can be passed to bpf() syscall for - * commands like BPF_PROG_LOAD, BPF_MAP_CREATE, etc. - * - * @param bpffs_fd FD for BPF FS instance from which to derive a BPF token - * instance. - * @param opts optional BPF token creation options, can be NULL - * - * @return BPF token FD > 0, on success; negative error code, otherwise (errno - * is also set to the error code) - */ -LIBBPF_API int bpf_token_create(int bpffs_fd, - struct bpf_token_create_opts *opts); - #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 63033c334320..ee95fd379d4d 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1317,9 +1317,7 @@ struct btf *btf__parse_split(const char *path, struct btf *base_btf) static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian); -int btf_load_into_kernel(struct btf *btf, - char *log_buf, size_t log_sz, __u32 log_level, - int token_fd) +int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level) { LIBBPF_OPTS(bpf_btf_load_opts, opts); __u32 buf_sz = 0, raw_size; @@ -1369,7 +1367,6 @@ retry_load: opts.log_level = log_level; } - opts.token_fd = token_fd; btf->fd = bpf_btf_load(raw_data, raw_size, &opts); if (btf->fd < 0) { /* time to turn on verbose mode and try again */ @@ -1397,7 +1394,7 @@ done: int btf__load_into_kernel(struct btf *btf) { - return btf_load_into_kernel(btf, NULL, 0, 0, 0); + return btf_load_into_kernel(btf, NULL, 0, 0); } int btf__fd(const struct btf *btf) diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index c92e02394159..b02faec748a5 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -11,6 +11,8 @@ #include "libbpf_internal.h" #include "str_error.h" +#define STRERR_BUFSIZE 128 + /* A SHT_GNU_versym section holds 16-bit words. This bit is set if * the symbol is hidden and can only be seen when referenced using an * explicit version number. This is a GNU extension. diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c deleted file mode 100644 index ce98a334be21..000000000000 --- a/tools/lib/bpf/features.c +++ /dev/null @@ -1,478 +0,0 @@ -// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ -#include -#include -#include "bpf.h" -#include "libbpf.h" -#include "libbpf_common.h" -#include "libbpf_internal.h" -#include "str_error.h" - -static inline __u64 ptr_to_u64(const void *ptr) -{ - return (__u64)(unsigned long)ptr; -} - -static int probe_fd(int fd) -{ - if (fd >= 0) - close(fd); - return fd >= 0; -} - -static int probe_kern_prog_name(int token_fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, prog_name); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - union bpf_attr attr; - int ret; - - memset(&attr, 0, attr_sz); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.license = ptr_to_u64("GPL"); - attr.insns = ptr_to_u64(insns); - attr.insn_cnt = (__u32)ARRAY_SIZE(insns); - attr.prog_token_fd = token_fd; - libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); - - /* make sure loading with name works */ - ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); - return probe_fd(ret); -} - -static int probe_kern_global_data(int token_fd) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), - BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_map_create_opts, map_opts, .token_fd = token_fd); - LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, .token_fd = token_fd); - int ret, map, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - insns[0].imm = map; - - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); - close(map); - return probe_fd(ret); -} - -static int probe_kern_btf(int token_fd) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_func(int token_fd) -{ - static const char strs[] = "\0int\0x\0a"; - /* void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_func_global(int token_fd) -{ - static const char strs[] = "\0int\0x\0a"; - /* static void x(int a) {} */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* FUNC_PROTO */ /* [2] */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), - BTF_PARAM_ENC(7, 1), - /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ - BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_datasec(int token_fd) -{ - static const char strs[] = "\0x\0.data"; - /* static int a; */ - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* DATASEC val */ /* [3] */ - BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), - BTF_VAR_SECINFO_ENC(2, 0, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_float(int token_fd) -{ - static const char strs[] = "\0float"; - __u32 types[] = { - /* float */ - BTF_TYPE_FLOAT_ENC(1, 4), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_decl_tag(int token_fd) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* VAR x */ /* [2] */ - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), - BTF_VAR_STATIC, - /* attr */ - BTF_TYPE_DECL_TAG_ENC(1, 2, -1), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_btf_type_tag(int token_fd) -{ - static const char strs[] = "\0tag"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ - /* attr */ - BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ - /* ptr */ - BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -static int probe_kern_array_mmap(int token_fd) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, - .map_flags = BPF_F_MMAPABLE, - .token_fd = token_fd, - ); - int fd; - - fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); - return probe_fd(fd); -} - -static int probe_kern_exp_attach_type(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE, - .token_fd = token_fd, - ); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - /* use any valid combination of program type and (optional) - * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) - * to see if kernel supports expected_attach_type field for - * BPF_PROG_LOAD command - */ - fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(fd); -} - -static int probe_kern_probe_read_kernel(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); - struct bpf_insn insns[] = { - BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ - BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ - BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), - BPF_EXIT_INSN(), - }; - int fd, insn_cnt = ARRAY_SIZE(insns); - - fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(fd); -} - -static int probe_prog_bind_map(int token_fd) -{ - char *cp, errmsg[STRERR_BUFSIZE]; - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_map_create_opts, map_opts, .token_fd = token_fd); - LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, .token_fd = token_fd); - int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); - - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts); - if (map < 0) { - ret = -errno; - cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); - pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", - __func__, cp, -ret); - return ret; - } - - prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); - if (prog < 0) { - close(map); - return 0; - } - - ret = bpf_prog_bind_map(prog, map, NULL); - - close(map); - close(prog); - - return ret >= 0; -} - -static int probe_module_btf(int token_fd) -{ - static const char strs[] = "\0int"; - __u32 types[] = { - /* int */ - BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), - }; - struct bpf_btf_info info; - __u32 len = sizeof(info); - char name[16]; - int fd, err; - - fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); - if (fd < 0) - return 0; /* BTF not supported at all */ - - memset(&info, 0, sizeof(info)); - info.name = ptr_to_u64(name); - info.name_len = sizeof(name); - - /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; - * kernel's module BTF support coincides with support for - * name/name_len fields in struct bpf_btf_info. - */ - err = bpf_btf_get_info_by_fd(fd, &info, &len); - close(fd); - return !err; -} - -static int probe_perf_link(int token_fd) -{ - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); - int prog_fd, link_fd, err; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", - insns, ARRAY_SIZE(insns), &opts); - if (prog_fd < 0) - return -errno; - - /* use invalid perf_event FD to get EBADF, if link is supported; - * otherwise EINVAL should be returned - */ - link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_uprobe_multi_link(int token_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, load_opts, - .expected_attach_type = BPF_TRACE_UPROBE_MULTI, - .token_fd = token_fd, - ); - LIBBPF_OPTS(bpf_link_create_opts, link_opts); - struct bpf_insn insns[] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - int prog_fd, link_fd, err; - unsigned long offset = 0; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", - insns, ARRAY_SIZE(insns), &load_opts); - if (prog_fd < 0) - return -errno; - - /* Creating uprobe in '/' binary should fail with -EBADF. */ - link_opts.uprobe_multi.path = "/"; - link_opts.uprobe_multi.offsets = &offset; - link_opts.uprobe_multi.cnt = 1; - - link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); - err = -errno; /* close() can clobber errno */ - - if (link_fd >= 0) - close(link_fd); - close(prog_fd); - - return link_fd < 0 && err == -EBADF; -} - -static int probe_kern_bpf_cookie(int token_fd) -{ - struct bpf_insn insns[] = { - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), - BPF_EXIT_INSN(), - }; - LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = token_fd); - int ret, insn_cnt = ARRAY_SIZE(insns); - - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); - return probe_fd(ret); -} - -static int probe_kern_btf_enum64(int token_fd) -{ - static const char strs[] = "\0enum64"; - __u32 types[] = { - BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), - }; - - return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), token_fd)); -} - -typedef int (*feature_probe_fn)(int /* token_fd */); - -static struct kern_feature_cache feature_cache; - -static struct kern_feature_desc { - const char *desc; - feature_probe_fn probe; -} feature_probes[__FEAT_CNT] = { - [FEAT_PROG_NAME] = { - "BPF program name", probe_kern_prog_name, - }, - [FEAT_GLOBAL_DATA] = { - "global variables", probe_kern_global_data, - }, - [FEAT_BTF] = { - "minimal BTF", probe_kern_btf, - }, - [FEAT_BTF_FUNC] = { - "BTF functions", probe_kern_btf_func, - }, - [FEAT_BTF_GLOBAL_FUNC] = { - "BTF global function", probe_kern_btf_func_global, - }, - [FEAT_BTF_DATASEC] = { - "BTF data section and variable", probe_kern_btf_datasec, - }, - [FEAT_ARRAY_MMAP] = { - "ARRAY map mmap()", probe_kern_array_mmap, - }, - [FEAT_EXP_ATTACH_TYPE] = { - "BPF_PROG_LOAD expected_attach_type attribute", - probe_kern_exp_attach_type, - }, - [FEAT_PROBE_READ_KERN] = { - "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, - }, - [FEAT_PROG_BIND_MAP] = { - "BPF_PROG_BIND_MAP support", probe_prog_bind_map, - }, - [FEAT_MODULE_BTF] = { - "module BTF support", probe_module_btf, - }, - [FEAT_BTF_FLOAT] = { - "BTF_KIND_FLOAT support", probe_kern_btf_float, - }, - [FEAT_PERF_LINK] = { - "BPF perf link support", probe_perf_link, - }, - [FEAT_BTF_DECL_TAG] = { - "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, - }, - [FEAT_BTF_TYPE_TAG] = { - "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, - }, - [FEAT_MEMCG_ACCOUNT] = { - "memcg-based memory accounting", probe_memcg_account, - }, - [FEAT_BPF_COOKIE] = { - "BPF cookie support", probe_kern_bpf_cookie, - }, - [FEAT_BTF_ENUM64] = { - "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, - }, - [FEAT_SYSCALL_WRAPPER] = { - "Kernel using syscall wrapper", probe_kern_syscall_wrapper, - }, - [FEAT_UPROBE_MULTI_LINK] = { - "BPF multi-uprobe link support", probe_uprobe_multi_link, - }, -}; - -bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) -{ - struct kern_feature_desc *feat = &feature_probes[feat_id]; - int ret; - - /* assume global feature cache, unless custom one is provided */ - if (!cache) - cache = &feature_cache; - - if (READ_ONCE(cache->res[feat_id]) == FEAT_UNKNOWN) { - ret = feat->probe(cache->token_fd); - if (ret > 0) { - WRITE_ONCE(cache->res[feat_id], FEAT_SUPPORTED); - } else if (ret == 0) { - WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); - } else { - pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); - WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); - } - } - - return READ_ONCE(cache->res[feat_id]) == FEAT_SUPPORTED; -} diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 4b5ff9508e18..ac54ebc0629f 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -59,8 +59,6 @@ #define BPF_FS_MAGIC 0xcafe4a11 #endif -#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" - #define BPF_INSN_SZ (sizeof(struct bpf_insn)) /* vsprintf() in __base_pr() uses nonliteral format string. It may break @@ -695,10 +693,6 @@ struct bpf_object { struct usdt_manager *usdt_man; - struct kern_feature_cache *feat_cache; - char *token_path; - int token_fd; - char path[]; }; @@ -2198,7 +2192,7 @@ static int build_map_pin_path(struct bpf_map *map, const char *path) int err; if (!path) - path = BPF_FS_DEFAULT_PATH; + path = "/sys/fs/bpf"; err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); if (err) @@ -3285,7 +3279,7 @@ skip_exception_cb: } else { /* currently BPF_BTF_LOAD only supports log_level 1 */ err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, - obj->log_level ? 1 : 0, obj->token_fd); + obj->log_level ? 1 : 0); } if (sanitize) { if (!err) { @@ -4608,63 +4602,6 @@ int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) return 0; } -static int bpf_object_prepare_token(struct bpf_object *obj) -{ - const char *bpffs_path; - int bpffs_fd = -1, token_fd, err; - bool mandatory; - enum libbpf_print_level level; - - /* token is already set up */ - if (obj->token_fd > 0) - return 0; - /* token is explicitly prevented */ - if (obj->token_fd < 0) { - pr_debug("object '%s': token is prevented, skipping...\n", obj->name); - /* reset to zero to avoid extra checks during map_create and prog_load steps */ - obj->token_fd = 0; - return 0; - } - - mandatory = obj->token_path != NULL; - level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG; - - bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; - bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR); - if (bpffs_fd < 0) { - err = -errno; - __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n", - obj->name, err, bpffs_path, - mandatory ? "" : ", skipping optional step..."); - return mandatory ? err : 0; - } - - token_fd = bpf_token_create(bpffs_fd, 0); - close(bpffs_fd); - if (token_fd < 0) { - if (!mandatory && token_fd == -ENOENT) { - pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n", - obj->name, bpffs_path); - return 0; - } - __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n", - obj->name, token_fd, bpffs_path, - mandatory ? "" : ", skipping optional step..."); - return mandatory ? token_fd : 0; - } - - obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); - if (!obj->feat_cache) { - close(token_fd); - return -ENOMEM; - } - - obj->token_fd = token_fd; - obj->feat_cache->token_fd = token_fd; - - return 0; -} - static int bpf_object__probe_loading(struct bpf_object *obj) { @@ -4674,7 +4611,6 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, .token_fd = obj->token_fd); if (obj->gen_loader) return 0; @@ -4684,9 +4620,9 @@ bpf_object__probe_loading(struct bpf_object *obj) pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); @@ -4701,18 +4637,462 @@ bpf_object__probe_loading(struct bpf_object *obj) return 0; } +static int probe_fd(int fd) +{ + if (fd >= 0) + close(fd); + return fd >= 0; +} + +static int probe_kern_prog_name(void) +{ + const size_t attr_sz = offsetofend(union bpf_attr, prog_name); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + union bpf_attr attr; + int ret; + + memset(&attr, 0, attr_sz); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.license = ptr_to_u64("GPL"); + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = (__u32)ARRAY_SIZE(insns); + libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); + + /* make sure loading with name works */ + ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); + return probe_fd(ret); +} + +static int probe_kern_global_data(void) +{ + char *cp, errmsg[STRERR_BUFSIZE]; + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL); + if (map < 0) { + ret = -errno; + cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); + pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", + __func__, cp, -ret); + return ret; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); + close(map); + return probe_fd(ret); +} + +static int probe_kern_btf(void) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_func(void) +{ + static const char strs[] = "\0int\0x\0a"; + /* void x(int a) {} */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* FUNC_PROTO */ /* [2] */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), + BTF_PARAM_ENC(7, 1), + /* FUNC x */ /* [3] */ + BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_func_global(void) +{ + static const char strs[] = "\0int\0x\0a"; + /* static void x(int a) {} */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* FUNC_PROTO */ /* [2] */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), + BTF_PARAM_ENC(7, 1), + /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ + BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_datasec(void) +{ + static const char strs[] = "\0x\0.data"; + /* static int a; */ + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* VAR x */ /* [2] */ + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), + BTF_VAR_STATIC, + /* DATASEC val */ /* [3] */ + BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), + BTF_VAR_SECINFO_ENC(2, 0, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_float(void) +{ + static const char strs[] = "\0float"; + __u32 types[] = { + /* float */ + BTF_TYPE_FLOAT_ENC(1, 4), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_decl_tag(void) +{ + static const char strs[] = "\0tag"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* VAR x */ /* [2] */ + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), + BTF_VAR_STATIC, + /* attr */ + BTF_TYPE_DECL_TAG_ENC(1, 2, -1), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_btf_type_tag(void) +{ + static const char strs[] = "\0tag"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* attr */ + BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ + /* ptr */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_array_mmap(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); + return probe_fd(fd); +} + +static int probe_kern_exp_attach_type(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int fd, insn_cnt = ARRAY_SIZE(insns); + + /* use any valid combination of program type and (optional) + * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) + * to see if kernel supports expected_attach_type field for + * BPF_PROG_LOAD command + */ + fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); + return probe_fd(fd); +} + +static int probe_kern_probe_read_kernel(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ + BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ + BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ + BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), + BPF_EXIT_INSN(), + }; + int fd, insn_cnt = ARRAY_SIZE(insns); + + fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); + return probe_fd(fd); +} + +static int probe_prog_bind_map(void) +{ + char *cp, errmsg[STRERR_BUFSIZE]; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL); + if (map < 0) { + ret = -errno; + cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); + pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", + __func__, cp, -ret); + return ret; + } + + prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); + if (prog < 0) { + close(map); + return 0; + } + + ret = bpf_prog_bind_map(prog, map, NULL); + + close(map); + close(prog); + + return ret >= 0; +} + +static int probe_module_btf(void) +{ + static const char strs[] = "\0int"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }; + struct bpf_btf_info info; + __u32 len = sizeof(info); + char name[16]; + int fd, err; + + fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); + if (fd < 0) + return 0; /* BTF not supported at all */ + + memset(&info, 0, sizeof(info)); + info.name = ptr_to_u64(name); + info.name_len = sizeof(name); + + /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; + * kernel's module BTF support coincides with support for + * name/name_len fields in struct bpf_btf_info. + */ + err = bpf_btf_get_info_by_fd(fd, &info, &len); + close(fd); + return !err; +} + +static int probe_perf_link(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int prog_fd, link_fd, err; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", + insns, ARRAY_SIZE(insns), NULL); + if (prog_fd < 0) + return -errno; + + /* use invalid perf_event FD to get EBADF, if link is supported; + * otherwise EINVAL should be returned + */ + link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); + err = -errno; /* close() can clobber errno */ + + if (link_fd >= 0) + close(link_fd); + close(prog_fd); + + return link_fd < 0 && err == -EBADF; +} + +static int probe_uprobe_multi_link(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, load_opts, + .expected_attach_type = BPF_TRACE_UPROBE_MULTI, + ); + LIBBPF_OPTS(bpf_link_create_opts, link_opts); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int prog_fd, link_fd, err; + unsigned long offset = 0; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", + insns, ARRAY_SIZE(insns), &load_opts); + if (prog_fd < 0) + return -errno; + + /* Creating uprobe in '/' binary should fail with -EBADF. */ + link_opts.uprobe_multi.path = "/"; + link_opts.uprobe_multi.offsets = &offset; + link_opts.uprobe_multi.cnt = 1; + + link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); + err = -errno; /* close() can clobber errno */ + + if (link_fd >= 0) + close(link_fd); + close(prog_fd); + + return link_fd < 0 && err == -EBADF; +} + +static int probe_kern_bpf_cookie(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), + BPF_EXIT_INSN(), + }; + int ret, insn_cnt = ARRAY_SIZE(insns); + + ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); + return probe_fd(ret); +} + +static int probe_kern_btf_enum64(void) +{ + static const char strs[] = "\0enum64"; + __u32 types[] = { + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_syscall_wrapper(void); + +enum kern_feature_result { + FEAT_UNKNOWN = 0, + FEAT_SUPPORTED = 1, + FEAT_MISSING = 2, +}; + +typedef int (*feature_probe_fn)(void); + +static struct kern_feature_desc { + const char *desc; + feature_probe_fn probe; + enum kern_feature_result res; +} feature_probes[__FEAT_CNT] = { + [FEAT_PROG_NAME] = { + "BPF program name", probe_kern_prog_name, + }, + [FEAT_GLOBAL_DATA] = { + "global variables", probe_kern_global_data, + }, + [FEAT_BTF] = { + "minimal BTF", probe_kern_btf, + }, + [FEAT_BTF_FUNC] = { + "BTF functions", probe_kern_btf_func, + }, + [FEAT_BTF_GLOBAL_FUNC] = { + "BTF global function", probe_kern_btf_func_global, + }, + [FEAT_BTF_DATASEC] = { + "BTF data section and variable", probe_kern_btf_datasec, + }, + [FEAT_ARRAY_MMAP] = { + "ARRAY map mmap()", probe_kern_array_mmap, + }, + [FEAT_EXP_ATTACH_TYPE] = { + "BPF_PROG_LOAD expected_attach_type attribute", + probe_kern_exp_attach_type, + }, + [FEAT_PROBE_READ_KERN] = { + "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, + }, + [FEAT_PROG_BIND_MAP] = { + "BPF_PROG_BIND_MAP support", probe_prog_bind_map, + }, + [FEAT_MODULE_BTF] = { + "module BTF support", probe_module_btf, + }, + [FEAT_BTF_FLOAT] = { + "BTF_KIND_FLOAT support", probe_kern_btf_float, + }, + [FEAT_PERF_LINK] = { + "BPF perf link support", probe_perf_link, + }, + [FEAT_BTF_DECL_TAG] = { + "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, + }, + [FEAT_BTF_TYPE_TAG] = { + "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, + }, + [FEAT_MEMCG_ACCOUNT] = { + "memcg-based memory accounting", probe_memcg_account, + }, + [FEAT_BPF_COOKIE] = { + "BPF cookie support", probe_kern_bpf_cookie, + }, + [FEAT_BTF_ENUM64] = { + "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, + }, + [FEAT_SYSCALL_WRAPPER] = { + "Kernel using syscall wrapper", probe_kern_syscall_wrapper, + }, + [FEAT_UPROBE_MULTI_LINK] = { + "BPF multi-uprobe link support", probe_uprobe_multi_link, + }, +}; + bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) { + struct kern_feature_desc *feat = &feature_probes[feat_id]; + int ret; + if (obj && obj->gen_loader) /* To generate loader program assume the latest kernel * to avoid doing extra prog_load, map_create syscalls. */ return true; - if (obj->token_fd) - return feat_supported(obj->feat_cache, feat_id); + if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { + ret = feat->probe(); + if (ret > 0) { + WRITE_ONCE(feat->res, FEAT_SUPPORTED); + } else if (ret == 0) { + WRITE_ONCE(feat->res, FEAT_MISSING); + } else { + pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); + WRITE_ONCE(feat->res, FEAT_MISSING); + } + } - return feat_supported(NULL, feat_id); + return READ_ONCE(feat->res) == FEAT_SUPPORTED; } static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) @@ -4831,7 +5211,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b create_attr.map_flags = def->map_flags; create_attr.numa_node = map->numa_node; create_attr.map_extra = map->map_extra; - create_attr.token_fd = obj->token_fd; if (bpf_map__is_struct_ops(map)) create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; @@ -6667,7 +7046,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog load_attr.attach_btf_id = prog->attach_btf_id; load_attr.kern_version = kern_version; load_attr.prog_ifindex = prog->prog_ifindex; - load_attr.token_fd = obj->token_fd; /* specify func_info/line_info only if kernel supports them */ btf_fd = bpf_object__btf_fd(obj); @@ -7129,10 +7507,10 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts) { - const char *obj_name, *kconfig, *btf_tmp_path, *token_path; + const char *obj_name, *kconfig, *btf_tmp_path; struct bpf_object *obj; char tmp_name[64]; - int err, token_fd; + int err; char *log_buf; size_t log_size; __u32 log_level; @@ -7166,28 +7544,6 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, if (log_size && !log_buf) return ERR_PTR(-EINVAL); - token_path = OPTS_GET(opts, bpf_token_path, NULL); - token_fd = OPTS_GET(opts, bpf_token_fd, -1); - /* non-empty token path can't be combined with invalid token FD */ - if (token_path && token_path[0] != '\0' && token_fd < 0) - return ERR_PTR(-EINVAL); - /* empty token path can't be combined with valid token FD */ - if (token_path && token_path[0] == '\0' && token_fd > 0) - return ERR_PTR(-EINVAL); - /* if user didn't specify bpf_token_path/bpf_token_fd explicitly, - * check if LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as - * bpf_token_path option - */ - if (token_fd == 0 && !token_path) - token_path = getenv("LIBBPF_BPF_TOKEN_PATH"); - /* empty token_path is equivalent to invalid token_fd */ - if (token_path && token_path[0] == '\0') { - token_path = NULL; - token_fd = -1; - } - if (token_path && strlen(token_path) >= PATH_MAX) - return ERR_PTR(-ENAMETOOLONG); - obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); if (IS_ERR(obj)) return obj; @@ -7196,19 +7552,6 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, obj->log_size = log_size; obj->log_level = log_level; - obj->token_fd = token_fd <= 0 ? token_fd : dup_good_fd(token_fd); - if (token_fd > 0 && obj->token_fd < 0) { - err = -errno; - goto out; - } - if (token_path) { - obj->token_path = strdup(token_path); - if (!obj->token_path) { - err = -ENOMEM; - goto out; - } - } - btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); if (btf_tmp_path) { if (strlen(btf_tmp_path) >= PATH_MAX) { @@ -7719,8 +8062,7 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch if (obj->gen_loader) bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); - err = bpf_object_prepare_token(obj); - err = err ? : bpf_object__probe_loading(obj); + err = bpf_object__probe_loading(obj); err = err ? : bpf_object__load_vmlinux_btf(obj, false); err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); err = err ? : bpf_object__sanitize_and_load_btf(obj); @@ -8257,11 +8599,6 @@ void bpf_object__close(struct bpf_object *obj) } zfree(&obj->programs); - zfree(&obj->feat_cache); - zfree(&obj->token_path); - if (obj->token_fd > 0) - close(obj->token_fd); - free(obj); } @@ -10275,7 +10612,7 @@ static const char *arch_specific_syscall_pfx(void) #endif } -int probe_kern_syscall_wrapper(int token_fd) +static int probe_kern_syscall_wrapper(void) { char syscall_name[64]; const char *ksys_pfx; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 916904bd2a7a..6cd9c501624f 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -177,45 +177,10 @@ struct bpf_object_open_opts { * logs through its print callback. */ __u32 kernel_log_level; - /* FD of a BPF token instantiated by user through bpf_token_create() - * API. BPF object will keep dup()'ed FD internally, so passed token - * FD can be closed after BPF object/skeleton open step. - * - * Setting bpf_token_fd to negative value disables libbpf's automatic - * attempt to create BPF token from default BPF FS mount point - * (/sys/fs/bpf), in case this default behavior is undesirable. - * - * If bpf_token_path and bpf_token_fd are not specified, libbpf will - * consult LIBBPF_BPF_TOKEN_PATH environment variable. If set, it will - * be taken as a value of bpf_token_path option and will force libbpf - * to either create BPF token from provided custom BPF FS path, or - * will disable implicit BPF token creation, if envvar value is an - * empty string. - * - * bpf_token_path and bpf_token_fd are mutually exclusive and only one - * of those options should be set. Either of them overrides - * LIBBPF_BPF_TOKEN_PATH envvar. - */ - int bpf_token_fd; - /* Path to BPF FS mount point to derive BPF token from. - * - * Created BPF token will be used for all bpf() syscall operations - * that accept BPF token (e.g., map creation, BTF and program loads, - * etc) automatically within instantiated BPF object. - * - * Setting bpf_token_path option to empty string disables libbpf's - * automatic attempt to create BPF token from default BPF FS mount - * point (/sys/fs/bpf), in case this default behavior is undesirable. - * - * bpf_token_path and bpf_token_fd are mutually exclusive and only one - * of those options should be set. Either of them overrides - * LIBBPF_BPF_TOKEN_PATH envvar. - */ - const char *bpf_token_path; size_t :0; }; -#define bpf_object_open_opts__last_field bpf_token_path +#define bpf_object_open_opts__last_field kernel_log_level /** * @brief **bpf_object__open()** creates a bpf_object by opening diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index df7657b65c47..91c5aef7dae7 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -401,7 +401,6 @@ LIBBPF_1.3.0 { bpf_program__attach_netkit; bpf_program__attach_tcx; bpf_program__attach_uprobe_multi; - bpf_token_create; ring__avail_data_size; ring__consume; ring__consumer_pos; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4cda32298c49..b5d334754e5d 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -360,32 +360,15 @@ enum kern_feature_id { __FEAT_CNT, }; -enum kern_feature_result { - FEAT_UNKNOWN = 0, - FEAT_SUPPORTED = 1, - FEAT_MISSING = 2, -}; - -struct kern_feature_cache { - enum kern_feature_result res[__FEAT_CNT]; - int token_fd; -}; - -bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); +int probe_memcg_account(void); bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); - -int probe_kern_syscall_wrapper(int token_fd); -int probe_memcg_account(int token_fd); int bump_rlimit_memlock(void); int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len, - int token_fd); -int btf_load_into_kernel(struct btf *btf, - char *log_buf, size_t log_sz, __u32 log_level, - int token_fd); + const char *str_sec, size_t str_len); +int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level); struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, @@ -549,17 +532,6 @@ static inline bool is_ldimm64_insn(struct bpf_insn *insn) return insn->code == (BPF_LD | BPF_IMM | BPF_DW); } -/* Unconditionally dup FD, ensuring it doesn't use [0, 2] range. - * Original FD is not closed or altered in any other way. - * Preserves original FD value, if it's invalid (negative). - */ -static inline int dup_good_fd(int fd) -{ - if (fd < 0) - return fd; - return fcntl(fd, F_DUPFD_CLOEXEC, 3); -} - /* if fd is stdin, stdout, or stderr, dup to a fd greater than 2 * Takes ownership of the fd passed in, and closes it if calling * fcntl(fd, F_DUPFD_CLOEXEC, 3). @@ -571,7 +543,7 @@ static inline int ensure_good_fd(int fd) if (fd < 0) return fd; if (fd < 3) { - fd = dup_good_fd(fd); + fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); saved_errno = errno; close(old_fd); errno = saved_errno; diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 8e7437006639..9c4db90b92b6 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -219,8 +219,7 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) } int libbpf__load_raw_btf(const char *raw_types, size_t types_len, - const char *str_sec, size_t str_len, - int token_fd) + const char *str_sec, size_t str_len) { struct btf_header hdr = { .magic = BTF_MAGIC, @@ -230,7 +229,6 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, .str_off = types_len, .str_len = str_len, }; - LIBBPF_OPTS(bpf_btf_load_opts, opts, .token_fd = token_fd); int btf_fd, btf_len; __u8 *raw_btf; @@ -243,7 +241,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len); memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len); - btf_fd = bpf_btf_load(raw_btf, btf_len, &opts); + btf_fd = bpf_btf_load(raw_btf, btf_len, NULL); free(raw_btf); return btf_fd; @@ -273,7 +271,7 @@ static int load_local_storage_btf(void) }; return libbpf__load_raw_btf((char *)types, sizeof(types), - strs, sizeof(strs), 0); + strs, sizeof(strs)); } static int probe_map_create(enum bpf_map_type map_type) diff --git a/tools/lib/bpf/str_error.h b/tools/lib/bpf/str_error.h index 626d7ffb03d6..a139334d57b6 100644 --- a/tools/lib/bpf/str_error.h +++ b/tools/lib/bpf/str_error.h @@ -2,8 +2,5 @@ #ifndef __LIBBPF_STR_ERROR_H #define __LIBBPF_STR_ERROR_H -#define STRERR_BUFSIZE 128 - char *libbpf_strerror_r(int err, char *dst, int len); - #endif /* __LIBBPF_STR_ERROR_H */ diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c index 4ed46ed58a7b..9f766ddd946a 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_probes.c @@ -30,8 +30,6 @@ void test_libbpf_probe_prog_types(void) if (prog_type == BPF_PROG_TYPE_UNSPEC) continue; - if (strcmp(prog_type_name, "__MAX_BPF_PROG_TYPE") == 0) - continue; if (!test__start_subtest(prog_type_name)) continue; @@ -70,8 +68,6 @@ void test_libbpf_probe_map_types(void) if (map_type == BPF_MAP_TYPE_UNSPEC) continue; - if (strcmp(map_type_name, "__MAX_BPF_MAP_TYPE") == 0) - continue; if (!test__start_subtest(map_type_name)) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c index 62ea855ec4d0..eb34d612d6f8 100644 --- a/tools/testing/selftests/bpf/prog_tests/libbpf_str.c +++ b/tools/testing/selftests/bpf/prog_tests/libbpf_str.c @@ -132,9 +132,6 @@ static void test_libbpf_bpf_map_type_str(void) const char *map_type_str; char buf[256]; - if (map_type == __MAX_BPF_MAP_TYPE) - continue; - map_type_name = btf__str_by_offset(btf, e->name_off); map_type_str = libbpf_bpf_map_type_str(map_type); ASSERT_OK_PTR(map_type_str, map_type_name); @@ -189,9 +186,6 @@ static void test_libbpf_bpf_prog_type_str(void) const char *prog_type_str; char buf[256]; - if (prog_type == __MAX_BPF_PROG_TYPE) - continue; - prog_type_name = btf__str_by_offset(btf, e->name_off); prog_type_str = libbpf_bpf_prog_type_str(prog_type); ASSERT_OK_PTR(prog_type_str, prog_type_name); diff --git a/tools/testing/selftests/bpf/prog_tests/token.c b/tools/testing/selftests/bpf/prog_tests/token.c deleted file mode 100644 index b5dce630e0e1..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/token.c +++ /dev/null @@ -1,1031 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ -#define _GNU_SOURCE -#include -#include -#include "cap_helpers.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "priv_map.skel.h" -#include "priv_prog.skel.h" -#include "dummy_st_ops_success.skel.h" - -static inline int sys_mount(const char *dev_name, const char *dir_name, - const char *type, unsigned long flags, - const void *data) -{ - return syscall(__NR_mount, dev_name, dir_name, type, flags, data); -} - -static inline int sys_fsopen(const char *fsname, unsigned flags) -{ - return syscall(__NR_fsopen, fsname, flags); -} - -static inline int sys_fspick(int dfd, const char *path, unsigned flags) -{ - return syscall(__NR_fspick, dfd, path, flags); -} - -static inline int sys_fsconfig(int fs_fd, unsigned cmd, const char *key, const void *val, int aux) -{ - return syscall(__NR_fsconfig, fs_fd, cmd, key, val, aux); -} - -static inline int sys_fsmount(int fs_fd, unsigned flags, unsigned ms_flags) -{ - return syscall(__NR_fsmount, fs_fd, flags, ms_flags); -} - -static inline int sys_move_mount(int from_dfd, const char *from_path, - int to_dfd, const char *to_path, - unsigned flags) -{ - return syscall(__NR_move_mount, from_dfd, from_path, to_dfd, to_path, flags); -} - -static int drop_priv_caps(__u64 *old_caps) -{ - return cap_disable_effective((1ULL << CAP_BPF) | - (1ULL << CAP_PERFMON) | - (1ULL << CAP_NET_ADMIN) | - (1ULL << CAP_SYS_ADMIN), old_caps); -} - -static int restore_priv_caps(__u64 old_caps) -{ - return cap_enable_effective(old_caps, NULL); -} - -static int set_delegate_mask(int fs_fd, const char *key, __u64 mask, const char *mask_str) -{ - char buf[32]; - int err; - - if (!mask_str) { - if (mask == ~0ULL) { - mask_str = "any"; - } else { - snprintf(buf, sizeof(buf), "0x%llx", (unsigned long long)mask); - mask_str = buf; - } - } - - err = sys_fsconfig(fs_fd, FSCONFIG_SET_STRING, key, - mask_str, 0); - if (err < 0) - err = -errno; - return err; -} - -#define zclose(fd) do { if (fd >= 0) close(fd); fd = -1; } while (0) - -struct bpffs_opts { - __u64 cmds; - __u64 maps; - __u64 progs; - __u64 attachs; - const char *cmds_str; - const char *maps_str; - const char *progs_str; - const char *attachs_str; -}; - -static int create_bpffs_fd(void) -{ - int fs_fd; - - /* create VFS context */ - fs_fd = sys_fsopen("bpf", 0); - ASSERT_GE(fs_fd, 0, "fs_fd"); - - return fs_fd; -} - -static int materialize_bpffs_fd(int fs_fd, struct bpffs_opts *opts) -{ - int mnt_fd, err; - - /* set up token delegation mount options */ - err = set_delegate_mask(fs_fd, "delegate_cmds", opts->cmds, opts->cmds_str); - if (!ASSERT_OK(err, "fs_cfg_cmds")) - return err; - err = set_delegate_mask(fs_fd, "delegate_maps", opts->maps, opts->maps_str); - if (!ASSERT_OK(err, "fs_cfg_maps")) - return err; - err = set_delegate_mask(fs_fd, "delegate_progs", opts->progs, opts->progs_str); - if (!ASSERT_OK(err, "fs_cfg_progs")) - return err; - err = set_delegate_mask(fs_fd, "delegate_attachs", opts->attachs, opts->attachs_str); - if (!ASSERT_OK(err, "fs_cfg_attachs")) - return err; - - /* instantiate FS object */ - err = sys_fsconfig(fs_fd, FSCONFIG_CMD_CREATE, NULL, NULL, 0); - if (err < 0) - return -errno; - - /* create O_PATH fd for detached mount */ - mnt_fd = sys_fsmount(fs_fd, 0, 0); - if (err < 0) - return -errno; - - return mnt_fd; -} - -/* send FD over Unix domain (AF_UNIX) socket */ -static int sendfd(int sockfd, int fd) -{ - struct msghdr msg = {}; - struct cmsghdr *cmsg; - int fds[1] = { fd }, err; - char iobuf[1]; - struct iovec io = { - .iov_base = iobuf, - .iov_len = sizeof(iobuf), - }; - union { - char buf[CMSG_SPACE(sizeof(fds))]; - struct cmsghdr align; - } u; - - msg.msg_iov = &io; - msg.msg_iovlen = 1; - msg.msg_control = u.buf; - msg.msg_controllen = sizeof(u.buf); - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(fds)); - memcpy(CMSG_DATA(cmsg), fds, sizeof(fds)); - - err = sendmsg(sockfd, &msg, 0); - if (err < 0) - err = -errno; - if (!ASSERT_EQ(err, 1, "sendmsg")) - return -EINVAL; - - return 0; -} - -/* receive FD over Unix domain (AF_UNIX) socket */ -static int recvfd(int sockfd, int *fd) -{ - struct msghdr msg = {}; - struct cmsghdr *cmsg; - int fds[1], err; - char iobuf[1]; - struct iovec io = { - .iov_base = iobuf, - .iov_len = sizeof(iobuf), - }; - union { - char buf[CMSG_SPACE(sizeof(fds))]; - struct cmsghdr align; - } u; - - msg.msg_iov = &io; - msg.msg_iovlen = 1; - msg.msg_control = u.buf; - msg.msg_controllen = sizeof(u.buf); - - err = recvmsg(sockfd, &msg, 0); - if (err < 0) - err = -errno; - if (!ASSERT_EQ(err, 1, "recvmsg")) - return -EINVAL; - - cmsg = CMSG_FIRSTHDR(&msg); - if (!ASSERT_OK_PTR(cmsg, "cmsg_null") || - !ASSERT_EQ(cmsg->cmsg_len, CMSG_LEN(sizeof(fds)), "cmsg_len") || - !ASSERT_EQ(cmsg->cmsg_level, SOL_SOCKET, "cmsg_level") || - !ASSERT_EQ(cmsg->cmsg_type, SCM_RIGHTS, "cmsg_type")) - return -EINVAL; - - memcpy(fds, CMSG_DATA(cmsg), sizeof(fds)); - *fd = fds[0]; - - return 0; -} - -static ssize_t write_nointr(int fd, const void *buf, size_t count) -{ - ssize_t ret; - - do { - ret = write(fd, buf, count); - } while (ret < 0 && errno == EINTR); - - return ret; -} - -static int write_file(const char *path, const void *buf, size_t count) -{ - int fd; - ssize_t ret; - - fd = open(path, O_WRONLY | O_CLOEXEC | O_NOCTTY | O_NOFOLLOW); - if (fd < 0) - return -1; - - ret = write_nointr(fd, buf, count); - close(fd); - if (ret < 0 || (size_t)ret != count) - return -1; - - return 0; -} - -static int create_and_enter_userns(void) -{ - uid_t uid; - gid_t gid; - char map[100]; - - uid = getuid(); - gid = getgid(); - - if (unshare(CLONE_NEWUSER)) - return -1; - - if (write_file("/proc/self/setgroups", "deny", sizeof("deny") - 1) && - errno != ENOENT) - return -1; - - snprintf(map, sizeof(map), "0 %d 1", uid); - if (write_file("/proc/self/uid_map", map, strlen(map))) - return -1; - - - snprintf(map, sizeof(map), "0 %d 1", gid); - if (write_file("/proc/self/gid_map", map, strlen(map))) - return -1; - - if (setgid(0)) - return -1; - - if (setuid(0)) - return -1; - - return 0; -} - -typedef int (*child_callback_fn)(int); - -static void child(int sock_fd, struct bpffs_opts *opts, child_callback_fn callback) -{ - LIBBPF_OPTS(bpf_map_create_opts, map_opts); - int mnt_fd = -1, fs_fd = -1, err = 0, bpffs_fd = -1; - - /* setup userns with root mappings */ - err = create_and_enter_userns(); - if (!ASSERT_OK(err, "create_and_enter_userns")) - goto cleanup; - - /* setup mountns to allow creating BPF FS (fsopen("bpf")) from unpriv process */ - err = unshare(CLONE_NEWNS); - if (!ASSERT_OK(err, "create_mountns")) - goto cleanup; - - err = sys_mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, 0); - if (!ASSERT_OK(err, "remount_root")) - goto cleanup; - - fs_fd = create_bpffs_fd(); - if (!ASSERT_GE(fs_fd, 0, "create_bpffs_fd")) { - err = -EINVAL; - goto cleanup; - } - - /* ensure unprivileged child cannot set delegation options */ - err = set_delegate_mask(fs_fd, "delegate_cmds", 0x1, NULL); - ASSERT_EQ(err, -EPERM, "delegate_cmd_eperm"); - err = set_delegate_mask(fs_fd, "delegate_maps", 0x1, NULL); - ASSERT_EQ(err, -EPERM, "delegate_maps_eperm"); - err = set_delegate_mask(fs_fd, "delegate_progs", 0x1, NULL); - ASSERT_EQ(err, -EPERM, "delegate_progs_eperm"); - err = set_delegate_mask(fs_fd, "delegate_attachs", 0x1, NULL); - ASSERT_EQ(err, -EPERM, "delegate_attachs_eperm"); - - /* pass BPF FS context object to parent */ - err = sendfd(sock_fd, fs_fd); - if (!ASSERT_OK(err, "send_fs_fd")) - goto cleanup; - zclose(fs_fd); - - /* avoid mucking around with mount namespaces and mounting at - * well-known path, just get detach-mounted BPF FS fd back from parent - */ - err = recvfd(sock_fd, &mnt_fd); - if (!ASSERT_OK(err, "recv_mnt_fd")) - goto cleanup; - - /* try to fspick() BPF FS and try to add some delegation options */ - fs_fd = sys_fspick(mnt_fd, "", FSPICK_EMPTY_PATH); - if (!ASSERT_GE(fs_fd, 0, "bpffs_fspick")) { - err = -EINVAL; - goto cleanup; - } - - /* ensure unprivileged child cannot reconfigure to set delegation options */ - err = set_delegate_mask(fs_fd, "delegate_cmds", 0, "any"); - if (!ASSERT_EQ(err, -EPERM, "delegate_cmd_eperm_reconfig")) { - err = -EINVAL; - goto cleanup; - } - err = set_delegate_mask(fs_fd, "delegate_maps", 0, "any"); - if (!ASSERT_EQ(err, -EPERM, "delegate_maps_eperm_reconfig")) { - err = -EINVAL; - goto cleanup; - } - err = set_delegate_mask(fs_fd, "delegate_progs", 0, "any"); - if (!ASSERT_EQ(err, -EPERM, "delegate_progs_eperm_reconfig")) { - err = -EINVAL; - goto cleanup; - } - err = set_delegate_mask(fs_fd, "delegate_attachs", 0, "any"); - if (!ASSERT_EQ(err, -EPERM, "delegate_attachs_eperm_reconfig")) { - err = -EINVAL; - goto cleanup; - } - zclose(fs_fd); - - bpffs_fd = openat(mnt_fd, ".", 0, O_RDWR); - if (!ASSERT_GE(bpffs_fd, 0, "bpffs_open")) { - err = -EINVAL; - goto cleanup; - } - - /* do custom test logic with customly set up BPF FS instance */ - err = callback(bpffs_fd); - if (!ASSERT_OK(err, "test_callback")) - goto cleanup; - - err = 0; -cleanup: - zclose(sock_fd); - zclose(mnt_fd); - zclose(fs_fd); - zclose(bpffs_fd); - - exit(-err); -} - -static int wait_for_pid(pid_t pid) -{ - int status, ret; - -again: - ret = waitpid(pid, &status, 0); - if (ret == -1) { - if (errno == EINTR) - goto again; - - return -1; - } - - if (!WIFEXITED(status)) - return -1; - - return WEXITSTATUS(status); -} - -static void parent(int child_pid, struct bpffs_opts *bpffs_opts, int sock_fd) -{ - int fs_fd = -1, mnt_fd = -1, err; - - err = recvfd(sock_fd, &fs_fd); - if (!ASSERT_OK(err, "recv_bpffs_fd")) - goto cleanup; - - mnt_fd = materialize_bpffs_fd(fs_fd, bpffs_opts); - if (!ASSERT_GE(mnt_fd, 0, "materialize_bpffs_fd")) { - err = -EINVAL; - goto cleanup; - } - zclose(fs_fd); - - /* pass BPF FS context object to parent */ - err = sendfd(sock_fd, mnt_fd); - if (!ASSERT_OK(err, "send_mnt_fd")) - goto cleanup; - zclose(mnt_fd); - - err = wait_for_pid(child_pid); - ASSERT_OK(err, "waitpid_child"); - -cleanup: - zclose(sock_fd); - zclose(fs_fd); - zclose(mnt_fd); - - if (child_pid > 0) - (void)kill(child_pid, SIGKILL); -} - -static void subtest_userns(struct bpffs_opts *bpffs_opts, child_callback_fn cb) -{ - int sock_fds[2] = { -1, -1 }; - int child_pid = 0, err; - - err = socketpair(AF_UNIX, SOCK_STREAM, 0, sock_fds); - if (!ASSERT_OK(err, "socketpair")) - goto cleanup; - - child_pid = fork(); - if (!ASSERT_GE(child_pid, 0, "fork")) - goto cleanup; - - if (child_pid == 0) { - zclose(sock_fds[0]); - return child(sock_fds[1], bpffs_opts, cb); - - } else { - zclose(sock_fds[1]); - return parent(child_pid, bpffs_opts, sock_fds[0]); - } - -cleanup: - zclose(sock_fds[0]); - zclose(sock_fds[1]); - if (child_pid > 0) - (void)kill(child_pid, SIGKILL); -} - -static int userns_map_create(int mnt_fd) -{ - LIBBPF_OPTS(bpf_map_create_opts, map_opts); - int err, token_fd = -1, map_fd = -1; - __u64 old_caps = 0; - - /* create BPF token from BPF FS mount */ - token_fd = bpf_token_create(mnt_fd, NULL); - if (!ASSERT_GT(token_fd, 0, "token_create")) { - err = -EINVAL; - goto cleanup; - } - - /* while inside non-init userns, we need both a BPF token *and* - * CAP_BPF inside current userns to create privileged map; let's test - * that neither BPF token alone nor namespaced CAP_BPF is sufficient - */ - err = drop_priv_caps(&old_caps); - if (!ASSERT_OK(err, "drop_caps")) - goto cleanup; - - /* no token, no CAP_BPF -> fail */ - map_opts.token_fd = 0; - map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "wo_token_wo_bpf", 0, 8, 1, &map_opts); - if (!ASSERT_LT(map_fd, 0, "stack_map_wo_token_wo_cap_bpf_should_fail")) { - err = -EINVAL; - goto cleanup; - } - - /* token without CAP_BPF -> fail */ - map_opts.token_fd = token_fd; - map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "w_token_wo_bpf", 0, 8, 1, &map_opts); - if (!ASSERT_LT(map_fd, 0, "stack_map_w_token_wo_cap_bpf_should_fail")) { - err = -EINVAL; - goto cleanup; - } - - /* get back effective local CAP_BPF (and CAP_SYS_ADMIN) */ - err = restore_priv_caps(old_caps); - if (!ASSERT_OK(err, "restore_caps")) - goto cleanup; - - /* CAP_BPF without token -> fail */ - map_opts.token_fd = 0; - map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "wo_token_w_bpf", 0, 8, 1, &map_opts); - if (!ASSERT_LT(map_fd, 0, "stack_map_wo_token_w_cap_bpf_should_fail")) { - err = -EINVAL; - goto cleanup; - } - - /* finally, namespaced CAP_BPF + token -> success */ - map_opts.token_fd = token_fd; - map_fd = bpf_map_create(BPF_MAP_TYPE_STACK, "w_token_w_bpf", 0, 8, 1, &map_opts); - if (!ASSERT_GT(map_fd, 0, "stack_map_w_token_w_cap_bpf")) { - err = -EINVAL; - goto cleanup; - } - -cleanup: - zclose(token_fd); - zclose(map_fd); - return err; -} - -static int userns_btf_load(int mnt_fd) -{ - LIBBPF_OPTS(bpf_btf_load_opts, btf_opts); - int err, token_fd = -1, btf_fd = -1; - const void *raw_btf_data; - struct btf *btf = NULL; - __u32 raw_btf_size; - __u64 old_caps = 0; - - /* create BPF token from BPF FS mount */ - token_fd = bpf_token_create(mnt_fd, NULL); - if (!ASSERT_GT(token_fd, 0, "token_create")) { - err = -EINVAL; - goto cleanup; - } - - /* while inside non-init userns, we need both a BPF token *and* - * CAP_BPF inside current userns to create privileged map; let's test - * that neither BPF token alone nor namespaced CAP_BPF is sufficient - */ - err = drop_priv_caps(&old_caps); - if (!ASSERT_OK(err, "drop_caps")) - goto cleanup; - - /* setup a trivial BTF data to load to the kernel */ - btf = btf__new_empty(); - if (!ASSERT_OK_PTR(btf, "empty_btf")) - goto cleanup; - - ASSERT_GT(btf__add_int(btf, "int", 4, 0), 0, "int_type"); - - raw_btf_data = btf__raw_data(btf, &raw_btf_size); - if (!ASSERT_OK_PTR(raw_btf_data, "raw_btf_data")) - goto cleanup; - - /* no token + no CAP_BPF -> failure */ - btf_opts.token_fd = 0; - btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); - if (!ASSERT_LT(btf_fd, 0, "no_token_no_cap_should_fail")) - goto cleanup; - - /* token + no CAP_BPF -> failure */ - btf_opts.token_fd = token_fd; - btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); - if (!ASSERT_LT(btf_fd, 0, "token_no_cap_should_fail")) - goto cleanup; - - /* get back effective local CAP_BPF (and CAP_SYS_ADMIN) */ - err = restore_priv_caps(old_caps); - if (!ASSERT_OK(err, "restore_caps")) - goto cleanup; - - /* token + CAP_BPF -> success */ - btf_opts.token_fd = token_fd; - btf_fd = bpf_btf_load(raw_btf_data, raw_btf_size, &btf_opts); - if (!ASSERT_GT(btf_fd, 0, "token_and_cap_success")) - goto cleanup; - - err = 0; -cleanup: - btf__free(btf); - zclose(btf_fd); - zclose(token_fd); - return err; -} - -static int userns_prog_load(int mnt_fd) -{ - LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); - int err, token_fd = -1, prog_fd = -1; - struct bpf_insn insns[] = { - /* bpf_jiffies64() requires CAP_BPF */ - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_jiffies64), - /* bpf_get_current_task() requires CAP_PERFMON */ - BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task), - /* r0 = 0; exit; */ - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }; - size_t insn_cnt = ARRAY_SIZE(insns); - __u64 old_caps = 0; - - /* create BPF token from BPF FS mount */ - token_fd = bpf_token_create(mnt_fd, NULL); - if (!ASSERT_GT(token_fd, 0, "token_create")) { - err = -EINVAL; - goto cleanup; - } - - /* validate we can successfully load BPF program with token; this - * being XDP program (CAP_NET_ADMIN) using bpf_jiffies64() (CAP_BPF) - * and bpf_get_current_task() (CAP_PERFMON) helpers validates we have - * BPF token wired properly in a bunch of places in the kernel - */ - prog_opts.token_fd = token_fd; - prog_opts.expected_attach_type = BPF_XDP; - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", - insns, insn_cnt, &prog_opts); - if (!ASSERT_GT(prog_fd, 0, "prog_fd")) { - err = -EPERM; - goto cleanup; - } - - /* no token + caps -> failure */ - prog_opts.token_fd = 0; - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", - insns, insn_cnt, &prog_opts); - if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { - err = -EPERM; - goto cleanup; - } - - err = drop_priv_caps(&old_caps); - if (!ASSERT_OK(err, "drop_caps")) - goto cleanup; - - /* no caps + token -> failure */ - prog_opts.token_fd = token_fd; - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", - insns, insn_cnt, &prog_opts); - if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { - err = -EPERM; - goto cleanup; - } - - /* no caps + no token -> definitely a failure */ - prog_opts.token_fd = 0; - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, "token_prog", "GPL", - insns, insn_cnt, &prog_opts); - if (!ASSERT_EQ(prog_fd, -EPERM, "prog_fd_eperm")) { - err = -EPERM; - goto cleanup; - } - - err = 0; -cleanup: - zclose(prog_fd); - zclose(token_fd); - return err; -} - -static int userns_obj_priv_map(int mnt_fd) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts); - char buf[256]; - struct priv_map *skel; - int err, token_fd; - - skel = priv_map__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { - priv_map__destroy(skel); - return -EINVAL; - } - - /* use bpf_token_path to provide BPF FS path */ - snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); - opts.bpf_token_path = buf; - skel = priv_map__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) - return -EINVAL; - - err = priv_map__load(skel); - priv_map__destroy(skel); - if (!ASSERT_OK(err, "obj_token_path_load")) - return -EINVAL; - - /* create token and pass it through bpf_token_fd */ - token_fd = bpf_token_create(mnt_fd, NULL); - if (!ASSERT_GT(token_fd, 0, "create_token")) - return -EINVAL; - - opts.bpf_token_path = NULL; - opts.bpf_token_fd = token_fd; - skel = priv_map__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_token_fd_open")) - return -EINVAL; - - /* we can close our token FD, bpf_object owns dup()'ed FD now */ - close(token_fd); - - err = priv_map__load(skel); - priv_map__destroy(skel); - if (!ASSERT_OK(err, "obj_token_fd_load")) - return -EINVAL; - - return 0; -} - -static int userns_obj_priv_prog(int mnt_fd) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts); - char buf[256]; - struct priv_prog *skel; - int err; - - skel = priv_prog__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { - priv_prog__destroy(skel); - return -EINVAL; - } - - /* use bpf_token_path to provide BPF FS path */ - snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); - opts.bpf_token_path = buf; - skel = priv_prog__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) - return -EINVAL; - - err = priv_prog__load(skel); - priv_prog__destroy(skel); - if (!ASSERT_OK(err, "obj_token_path_load")) - return -EINVAL; - - return 0; -} - -/* this test is called with BPF FS that doesn't delegate BPF_BTF_LOAD command, - * which should cause struct_ops application to fail, as BTF won't be uploaded - * into the kernel, even if STRUCT_OPS programs themselves are allowed - */ -static int validate_struct_ops_load(int mnt_fd, bool expect_success) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts); - char buf[256]; - struct dummy_st_ops_success *skel; - int err; - - snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); - opts.bpf_token_path = buf; - skel = dummy_st_ops_success__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_token_path_open")) - return -EINVAL; - - err = dummy_st_ops_success__load(skel); - dummy_st_ops_success__destroy(skel); - if (expect_success) { - if (!ASSERT_OK(err, "obj_token_path_load")) - return -EINVAL; - } else /* expect failure */ { - if (!ASSERT_ERR(err, "obj_token_path_load")) - return -EINVAL; - } - - return 0; -} - -static int userns_obj_priv_btf_fail(int mnt_fd) -{ - return validate_struct_ops_load(mnt_fd, false /* should fail */); -} - -static int userns_obj_priv_btf_success(int mnt_fd) -{ - return validate_struct_ops_load(mnt_fd, true /* should succeed */); -} - -#define TOKEN_ENVVAR "LIBBPF_BPF_TOKEN_PATH" -#define TOKEN_BPFFS_CUSTOM "/bpf-token-fs" - -static int userns_obj_priv_implicit_token(int mnt_fd) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts); - struct dummy_st_ops_success *skel; - int err; - - /* before we mount BPF FS with token delegation, struct_ops skeleton - * should fail to load - */ - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { - dummy_st_ops_success__destroy(skel); - return -EINVAL; - } - - /* mount custom BPF FS over /sys/fs/bpf so that libbpf can create BPF - * token automatically and implicitly - */ - err = sys_move_mount(mnt_fd, "", AT_FDCWD, "/sys/fs/bpf", MOVE_MOUNT_F_EMPTY_PATH); - if (!ASSERT_OK(err, "move_mount_bpffs")) - return -EINVAL; - - /* disable implicit BPF token creation by setting - * LIBBPF_BPF_TOKEN_PATH envvar to empty value, load should fail - */ - err = setenv(TOKEN_ENVVAR, "", 1 /*overwrite*/); - if (!ASSERT_OK(err, "setenv_token_path")) - return -EINVAL; - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_token_envvar_disabled_load")) { - unsetenv(TOKEN_ENVVAR); - dummy_st_ops_success__destroy(skel); - return -EINVAL; - } - unsetenv(TOKEN_ENVVAR); - - /* now the same struct_ops skeleton should succeed thanks to libppf - * creating BPF token from /sys/fs/bpf mount point - */ - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_OK_PTR(skel, "obj_implicit_token_load")) - return -EINVAL; - - dummy_st_ops_success__destroy(skel); - - /* now disable implicit token through empty bpf_token_path, should fail */ - opts.bpf_token_path = ""; - skel = dummy_st_ops_success__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_empty_token_path_open")) - return -EINVAL; - - err = dummy_st_ops_success__load(skel); - dummy_st_ops_success__destroy(skel); - if (!ASSERT_ERR(err, "obj_empty_token_path_load")) - return -EINVAL; - - /* now disable implicit token through negative bpf_token_fd, should fail */ - opts.bpf_token_path = NULL; - opts.bpf_token_fd = -1; - skel = dummy_st_ops_success__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_neg_token_fd_open")) - return -EINVAL; - - err = dummy_st_ops_success__load(skel); - dummy_st_ops_success__destroy(skel); - if (!ASSERT_ERR(err, "obj_neg_token_fd_load")) - return -EINVAL; - - return 0; -} - -static int userns_obj_priv_implicit_token_envvar(int mnt_fd) -{ - LIBBPF_OPTS(bpf_object_open_opts, opts); - struct dummy_st_ops_success *skel; - int err; - - /* before we mount BPF FS with token delegation, struct_ops skeleton - * should fail to load - */ - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load")) { - dummy_st_ops_success__destroy(skel); - return -EINVAL; - } - - /* mount custom BPF FS over custom location, so libbpf can't create - * BPF token implicitly, unless pointed to it through - * LIBBPF_BPF_TOKEN_PATH envvar - */ - rmdir(TOKEN_BPFFS_CUSTOM); - if (!ASSERT_OK(mkdir(TOKEN_BPFFS_CUSTOM, 0777), "mkdir_bpffs_custom")) - goto err_out; - err = sys_move_mount(mnt_fd, "", AT_FDCWD, TOKEN_BPFFS_CUSTOM, MOVE_MOUNT_F_EMPTY_PATH); - if (!ASSERT_OK(err, "move_mount_bpffs")) - goto err_out; - - /* even though we have BPF FS with delegation, it's not at default - * /sys/fs/bpf location, so we still fail to load until envvar is set up - */ - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_ERR_PTR(skel, "obj_tokenless_load2")) { - dummy_st_ops_success__destroy(skel); - goto err_out; - } - - err = setenv(TOKEN_ENVVAR, TOKEN_BPFFS_CUSTOM, 1 /*overwrite*/); - if (!ASSERT_OK(err, "setenv_token_path")) - goto err_out; - - /* now the same struct_ops skeleton should succeed thanks to libppf - * creating BPF token from custom mount point - */ - skel = dummy_st_ops_success__open_and_load(); - if (!ASSERT_OK_PTR(skel, "obj_implicit_token_load")) - goto err_out; - - dummy_st_ops_success__destroy(skel); - - /* now disable implicit token through empty bpf_token_path, envvar - * will be ignored, should fail - */ - opts.bpf_token_path = ""; - skel = dummy_st_ops_success__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_empty_token_path_open")) - goto err_out; - - err = dummy_st_ops_success__load(skel); - dummy_st_ops_success__destroy(skel); - if (!ASSERT_ERR(err, "obj_empty_token_path_load")) - goto err_out; - - /* now disable implicit token through negative bpf_token_fd, envvar - * will be ignored, should fail - */ - opts.bpf_token_path = NULL; - opts.bpf_token_fd = -1; - skel = dummy_st_ops_success__open_opts(&opts); - if (!ASSERT_OK_PTR(skel, "obj_neg_token_fd_open")) - goto err_out; - - err = dummy_st_ops_success__load(skel); - dummy_st_ops_success__destroy(skel); - if (!ASSERT_ERR(err, "obj_neg_token_fd_load")) - goto err_out; - - rmdir(TOKEN_BPFFS_CUSTOM); - unsetenv(TOKEN_ENVVAR); - return 0; -err_out: - rmdir(TOKEN_BPFFS_CUSTOM); - unsetenv(TOKEN_ENVVAR); - return -EINVAL; -} - -#define bit(n) (1ULL << (n)) - -void test_token(void) -{ - if (test__start_subtest("map_token")) { - struct bpffs_opts opts = { - .cmds_str = "map_create", - .maps_str = "stack", - }; - - subtest_userns(&opts, userns_map_create); - } - if (test__start_subtest("btf_token")) { - struct bpffs_opts opts = { - .cmds = 1ULL << BPF_BTF_LOAD, - }; - - subtest_userns(&opts, userns_btf_load); - } - if (test__start_subtest("prog_token")) { - struct bpffs_opts opts = { - .cmds_str = "PROG_LOAD", - .progs_str = "XDP", - .attachs_str = "xdp", - }; - - subtest_userns(&opts, userns_prog_load); - } - if (test__start_subtest("obj_priv_map")) { - struct bpffs_opts opts = { - .cmds = bit(BPF_MAP_CREATE), - .maps = bit(BPF_MAP_TYPE_QUEUE), - }; - - subtest_userns(&opts, userns_obj_priv_map); - } - if (test__start_subtest("obj_priv_prog")) { - struct bpffs_opts opts = { - .cmds = bit(BPF_PROG_LOAD), - .progs = bit(BPF_PROG_TYPE_KPROBE), - .attachs = ~0ULL, - }; - - subtest_userns(&opts, userns_obj_priv_prog); - } - if (test__start_subtest("obj_priv_btf_fail")) { - struct bpffs_opts opts = { - /* disallow BTF loading */ - .cmds = bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), - .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), - .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), - .attachs = ~0ULL, - }; - - subtest_userns(&opts, userns_obj_priv_btf_fail); - } - if (test__start_subtest("obj_priv_btf_success")) { - struct bpffs_opts opts = { - /* allow BTF loading */ - .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), - .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), - .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), - .attachs = ~0ULL, - }; - - subtest_userns(&opts, userns_obj_priv_btf_success); - } - if (test__start_subtest("obj_priv_implicit_token")) { - struct bpffs_opts opts = { - /* allow BTF loading */ - .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), - .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), - .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), - .attachs = ~0ULL, - }; - - subtest_userns(&opts, userns_obj_priv_implicit_token); - } - if (test__start_subtest("obj_priv_implicit_token_envvar")) { - struct bpffs_opts opts = { - /* allow BTF loading */ - .cmds = bit(BPF_BTF_LOAD) | bit(BPF_MAP_CREATE) | bit(BPF_PROG_LOAD), - .maps = bit(BPF_MAP_TYPE_STRUCT_OPS), - .progs = bit(BPF_PROG_TYPE_STRUCT_OPS), - .attachs = ~0ULL, - }; - - subtest_userns(&opts, userns_obj_priv_implicit_token_envvar); - } -} diff --git a/tools/testing/selftests/bpf/progs/priv_map.c b/tools/testing/selftests/bpf/progs/priv_map.c deleted file mode 100644 index 9085be50f03b..000000000000 --- a/tools/testing/selftests/bpf/progs/priv_map.c +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ - -#include "vmlinux.h" -#include - -char _license[] SEC("license") = "GPL"; - -struct { - __uint(type, BPF_MAP_TYPE_QUEUE); - __uint(max_entries, 1); - __type(value, __u32); -} priv_map SEC(".maps"); diff --git a/tools/testing/selftests/bpf/progs/priv_prog.c b/tools/testing/selftests/bpf/progs/priv_prog.c deleted file mode 100644 index 3c7b2b618c8a..000000000000 --- a/tools/testing/selftests/bpf/progs/priv_prog.c +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ - -#include "vmlinux.h" -#include - -char _license[] SEC("license") = "GPL"; - -SEC("kprobe") -int kprobe_prog(void *ctx) -{ - return 1; -} -- cgit v1.2.3 From 4ba1d0f23414135e4f426dae4cb5cdc2ce246f89 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 14 Dec 2023 17:13:25 -0800 Subject: bpf: abstract away global subprog arg preparation logic from reg state setup btf_prepare_func_args() is used to understand expectations and restrictions on global subprog arguments. But current implementation is hard to extend, as it intermixes BTF-based func prototype parsing and interpretation logic with setting up register state at subprog entry. Worse still, those registers are not completely set up inside btf_prepare_func_args(), requiring some more logic later in do_check_common(). Like calling mark_reg_unknown() and similar initialization operations. This intermixing of BTF interpretation and register state setup is problematic. First, it causes duplication of BTF parsing logic for global subprog verification (to set up initial state of global subprog) and global subprog call sites analysis (when we need to check that whatever is being passed into global subprog matches expectations), performed in btf_check_subprog_call(). Given we want to extend global func argument with tags later, this duplication is problematic. So refactor btf_prepare_func_args() to do only BTF-based func proto and args parsing, returning high-level argument "expectations" only, with no regard to specifics of register state. I.e., if it's a context argument, instead of setting register state to PTR_TO_CTX, we return ARG_PTR_TO_CTX enum for that argument as "an argument specification" for further processing inside do_check_common(). Similarly for SCALAR arguments, PTR_TO_MEM, etc. This allows to reuse btf_prepare_func_args() in following patches at global subprog call site analysis time. It also keeps register setup code consistently in one place, do_check_common(). Besides all this, we cache this argument specs information inside env->subprog_info, eliminating the need to redo these potentially expensive BTF traversals, especially if BPF program's BTF is big and/or there are lots of global subprog calls. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231215011334.2307144-2-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 3 +-- include/linux/bpf_verifier.h | 16 ++++++++++++++++ kernel/bpf/btf.c | 38 ++++++++++++++++++++------------------ kernel/bpf/verifier.c | 43 +++++++++++++++++++++++++++---------------- 4 files changed, 64 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7a8d4c81a39a..c050c82cc9a5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2470,8 +2470,7 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); -int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *reg, u32 *nargs); +int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index c2819a6579a5..5742e9c0a7b8 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -606,6 +606,13 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) #define BPF_MAX_SUBPROGS 256 +struct bpf_subprog_arg_info { + enum bpf_arg_type arg_type; + union { + u32 mem_size; + }; +}; + struct bpf_subprog_info { /* 'start' has to be the first field otherwise find_subprog() won't work */ u32 start; /* insn idx of function entry point */ @@ -617,6 +624,10 @@ struct bpf_subprog_info { bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; + bool args_cached: 1; + + u8 arg_cnt; + struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS]; }; struct bpf_verifier_env; @@ -727,6 +738,11 @@ struct bpf_verifier_env { char tmp_str_buf[TMP_STR_BUF_LEN]; }; +static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog) +{ + return &env->subprog_info[subprog]; +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d56433bf8aba..be2104e5f2f5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6948,16 +6948,17 @@ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, return err; } -/* Convert BTF of a function into bpf_reg_state if possible +/* Process BTF of a function to produce high-level expectation of function + * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information + * is cached in subprog info for reuse. * Returns: * EFAULT - there is a verifier bug. Abort verification. * EINVAL - cannot convert BTF. - * 0 - Successfully converted BTF into bpf_reg_state - * (either PTR_TO_CTX or SCALAR_VALUE). + * 0 - Successfully processed BTF and constructed argument expectations. */ -int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs, u32 *arg_cnt) +int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) { + struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_verifier_log *log = &env->log; struct bpf_prog *prog = env->prog; enum bpf_prog_type prog_type = prog->type; @@ -6967,6 +6968,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, u32 i, nargs, btf_id; const char *tname; + if (sub->args_cached) + return 0; + if (!prog->aux->func_info || prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) { bpf_log(log, "Verifier bug\n"); @@ -6990,10 +6994,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, } tname = btf_name_by_offset(btf, t->name_off); - if (log->level & BPF_LOG_LEVEL) - bpf_log(log, "Validating %s() func#%d...\n", - tname, subprog); - if (prog->aux->func_info_aux[subprog].unreliable) { bpf_log(log, "Verifier bug in function %s()\n", tname); return -EFAULT; @@ -7013,7 +7013,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, tname, nargs, MAX_BPF_FUNC_REG_ARGS); return -EINVAL; } - *arg_cnt = nargs; /* check that function returns int, exception cb also requires this */ t = btf_type_by_id(btf, t->type); while (btf_type_is_modifier(t)) @@ -7028,24 +7027,24 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, * Only PTR_TO_CTX and SCALAR are supported atm. */ for (i = 0; i < nargs; i++) { - struct bpf_reg_state *reg = ®s[i + 1]; - t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); if (btf_type_is_int(t) || btf_is_any_enum(t)) { - reg->type = SCALAR_VALUE; + sub->args[i].arg_type = ARG_ANYTHING; continue; } if (btf_type_is_ptr(t)) { + u32 mem_size; + if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { - reg->type = PTR_TO_CTX; + sub->args[i].arg_type = ARG_PTR_TO_CTX; continue; } t = btf_type_skip_modifiers(btf, t->type, NULL); - ref_t = btf_resolve_size(btf, t, ®->mem_size); + ref_t = btf_resolve_size(btf, t, &mem_size); if (IS_ERR(ref_t)) { bpf_log(log, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", @@ -7054,15 +7053,18 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, return -EINVAL; } - reg->type = PTR_TO_MEM | PTR_MAYBE_NULL; - reg->id = ++env->id_gen; - + sub->args[i].arg_type = ARG_PTR_TO_MEM_OR_NULL; + sub->args[i].mem_size = mem_size; continue; } bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", i, btf_type_str(t), tname); return -EINVAL; } + + sub->arg_cnt = nargs; + sub->args_cached = true; + return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index df1cae459c77..6555785b9f63 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -442,11 +442,6 @@ static struct bpf_func_info_aux *subprog_aux(const struct bpf_verifier_env *env, return &env->prog->aux->func_info_aux[subprog]; } -static struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog) -{ - return &env->subprog_info[subprog]; -} - static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) { struct bpf_subprog_info *info = subprog_info(env, subprog); @@ -19937,34 +19932,50 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - u32 nargs; + struct bpf_subprog_info *sub = subprog_info(env, subprog); + const char *sub_name = subprog_name(env, subprog); + struct bpf_subprog_arg_info *arg; + struct bpf_reg_state *reg; - ret = btf_prepare_func_args(env, subprog, regs, &nargs); + verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); + ret = btf_prepare_func_args(env, subprog); if (ret) goto out; + if (subprog_is_exc_cb(env, subprog)) { state->frame[0]->in_exception_callback_fn = true; /* We have already ensured that the callback returns an integer, just * like all global subprogs. We need to determine it only has a single * scalar argument. */ - if (nargs != 1 || regs[BPF_REG_1].type != SCALAR_VALUE) { + if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= BPF_REG_5; i++) { - if (regs[i].type == PTR_TO_CTX) + for (i = BPF_REG_1; i <= sub->arg_cnt; i++) { + arg = &sub->args[i - BPF_REG_1]; + reg = ®s[i]; + + if (arg->arg_type == ARG_PTR_TO_CTX) { + reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); - else if (regs[i].type == SCALAR_VALUE) + } else if (arg->arg_type == ARG_ANYTHING) { + reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); - else if (base_type(regs[i].type) == PTR_TO_MEM) { - const u32 mem_size = regs[i].mem_size; - + } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { + reg->type = PTR_TO_MEM; + if (arg->arg_type & PTR_MAYBE_NULL) + reg->type |= PTR_MAYBE_NULL; mark_reg_known_zero(env, regs, i); - regs[i].mem_size = mem_size; - regs[i].id = ++env->id_gen; + reg->mem_size = arg->mem_size; + reg->id = ++env->id_gen; + } else { + WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n", + i - BPF_REG_1, arg->arg_type); + ret = -EFAULT; + goto out; } } } else { -- cgit v1.2.3 From 5eccd2db42d77e3570619c32d39e39bf486607cf Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 14 Dec 2023 17:13:26 -0800 Subject: bpf: reuse btf_prepare_func_args() check for main program BTF validation Instead of btf_check_subprog_arg_match(), use btf_prepare_func_args() logic to validate "trustworthiness" of main BPF program's BTF information, if it is present. We ignored results of original BTF check anyway, often times producing confusing and ominously-sounding "reg type unsupported for arg#0 function" message, which has no apparent effect on program correctness and verification process. All the -EFAULT returning sanity checks are already performed in check_btf_info_early(), so there is zero reason to have this duplication of logic between btf_check_subprog_call() and btf_check_subprog_arg_match(). Dropping btf_check_subprog_arg_match() simplifies btf_check_func_arg_match() further removing `bool processing_call` flag. One subtle bit that was done by btf_check_subprog_arg_match() was potentially marking main program's BTF as unreliable. We do this explicitly now with a dedicated simple check, preserving the original behavior, but now based on well factored btf_prepare_func_args() logic. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231215011334.2307144-3-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 - kernel/bpf/btf.c | 50 ++-------------------- kernel/bpf/verifier.c | 25 ++++++----- tools/testing/selftests/bpf/prog_tests/log_fixup.c | 4 +- .../selftests/bpf/progs/cgrp_kfunc_failure.c | 2 +- .../selftests/bpf/progs/task_kfunc_failure.c | 2 +- 6 files changed, 19 insertions(+), 66 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c050c82cc9a5..d0d7eff22b8a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2466,8 +2466,6 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf_func_model *m); struct bpf_reg_state; -int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs); int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index be2104e5f2f5..33d9a1c73f6e 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6768,8 +6768,7 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr static int btf_check_func_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, - bool ptr_to_mem_ok, - bool processing_call) + bool ptr_to_mem_ok) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); struct bpf_verifier_log *log = &env->log; @@ -6842,7 +6841,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, i, btf_type_str(t)); return -EINVAL; } - } else if (ptr_to_mem_ok && processing_call) { + } else if (ptr_to_mem_ok) { const struct btf_type *resolve_ret; u32 type_size; @@ -6867,55 +6866,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return 0; } -/* Compare BTF of a function declaration with given bpf_reg_state. - * Returns: - * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. - * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. - */ -int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) -{ - struct bpf_prog *prog = env->prog; - struct btf *btf = prog->aux->btf; - bool is_global; - u32 btf_id; - int err; - - if (!prog->aux->func_info) - return -EINVAL; - - btf_id = prog->aux->func_info[subprog].type_id; - if (!btf_id) - return -EFAULT; - - if (prog->aux->func_info_aux[subprog].unreliable) - return -EINVAL; - - is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, false); - - /* Compiler optimizations can remove arguments from static functions - * or mismatched type can be passed into a global function. - * In such cases mark the function as unreliable from BTF point of view. - */ - if (err) - prog->aux->func_info_aux[subprog].unreliable = true; - return err; -} - /* Compare BTF of a function call with given bpf_reg_state. * Returns: * EFAULT - there is a verifier bug. Abort verification. * EINVAL - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. - * - * NOTE: the code is duplicated from btf_check_subprog_arg_match() - * because btf_check_func_arg_match() is still doing both. Once that - * function is split in 2, we can call from here btf_check_subprog_arg_match() - * first, and then treat the calling part in a new code path. */ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs) @@ -6937,7 +6893,7 @@ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, return -EINVAL; is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, true); + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6555785b9f63..c26e9ab5226c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19904,6 +19904,7 @@ static void free_states(struct bpf_verifier_env *env) static int do_check_common(struct bpf_verifier_env *env, int subprog) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); + struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_verifier_state *state; struct bpf_reg_state *regs; int ret, i; @@ -19930,9 +19931,9 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->first_insn_idx = env->subprog_info[subprog].start; state->last_insn_idx = -1; + regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - struct bpf_subprog_info *sub = subprog_info(env, subprog); const char *sub_name = subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -19979,21 +19980,19 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) } } } else { + /* if main BPF program has associated BTF info, validate that + * it's matching expected signature, and otherwise mark BTF + * info for main program as unreliable + */ + if (env->prog->aux->func_info_aux) { + ret = btf_prepare_func_args(env, 0); + if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) + env->prog->aux->func_info_aux[0].unreliable = true; + } + /* 1st arg to a function */ regs[BPF_REG_1].type = PTR_TO_CTX; mark_reg_known_zero(env, regs, BPF_REG_1); - ret = btf_check_subprog_arg_match(env, subprog, regs); - if (ret == -EFAULT) - /* unlikely verifier bug. abort. - * ret == 0 and ret < 0 are sadly acceptable for - * main() function due to backward compatibility. - * Like socket filter program may be written as: - * int bpf_prog(struct pt_regs *ctx) - * and never dereference that ctx in the program. - * 'struct pt_regs' is a type mismatch for socket - * filter that should be using 'struct __sk_buff'. - */ - goto out; } ret = do_check(env); diff --git a/tools/testing/selftests/bpf/prog_tests/log_fixup.c b/tools/testing/selftests/bpf/prog_tests/log_fixup.c index effd78b2a657..7a3fa2ff567b 100644 --- a/tools/testing/selftests/bpf/prog_tests/log_fixup.c +++ b/tools/testing/selftests/bpf/prog_tests/log_fixup.c @@ -169,9 +169,9 @@ void test_log_fixup(void) if (test__start_subtest("bad_core_relo_trunc_none")) bad_core_relo(0, TRUNC_NONE /* full buf */); if (test__start_subtest("bad_core_relo_trunc_partial")) - bad_core_relo(300, TRUNC_PARTIAL /* truncate original log a bit */); + bad_core_relo(280, TRUNC_PARTIAL /* truncate original log a bit */); if (test__start_subtest("bad_core_relo_trunc_full")) - bad_core_relo(210, TRUNC_FULL /* truncate also libbpf's message patch */); + bad_core_relo(220, TRUNC_FULL /* truncate also libbpf's message patch */); if (test__start_subtest("bad_core_relo_subprog")) bad_core_relo_subprog(); if (test__start_subtest("missing_map")) diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index 0fa564a5cc5b..9fe9c4a4e8f6 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -78,7 +78,7 @@ int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) } SEC("kretprobe/cgroup_destroy_locked") -__failure __msg("reg type unsupported for arg#0 function") +__failure __msg("calling kernel function bpf_cgroup_acquire is not allowed") int BPF_PROG(cgrp_kfunc_acquire_unsafe_kretprobe, struct cgroup *cgrp) { struct cgroup *acquired; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index dcdea3127086..ad88a3796ddf 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -248,7 +248,7 @@ int BPF_PROG(task_kfunc_from_pid_no_null_check, struct task_struct *task, u64 cl } SEC("lsm/task_free") -__failure __msg("reg type unsupported for arg#0 function") +__failure __msg("R1 must be a rcu pointer") int BPF_PROG(task_kfunc_from_lsm_task_free, struct task_struct *task) { struct task_struct *acquired; -- cgit v1.2.3 From e26080d0da87f20222ca6712b65f95a856fadee0 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 14 Dec 2023 17:13:27 -0800 Subject: bpf: prepare btf_prepare_func_args() for handling static subprogs Generalize btf_prepare_func_args() to support both global and static subprogs. We are going to utilize this property in the next patch, reusing btf_prepare_func_args() for subprog call logic instead of reparsing BTF information in a completely separate implementation. btf_prepare_func_args() now detects whether subprog is global or static makes slight logic adjustments for static func cases, like not failing fatally (-EFAULT) for conditions that are allowable for static subprogs. Somewhat subtle (but major!) difference is the handling of pointer arguments. Both global and static functions need to handle special context arguments (which are pointers to predefined type names), but static subprogs give up on any other pointers, falling back to marking subprog as "unreliable", disabling the use of BTF type information altogether. For global functions, though, we are assuming that such pointers to unrecognized types are just pointers to fixed-sized memory region (or error out if size cannot be established, like for `void *` pointers). This patch accommodates these small differences and sets up a stage for refactoring in the next patch, eliminating a separate BTF-based parsing logic in btf_check_func_arg_match(). Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231215011334.2307144-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 5 +++++ kernel/bpf/btf.c | 18 +++++++++--------- kernel/bpf/verifier.c | 5 ----- 3 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5742e9c0a7b8..d3ea9ef04767 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -738,6 +738,11 @@ struct bpf_verifier_env { char tmp_str_buf[TMP_STR_BUF_LEN]; }; +static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) +{ + return &env->prog->aux->func_info_aux[subprog]; +} + static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env, int subprog) { return &env->subprog_info[subprog]; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 33d9a1c73f6e..d321340e16f1 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6914,6 +6914,7 @@ int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, */ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) { + bool is_global = subprog_aux(env, subprog)->linkage == BTF_FUNC_GLOBAL; struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_verifier_log *log = &env->log; struct bpf_prog *prog = env->prog; @@ -6927,14 +6928,15 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (sub->args_cached) return 0; - if (!prog->aux->func_info || - prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) { + if (!prog->aux->func_info) { bpf_log(log, "Verifier bug\n"); return -EFAULT; } btf_id = prog->aux->func_info[subprog].type_id; if (!btf_id) { + if (!is_global) /* not fatal for static funcs */ + return -EINVAL; bpf_log(log, "Global functions need valid BTF\n"); return -EFAULT; } @@ -6990,16 +6992,14 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) sub->args[i].arg_type = ARG_ANYTHING; continue; } - if (btf_type_is_ptr(t)) { + if (btf_type_is_ptr(t) && btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { + sub->args[i].arg_type = ARG_PTR_TO_CTX; + continue; + } + if (is_global && btf_type_is_ptr(t)) { u32 mem_size; - if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { - sub->args[i].arg_type = ARG_PTR_TO_CTX; - continue; - } - t = btf_type_skip_modifiers(btf, t->type, NULL); - ref_t = btf_resolve_size(btf, t, &mem_size); if (IS_ERR(ref_t)) { bpf_log(log, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c26e9ab5226c..6c9ecb86a8d9 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -437,11 +437,6 @@ static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) return btf_type_name(env->prog->aux->btf, info->type_id); } -static struct bpf_func_info_aux *subprog_aux(const struct bpf_verifier_env *env, int subprog) -{ - return &env->prog->aux->func_info_aux[subprog]; -} - static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) { struct bpf_subprog_info *info = subprog_info(env, subprog); -- cgit v1.2.3 From c5a7244759b1eeacc59d0426fb73859afa942d0d Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 14 Dec 2023 17:13:28 -0800 Subject: bpf: move subprog call logic back to verifier.c Subprog call logic in btf_check_subprog_call() currently has both a lot of BTF parsing logic (which is, presumably, what justified putting it into btf.c), but also a bunch of register state checks, some of each utilize deep verifier logic helpers, necessarily exported from verifier.c: check_ptr_off_reg(), check_func_arg_reg_off(), and check_mem_reg(). Going forward, btf_check_subprog_call() will have a minimum of BTF-related logic, but will get more internal verifier logic related to register state manipulation. So move it into verifier.c to minimize amount of verifier-specific logic exposed to btf.c. We do this move before refactoring btf_check_func_arg_match() to preserve as much history post-refactoring as possible. No functional changes. Acked-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20231215011334.2307144-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 2 - include/linux/bpf_verifier.h | 8 --- kernel/bpf/btf.c | 139 --------------------------------------- kernel/bpf/verifier.c | 153 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 146 insertions(+), 156 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d0d7eff22b8a..7671530d6e4e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2466,8 +2466,6 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf_func_model *m); struct bpf_reg_state; -int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index d3ea9ef04767..d07d857ca67f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -785,14 +785,6 @@ bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, void bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); -int check_ptr_off_reg(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno); -int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno, - enum bpf_arg_type arg_type); -int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - u32 regno, u32 mem_size); - /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */ static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog, struct btf *btf, u32 btf_id) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d321340e16f1..341811bcca53 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6765,145 +6765,6 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr return btf_check_func_type_match(log, btf1, t1, btf2, t2); } -static int btf_check_func_arg_match(struct bpf_verifier_env *env, - const struct btf *btf, u32 func_id, - struct bpf_reg_state *regs, - bool ptr_to_mem_ok) -{ - enum bpf_prog_type prog_type = resolve_prog_type(env->prog); - struct bpf_verifier_log *log = &env->log; - const char *func_name, *ref_tname; - const struct btf_type *t, *ref_t; - const struct btf_param *args; - u32 i, nargs, ref_id; - int ret; - - t = btf_type_by_id(btf, func_id); - if (!t || !btf_type_is_func(t)) { - /* These checks were already done by the verifier while loading - * struct bpf_func_info or in add_kfunc_call(). - */ - bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n", - func_id); - return -EFAULT; - } - func_name = btf_name_by_offset(btf, t->name_off); - - t = btf_type_by_id(btf, t->type); - if (!t || !btf_type_is_func_proto(t)) { - bpf_log(log, "Invalid BTF of func %s\n", func_name); - return -EFAULT; - } - args = (const struct btf_param *)(t + 1); - nargs = btf_type_vlen(t); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } - - /* check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < nargs; i++) { - enum bpf_arg_type arg_type = ARG_DONTCARE; - u32 regno = i + 1; - struct bpf_reg_state *reg = ®s[regno]; - - t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type == SCALAR_VALUE) - continue; - bpf_log(log, "R%d is not a scalar\n", regno); - return -EINVAL; - } - - if (!btf_type_is_ptr(t)) { - bpf_log(log, "Unrecognized arg#%d type %s\n", - i, btf_type_str(t)); - return -EINVAL; - } - - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - - ret = check_func_arg_reg_off(env, reg, regno, arg_type); - if (ret < 0) - return ret; - - if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { - /* If function expects ctx type in BTF check that caller - * is passing PTR_TO_CTX. - */ - if (reg->type != PTR_TO_CTX) { - bpf_log(log, - "arg#%d expected pointer to ctx, but got %s\n", - i, btf_type_str(t)); - return -EINVAL; - } - } else if (ptr_to_mem_ok) { - const struct btf_type *resolve_ret; - u32 type_size; - - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - bpf_log(log, - "arg#%d reference type('%s %s') size cannot be determined: %ld\n", - i, btf_type_str(ref_t), ref_tname, - PTR_ERR(resolve_ret)); - return -EINVAL; - } - - if (check_mem_reg(env, reg, regno, type_size)) - return -EINVAL; - } else { - bpf_log(log, "reg type unsupported for arg#%d function %s#%d\n", i, - func_name, func_id); - return -EINVAL; - } - } - - return 0; -} - -/* Compare BTF of a function call with given bpf_reg_state. - * Returns: - * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. - * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. - */ -int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) -{ - struct bpf_prog *prog = env->prog; - struct btf *btf = prog->aux->btf; - bool is_global; - u32 btf_id; - int err; - - if (!prog->aux->func_info) - return -EINVAL; - - btf_id = prog->aux->func_info[subprog].type_id; - if (!btf_id) - return -EFAULT; - - if (prog->aux->func_info_aux[subprog].unreliable) - return -EINVAL; - - is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global); - - /* Compiler optimizations can remove arguments from static functions - * or mismatched type can be passed into a global function. - * In such cases mark the function as unreliable from BTF point of view. - */ - if (err) - prog->aux->func_info_aux[subprog].unreliable = true; - return err; -} - /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6c9ecb86a8d9..f48e49f2d482 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5127,8 +5127,8 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, return 0; } -int check_ptr_off_reg(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno) +static int check_ptr_off_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, int regno) { return __check_ptr_off_reg(env, reg, regno, false); } @@ -7300,8 +7300,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, return err; } -int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - u32 regno, u32 mem_size) +static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 regno, u32 mem_size) { bool may_be_null = type_may_be_null(reg->type); struct bpf_reg_state saved_reg; @@ -8286,9 +8286,9 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, int regno, - enum bpf_arg_type arg_type) +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, int regno, + enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -9249,6 +9249,145 @@ err_out: return err; } +static int btf_check_func_arg_match(struct bpf_verifier_env *env, + const struct btf *btf, u32 func_id, + struct bpf_reg_state *regs, + bool ptr_to_mem_ok) +{ + enum bpf_prog_type prog_type = resolve_prog_type(env->prog); + struct bpf_verifier_log *log = &env->log; + const char *func_name, *ref_tname; + const struct btf_type *t, *ref_t; + const struct btf_param *args; + u32 i, nargs, ref_id; + int ret; + + t = btf_type_by_id(btf, func_id); + if (!t || !btf_type_is_func(t)) { + /* These checks were already done by the verifier while loading + * struct bpf_func_info or in add_kfunc_call(). + */ + bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n", + func_id); + return -EFAULT; + } + func_name = btf_name_by_offset(btf, t->name_off); + + t = btf_type_by_id(btf, t->type); + if (!t || !btf_type_is_func_proto(t)) { + bpf_log(log, "Invalid BTF of func %s\n", func_name); + return -EFAULT; + } + args = (const struct btf_param *)(t + 1); + nargs = btf_type_vlen(t); + if (nargs > MAX_BPF_FUNC_REG_ARGS) { + bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs, + MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + + /* check that BTF function arguments match actual types that the + * verifier sees. + */ + for (i = 0; i < nargs; i++) { + enum bpf_arg_type arg_type = ARG_DONTCARE; + u32 regno = i + 1; + struct bpf_reg_state *reg = ®s[regno]; + + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (btf_type_is_scalar(t)) { + if (reg->type == SCALAR_VALUE) + continue; + bpf_log(log, "R%d is not a scalar\n", regno); + return -EINVAL; + } + + if (!btf_type_is_ptr(t)) { + bpf_log(log, "Unrecognized arg#%d type %s\n", + i, btf_type_str(t)); + return -EINVAL; + } + + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + + ret = check_func_arg_reg_off(env, reg, regno, arg_type); + if (ret < 0) + return ret; + + if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { + /* If function expects ctx type in BTF check that caller + * is passing PTR_TO_CTX. + */ + if (reg->type != PTR_TO_CTX) { + bpf_log(log, + "arg#%d expected pointer to ctx, but got %s\n", + i, btf_type_str(t)); + return -EINVAL; + } + } else if (ptr_to_mem_ok) { + const struct btf_type *resolve_ret; + u32 type_size; + + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + bpf_log(log, + "arg#%d reference type('%s %s') size cannot be determined: %ld\n", + i, btf_type_str(ref_t), ref_tname, + PTR_ERR(resolve_ret)); + return -EINVAL; + } + + if (check_mem_reg(env, reg, regno, type_size)) + return -EINVAL; + } else { + bpf_log(log, "reg type unsupported for arg#%d function %s#%d\n", i, + func_name, func_id); + return -EINVAL; + } + } + + return 0; +} + +/* Compare BTF of a function call with given bpf_reg_state. + * Returns: + * EFAULT - there is a verifier bug. Abort verification. + * EINVAL - there is a type mismatch or BTF is not available. + * 0 - BTF matches with what bpf_reg_state expects. + * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. + */ +static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, + struct bpf_reg_state *regs) +{ + struct bpf_prog *prog = env->prog; + struct btf *btf = prog->aux->btf; + bool is_global; + u32 btf_id; + int err; + + if (!prog->aux->func_info) + return -EINVAL; + + btf_id = prog->aux->func_info[subprog].type_id; + if (!btf_id) + return -EFAULT; + + if (prog->aux->func_info_aux[subprog].unreliable) + return -EINVAL; + + is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global); + + /* Compiler optimizations can remove arguments from static functions + * or mismatched type can be passed into a global function. + * In such cases mark the function as unreliable from BTF point of view. + */ + if (err) + prog->aux->func_info_aux[subprog].unreliable = true; + return err; +} + static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, int subprog, set_callee_state_fn set_callee_state_cb) -- cgit v1.2.3 From 1a36e0f50f963465e9b2b980d250ab38b8fcd7a3 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Sun, 17 Dec 2023 10:32:38 +0200 Subject: net: Add MDB bulk deletion device operation Add MDB net device operation that will be invoked by rtnetlink code in response to received 'RTM_DELMDB' messages with the 'NLM_F_BULK' flag set. Subsequent patches will implement the operation in the bridge and VXLAN drivers. Signed-off-by: Ido Schimmel Reviewed-by: Petr Machata Acked-by: Nikolay Aleksandrov Signed-off-by: David S. Miller --- include/linux/netdevice.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1b935ee341b4..75c7725e5e4f 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1329,6 +1329,9 @@ struct netdev_net_notifier { * int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[], * struct netlink_ext_ack *extack); * Deletes the MDB entry from dev. + * int (*ndo_mdb_del_bulk)(struct net_device *dev, struct nlattr *tb[], + * struct netlink_ext_ack *extack); + * Bulk deletes MDB entries from dev. * int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb, * struct netlink_callback *cb); * Dumps MDB entries from dev. The first argument (marker) in the netlink @@ -1611,6 +1614,9 @@ struct net_device_ops { int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[], struct netlink_ext_ack *extack); + int (*ndo_mdb_del_bulk)(struct net_device *dev, + struct nlattr *tb[], + struct netlink_ext_ack *extack); int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb, struct netlink_callback *cb); -- cgit v1.2.3 From a7e7b40c4bc115dbf2a2bb453d7bbb2e0ea99703 Mon Sep 17 00:00:00 2001 From: Saeed Mahameed Date: Fri, 15 Dec 2023 19:31:14 -0800 Subject: net/mlx5e: Use the correct lag ports number when creating TISes The cited commit moved the code of mlx5e_create_tises() and changed the loop to create TISes over MLX5_MAX_PORTS constant value, instead of getting the correct lag ports supported by the device, which can cause FW errors on devices with less than MLX5_MAX_PORTS ports. Change that back to mlx5e_get_num_lag_ports(mdev). Also IPoIB interfaces create there own TISes, they don't use the eth TISes, pass a flag to indicate that. Fixes: b25bd37c859f ("net/mlx5: Move TISes from priv to mdev HW resources") Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en_common.c | 21 +++++++++++++-------- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 2 +- .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c | 2 +- include/linux/mlx5/driver.h | 1 + 5 files changed, 17 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 0bfe1ca8a364..55c6ace0acd5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1124,7 +1124,7 @@ static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) extern const struct ethtool_ops mlx5e_ethtool_ops; int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); -int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev); +int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises); void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev); int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb, bool enable_mc_lb); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 67f546683e85..6ed3a32b7e22 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -95,7 +95,7 @@ static void mlx5e_destroy_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PO { int tc, i; - for (i = 0; i < MLX5_MAX_PORTS; i++) + for (i = 0; i < mlx5e_get_num_lag_ports(mdev); i++) for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) mlx5e_destroy_tis(mdev, tisn[i][tc]); } @@ -110,7 +110,7 @@ static int mlx5e_create_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORT int tc, i; int err; - for (i = 0; i < MLX5_MAX_PORTS; i++) { + for (i = 0; i < mlx5e_get_num_lag_ports(mdev); i++) { for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; void *tisc; @@ -140,7 +140,7 @@ err_close_tises: return err; } -int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) +int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; int err; @@ -169,11 +169,15 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) goto err_destroy_mkey; } - err = mlx5e_create_tises(mdev, res->tisn); - if (err) { - mlx5_core_err(mdev, "alloc tises failed, %d\n", err); - goto err_destroy_bfreg; + if (create_tises) { + err = mlx5e_create_tises(mdev, res->tisn); + if (err) { + mlx5_core_err(mdev, "alloc tises failed, %d\n", err); + goto err_destroy_bfreg; + } + res->tisn_valid = true; } + INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); @@ -203,7 +207,8 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); mdev->mlx5e_res.dek_priv = NULL; - mlx5e_destroy_tises(mdev, res->tisn); + if (res->tisn_valid) + mlx5e_destroy_tises(mdev, res->tisn); mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b5f1c4ca38ba..c8e8f512803e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -5992,7 +5992,7 @@ static int mlx5e_resume(struct auxiliary_device *adev) if (netif_device_present(netdev)) return 0; - err = mlx5e_create_mdev_resources(mdev); + err = mlx5e_create_mdev_resources(mdev, true); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 58845121954c..d77be1b4dd9c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -783,7 +783,7 @@ static int mlx5_rdma_setup_rn(struct ib_device *ibdev, u32 port_num, } /* This should only be called once per mdev */ - err = mlx5e_create_mdev_resources(mdev); + err = mlx5e_create_mdev_resources(mdev, false); if (err) goto destroy_ht; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 7ee5b79ff3d6..aafb36c9e5d9 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -681,6 +681,7 @@ struct mlx5e_resources { struct mlx5_sq_bfreg bfreg; #define MLX5_MAX_NUM_TC 8 u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]; + bool tisn_valid; } hw_objs; struct net_device *uplink_netdev; struct mutex uplink_netdev_lock; -- cgit v1.2.3 From e04984a37398b3f4f5a79c993b94c6b1224184cc Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 19 Dec 2023 14:46:20 +0200 Subject: net/mlx5: Fix query of sd_group field The sd_group field moved in the HW spec from the MPIR register to the vport context. Align the query accordingly. Fixes: f5e956329960 ("net/mlx5: Expose Management PCIe Index Register (MPIR)") Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/vport.c | 21 +++++++++++++++++++++ include/linux/mlx5/mlx5_ifc.h | 10 +++++++--- include/linux/mlx5/vport.h | 1 + 3 files changed, 29 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 5a31fb47ffa5..c95a84b7db3a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -440,6 +440,27 @@ out: } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_system_image_guid); +int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group) +{ + int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); + u32 *out; + int err; + + out = kvzalloc(outlen, GFP_KERNEL); + if (!out) + return -ENOMEM; + + err = mlx5_query_nic_vport_context(mdev, 0, out); + if (err) + goto out; + + *sd_group = MLX5_GET(query_nic_vport_context_out, out, + nic_vport_context.sd_group); +out: + kvfree(out); + return err; +} + int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid) { u32 *out; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index fee20fc010c2..bf2d51952e48 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -4030,8 +4030,13 @@ struct mlx5_ifc_nic_vport_context_bits { u8 affiliation_criteria[0x4]; u8 affiliated_vhca_id[0x10]; - u8 reserved_at_60[0xd0]; + u8 reserved_at_60[0xa0]; + u8 reserved_at_100[0x1]; + u8 sd_group[0x3]; + u8 reserved_at_104[0x1c]; + + u8 reserved_at_120[0x10]; u8 mtu[0x10]; u8 system_image_guid[0x40]; @@ -10116,8 +10121,7 @@ struct mlx5_ifc_mpir_reg_bits { u8 reserved_at_20[0x20]; u8 local_port[0x8]; - u8 reserved_at_28[0x15]; - u8 sd_group[0x3]; + u8 reserved_at_28[0x18]; u8 reserved_at_60[0x20]; }; diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index fbb9bf447889..c36cc6d82926 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -72,6 +72,7 @@ int mlx5_query_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 *mtu); int mlx5_modify_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 mtu); int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev, u64 *system_image_guid); +int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group); int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid); int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, u16 vport, u64 node_guid); -- cgit v1.2.3 From c88c49ac9c18fb7c3fa431126de1d8f8f555e912 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Tue, 5 Dec 2023 23:54:21 +0200 Subject: net/mlx5: Enable SD feature Have an actual mlx5_sd instance in the core device, and fix the getter accordingly. This allows SD stuff to flow, the feature becomes supported only here. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h | 3 ++- include/linux/mlx5/driver.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 0810b92b48d0..37d5f445598c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -59,10 +59,11 @@ struct mlx5_sd; static inline struct mlx5_sd *mlx5_get_sd(struct mlx5_core_dev *dev) { - return NULL; + return dev->sd; } static inline void mlx5_set_sd(struct mlx5_core_dev *dev, struct mlx5_sd *sd) { + dev->sd = sd; } #endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index aafb36c9e5d9..cd286b681970 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -822,6 +822,7 @@ struct mlx5_core_dev { struct blocking_notifier_head macsec_nh; #endif u64 num_ipsec_offloads; + struct mlx5_sd *sd; }; struct mlx5_db { -- cgit v1.2.3 From 22c4640698a1d47606b5a4264a584e8046641784 Mon Sep 17 00:00:00 2001 From: Armen Ratner Date: Fri, 8 Sep 2023 14:53:09 -0500 Subject: net/mlx5: Implement management PF Ethernet profile Add management PF modules, which introduce support for the structures needed to create the resources for the MGMT PF to work. Also, add the necessary calls and functions to establish this functionality. Signed-off-by: Armen Ratner Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Reviewed-by: Daniel Jurgens --- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/dev.c | 3 + drivers/net/ethernet/mellanox/mlx5/core/ecpf.c | 6 + drivers/net/ethernet/mellanox/mlx5/core/en.h | 4 + .../net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c | 268 +++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 24 +- drivers/net/ethernet/mellanox/mlx5/core/eswitch.c | 2 +- include/linux/mlx5/driver.h | 8 + include/linux/mlx5/mlx5_ifc.h | 14 +- 9 files changed, 323 insertions(+), 8 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 76dc5a9b9648..f36232dead1a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -29,7 +29,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ - lib/crypto.o lib/sd.o + en/mgmt_pf.o lib/crypto.o lib/sd.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index cf0477f53dc4..aa1b471e13fa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -190,6 +190,9 @@ bool mlx5_rdma_supported(struct mlx5_core_dev *dev) if (is_mp_supported(dev)) return false; + if (mlx5_core_is_mgmt_pf(dev)) + return false; + return true; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c index d000236ddbac..aa397e3ebe6d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c @@ -75,6 +75,9 @@ int mlx5_ec_init(struct mlx5_core_dev *dev) if (!mlx5_core_is_ecpf(dev)) return 0; + if (mlx5_core_is_mgmt_pf(dev)) + return 0; + return mlx5_host_pf_init(dev); } @@ -85,6 +88,9 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev) if (!mlx5_core_is_ecpf(dev)) return; + if (mlx5_core_is_mgmt_pf(dev)) + return; + mlx5_host_pf_cleanup(dev); err = mlx5_wait_for_pages(dev, &dev->priv.page_counters[MLX5_HOST_PF]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 84db05fb9389..922b63c25154 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -63,6 +63,7 @@ #include "lib/sd.h" extern const struct net_device_ops mlx5e_netdev_ops; +extern const struct net_device_ops mlx5e_mgmt_netdev_ops; struct page_pool; #define MLX5E_METADATA_ETHER_TYPE (0x8CE4) @@ -1125,6 +1126,7 @@ static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) } extern const struct ethtool_ops mlx5e_ethtool_ops; +extern const struct mlx5e_profile mlx5e_mgmt_pf_nic_profile; int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises); @@ -1230,6 +1232,8 @@ netdev_features_t mlx5e_features_check(struct sk_buff *skb, struct net_device *netdev, netdev_features_t features); int mlx5e_set_features(struct net_device *netdev, netdev_features_t features); +void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv); + #ifdef CONFIG_MLX5_ESWITCH int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac); int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c b/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c new file mode 100644 index 000000000000..77b5805895b9 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include +#include "en/params.h" +#include "en/health.h" +#include "lib/eq.h" +#include "en/dcbnl.h" +#include "en_accel/ipsec.h" +#include "en_accel/en_accel.h" +#include "en/trap.h" +#include "en/monitor_stats.h" +#include "en/hv_vhca_stats.h" +#include "en_rep.h" +#include "en.h" + +static int mgmt_pf_async_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); + struct mlx5_eqe *eqe = data; + + if (event != MLX5_EVENT_TYPE_PORT_CHANGE) + return NOTIFY_DONE; + + switch (eqe->sub_type) { + case MLX5_PORT_CHANGE_SUBTYPE_DOWN: + case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: + queue_work(priv->wq, &priv->update_carrier_work); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static void mlx5e_mgmt_pf_enable_async_events(struct mlx5e_priv *priv) +{ + priv->events_nb.notifier_call = mgmt_pf_async_event; + mlx5_notifier_register(priv->mdev, &priv->events_nb); +} + +static void mlx5e_disable_mgmt_pf_async_events(struct mlx5e_priv *priv) +{ + mlx5_notifier_unregister(priv->mdev, &priv->events_nb); +} + +static void mlx5e_modify_mgmt_pf_admin_state(struct mlx5_core_dev *mdev, + enum mlx5_port_status state) +{ + struct mlx5_eswitch *esw = mdev->priv.eswitch; + int vport_admin_state; + + mlx5_set_port_admin_status(mdev, state); + + if (state == MLX5_PORT_UP) + vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; + else + vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; + + mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); +} + +static void mlx5e_build_mgmt_pf_nic_params(struct mlx5e_priv *priv, u16 mtu) +{ + struct mlx5e_params *params = &priv->channels.params; + struct mlx5_core_dev *mdev = priv->mdev; + u8 rx_cq_period_mode; + + params->sw_mtu = mtu; + params->hard_mtu = MLX5E_ETH_HARD_MTU; + params->num_channels = 1; + + /* SQ */ + params->log_sq_size = is_kdump_kernel() ? + MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : + MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; + MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); + + MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); + + /* RQ */ + mlx5e_build_rq_params(mdev, params); + + /* CQ moderation params */ + rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? + MLX5_CQ_PERIOD_MODE_START_FROM_CQE : + MLX5_CQ_PERIOD_MODE_START_FROM_EQE; + params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); + mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); + + /* TX inline */ + mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); +} + +static int mlx5e_mgmt_pf_init(struct mlx5_core_dev *mdev, + struct net_device *netdev) +{ + struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_flow_steering *fs; + int err; + + mlx5e_build_mgmt_pf_nic_params(priv, netdev->mtu); + + mlx5e_timestamp_init(priv); + + fs = mlx5e_fs_init(priv->profile, mdev, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state), + priv->dfs_root); + if (!fs) { + err = -ENOMEM; + mlx5_core_err(mdev, "FS initialization failed, %d\n", err); + return err; + } + priv->fs = fs; + + mlx5e_health_create_reporters(priv); + + return 0; +} + +static void mlx5e_mgmt_pf_cleanup(struct mlx5e_priv *priv) +{ + mlx5e_health_destroy_reporters(priv); + mlx5e_fs_cleanup(priv->fs); + priv->fs = NULL; +} + +static int mlx5e_mgmt_pf_init_rx(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + int err; + + priv->rx_res = mlx5e_rx_res_create(mdev, 0, priv->max_nch, priv->drop_rq.rqn, + &priv->channels.params.packet_merge, + priv->channels.params.num_channels); + if (!priv->rx_res) + return -ENOMEM; + + mlx5e_create_q_counters(priv); + + err = mlx5e_open_drop_rq(priv, &priv->drop_rq); + if (err) { + mlx5_core_err(mdev, "open drop rq failed, %d\n", err); + goto err_destroy_q_counters; + } + + err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, + priv->netdev); + if (err) { + mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); + goto err_destroy_rx_res; + } + + return 0; + +err_destroy_rx_res: + mlx5e_rx_res_destroy(priv->rx_res); + priv->rx_res = NULL; + mlx5e_close_drop_rq(&priv->drop_rq); +err_destroy_q_counters: + mlx5e_destroy_q_counters(priv); + return err; +} + +static void mlx5e_mgmt_pf_cleanup_rx(struct mlx5e_priv *priv) +{ + mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), + priv->profile); + mlx5e_rx_res_destroy(priv->rx_res); + priv->rx_res = NULL; + mlx5e_close_drop_rq(&priv->drop_rq); + mlx5e_destroy_q_counters(priv); +} + +static int mlx5e_mgmt_pf_init_tx(struct mlx5e_priv *priv) +{ + return 0; +} + +static void mlx5e_mgmt_pf_cleanup_tx(struct mlx5e_priv *priv) +{ +} + +static void mlx5e_mgmt_pf_enable(struct mlx5e_priv *priv) +{ + struct net_device *netdev = priv->netdev; + struct mlx5_core_dev *mdev = priv->mdev; + + mlx5e_fs_init_l2_addr(priv->fs, netdev); + + /* Marking the link as currently not needed by the Driver */ + if (!netif_running(netdev)) + mlx5e_modify_mgmt_pf_admin_state(mdev, MLX5_PORT_DOWN); + + mlx5e_set_netdev_mtu_boundaries(priv); + mlx5e_set_dev_port_mtu(priv); + + mlx5e_mgmt_pf_enable_async_events(priv); + if (mlx5e_monitor_counter_supported(priv)) + mlx5e_monitor_counter_init(priv); + + mlx5e_hv_vhca_stats_create(priv); + if (netdev->reg_state != NETREG_REGISTERED) + return; + mlx5e_dcbnl_init_app(priv); + + mlx5e_nic_set_rx_mode(priv); + + rtnl_lock(); + if (netif_running(netdev)) + mlx5e_open(netdev); + udp_tunnel_nic_reset_ntf(priv->netdev); + netif_device_attach(netdev); + rtnl_unlock(); +} + +static void mlx5e_mgmt_pf_disable(struct mlx5e_priv *priv) +{ + if (priv->netdev->reg_state == NETREG_REGISTERED) + mlx5e_dcbnl_delete_app(priv); + + rtnl_lock(); + if (netif_running(priv->netdev)) + mlx5e_close(priv->netdev); + netif_device_detach(priv->netdev); + rtnl_unlock(); + + mlx5e_nic_set_rx_mode(priv); + + mlx5e_hv_vhca_stats_destroy(priv); + if (mlx5e_monitor_counter_supported(priv)) + mlx5e_monitor_counter_cleanup(priv); + + mlx5e_disable_mgmt_pf_async_events(priv); + mlx5e_ipsec_cleanup(priv); +} + +static int mlx5e_mgmt_pf_update_rx(struct mlx5e_priv *priv) +{ + return mlx5e_refresh_tirs(priv, false, false); +} + +static int mlx5e_mgmt_pf_max_nch_limit(struct mlx5_core_dev *mdev) +{ + return 1; +} + +const struct mlx5e_profile mlx5e_mgmt_pf_nic_profile = { + .init = mlx5e_mgmt_pf_init, + .cleanup = mlx5e_mgmt_pf_cleanup, + .init_rx = mlx5e_mgmt_pf_init_rx, + .cleanup_rx = mlx5e_mgmt_pf_cleanup_rx, + .init_tx = mlx5e_mgmt_pf_init_tx, + .cleanup_tx = mlx5e_mgmt_pf_cleanup_tx, + .enable = mlx5e_mgmt_pf_enable, + .disable = mlx5e_mgmt_pf_disable, + .update_rx = mlx5e_mgmt_pf_update_rx, + .update_stats = mlx5e_stats_update_ndo_stats, + .update_carrier = mlx5e_update_carrier, + .rx_handlers = &mlx5e_rx_handlers_nic, + .max_tc = 1, + .max_nch_limit = mlx5e_mgmt_pf_max_nch_limit, + .stats_grps = mlx5e_nic_stats_grps, + .stats_grps_num = mlx5e_nic_stats_grps_num +}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b8f08d64f66b..40626b6108fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3799,7 +3799,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; } -static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) +void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) { if (mlx5e_is_uplink_rep(priv)) return; /* no rx mode for uplink rep */ @@ -5004,6 +5004,15 @@ const struct net_device_ops mlx5e_netdev_ops = { #endif }; +const struct net_device_ops mlx5e_mgmt_netdev_ops = { + .ndo_open = mlx5e_open, + .ndo_stop = mlx5e_close, + .ndo_start_xmit = mlx5e_xmit, + .ndo_get_stats64 = mlx5e_get_stats, + .ndo_change_mtu = mlx5e_change_nic_mtu, + .ndo_set_rx_mode = mlx5e_set_rx_mode, +}; + static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) { int i; @@ -5143,7 +5152,11 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) SET_NETDEV_DEV(netdev, mdev->device); - netdev->netdev_ops = &mlx5e_netdev_ops; + if (mlx5_core_is_mgmt_pf(mdev)) + netdev->netdev_ops = &mlx5e_mgmt_netdev_ops; + else + netdev->netdev_ops = &mlx5e_netdev_ops; + netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; @@ -6094,13 +6107,18 @@ static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) static int _mlx5e_probe(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); - const struct mlx5e_profile *profile = &mlx5e_nic_profile; struct mlx5_core_dev *mdev = edev->mdev; + const struct mlx5e_profile *profile; struct mlx5e_dev *mlx5e_dev; struct net_device *netdev; struct mlx5e_priv *priv; int err; + if (mlx5_core_is_mgmt_pf(mdev)) + profile = &mlx5e_mgmt_pf_nic_profile; + else + profile = &mlx5e_nic_profile; + mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); if (IS_ERR(mlx5e_dev)) return PTR_ERR(mlx5e_dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 3047d7015c52..3bf419d06d53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1665,7 +1665,7 @@ int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 * void *hca_caps; int err; - if (!mlx5_core_is_ecpf(dev)) { + if (!mlx5_core_is_ecpf(dev) || mlx5_core_is_mgmt_pf(dev)) { *max_sfs = 0; return 0; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index cd286b681970..2bba88c67f58 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1224,6 +1224,14 @@ static inline bool mlx5_core_is_ecpf(const struct mlx5_core_dev *dev) return dev->caps.embedded_cpu; } +static inline bool mlx5_core_is_mgmt_pf(const struct mlx5_core_dev *dev) +{ + if (!MLX5_CAP_GEN_2(dev, local_mng_port_valid)) + return false; + + return MLX5_CAP_GEN_2(dev, local_mng_port); +} + static inline bool mlx5_core_is_ecpf_esw_manager(const struct mlx5_core_dev *dev) { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index bf2d51952e48..586569209254 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1954,8 +1954,10 @@ enum { struct mlx5_ifc_cmd_hca_cap_2_bits { u8 reserved_at_0[0x80]; - u8 migratable[0x1]; - u8 reserved_at_81[0x1f]; + u8 migratable[0x1]; + u8 reserved_at_81[0x19]; + u8 local_mng_port[0x1]; + u8 reserved_at_9b[0x5]; u8 max_reformat_insert_size[0x8]; u8 max_reformat_insert_offset[0x8]; @@ -1973,7 +1975,13 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 allowed_object_for_other_vhca_access[0x40]; - u8 reserved_at_140[0x60]; + u8 reserved_at_140[0x20]; + + u8 reserved_at_160[0xa]; + u8 local_mng_port_valid[0x1]; + u8 reserved_at_16b[0x15]; + + u8 reserved_at_180[0x20]; u8 flow_table_type_2_type[0x8]; u8 reserved_at_1a8[0x3]; -- cgit v1.2.3 From 645f3d85129d8aac3b896ba685fbc20a31c2c036 Mon Sep 17 00:00:00 2001 From: Mukesh Sisodiya Date: Wed, 20 Dec 2023 13:41:38 +0200 Subject: wifi: cfg80211: handle UHB AP and STA power type UHB AP send supported power type(LPI, SP, VLP) in beacon and probe response IE and STA should connect to these AP only if their regulatory support the AP power type. Beacon/Probe response are reported to userspace with reason "STA regulatory not supporting to connect to AP based on transmitted power type" and it should not connect to AP. Signed-off-by: Mukesh Sisodiya Reviewed-by: Gregory Greenman Signed-off-by: Miri Korenblit Link: https://msgid.link/20231220133549.cbfbef9170a9.I432f78438de18aa9f5c9006be12e41dc34cc47c5@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 1 + include/net/cfg80211.h | 6 ++++++ include/uapi/linux/nl80211.h | 13 +++++++++++++ net/wireless/nl80211.c | 6 ++++++ net/wireless/reg.c | 4 ++++ net/wireless/scan.c | 38 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 8ad008591e32..2f5554482047 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2720,6 +2720,7 @@ static inline bool ieee80211_he_capa_size_ok(const u8 *data, u8 len) #define IEEE80211_6GHZ_CTRL_REG_LPI_AP 0 #define IEEE80211_6GHZ_CTRL_REG_SP_AP 1 +#define IEEE80211_6GHZ_CTRL_REG_VLP_AP 2 /** * struct ieee80211_he_6ghz_oper - HE 6 GHz operation Information field diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 745974d45ea4..cf79656ce09c 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -118,6 +118,10 @@ struct wiphy; * restrictions. * @IEEE80211_CHAN_NO_EHT: EHT operation is not permitted on this channel. * @IEEE80211_CHAN_DFS_CONCURRENT: See %NL80211_RRF_DFS_CONCURRENT + * @IEEE80211_CHAN_NO_UHB_VLP_CLIENT: Client connection with VLP AP + * not permitted using this channel + * @IEEE80211_CHAN_NO_UHB_AFC_CLIENT: Client connection with AFC AP + * not permitted using this channel */ enum ieee80211_channel_flags { IEEE80211_CHAN_DISABLED = 1<<0, @@ -142,6 +146,8 @@ enum ieee80211_channel_flags { IEEE80211_CHAN_NO_320MHZ = 1<<19, IEEE80211_CHAN_NO_EHT = 1<<20, IEEE80211_CHAN_DFS_CONCURRENT = 1<<21, + IEEE80211_CHAN_NO_UHB_VLP_CLIENT= 1<<22, + IEEE80211_CHAN_NO_UHB_AFC_CLIENT= 1<<23, }; #define IEEE80211_CHAN_NO_HT40 \ diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 466da830e65f..1ccdcae24372 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -4260,6 +4260,10 @@ enum nl80211_wmm_rule { * allowed for peer-to-peer or adhoc communication under the control * of a DFS master which operates on the same channel (FCC-594280 D01 * Section B.3). Should be used together with %NL80211_RRF_DFS only. + * @NL80211_FREQUENCY_ATTR_NO_UHB_VLP_CLIENT: Client connection to VLP AP + * not allowed using this channel + * @NL80211_FREQUENCY_ATTR_NO_UHB_AFC_CLIENT: Client connection to AFC AP + * not allowed using this channel * @NL80211_FREQUENCY_ATTR_MAX: highest frequency attribute number * currently defined * @__NL80211_FREQUENCY_ATTR_AFTER_LAST: internal use @@ -4300,6 +4304,8 @@ enum nl80211_frequency_attr { NL80211_FREQUENCY_ATTR_NO_EHT, NL80211_FREQUENCY_ATTR_PSD, NL80211_FREQUENCY_ATTR_DFS_CONCURRENT, + NL80211_FREQUENCY_ATTR_NO_UHB_VLP_CLIENT, + NL80211_FREQUENCY_ATTR_NO_UHB_AFC_CLIENT, /* keep last */ __NL80211_FREQUENCY_ATTR_AFTER_LAST, @@ -4509,6 +4515,8 @@ enum nl80211_sched_scan_match_attr { peer-to-peer or adhoc communication under the control of a DFS master which operates on the same channel (FCC-594280 D01 Section B.3). Should be used together with %NL80211_RRF_DFS only. + * @NL80211_RRF_NO_UHB_VLP_CLIENT: Client connection to VLP AP not allowed + * @NL80211_RRF_NO_UHB_AFC_CLIENT: Client connection to AFC AP not allowed */ enum nl80211_reg_rule_flags { NL80211_RRF_NO_OFDM = 1<<0, @@ -4531,6 +4539,8 @@ enum nl80211_reg_rule_flags { NL80211_RRF_NO_EHT = 1<<19, NL80211_RRF_PSD = 1<<20, NL80211_RRF_DFS_CONCURRENT = 1<<21, + NL80211_RRF_NO_UHB_VLP_CLIENT = 1<<22, + NL80211_RRF_NO_UHB_AFC_CLIENT = 1<<23, }; #define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR @@ -5086,9 +5096,12 @@ enum nl80211_bss_use_for { * BSS isn't possible * @NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY: NSTR nonprimary links aren't * supported by the device, and this BSS entry represents one. + * @NL80211_BSS_CANNOT_USE_UHB_PWR_MISMATCH: STA is not supporting + * the AP power type (SP, VLP, AP) that the AP uses. */ enum nl80211_bss_cannot_use_reasons { NL80211_BSS_CANNOT_USE_NSTR_NONPRIMARY = 1 << 0, + NL80211_BSS_CANNOT_USE_UHB_PWR_MISMATCH = 1 << 1, }; /** diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 534ef3fe0696..60877b532993 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -1204,6 +1204,12 @@ static int nl80211_msg_put_channel(struct sk_buff *msg, struct wiphy *wiphy, if ((chan->flags & IEEE80211_CHAN_DFS_CONCURRENT) && nla_put_flag(msg, NL80211_FREQUENCY_ATTR_DFS_CONCURRENT)) goto nla_put_failure; + if ((chan->flags & IEEE80211_CHAN_NO_UHB_VLP_CLIENT) && + nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_UHB_VLP_CLIENT)) + goto nla_put_failure; + if ((chan->flags & IEEE80211_CHAN_NO_UHB_AFC_CLIENT) && + nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_UHB_AFC_CLIENT)) + goto nla_put_failure; } if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_MAX_TX_POWER, diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 44684df64734..2741b626919a 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -1595,6 +1595,10 @@ static u32 map_regdom_flags(u32 rd_flags) channel_flags |= IEEE80211_CHAN_NO_EHT; if (rd_flags & NL80211_RRF_DFS_CONCURRENT) channel_flags |= IEEE80211_CHAN_DFS_CONCURRENT; + if (rd_flags & NL80211_RRF_NO_UHB_VLP_CLIENT) + channel_flags |= IEEE80211_CHAN_NO_UHB_VLP_CLIENT; + if (rd_flags & NL80211_RRF_NO_UHB_AFC_CLIENT) + channel_flags |= IEEE80211_CHAN_NO_UHB_AFC_CLIENT; if (rd_flags & NL80211_RRF_PSD) channel_flags |= IEEE80211_CHAN_PSD; return channel_flags; diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 3d260c99c348..a601f1c7f835 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -2848,6 +2848,36 @@ cfg80211_inform_bss_data(struct wiphy *wiphy, } EXPORT_SYMBOL(cfg80211_inform_bss_data); +static bool cfg80211_uhb_power_type_valid(const u8 *ie, + size_t ielen, + const u32 flags) +{ + const struct element *tmp; + struct ieee80211_he_operation *he_oper; + + tmp = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ie, ielen); + if (tmp && tmp->datalen >= sizeof(*he_oper) + 1) { + const struct ieee80211_he_6ghz_oper *he_6ghz_oper; + + he_oper = (void *)&tmp->data[1]; + he_6ghz_oper = ieee80211_he_6ghz_oper(he_oper); + + if (!he_6ghz_oper) + return false; + + switch (u8_get_bits(he_6ghz_oper->control, + IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO)) { + case IEEE80211_6GHZ_CTRL_REG_LPI_AP: + return true; + case IEEE80211_6GHZ_CTRL_REG_SP_AP: + return !(flags & IEEE80211_CHAN_NO_UHB_AFC_CLIENT); + case IEEE80211_6GHZ_CTRL_REG_VLP_AP: + return !(flags & IEEE80211_CHAN_NO_UHB_VLP_CLIENT); + } + } + return false; +} + /* cfg80211_inform_bss_width_frame helper */ static struct cfg80211_bss * cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, @@ -2906,6 +2936,14 @@ cfg80211_inform_single_bss_frame_data(struct wiphy *wiphy, if (!channel) return NULL; + if (channel->band == NL80211_BAND_6GHZ && + !cfg80211_uhb_power_type_valid(variable, ielen, channel->flags)) { + data->restrict_use = 1; + data->use_for = 0; + data->cannot_use_reasons = + NL80211_BSS_CANNOT_USE_UHB_PWR_MISMATCH; + } + if (ext) { const struct ieee80211_s1g_bcn_compat_ie *compat; const struct element *elem; -- cgit v1.2.3 From dcc3e46472d678f4af5ce1194a23649231c5d241 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 18 Dec 2023 17:26:26 -0700 Subject: net: skbuff: Remove some excess struct-member documentation Remove documentation for nonexistent structure members, addressing these warnings: ./include/linux/skbuff.h:1063: warning: Excess struct member 'sp' description in 'sk_buff' ./include/linux/skbuff.h:1063: warning: Excess struct member 'nf_bridge' description in 'sk_buff' Signed-off-by: Jonathan Corbet Reviewed-by: Randy Dunlap Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index ea5c8ab3ed00..50e92c8471dc 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -754,7 +754,6 @@ typedef unsigned char *sk_buff_data_t; * @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL * @cb: Control buffer. Free for use by every layer. Put private vars here * @_skb_refdst: destination entry (with norefcount bit) - * @sp: the security path, used for xfrm * @len: Length of actual data * @data_len: Data length * @mac_len: Length of link layer header @@ -788,7 +787,6 @@ typedef unsigned char *sk_buff_data_t; * @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue) * @_sk_redir: socket redirection information for skmsg * @_nfct: Associated connection, if any (with nfctinfo bits) - * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c * @skb_iif: ifindex of device we arrived on * @tc_index: Traffic control index * @hash: the packet hash -- cgit v1.2.3 From b3bf76024f645369e1fc45e0b08a2bd24f200d9b Mon Sep 17 00:00:00 2001 From: Wen Gu Date: Tue, 19 Dec 2023 22:26:16 +0800 Subject: net/smc: manage system EID in SMC stack instead of ISM driver The System EID (SEID) is an internal EID that is used by the SMCv2 software stack that has a predefined and constant value representing the s390 physical machine that the OS is executing on. So it should be managed by SMC stack instead of ISM driver and be consistent for all ISMv2 device (including virtual ISM devices) on s390 architecture. Suggested-by: Alexandra Winter Signed-off-by: Wen Gu Reviewed-and-tested-by: Wenjia Zhang Reviewed-by: Alexandra Winter Signed-off-by: David S. Miller --- drivers/s390/net/ism.h | 7 ------- drivers/s390/net/ism_drv.c | 38 ++++++-------------------------------- include/linux/ism.h | 1 - include/net/smc.h | 1 - net/smc/smc_ism.c | 33 ++++++++++++++++++++++++--------- net/smc/smc_ism.h | 7 +++++++ 6 files changed, 37 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/drivers/s390/net/ism.h b/drivers/s390/net/ism.h index 70c5bbda0fea..047fa6101555 100644 --- a/drivers/s390/net/ism.h +++ b/drivers/s390/net/ism.h @@ -16,7 +16,6 @@ */ #define ISM_DMB_WORD_OFFSET 1 #define ISM_DMB_BIT_OFFSET (ISM_DMB_WORD_OFFSET * 32) -#define ISM_IDENT_MASK 0x00FFFF #define ISM_REG_SBA 0x1 #define ISM_REG_IEQ 0x2 @@ -192,12 +191,6 @@ struct ism_sba { #define ISM_CREATE_REQ(dmb, idx, sf, offset) \ ((dmb) | (idx) << 24 | (sf) << 23 | (offset)) -struct ism_systemeid { - u8 seid_string[24]; - u8 serial_number[4]; - u8 type[4]; -}; - static inline void __ism_read_cmd(struct ism_dev *ism, void *data, unsigned long offset, unsigned long len) { diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index 34dd06324e38..2c8e964425dc 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -36,6 +36,7 @@ static struct ism_client *clients[MAX_CLIENTS]; /* use an array rather than */ /* a list for fast mapping */ static u8 max_client; static DEFINE_MUTEX(clients_lock); +static bool ism_v2_capable; struct ism_dev_list { struct list_head list; struct mutex mutex; /* protects ism device list */ @@ -443,32 +444,6 @@ int ism_move(struct ism_dev *ism, u64 dmb_tok, unsigned int idx, bool sf, } EXPORT_SYMBOL_GPL(ism_move); -static struct ism_systemeid SYSTEM_EID = { - .seid_string = "IBM-SYSZ-ISMSEID00000000", - .serial_number = "0000", - .type = "0000", -}; - -static void ism_create_system_eid(void) -{ - struct cpuid id; - u16 ident_tail; - char tmp[5]; - - get_cpu_id(&id); - ident_tail = (u16)(id.ident & ISM_IDENT_MASK); - snprintf(tmp, 5, "%04X", ident_tail); - memcpy(&SYSTEM_EID.serial_number, tmp, 4); - snprintf(tmp, 5, "%04X", id.machine); - memcpy(&SYSTEM_EID.type, tmp, 4); -} - -u8 *ism_get_seid(void) -{ - return SYSTEM_EID.seid_string; -} -EXPORT_SYMBOL_GPL(ism_get_seid); - static void ism_handle_event(struct ism_dev *ism) { struct ism_event *entry; @@ -560,7 +535,9 @@ static int ism_dev_init(struct ism_dev *ism) if (!ism_add_vlan_id(ism, ISM_RESERVED_VLANID)) /* hardware is V2 capable */ - ism_create_system_eid(); + ism_v2_capable = true; + else + ism_v2_capable = false; mutex_lock(&ism_dev_list.mutex); mutex_lock(&clients_lock); @@ -665,8 +642,7 @@ static void ism_dev_exit(struct ism_dev *ism) } mutex_unlock(&clients_lock); - if (SYSTEM_EID.serial_number[0] != '0' || - SYSTEM_EID.type[0] != '0') + if (ism_v2_capable) ism_del_vlan_id(ism, ISM_RESERVED_VLANID); unregister_ieq(ism); unregister_sba(ism); @@ -813,8 +789,7 @@ static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx, static int smcd_supports_v2(void) { - return SYSTEM_EID.serial_number[0] != '0' || - SYSTEM_EID.type[0] != '0'; + return ism_v2_capable; } static u64 ism_get_local_gid(struct ism_dev *ism) @@ -860,7 +835,6 @@ static const struct smcd_ops ism_ops = { .signal_event = smcd_signal_ieq, .move_data = smcd_move, .supports_v2 = smcd_supports_v2, - .get_system_eid = ism_get_seid, .get_local_gid = smcd_get_local_gid, .get_chid = smcd_get_chid, .get_dev = smcd_get_dev, diff --git a/include/linux/ism.h b/include/linux/ism.h index 9a4c204df3da..5428edd90982 100644 --- a/include/linux/ism.h +++ b/include/linux/ism.h @@ -86,7 +86,6 @@ int ism_register_dmb(struct ism_dev *dev, struct ism_dmb *dmb, int ism_unregister_dmb(struct ism_dev *dev, struct ism_dmb *dmb); int ism_move(struct ism_dev *dev, u64 dmb_tok, unsigned int idx, bool sf, unsigned int offset, void *data, unsigned int size); -u8 *ism_get_seid(void); const struct smcd_ops *ism_get_smcd_ops(void); diff --git a/include/net/smc.h b/include/net/smc.h index a0dc1187e96e..c9dcb30e3fd9 100644 --- a/include/net/smc.h +++ b/include/net/smc.h @@ -73,7 +73,6 @@ struct smcd_ops { bool sf, unsigned int offset, void *data, unsigned int size); int (*supports_v2)(void); - u8* (*get_system_eid)(void); void (*get_local_gid)(struct smcd_dev *dev, struct smcd_gid *gid); u16 (*get_chid)(struct smcd_dev *dev); struct device* (*get_dev)(struct smcd_dev *dev); diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c index a33f861cf7c1..ac88de2a06a0 100644 --- a/net/smc/smc_ism.c +++ b/net/smc/smc_ism.c @@ -43,6 +43,27 @@ static struct ism_client smc_ism_client = { }; #endif +static void smc_ism_create_system_eid(void) +{ + struct smc_ism_seid *seid = + (struct smc_ism_seid *)smc_ism_v2_system_eid; +#if IS_ENABLED(CONFIG_S390) + struct cpuid id; + u16 ident_tail; + char tmp[5]; + + memcpy(seid->seid_string, "IBM-SYSZ-ISMSEID00000000", 24); + get_cpu_id(&id); + ident_tail = (u16)(id.ident & SMC_ISM_IDENT_MASK); + snprintf(tmp, 5, "%04X", ident_tail); + memcpy(seid->serial_number, tmp, 4); + snprintf(tmp, 5, "%04X", id.machine); + memcpy(seid->type, tmp, 4); +#else + memset(seid, 0, SMC_MAX_EID_LEN); +#endif +} + /* Test if an ISM communication is possible - same CPC */ int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id, struct smcd_dev *smcd) @@ -431,14 +452,8 @@ static void smcd_register_dev(struct ism_dev *ism) mutex_lock(&smcd_dev_list.mutex); if (list_empty(&smcd_dev_list.list)) { - u8 *system_eid = NULL; - - system_eid = smcd->ops->get_system_eid(); - if (smcd->ops->supports_v2()) { + if (smcd->ops->supports_v2()) smc_ism_v2_capable = true; - memcpy(smc_ism_v2_system_eid, system_eid, - SMC_MAX_EID_LEN); - } } /* sort list: devices without pnetid before devices with pnetid */ if (smcd->pnetid[0]) @@ -542,10 +557,10 @@ int smc_ism_init(void) { int rc = 0; -#if IS_ENABLED(CONFIG_ISM) smc_ism_v2_capable = false; - memset(smc_ism_v2_system_eid, 0, SMC_MAX_EID_LEN); + smc_ism_create_system_eid(); +#if IS_ENABLED(CONFIG_ISM) rc = ism_register_client(&smc_ism_client); #endif return rc; diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h index 0e5e563099ec..ffff40c30a06 100644 --- a/net/smc/smc_ism.h +++ b/net/smc/smc_ism.h @@ -16,6 +16,7 @@ #include "smc.h" #define SMC_VIRTUAL_ISM_CHID_MASK 0xFF00 +#define SMC_ISM_IDENT_MASK 0x00FFFF struct smcd_dev_list { /* List of SMCD devices */ struct list_head list; @@ -30,6 +31,12 @@ struct smc_ism_vlanid { /* VLAN id set on ISM device */ refcount_t refcnt; /* Reference count */ }; +struct smc_ism_seid { + u8 seid_string[24]; + u8 serial_number[4]; + u8 type[4]; +}; + struct smcd_dev; int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id, -- cgit v1.2.3 From 90abde49ea85a8af9a56bbab8c419aefc77f919a Mon Sep 17 00:00:00 2001 From: "Radu Pirea (NXP OSS)" Date: Tue, 19 Dec 2023 16:53:25 +0200 Subject: net: rename dsa_realloc_skb to skb_ensure_writable_head_tail Rename dsa_realloc_skb to skb_ensure_writable_head_tail and move it to skbuff.c to use it as helper. Signed-off-by: Radu Pirea (NXP OSS) Signed-off-by: David S. Miller --- include/linux/skbuff.h | 1 + net/core/skbuff.c | 25 +++++++++++++++++++++++++ net/dsa/user.c | 29 +++-------------------------- 3 files changed, 29 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 50e92c8471dc..a5ae952454c8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4007,6 +4007,7 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features unsigned int offset); struct sk_buff *skb_vlan_untag(struct sk_buff *skb); int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len); +int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev); int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci); int skb_vlan_pop(struct sk_buff *skb); int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci); diff --git a/net/core/skbuff.c b/net/core/skbuff.c index ce5687ddb768..12d22c0b8551 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -5995,6 +5995,31 @@ int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len) } EXPORT_SYMBOL(skb_ensure_writable); +int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev) +{ + int needed_headroom = dev->needed_headroom; + int needed_tailroom = dev->needed_tailroom; + + /* For tail taggers, we need to pad short frames ourselves, to ensure + * that the tail tag does not fail at its role of being at the end of + * the packet, once the conduit interface pads the frame. Account for + * that pad length here, and pad later. + */ + if (unlikely(needed_tailroom && skb->len < ETH_ZLEN)) + needed_tailroom += ETH_ZLEN - skb->len; + /* skb_headroom() returns unsigned int... */ + needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0); + needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0); + + if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb))) + /* No reallocation needed, yay! */ + return 0; + + return pskb_expand_head(skb, needed_headroom, needed_tailroom, + GFP_ATOMIC); +} +EXPORT_SYMBOL(skb_ensure_writable_head_tail); + /* remove VLAN header from packet and update csum accordingly. * expects a non skb_vlan_tag_present skb with a vlan tag payload */ diff --git a/net/dsa/user.c b/net/dsa/user.c index d438884a4eb0..b738a466e2dc 100644 --- a/net/dsa/user.c +++ b/net/dsa/user.c @@ -920,30 +920,6 @@ netdev_tx_t dsa_enqueue_skb(struct sk_buff *skb, struct net_device *dev) } EXPORT_SYMBOL_GPL(dsa_enqueue_skb); -static int dsa_realloc_skb(struct sk_buff *skb, struct net_device *dev) -{ - int needed_headroom = dev->needed_headroom; - int needed_tailroom = dev->needed_tailroom; - - /* For tail taggers, we need to pad short frames ourselves, to ensure - * that the tail tag does not fail at its role of being at the end of - * the packet, once the conduit interface pads the frame. Account for - * that pad length here, and pad later. - */ - if (unlikely(needed_tailroom && skb->len < ETH_ZLEN)) - needed_tailroom += ETH_ZLEN - skb->len; - /* skb_headroom() returns unsigned int... */ - needed_headroom = max_t(int, needed_headroom - skb_headroom(skb), 0); - needed_tailroom = max_t(int, needed_tailroom - skb_tailroom(skb), 0); - - if (likely(!needed_headroom && !needed_tailroom && !skb_cloned(skb))) - /* No reallocation needed, yay! */ - return 0; - - return pskb_expand_head(skb, needed_headroom, needed_tailroom, - GFP_ATOMIC); -} - static netdev_tx_t dsa_user_xmit(struct sk_buff *skb, struct net_device *dev) { struct dsa_user_priv *p = netdev_priv(dev); @@ -956,13 +932,14 @@ static netdev_tx_t dsa_user_xmit(struct sk_buff *skb, struct net_device *dev) /* Handle tx timestamp if any */ dsa_skb_tx_timestamp(p, skb); - if (dsa_realloc_skb(skb, dev)) { + if (skb_ensure_writable_head_tail(skb, dev)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* needed_tailroom should still be 'warm' in the cache line from - * dsa_realloc_skb(), which has also ensured that padding is safe. + * skb_ensure_writable_head_tail(), which has also ensured that + * padding is safe. */ if (dev->needed_tailroom) eth_skb_pad(skb); -- cgit v1.2.3 From 1271ca00aa7f9bb3fd94cb7ac8f654de71099580 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 19 Dec 2023 16:55:31 -0700 Subject: ethtool: reformat kerneldoc for struct ethtool_fec_stats The kerneldoc comment for struct ethtool_fec_stats attempts to describe the "total" and "lanes" fields of the ethtool_fec_stat substructure in a way leading to these warnings: ./include/linux/ethtool.h:424: warning: Excess struct member 'lane' description in 'ethtool_fec_stats' ./include/linux/ethtool.h:424: warning: Excess struct member 'total' description in 'ethtool_fec_stats' Reformat the comment to retain the information while eliminating the warnings. Signed-off-by: Jonathan Corbet Reviewed-by: Randy Dunlap Signed-off-by: David S. Miller --- include/linux/ethtool.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index cfcd952a1d4f..325e0778e937 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -411,8 +411,10 @@ struct ethtool_pause_stats { * not entire FEC data blocks. This is a non-standard statistic. * Reported to user space as %ETHTOOL_A_FEC_STAT_CORR_BITS. * - * @lane: per-lane/PCS-instance counts as defined by the standard - * @total: error counts for the entire port, for drivers incapable of reporting + * For each of the above fields, the two substructure members are: + * + * - @lanes: per-lane/PCS-instance counts as defined by the standard + * - @total: error counts for the entire port, for drivers incapable of reporting * per-lane stats * * Drivers should fill in either only total or per-lane statistics, core -- cgit v1.2.3 From cff9c565e65f3622e8dc1dcc21c1520a083dff35 Mon Sep 17 00:00:00 2001 From: Luiz Angelo Daros de Luca Date: Wed, 20 Dec 2023 01:52:29 -0300 Subject: net: mdio: get/put device node during (un)registration The __of_mdiobus_register() function was storing the device node in dev.of_node without increasing its reference count. It implicitly relied on the caller to maintain the allocated node until the mdiobus was unregistered. Now, __of_mdiobus_register() will acquire the node before assigning it, and of_mdiobus_unregister_callback() will be called at the end of mdio_unregister(). Drivers can now release the node immediately after MDIO registration. Some of them are already doing that even before this patch. Signed-off-by: Luiz Angelo Daros de Luca Signed-off-by: David S. Miller --- drivers/net/mdio/of_mdio.c | 12 +++++++++++- drivers/net/phy/mdio_bus.c | 3 +++ include/linux/phy.h | 3 +++ 3 files changed, 17 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 64ebcb6d235c..9b6cab6154e0 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -139,6 +139,11 @@ bool of_mdiobus_child_is_phy(struct device_node *child) } EXPORT_SYMBOL(of_mdiobus_child_is_phy); +static void __of_mdiobus_unregister_callback(struct mii_bus *mdio) +{ + of_node_put(mdio->dev.of_node); +} + /** * __of_mdiobus_register - Register mii_bus and create PHYs from the device tree * @mdio: pointer to mii_bus structure @@ -166,6 +171,8 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, * the device tree are populated after the bus has been registered */ mdio->phy_mask = ~0; + mdio->__unregister_callback = __of_mdiobus_unregister_callback; + of_node_get(np); device_set_node(&mdio->dev, of_fwnode_handle(np)); /* Get bus level PHY reset GPIO details */ @@ -177,7 +184,7 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, /* Register the MDIO bus */ rc = __mdiobus_register(mdio, owner); if (rc) - return rc; + goto put_node; /* Loop over the child nodes and register a phy_device for each phy */ for_each_available_child_of_node(np, child) { @@ -237,6 +244,9 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, unregister: of_node_put(child); mdiobus_unregister(mdio); + +put_node: + of_node_put(np); return rc; } EXPORT_SYMBOL(__of_mdiobus_register); diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 6cf73c15635b..4a30757c4ff8 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -787,6 +787,9 @@ void mdiobus_unregister(struct mii_bus *bus) gpiod_set_value_cansleep(bus->reset_gpiod, 1); device_del(&bus->dev); + + if (bus->__unregister_callback) + bus->__unregister_callback(bus); } EXPORT_SYMBOL(mdiobus_unregister); diff --git a/include/linux/phy.h b/include/linux/phy.h index e9e85d347587..ede891776d8b 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -434,6 +434,9 @@ struct mii_bus { /** @shared: shared state across different PHYs */ struct phy_package_shared *shared[PHY_MAX_ADDR]; + + /** @__unregister_callback: called at the last step of unregistration */ + void (*__unregister_callback)(struct mii_bus *bus); }; #define to_mii_bus(d) container_of(d, struct mii_bus, dev) -- cgit v1.2.3 From 02018c544ef113e980a2349eba89003d6f399d22 Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 21 Dec 2023 19:00:34 +0100 Subject: net: phy: Introduce ethernet link topology representation Link topologies containing multiple network PHYs attached to the same net_device can be found when using a PHY as a media converter for use with an SFP connector, on which an SFP transceiver containing a PHY can be used. With the current model, the transceiver's PHY can't be used for operations such as cable testing, timestamping, macsec offload, etc. The reason being that most of the logic for these configuration, coming from either ethtool netlink or ioctls tend to use netdev->phydev, which in multi-phy systems will reference the PHY closest to the MAC. Introduce a numbering scheme allowing to enumerate PHY devices that belong to any netdev, which can in turn allow userspace to take more precise decisions with regard to each PHY's configuration. The numbering is maintained per-netdev, in a phy_device_list. The numbering works similarly to a netdevice's ifindex, with identifiers that are only recycled once INT_MAX has been reached. This prevents races that could occur between PHY listing and SFP transceiver removal/insertion. The identifiers are assigned at phy_attach time, as the numbering depends on the netdevice the phy is attached to. Signed-off-by: Maxime Chevallier Signed-off-by: David S. Miller --- MAINTAINERS | 2 + drivers/net/phy/Makefile | 2 +- drivers/net/phy/phy_device.c | 7 ++++ drivers/net/phy/phy_link_topology.c | 66 +++++++++++++++++++++++++++++++++ include/linux/netdevice.h | 4 +- include/linux/phy.h | 4 ++ include/linux/phy_link_topology.h | 67 ++++++++++++++++++++++++++++++++++ include/linux/phy_link_topology_core.h | 19 ++++++++++ include/uapi/linux/ethtool.h | 16 ++++++++ net/core/dev.c | 3 ++ 10 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 drivers/net/phy/phy_link_topology.c create mode 100644 include/linux/phy_link_topology.h create mode 100644 include/linux/phy_link_topology_core.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 2b916990d7f0..79ac49b113dc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7871,6 +7871,8 @@ F: include/linux/mii.h F: include/linux/of_net.h F: include/linux/phy.h F: include/linux/phy_fixed.h +F: include/linux/phy_link_topology.h +F: include/linux/phy_link_topology_core.h F: include/linux/phylib_stubs.h F: include/linux/platform_data/mdio-bcm-unimac.h F: include/linux/platform_data/mdio-gpio.h diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile index 6097afd44392..f218954fd7a8 100644 --- a/drivers/net/phy/Makefile +++ b/drivers/net/phy/Makefile @@ -2,7 +2,7 @@ # Makefile for Linux PHY drivers libphy-y := phy.o phy-c45.o phy-core.o phy_device.o \ - linkmode.o + linkmode.o phy_link_topology.o mdio-bus-y += mdio_bus.o mdio_device.o ifdef CONFIG_MDIO_DEVICE diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 3611ea64875e..ab8ae976a2f8 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1491,6 +1492,11 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, if (phydev->sfp_bus_attached) dev->sfp_bus = phydev->sfp_bus; + + err = phy_link_topo_add_phy(&dev->link_topo, phydev, + PHY_UPSTREAM_MAC, dev); + if (err) + goto error; } /* Some Ethernet drivers try to connect to a PHY device before @@ -1820,6 +1826,7 @@ void phy_detach(struct phy_device *phydev) if (dev) { phydev->attached_dev->phydev = NULL; phydev->attached_dev = NULL; + phy_link_topo_del_phy(&dev->link_topo, phydev); } phydev->phylink = NULL; diff --git a/drivers/net/phy/phy_link_topology.c b/drivers/net/phy/phy_link_topology.c new file mode 100644 index 000000000000..34e7e08fbfc3 --- /dev/null +++ b/drivers/net/phy/phy_link_topology.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Infrastructure to handle all PHY devices connected to a given netdev, + * either directly or indirectly attached. + * + * Copyright (c) 2023 Maxime Chevallier + */ + +#include +#include +#include +#include +#include + +int phy_link_topo_add_phy(struct phy_link_topology *topo, + struct phy_device *phy, + enum phy_upstream upt, void *upstream) +{ + struct phy_device_node *pdn; + int ret; + + pdn = kzalloc(sizeof(*pdn), GFP_KERNEL); + if (!pdn) + return -ENOMEM; + + pdn->phy = phy; + switch (upt) { + case PHY_UPSTREAM_MAC: + pdn->upstream.netdev = (struct net_device *)upstream; + if (phy_on_sfp(phy)) + pdn->parent_sfp_bus = pdn->upstream.netdev->sfp_bus; + break; + case PHY_UPSTREAM_PHY: + pdn->upstream.phydev = (struct phy_device *)upstream; + if (phy_on_sfp(phy)) + pdn->parent_sfp_bus = pdn->upstream.phydev->sfp_bus; + break; + default: + ret = -EINVAL; + goto err; + } + pdn->upstream_type = upt; + + ret = xa_alloc_cyclic(&topo->phys, &phy->phyindex, pdn, xa_limit_32b, + &topo->next_phy_index, GFP_KERNEL); + if (ret) + goto err; + + return 0; + +err: + kfree(pdn); + return ret; +} +EXPORT_SYMBOL_GPL(phy_link_topo_add_phy); + +void phy_link_topo_del_phy(struct phy_link_topology *topo, + struct phy_device *phy) +{ + struct phy_device_node *pdn = xa_erase(&topo->phys, phy->phyindex); + + phy->phyindex = 0; + + kfree(pdn); +} +EXPORT_SYMBOL_GPL(phy_link_topo_del_phy); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 75c7725e5e4f..5baa5517f533 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -40,7 +40,6 @@ #include #endif #include - #include #include #include @@ -52,6 +51,7 @@ #include #include #include +#include struct netpoll_info; struct device; @@ -2047,6 +2047,7 @@ enum netdev_stat_type { * @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp * * @priomap: XXX: need comments on this one + * @link_topo: Physical link topology tracking attached PHYs * @phydev: Physical device may attach itself * for hardware timestamping * @sfp_bus: attached &struct sfp_bus structure. @@ -2441,6 +2442,7 @@ struct net_device { #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) struct netprio_map __rcu *priomap; #endif + struct phy_link_topology link_topo; struct phy_device *phydev; struct sfp_bus *sfp_bus; struct lock_class_key *qdisc_tx_busylock; diff --git a/include/linux/phy.h b/include/linux/phy.h index ede891776d8b..ea9416797b89 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -547,6 +547,9 @@ struct macsec_ops; * @drv: Pointer to the driver for this PHY instance * @devlink: Create a link between phy dev and mac dev, if the external phy * used by current mac interface is managed by another mac interface. + * @phyindex: Unique id across the phy's parent tree of phys to address the PHY + * from userspace, similar to ifindex. A zero index means the PHY + * wasn't assigned an id yet. * @phy_id: UID for this device found during discovery * @c45_ids: 802.3-c45 Device Identifiers if is_c45. * @is_c45: Set to true if this PHY uses clause 45 addressing. @@ -646,6 +649,7 @@ struct phy_device { struct device_link *devlink; + u32 phyindex; u32 phy_id; struct phy_c45_device_ids c45_ids; diff --git a/include/linux/phy_link_topology.h b/include/linux/phy_link_topology.h new file mode 100644 index 000000000000..91902263ec0e --- /dev/null +++ b/include/linux/phy_link_topology.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PHY device list allow maintaining a list of PHY devices that are + * part of a netdevice's link topology. PHYs can for example be chained, + * as is the case when using a PHY that exposes an SFP module, on which an + * SFP transceiver that embeds a PHY is connected. + * + * This list can then be used by userspace to leverage individual PHY + * capabilities. + */ +#ifndef __PHY_LINK_TOPOLOGY_H +#define __PHY_LINK_TOPOLOGY_H + +#include +#include + +struct xarray; +struct phy_device; +struct net_device; +struct sfp_bus; + +struct phy_device_node { + enum phy_upstream upstream_type; + + union { + struct net_device *netdev; + struct phy_device *phydev; + } upstream; + + struct sfp_bus *parent_sfp_bus; + + struct phy_device *phy; +}; + +static inline struct phy_device * +phy_link_topo_get_phy(struct phy_link_topology *topo, u32 phyindex) +{ + struct phy_device_node *pdn = xa_load(&topo->phys, phyindex); + + if (pdn) + return pdn->phy; + + return NULL; +} + +#if IS_ENABLED(CONFIG_PHYLIB) +int phy_link_topo_add_phy(struct phy_link_topology *topo, + struct phy_device *phy, + enum phy_upstream upt, void *upstream); + +void phy_link_topo_del_phy(struct phy_link_topology *lt, struct phy_device *phy); + +#else +static inline int phy_link_topo_add_phy(struct phy_link_topology *topo, + struct phy_device *phy, + enum phy_upstream upt, void *upstream) +{ + return 0; +} + +static inline void phy_link_topo_del_phy(struct phy_link_topology *topo, + struct phy_device *phy) +{ +} +#endif + +#endif /* __PHY_LINK_TOPOLOGY_H */ diff --git a/include/linux/phy_link_topology_core.h b/include/linux/phy_link_topology_core.h new file mode 100644 index 000000000000..78c75f909489 --- /dev/null +++ b/include/linux/phy_link_topology_core.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PHY_LINK_TOPOLOGY_CORE_H +#define __PHY_LINK_TOPOLOGY_CORE_H + +struct xarray; + +struct phy_link_topology { + struct xarray phys; + + u32 next_phy_index; +}; + +static inline void phy_link_topo_init(struct phy_link_topology *topo) +{ + xa_init_flags(&topo->phys, XA_FLAGS_ALLOC1); + topo->next_phy_index = 1; +} + +#endif /* __PHY_LINK_TOPOLOGY_CORE_H */ diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index 85c412c23ab5..60801df9d8c0 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -2219,4 +2219,20 @@ struct ethtool_link_settings { * __u32 map_lp_advertising[link_mode_masks_nwords]; */ }; + +/** + * enum phy_upstream - Represents the upstream component a given PHY device + * is connected to, as in what is on the other end of the MII bus. Most PHYs + * will be attached to an Ethernet MAC controller, but in some cases, there's + * an intermediate PHY used as a media-converter, which will driver another + * MII interface as its output. + * @PHY_UPSTREAM_MAC: Upstream component is a MAC (a switch port, + * or ethernet controller) + * @PHY_UPSTREAM_PHY: Upstream component is a PHY (likely a media converter) + */ +enum phy_upstream { + PHY_UPSTREAM_MAC, + PHY_UPSTREAM_PHY, +}; + #endif /* _UAPI_LINUX_ETHTOOL_H */ diff --git a/net/core/dev.c b/net/core/dev.c index f9d4b550ef4b..df04cbf77551 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -153,6 +153,7 @@ #include #include #include +#include #include "dev.h" #include "net-sysfs.h" @@ -10875,6 +10876,8 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, #ifdef CONFIG_NET_SCHED hash_init(dev->qdisc_hash); #endif + phy_link_topo_init(&dev->link_topo); + dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; setup(dev); -- cgit v1.2.3 From 9c5625f559ad6fe9f6f733c11475bf470e637d34 Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 21 Dec 2023 19:00:35 +0100 Subject: net: sfp: pass the phy_device when disconnecting an sfp module's PHY Pass the phy_device as a parameter to the sfp upstream .disconnect_phy operation. This is preparatory work to help track phy devices across a net_device's link. Signed-off-by: Maxime Chevallier Signed-off-by: David S. Miller --- drivers/net/phy/phy_device.c | 8 ++++++++ drivers/net/phy/phylink.c | 3 ++- drivers/net/phy/sfp-bus.c | 4 ++-- include/linux/sfp.h | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index ab8ae976a2f8..711629c49c11 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -266,6 +266,14 @@ static void phy_mdio_device_remove(struct mdio_device *mdiodev) static struct phy_driver genphy_driver; +static struct phy_link_topology *phy_get_link_topology(struct phy_device *phydev) +{ + if (phydev->attached_dev) + return &phydev->attached_dev->link_topo; + + return NULL; +} + static LIST_HEAD(phy_fixup_list); static DEFINE_MUTEX(phy_fixup_lock); diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 298dfd6982a5..3d25a4a6212b 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -3319,7 +3319,8 @@ static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy) return ret; } -static void phylink_sfp_disconnect_phy(void *upstream) +static void phylink_sfp_disconnect_phy(void *upstream, + struct phy_device *phydev) { phylink_disconnect_phy(upstream); } diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c index 6fa679b36290..3a86c41e1235 100644 --- a/drivers/net/phy/sfp-bus.c +++ b/drivers/net/phy/sfp-bus.c @@ -486,7 +486,7 @@ static void sfp_unregister_bus(struct sfp_bus *bus) bus->socket_ops->stop(bus->sfp); bus->socket_ops->detach(bus->sfp); if (bus->phydev && ops && ops->disconnect_phy) - ops->disconnect_phy(bus->upstream); + ops->disconnect_phy(bus->upstream, bus->phydev); } bus->registered = false; } @@ -742,7 +742,7 @@ void sfp_remove_phy(struct sfp_bus *bus) const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus); if (ops && ops->disconnect_phy) - ops->disconnect_phy(bus->upstream); + ops->disconnect_phy(bus->upstream, bus->phydev); bus->phydev = NULL; } EXPORT_SYMBOL_GPL(sfp_remove_phy); diff --git a/include/linux/sfp.h b/include/linux/sfp.h index 9346cd44814d..0573e53b0c11 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -544,7 +544,7 @@ struct sfp_upstream_ops { void (*link_down)(void *priv); void (*link_up)(void *priv); int (*connect_phy)(void *priv, struct phy_device *); - void (*disconnect_phy)(void *priv); + void (*disconnect_phy)(void *priv, struct phy_device *); }; #if IS_ENABLED(CONFIG_SFP) -- cgit v1.2.3 From 034fcc210349b873ece7356905be5c6ca11eef2a Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 21 Dec 2023 19:00:36 +0100 Subject: net: phy: add helpers to handle sfp phy connect/disconnect There are a few PHY drivers that can handle SFP modules through their sfp_upstream_ops. Introduce Phylib helpers to keep track of connected SFP PHYs in a netdevice's namespace, by adding the SFP PHY to the upstream PHY's netdev's namespace. By doing so, these SFP PHYs can be enumerated and exposed to users, which will be able to use their capabilities. Signed-off-by: Maxime Chevallier Signed-off-by: David S. Miller --- drivers/net/phy/at803x.c | 2 ++ drivers/net/phy/marvell-88x2222.c | 2 ++ drivers/net/phy/marvell.c | 2 ++ drivers/net/phy/marvell10g.c | 2 ++ drivers/net/phy/phy_device.c | 40 +++++++++++++++++++++++++++++++++++++++ include/linux/phy.h | 2 ++ 6 files changed, 50 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index 19cfbf36fe80..aaf6c654aaed 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -1452,6 +1452,8 @@ static const struct sfp_upstream_ops at8031_sfp_ops = { .attach = phy_sfp_attach, .detach = phy_sfp_detach, .module_insert = at8031_sfp_insert, + .connect_phy = phy_sfp_connect_phy, + .disconnect_phy = phy_sfp_disconnect_phy, }; static int at8031_parse_dt(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c index e3aa30dad2e6..3f77bbc7e04f 100644 --- a/drivers/net/phy/marvell-88x2222.c +++ b/drivers/net/phy/marvell-88x2222.c @@ -555,6 +555,8 @@ static const struct sfp_upstream_ops sfp_phy_ops = { .link_down = mv2222_sfp_link_down, .attach = phy_sfp_attach, .detach = phy_sfp_detach, + .connect_phy = phy_sfp_connect_phy, + .disconnect_phy = phy_sfp_disconnect_phy, }; static int mv2222_probe(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index eba652a4c1d8..674e29bce2cc 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -3254,6 +3254,8 @@ static const struct sfp_upstream_ops m88e1510_sfp_ops = { .module_remove = m88e1510_sfp_remove, .attach = phy_sfp_attach, .detach = phy_sfp_detach, + .connect_phy = phy_sfp_connect_phy, + .disconnect_phy = phy_sfp_disconnect_phy, }; static int m88e1510_probe(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c index ad43e280930c..6642eb642d4b 100644 --- a/drivers/net/phy/marvell10g.c +++ b/drivers/net/phy/marvell10g.c @@ -503,6 +503,8 @@ static int mv3310_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) static const struct sfp_upstream_ops mv3310_sfp_ops = { .attach = phy_sfp_attach, .detach = phy_sfp_detach, + .connect_phy = phy_sfp_connect_phy, + .disconnect_phy = phy_sfp_disconnect_phy, .module_insert = mv3310_sfp_insert, }; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 711629c49c11..1e595762afea 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1363,6 +1363,46 @@ phy_standalone_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(phy_standalone); +/** + * phy_sfp_connect_phy - Connect the SFP module's PHY to the upstream PHY + * @upstream: pointer to the upstream phy device + * @phy: pointer to the SFP module's phy device + * + * This helper allows keeping track of PHY devices on the link. It adds the + * SFP module's phy to the phy namespace of the upstream phy + */ +int phy_sfp_connect_phy(void *upstream, struct phy_device *phy) +{ + struct phy_device *phydev = upstream; + struct phy_link_topology *topo = phy_get_link_topology(phydev); + + if (topo) + return phy_link_topo_add_phy(topo, phy, PHY_UPSTREAM_PHY, phydev); + + return 0; +} +EXPORT_SYMBOL(phy_sfp_connect_phy); + +/** + * phy_sfp_disconnect_phy - Disconnect the SFP module's PHY from the upstream PHY + * @upstream: pointer to the upstream phy device + * @phy: pointer to the SFP module's phy device + * + * This helper allows keeping track of PHY devices on the link. It removes the + * SFP module's phy to the phy namespace of the upstream phy. As the module phy + * will be destroyed, re-inserting the same module will add a new phy with a + * new index. + */ +void phy_sfp_disconnect_phy(void *upstream, struct phy_device *phy) +{ + struct phy_device *phydev = upstream; + struct phy_link_topology *topo = phy_get_link_topology(phydev); + + if (topo) + phy_link_topo_del_phy(topo, phy); +} +EXPORT_SYMBOL(phy_sfp_disconnect_phy); + /** * phy_sfp_attach - attach the SFP bus to the PHY upstream network device * @upstream: pointer to the phy device diff --git a/include/linux/phy.h b/include/linux/phy.h index ea9416797b89..ac22b8e28a85 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1729,6 +1729,8 @@ int phy_suspend(struct phy_device *phydev); int phy_resume(struct phy_device *phydev); int __phy_resume(struct phy_device *phydev); int phy_loopback(struct phy_device *phydev, bool enable); +int phy_sfp_connect_phy(void *upstream, struct phy_device *phy); +void phy_sfp_disconnect_phy(void *upstream, struct phy_device *phy); void phy_sfp_attach(void *upstream, struct sfp_bus *bus); void phy_sfp_detach(void *upstream, struct sfp_bus *bus); int phy_sfp_probe(struct phy_device *phydev, -- cgit v1.2.3 From dedd702a35793ab462fce4c737eeba0badf9718e Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 21 Dec 2023 19:00:37 +0100 Subject: net: sfp: Add helper to return the SFP bus name Knowing the bus name is helpful when we want to expose the link topology to userspace, add a helper to return the SFP bus name. Signed-off-by: Maxime Chevallier Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/phy/sfp-bus.c | 11 +++++++++++ include/linux/sfp.h | 6 ++++++ 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c index 3a86c41e1235..fb1c102714b5 100644 --- a/drivers/net/phy/sfp-bus.c +++ b/drivers/net/phy/sfp-bus.c @@ -859,3 +859,14 @@ void sfp_unregister_socket(struct sfp_bus *bus) sfp_bus_put(bus); } EXPORT_SYMBOL_GPL(sfp_unregister_socket); + +const char *sfp_get_name(struct sfp_bus *bus) +{ + ASSERT_RTNL(); + + if (bus->sfp_dev) + return dev_name(bus->sfp_dev); + + return NULL; +} +EXPORT_SYMBOL_GPL(sfp_get_name); diff --git a/include/linux/sfp.h b/include/linux/sfp.h index 0573e53b0c11..55c0ab17c9e2 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -570,6 +570,7 @@ struct sfp_bus *sfp_bus_find_fwnode(const struct fwnode_handle *fwnode); int sfp_bus_add_upstream(struct sfp_bus *bus, void *upstream, const struct sfp_upstream_ops *ops); void sfp_bus_del_upstream(struct sfp_bus *bus); +const char *sfp_get_name(struct sfp_bus *bus); #else static inline int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id, @@ -648,6 +649,11 @@ static inline int sfp_bus_add_upstream(struct sfp_bus *bus, void *upstream, static inline void sfp_bus_del_upstream(struct sfp_bus *bus) { } + +static inline const char *sfp_get_name(struct sfp_bus *bus) +{ + return NULL; +} #endif #endif -- cgit v1.2.3 From 993498e537af9260e697219ce41b41b22b6199cc Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 21 Dec 2023 14:07:47 +0000 Subject: net-device: move gso_partial_features to net_device_read_tx dev->gso_partial_features is read from tx fast path for GSO packets. Move it to appropriate section to avoid a cache line miss. Fixes: 43a71cd66b9c ("net-device: reorganize net_device fast path variables") Signed-off-by: Eric Dumazet Cc: Coco Li Cc: David Ahern Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- Documentation/networking/net_cachelines/net_device.rst | 2 +- include/linux/netdevice.h | 2 +- net/core/dev.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 6cab1b797739..2dd8d8f20da2 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -38,7 +38,7 @@ netdev_features_t wanted_features netdev_features_t vlan_features netdev_features_t hw_enc_features - - netif_skb_features netdev_features_t mpls_features -netdev_features_t gso_partial_features +netdev_features_t gso_partial_features read_mostly gso_features_check unsigned_int min_mtu unsigned_int max_mtu unsigned_short type diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5baa5517f533..d59db9adcc96 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2115,6 +2115,7 @@ struct net_device { const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; struct netdev_queue *_tx; + netdev_features_t gso_partial_features; unsigned int real_num_tx_queues; unsigned int gso_max_size; unsigned int gso_ipv4_max_size; @@ -2211,7 +2212,6 @@ struct net_device { netdev_features_t vlan_features; netdev_features_t hw_enc_features; netdev_features_t mpls_features; - netdev_features_t gso_partial_features; unsigned int min_mtu; unsigned int max_mtu; diff --git a/net/core/dev.c b/net/core/dev.c index df04cbf77551..31588a50b757 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -11629,6 +11629,7 @@ static void __init net_dev_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs); + CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu); CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom); @@ -11642,7 +11643,7 @@ static void __init net_dev_struct_check(void) #ifdef CONFIG_NET_XGRESS CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress); #endif - CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 152); + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160); /* TXRX read-mostly hotpath */ CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags); -- cgit v1.2.3 From 3e64db35bc37edbe9e37aaa987df92cde12ddb6c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 2 Jan 2024 14:23:34 -0800 Subject: Revert "net: mdio: get/put device node during (un)registration" This reverts commit cff9c565e65f3622e8dc1dcc21c1520a083dff35. Revert based on feedback from Russell. Link: https://lore.kernel.org/all/ZZPtUIRerqTI2%2Fyh@shell.armlinux.org.uk/ Signed-off-by: Jakub Kicinski --- drivers/net/mdio/of_mdio.c | 12 +----------- drivers/net/phy/mdio_bus.c | 3 --- include/linux/phy.h | 3 --- 3 files changed, 1 insertion(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 9b6cab6154e0..64ebcb6d235c 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -139,11 +139,6 @@ bool of_mdiobus_child_is_phy(struct device_node *child) } EXPORT_SYMBOL(of_mdiobus_child_is_phy); -static void __of_mdiobus_unregister_callback(struct mii_bus *mdio) -{ - of_node_put(mdio->dev.of_node); -} - /** * __of_mdiobus_register - Register mii_bus and create PHYs from the device tree * @mdio: pointer to mii_bus structure @@ -171,8 +166,6 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, * the device tree are populated after the bus has been registered */ mdio->phy_mask = ~0; - mdio->__unregister_callback = __of_mdiobus_unregister_callback; - of_node_get(np); device_set_node(&mdio->dev, of_fwnode_handle(np)); /* Get bus level PHY reset GPIO details */ @@ -184,7 +177,7 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, /* Register the MDIO bus */ rc = __mdiobus_register(mdio, owner); if (rc) - goto put_node; + return rc; /* Loop over the child nodes and register a phy_device for each phy */ for_each_available_child_of_node(np, child) { @@ -244,9 +237,6 @@ int __of_mdiobus_register(struct mii_bus *mdio, struct device_node *np, unregister: of_node_put(child); mdiobus_unregister(mdio); - -put_node: - of_node_put(np); return rc; } EXPORT_SYMBOL(__of_mdiobus_register); diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 4a30757c4ff8..6cf73c15635b 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -787,9 +787,6 @@ void mdiobus_unregister(struct mii_bus *bus) gpiod_set_value_cansleep(bus->reset_gpiod, 1); device_del(&bus->dev); - - if (bus->__unregister_callback) - bus->__unregister_callback(bus); } EXPORT_SYMBOL(mdiobus_unregister); diff --git a/include/linux/phy.h b/include/linux/phy.h index ac22b8e28a85..6cb9d843aee9 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -434,9 +434,6 @@ struct mii_bus { /** @shared: shared state across different PHYs */ struct phy_package_shared *shared[PHY_MAX_ADDR]; - - /** @__unregister_callback: called at the last step of unregistration */ - void (*__unregister_callback)(struct mii_bus *bus); }; #define to_mii_bus(d) container_of(d, struct mii_bus, dev) -- cgit v1.2.3 From 7865dfb1eb941ddd25802a9e13b6ff5f3f4dc02f Mon Sep 17 00:00:00 2001 From: John Fastabend Date: Thu, 21 Dec 2023 15:23:24 -0800 Subject: bpf: sockmap, added comments describing update proto rules Add a comment describing that the psock update proto callbback can be called multiple times and this must be safe. Signed-off-by: John Fastabend Signed-off-by: Martin KaFai Lau Reviewed-by: Jakub Sitnicki Link: https://lore.kernel.org/r/20231221232327.43678-3-john.fastabend@gmail.com --- include/linux/skmsg.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index c953b8c0d2f4..888a4b217829 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -100,6 +100,11 @@ struct sk_psock { void (*saved_close)(struct sock *sk, long timeout); void (*saved_write_space)(struct sock *sk); void (*saved_data_ready)(struct sock *sk); + /* psock_update_sk_prot may be called with restore=false many times + * so the handler must be safe for this case. It will be called + * exactly once with restore=true when the psock is being destroyed + * and psock refcnt is zero, but before an RCU grace period. + */ int (*psock_update_sk_prot)(struct sock *sk, struct sk_psock *psock, bool restore); struct proto *sk_proto; -- cgit v1.2.3 From d3d344a1ca69d8fb2413e29e6400f3ad58a05c06 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 2 Jan 2024 16:22:20 +0000 Subject: net-device: move xdp_prog to net_device_read_rx xdp_prog is used in receive path, both from XDP enabled drivers and from netif_elide_gro(). This patch also removes two 4-bytes holes. Fixes: 43a71cd66b9c ("net-device: reorganize net_device fast path variables") Signed-off-by: Eric Dumazet Cc: Coco Li Cc: Simon Horman Link: https://lore.kernel.org/r/20240102162220.750823-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- Documentation/networking/net_cachelines/net_device.rst | 2 +- include/linux/netdevice.h | 2 +- net/core/dev.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst index 2dd8d8f20da2..e75a53593bb9 100644 --- a/Documentation/networking/net_cachelines/net_device.rst +++ b/Documentation/networking/net_cachelines/net_device.rst @@ -96,7 +96,7 @@ unsigned_char* dev_addr struct_netdev_queue* _rx read_mostly - netdev_get_rx_queue(rx) unsigned_int num_rx_queues unsigned_int real_num_rx_queues - read_mostly get_rps_cpu -struct_bpf_prog* xdp_prog +struct_bpf_prog* xdp_prog - read_mostly netif_elide_gro() unsigned_long gro_flush_timeout - read_mostly napi_complete_done int napi_defer_hard_irqs - read_mostly napi_complete_done unsigned_int gro_max_size - read_mostly skb_gro_receive diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index d59db9adcc96..e265aa1f2169 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2150,6 +2150,7 @@ struct net_device { /* RX read-mostly hotpath */ __cacheline_group_begin(net_device_read_rx); + struct bpf_prog __rcu *xdp_prog; struct list_head ptype_specific; int ifindex; unsigned int real_num_rx_queues; @@ -2325,7 +2326,6 @@ struct net_device { const unsigned char *dev_addr; unsigned int num_rx_queues; - struct bpf_prog __rcu *xdp_prog; #define GRO_LEGACY_MAX_SIZE 65536u /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE), * and shinfo->gso_segs is a 16bit field. diff --git a/net/core/dev.c b/net/core/dev.c index 31588a50b757..bc4ac49d4643 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -11670,7 +11670,7 @@ static void __init net_dev_struct_check(void) #ifdef CONFIG_NET_XGRESS CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress); #endif - CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 96); + CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 104); } /* -- cgit v1.2.3 From 9fc8e802048ad150e8032c4f3dbf40112160cfe9 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Thu, 21 Dec 2023 19:17:39 -0800 Subject: bpf: Add objcg to bpf_mem_alloc The objcg is a bpf_mem_alloc level property since all bpf_mem_cache's are with the same objcg. This patch made such a property explicit. The next patch will use this property to save and restore objcg for percpu unit allocator. Acked-by: Hou Tao Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20231222031739.1288590-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_mem_alloc.h | 1 + kernel/bpf/memalloc.c | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h index bb1223b21308..acef8c808599 100644 --- a/include/linux/bpf_mem_alloc.h +++ b/include/linux/bpf_mem_alloc.h @@ -11,6 +11,7 @@ struct bpf_mem_caches; struct bpf_mem_alloc { struct bpf_mem_caches __percpu *caches; struct bpf_mem_cache __percpu *cache; + struct obj_cgroup *objcg; bool percpu; struct work_struct work; }; diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 288ec4a967d0..4a21050f0359 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -523,6 +523,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) if (memcg_bpf_enabled()) objcg = get_obj_cgroup_from_current(); #endif + ma->objcg = objcg; for_each_possible_cpu(cpu) { c = per_cpu_ptr(pc, cpu); c->unit_size = unit_size; @@ -542,6 +543,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) #ifdef CONFIG_MEMCG_KMEM objcg = get_obj_cgroup_from_current(); #endif + ma->objcg = objcg; for_each_possible_cpu(cpu) { cc = per_cpu_ptr(pcc, cpu); for (i = 0; i < NUM_CACHES; i++) { @@ -691,9 +693,8 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } - /* objcg is the same across cpus */ - if (c->objcg) - obj_cgroup_put(c->objcg); + if (ma->objcg) + obj_cgroup_put(ma->objcg); destroy_mem_alloc(ma, rcu_in_progress); } if (ma->caches) { @@ -709,8 +710,8 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } } - if (c->objcg) - obj_cgroup_put(c->objcg); + if (ma->objcg) + obj_cgroup_put(ma->objcg); destroy_mem_alloc(ma, rcu_in_progress); } } -- cgit v1.2.3 From c39aa3b289e9c10d0d246cd919b06809f13b72b8 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Thu, 21 Dec 2023 19:17:45 -0800 Subject: bpf: Allow per unit prefill for non-fix-size percpu memory allocator Commit 41a5db8d8161 ("Add support for non-fix-size percpu mem allocation") added support for non-fix-size percpu memory allocation. Such allocation will allocate percpu memory for all buckets on all cpus and the memory consumption is in the order to quadratic. For example, let us say, 4 cpus, unit size 16 bytes, so each cpu has 16 * 4 = 64 bytes, with 4 cpus, total will be 64 * 4 = 256 bytes. Then let us say, 8 cpus with the same unit size, each cpu has 16 * 8 = 128 bytes, with 8 cpus, total will be 128 * 8 = 1024 bytes. So if the number of cpus doubles, the number of memory consumption will be 4 times. So for a system with large number of cpus, the memory consumption goes up quickly with quadratic order. For example, for 4KB percpu allocation, 128 cpus. The total memory consumption will 4KB * 128 * 128 = 64MB. Things will become worse if the number of cpus is bigger (e.g., 512, 1024, etc.) In Commit 41a5db8d8161, the non-fix-size percpu memory allocation is done in boot time, so for system with large number of cpus, the initial percpu memory consumption is very visible. For example, for 128 cpu system, the total percpu memory allocation will be at least (16 + 32 + 64 + 96 + 128 + 196 + 256 + 512 + 1024 + 2048 + 4096) * 128 * 128 = ~138MB. which is pretty big. It will be even bigger for larger number of cpus. Note that the current prefill also allocates 4 entries if the unit size is less than 256. So on top of 138MB memory consumption, this will add more consumption with 3 * (16 + 32 + 64 + 96 + 128 + 196 + 256) * 128 * 128 = ~38MB. Next patch will try to reduce this memory consumption. Later on, Commit 1fda5bb66ad8 ("bpf: Do not allocate percpu memory at init stage") moved the non-fix-size percpu memory allocation to bpf verificaiton stage. Once a particular bpf_percpu_obj_new() is called by bpf program, the memory allocator will try to fill in the cache with all sizes, causing the same amount of percpu memory consumption as in the boot stage. To reduce the initial percpu memory consumption for non-fix-size percpu memory allocation, instead of filling the cache with all supported allocation sizes, this patch intends to fill the cache only for the requested size. As typically users will not use large percpu data structure, this can save memory significantly. For example, the allocation size is 64 bytes with 128 cpus. Then total percpu memory amount will be 64 * 128 * 128 = 1MB, much less than previous 138MB. Signed-off-by: Yonghong Song Acked-by: Hou Tao Link: https://lore.kernel.org/r/20231222031745.1289082-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_mem_alloc.h | 7 ++++++ kernel/bpf/memalloc.c | 57 ++++++++++++++++++++++++++++++++++++++++++- kernel/bpf/verifier.c | 37 +++++++++++++++++----------- 3 files changed, 86 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h index acef8c808599..aaf004d94322 100644 --- a/include/linux/bpf_mem_alloc.h +++ b/include/linux/bpf_mem_alloc.h @@ -22,8 +22,15 @@ struct bpf_mem_alloc { * 'size = 0' is for bpf_mem_alloc which manages many fixed-size objects. * Alloc and free are done with bpf_mem_{alloc,free}() and the size of * the returned object is given by the size argument of bpf_mem_alloc(). + * If percpu equals true, error will be returned in order to avoid + * large memory consumption and the below bpf_mem_alloc_percpu_unit_init() + * should be used to do on-demand per-cpu allocation for each size. */ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu); +/* Initialize a non-fix-size percpu memory allocator */ +int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg); +/* The percpu allocation with a specific unit size. */ +int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size); void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma); /* kmalloc/kfree equivalent: */ diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 4a21050f0359..f71da07eb8a0 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -121,6 +121,8 @@ struct bpf_mem_caches { struct bpf_mem_cache cache[NUM_CACHES]; }; +static const u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096}; + static struct llist_node notrace *__llist_del_first(struct llist_head *head) { struct llist_node *entry, *next; @@ -499,12 +501,14 @@ static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) */ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) { - static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096}; struct bpf_mem_caches *cc, __percpu *pcc; struct bpf_mem_cache *c, __percpu *pc; struct obj_cgroup *objcg = NULL; int cpu, i, unit_size, percpu_size = 0; + if (percpu && size == 0) + return -EINVAL; + /* room for llist_node and per-cpu pointer */ if (percpu) percpu_size = LLIST_NODE_SZ + sizeof(void *); @@ -524,6 +528,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) objcg = get_obj_cgroup_from_current(); #endif ma->objcg = objcg; + for_each_possible_cpu(cpu) { c = per_cpu_ptr(pc, cpu); c->unit_size = unit_size; @@ -562,6 +567,56 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) return 0; } +int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg) +{ + struct bpf_mem_caches __percpu *pcc; + + pcc = __alloc_percpu_gfp(sizeof(struct bpf_mem_caches), 8, GFP_KERNEL); + if (!pcc) + return -ENOMEM; + + ma->caches = pcc; + ma->objcg = objcg; + ma->percpu = true; + return 0; +} + +int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size) +{ + struct bpf_mem_caches *cc, __percpu *pcc; + int cpu, i, unit_size, percpu_size; + struct obj_cgroup *objcg; + struct bpf_mem_cache *c; + + i = bpf_mem_cache_idx(size); + if (i < 0) + return -EINVAL; + + /* room for llist_node and per-cpu pointer */ + percpu_size = LLIST_NODE_SZ + sizeof(void *); + + unit_size = sizes[i]; + objcg = ma->objcg; + pcc = ma->caches; + + for_each_possible_cpu(cpu) { + cc = per_cpu_ptr(pcc, cpu); + c = &cc->cache[i]; + if (cpu == 0 && c->unit_size) + break; + + c->unit_size = unit_size; + c->objcg = objcg; + c->percpu_size = percpu_size; + c->tgt = c; + + init_refill_work(c); + prefill_mem_cache(c, cpu); + } + + return 0; +} + static void drain_mem_cache(struct bpf_mem_cache *c) { bool percpu = !!c->percpu_size; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d4e31f61de0e..e9699a2cfe4f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12139,20 +12139,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set) return -ENOMEM; - if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { - if (!bpf_global_percpu_ma_set) { - mutex_lock(&bpf_percpu_ma_lock); - if (!bpf_global_percpu_ma_set) { - err = bpf_mem_alloc_init(&bpf_global_percpu_ma, 0, true); - if (!err) - bpf_global_percpu_ma_set = true; - } - mutex_unlock(&bpf_percpu_ma_lock); - if (err) - return err; - } - } - if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); return -EINVAL; @@ -12173,6 +12159,29 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EINVAL; } + if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { + if (!bpf_global_percpu_ma_set) { + mutex_lock(&bpf_percpu_ma_lock); + if (!bpf_global_percpu_ma_set) { + /* Charge memory allocated with bpf_global_percpu_ma to + * root memcg. The obj_cgroup for root memcg is NULL. + */ + err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL); + if (!err) + bpf_global_percpu_ma_set = true; + } + mutex_unlock(&bpf_percpu_ma_lock); + if (err) + return err; + } + + mutex_lock(&bpf_percpu_ma_lock); + err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size); + mutex_unlock(&bpf_percpu_ma_lock); + if (err) + return err; + } + struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { -- cgit v1.2.3 From 5e5401d6612ef599ad45785b941eebda7effc90f Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 4 Jan 2024 09:47:36 +0000 Subject: net: phylink: move phylink_pcs_neg_mode() into phylink.c Move phylink_pcs_neg_mode() from the header file into the .c file since nothing should be using it. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/phy/phylink.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/phylink.h | 66 ----------------------------------------------- 2 files changed, 66 insertions(+), 66 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 3d25a4a6212b..a816391add12 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -1074,6 +1074,72 @@ static void phylink_pcs_an_restart(struct phylink *pl) pl->pcs->ops->pcs_an_restart(pl->pcs); } +/** + * phylink_pcs_neg_mode() - helper to determine PCS inband mode + * @mode: one of %MLO_AN_FIXED, %MLO_AN_PHY, %MLO_AN_INBAND. + * @interface: interface mode to be used + * @advertising: adertisement ethtool link mode mask + * + * Determines the negotiation mode to be used by the PCS, and returns + * one of: + * + * - %PHYLINK_PCS_NEG_NONE: interface mode does not support inband + * - %PHYLINK_PCS_NEG_OUTBAND: an out of band mode (e.g. reading the PHY) + * will be used. + * - %PHYLINK_PCS_NEG_INBAND_DISABLED: inband mode selected but autoneg + * disabled + * - %PHYLINK_PCS_NEG_INBAND_ENABLED: inband mode selected and autoneg enabled + * + * Note: this is for cases where the PCS itself is involved in negotiation + * (e.g. Clause 37, SGMII and similar) not Clause 73. + */ +static unsigned int phylink_pcs_neg_mode(unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising) +{ + unsigned int neg_mode; + + switch (interface) { + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: + case PHY_INTERFACE_MODE_USXGMII: + /* These protocols are designed for use with a PHY which + * communicates its negotiation result back to the MAC via + * inband communication. Note: there exist PHYs that run + * with SGMII but do not send the inband data. + */ + if (!phylink_autoneg_inband(mode)) + neg_mode = PHYLINK_PCS_NEG_OUTBAND; + else + neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; + break; + + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + /* 1000base-X is designed for use media-side for Fibre + * connections, and thus the Autoneg bit needs to be + * taken into account. We also do this for 2500base-X + * as well, but drivers may not support this, so may + * need to override this. + */ + if (!phylink_autoneg_inband(mode)) + neg_mode = PHYLINK_PCS_NEG_OUTBAND; + else if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising)) + neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; + else + neg_mode = PHYLINK_PCS_NEG_INBAND_DISABLED; + break; + + default: + neg_mode = PHYLINK_PCS_NEG_NONE; + break; + } + + return neg_mode; +} + static void phylink_major_config(struct phylink *pl, bool restart, const struct phylink_link_state *state) { diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 875439ab45de..d589f89c612c 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -98,72 +98,6 @@ static inline bool phylink_autoneg_inband(unsigned int mode) return mode == MLO_AN_INBAND; } -/** - * phylink_pcs_neg_mode() - helper to determine PCS inband mode - * @mode: one of %MLO_AN_FIXED, %MLO_AN_PHY, %MLO_AN_INBAND. - * @interface: interface mode to be used - * @advertising: adertisement ethtool link mode mask - * - * Determines the negotiation mode to be used by the PCS, and returns - * one of: - * - * - %PHYLINK_PCS_NEG_NONE: interface mode does not support inband - * - %PHYLINK_PCS_NEG_OUTBAND: an out of band mode (e.g. reading the PHY) - * will be used. - * - %PHYLINK_PCS_NEG_INBAND_DISABLED: inband mode selected but autoneg - * disabled - * - %PHYLINK_PCS_NEG_INBAND_ENABLED: inband mode selected and autoneg enabled - * - * Note: this is for cases where the PCS itself is involved in negotiation - * (e.g. Clause 37, SGMII and similar) not Clause 73. - */ -static inline unsigned int phylink_pcs_neg_mode(unsigned int mode, - phy_interface_t interface, - const unsigned long *advertising) -{ - unsigned int neg_mode; - - switch (interface) { - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_QSGMII: - case PHY_INTERFACE_MODE_QUSGMII: - case PHY_INTERFACE_MODE_USXGMII: - /* These protocols are designed for use with a PHY which - * communicates its negotiation result back to the MAC via - * inband communication. Note: there exist PHYs that run - * with SGMII but do not send the inband data. - */ - if (!phylink_autoneg_inband(mode)) - neg_mode = PHYLINK_PCS_NEG_OUTBAND; - else - neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; - break; - - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_2500BASEX: - /* 1000base-X is designed for use media-side for Fibre - * connections, and thus the Autoneg bit needs to be - * taken into account. We also do this for 2500base-X - * as well, but drivers may not support this, so may - * need to override this. - */ - if (!phylink_autoneg_inband(mode)) - neg_mode = PHYLINK_PCS_NEG_OUTBAND; - else if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, - advertising)) - neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; - else - neg_mode = PHYLINK_PCS_NEG_INBAND_DISABLED; - break; - - default: - neg_mode = PHYLINK_PCS_NEG_NONE; - break; - } - - return neg_mode; -} - /** * struct phylink_link_state - link state structure * @advertising: ethtool bitmask containing advertised link modes -- cgit v1.2.3 From 98e20e5e13d2811898921f999288be7151a11954 Mon Sep 17 00:00:00 2001 From: Quentin Deslandes Date: Tue, 26 Dec 2023 14:07:42 +0100 Subject: bpfilter: remove bpfilter bpfilter was supposed to convert iptables filtering rules into BPF programs on the fly, from the kernel, through a usermode helper. The base code for the UMH was introduced in 2018, and couple of attempts (2, 3) tried to introduce the BPF program generate features but were abandoned. bpfilter now sits in a kernel tree unused and unusable, occasionally causing confusion amongst Linux users (4, 5). As bpfilter is now developed in a dedicated repository on GitHub (6), it was suggested a couple of times this year (LSFMM/BPF 2023, LPC 2023) to remove the deprecated kernel part of the project. This is the purpose of this patch. [1]: https://lore.kernel.org/lkml/20180522022230.2492505-1-ast@kernel.org/ [2]: https://lore.kernel.org/bpf/20210829183608.2297877-1-me@ubique.spb.ru/#t [3]: https://lore.kernel.org/lkml/20221224000402.476079-1-qde@naccy.de/ [4]: https://dxuuu.xyz/bpfilter.html [5]: https://github.com/linuxkit/linuxkit/pull/3904 [6]: https://github.com/facebook/bpfilter Signed-off-by: Quentin Deslandes Link: https://lore.kernel.org/r/20231226130745.465988-1-qde@naccy.de Signed-off-by: Alexei Starovoitov --- arch/loongarch/configs/loongson3_defconfig | 1 - include/linux/bpfilter.h | 24 ----- include/uapi/linux/bpfilter.h | 21 ----- net/Kconfig | 2 - net/Makefile | 1 - net/bpfilter/.gitignore | 2 - net/bpfilter/Kconfig | 23 ----- net/bpfilter/Makefile | 20 ----- net/bpfilter/bpfilter_kern.c | 136 ----------------------------- net/bpfilter/bpfilter_umh_blob.S | 7 -- net/bpfilter/main.c | 64 -------------- net/bpfilter/msgfmt.h | 17 ---- net/ipv4/Makefile | 2 - net/ipv4/bpfilter/Makefile | 2 - net/ipv4/bpfilter/sockopt.c | 71 --------------- net/ipv4/ip_sockglue.c | 12 --- tools/bpf/bpftool/feature.c | 4 - tools/testing/selftests/bpf/config.aarch64 | 1 - tools/testing/selftests/bpf/config.s390x | 1 - tools/testing/selftests/bpf/config.x86_64 | 1 - tools/testing/selftests/hid/config | 1 - 21 files changed, 413 deletions(-) delete mode 100644 include/linux/bpfilter.h delete mode 100644 include/uapi/linux/bpfilter.h delete mode 100644 net/bpfilter/.gitignore delete mode 100644 net/bpfilter/Kconfig delete mode 100644 net/bpfilter/Makefile delete mode 100644 net/bpfilter/bpfilter_kern.c delete mode 100644 net/bpfilter/bpfilter_umh_blob.S delete mode 100644 net/bpfilter/main.c delete mode 100644 net/bpfilter/msgfmt.h delete mode 100644 net/ipv4/bpfilter/Makefile delete mode 100644 net/ipv4/bpfilter/sockopt.c (limited to 'include/linux') diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 9c333d133c30..60e331af9839 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -276,7 +276,6 @@ CONFIG_BRIDGE_EBT_T_NAT=m CONFIG_BRIDGE_EBT_ARP=m CONFIG_BRIDGE_EBT_IP=m CONFIG_BRIDGE_EBT_IP6=m -CONFIG_BPFILTER=y CONFIG_IP_SCTP=m CONFIG_RDS=y CONFIG_L2TP=m diff --git a/include/linux/bpfilter.h b/include/linux/bpfilter.h deleted file mode 100644 index 736ded4905e0..000000000000 --- a/include/linux/bpfilter.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_BPFILTER_H -#define _LINUX_BPFILTER_H - -#include -#include -#include - -struct sock; -int bpfilter_ip_set_sockopt(struct sock *sk, int optname, sockptr_t optval, - unsigned int optlen); -int bpfilter_ip_get_sockopt(struct sock *sk, int optname, char __user *optval, - int __user *optlen); - -struct bpfilter_umh_ops { - struct umd_info info; - /* since ip_getsockopt() can run in parallel, serialize access to umh */ - struct mutex lock; - int (*sockopt)(struct sock *sk, int optname, sockptr_t optval, - unsigned int optlen, bool is_set); - int (*start)(void); -}; -extern struct bpfilter_umh_ops bpfilter_ops; -#endif diff --git a/include/uapi/linux/bpfilter.h b/include/uapi/linux/bpfilter.h deleted file mode 100644 index cbc1f5813f50..000000000000 --- a/include/uapi/linux/bpfilter.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_LINUX_BPFILTER_H -#define _UAPI_LINUX_BPFILTER_H - -#include - -enum { - BPFILTER_IPT_SO_SET_REPLACE = 64, - BPFILTER_IPT_SO_SET_ADD_COUNTERS = 65, - BPFILTER_IPT_SET_MAX, -}; - -enum { - BPFILTER_IPT_SO_GET_INFO = 64, - BPFILTER_IPT_SO_GET_ENTRIES = 65, - BPFILTER_IPT_SO_GET_REVISION_MATCH = 66, - BPFILTER_IPT_SO_GET_REVISION_TARGET = 67, - BPFILTER_IPT_GET_MAX, -}; - -#endif /* _UAPI_LINUX_BPFILTER_H */ diff --git a/net/Kconfig b/net/Kconfig index 3ec6bc98fa05..4adc47d0c9c2 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -233,8 +233,6 @@ source "net/bridge/netfilter/Kconfig" endif -source "net/bpfilter/Kconfig" - source "net/dccp/Kconfig" source "net/sctp/Kconfig" source "net/rds/Kconfig" diff --git a/net/Makefile b/net/Makefile index 4c4dc535453d..b06b5539e7a6 100644 --- a/net/Makefile +++ b/net/Makefile @@ -19,7 +19,6 @@ obj-$(CONFIG_TLS) += tls/ obj-$(CONFIG_XFRM) += xfrm/ obj-$(CONFIG_UNIX_SCM) += unix/ obj-y += ipv6/ -obj-$(CONFIG_BPFILTER) += bpfilter/ obj-$(CONFIG_PACKET) += packet/ obj-$(CONFIG_NET_KEY) += key/ obj-$(CONFIG_BRIDGE) += bridge/ diff --git a/net/bpfilter/.gitignore b/net/bpfilter/.gitignore deleted file mode 100644 index f34e85ee8204..000000000000 --- a/net/bpfilter/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -bpfilter_umh diff --git a/net/bpfilter/Kconfig b/net/bpfilter/Kconfig deleted file mode 100644 index 3d4a21462458..000000000000 --- a/net/bpfilter/Kconfig +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -menuconfig BPFILTER - bool "BPF based packet filtering framework (BPFILTER)" - depends on BPF && INET - select USERMODE_DRIVER - help - This builds experimental bpfilter framework that is aiming to - provide netfilter compatible functionality via BPF - -if BPFILTER -config BPFILTER_UMH - tristate "bpfilter kernel module with user mode helper" - depends on CC_CAN_LINK - depends on m || CC_CAN_LINK_STATIC - default m - help - This builds bpfilter kernel module with embedded user mode helper - - Note: To compile this as built-in, your toolchain must support - building static binaries, since rootfs isn't mounted at the time - when __init functions are called and do_execv won't be able to find - the elf interpreter. -endif diff --git a/net/bpfilter/Makefile b/net/bpfilter/Makefile deleted file mode 100644 index cdac82b8c53a..000000000000 --- a/net/bpfilter/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for the Linux BPFILTER layer. -# - -userprogs := bpfilter_umh -bpfilter_umh-objs := main.o -userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi - -ifeq ($(CONFIG_BPFILTER_UMH), y) -# builtin bpfilter_umh should be linked with -static -# since rootfs isn't mounted at the time of __init -# function is called and do_execv won't find elf interpreter -userldflags += -static -endif - -$(obj)/bpfilter_umh_blob.o: $(obj)/bpfilter_umh - -obj-$(CONFIG_BPFILTER_UMH) += bpfilter.o -bpfilter-objs += bpfilter_kern.o bpfilter_umh_blob.o diff --git a/net/bpfilter/bpfilter_kern.c b/net/bpfilter/bpfilter_kern.c deleted file mode 100644 index 97e129e3f31c..000000000000 --- a/net/bpfilter/bpfilter_kern.c +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include "msgfmt.h" - -extern char bpfilter_umh_start; -extern char bpfilter_umh_end; - -static void shutdown_umh(void) -{ - struct umd_info *info = &bpfilter_ops.info; - struct pid *tgid = info->tgid; - - if (tgid) { - kill_pid(tgid, SIGKILL, 1); - wait_event(tgid->wait_pidfd, thread_group_exited(tgid)); - umd_cleanup_helper(info); - } -} - -static void __stop_umh(void) -{ - if (IS_ENABLED(CONFIG_INET)) - shutdown_umh(); -} - -static int bpfilter_send_req(struct mbox_request *req) -{ - struct mbox_reply reply; - loff_t pos = 0; - ssize_t n; - - if (!bpfilter_ops.info.tgid) - return -EFAULT; - pos = 0; - n = kernel_write(bpfilter_ops.info.pipe_to_umh, req, sizeof(*req), - &pos); - if (n != sizeof(*req)) { - pr_err("write fail %zd\n", n); - goto stop; - } - pos = 0; - n = kernel_read(bpfilter_ops.info.pipe_from_umh, &reply, sizeof(reply), - &pos); - if (n != sizeof(reply)) { - pr_err("read fail %zd\n", n); - goto stop; - } - return reply.status; -stop: - __stop_umh(); - return -EFAULT; -} - -static int bpfilter_process_sockopt(struct sock *sk, int optname, - sockptr_t optval, unsigned int optlen, - bool is_set) -{ - struct mbox_request req = { - .is_set = is_set, - .pid = current->pid, - .cmd = optname, - .addr = (uintptr_t)optval.user, - .len = optlen, - }; - if (sockptr_is_kernel(optval)) { - pr_err("kernel access not supported\n"); - return -EFAULT; - } - return bpfilter_send_req(&req); -} - -static int start_umh(void) -{ - struct mbox_request req = { .pid = current->pid }; - int err; - - /* fork usermode process */ - err = fork_usermode_driver(&bpfilter_ops.info); - if (err) - return err; - pr_info("Loaded bpfilter_umh pid %d\n", pid_nr(bpfilter_ops.info.tgid)); - - /* health check that usermode process started correctly */ - if (bpfilter_send_req(&req) != 0) { - shutdown_umh(); - return -EFAULT; - } - - return 0; -} - -static int __init load_umh(void) -{ - int err; - - err = umd_load_blob(&bpfilter_ops.info, - &bpfilter_umh_start, - &bpfilter_umh_end - &bpfilter_umh_start); - if (err) - return err; - - mutex_lock(&bpfilter_ops.lock); - err = start_umh(); - if (!err && IS_ENABLED(CONFIG_INET)) { - bpfilter_ops.sockopt = &bpfilter_process_sockopt; - bpfilter_ops.start = &start_umh; - } - mutex_unlock(&bpfilter_ops.lock); - if (err) - umd_unload_blob(&bpfilter_ops.info); - return err; -} - -static void __exit fini_umh(void) -{ - mutex_lock(&bpfilter_ops.lock); - if (IS_ENABLED(CONFIG_INET)) { - shutdown_umh(); - bpfilter_ops.start = NULL; - bpfilter_ops.sockopt = NULL; - } - mutex_unlock(&bpfilter_ops.lock); - - umd_unload_blob(&bpfilter_ops.info); -} -module_init(load_umh); -module_exit(fini_umh); -MODULE_LICENSE("GPL"); diff --git a/net/bpfilter/bpfilter_umh_blob.S b/net/bpfilter/bpfilter_umh_blob.S deleted file mode 100644 index 40311d10d2f2..000000000000 --- a/net/bpfilter/bpfilter_umh_blob.S +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - .section .init.rodata, "a" - .global bpfilter_umh_start -bpfilter_umh_start: - .incbin "net/bpfilter/bpfilter_umh" - .global bpfilter_umh_end -bpfilter_umh_end: diff --git a/net/bpfilter/main.c b/net/bpfilter/main.c deleted file mode 100644 index 291a92546246..000000000000 --- a/net/bpfilter/main.c +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include "../../include/uapi/linux/bpf.h" -#include -#include "msgfmt.h" - -FILE *debug_f; - -static int handle_get_cmd(struct mbox_request *cmd) -{ - switch (cmd->cmd) { - case 0: - return 0; - default: - break; - } - return -ENOPROTOOPT; -} - -static int handle_set_cmd(struct mbox_request *cmd) -{ - return -ENOPROTOOPT; -} - -static void loop(void) -{ - while (1) { - struct mbox_request req; - struct mbox_reply reply; - int n; - - n = read(0, &req, sizeof(req)); - if (n != sizeof(req)) { - fprintf(debug_f, "invalid request %d\n", n); - return; - } - - reply.status = req.is_set ? - handle_set_cmd(&req) : - handle_get_cmd(&req); - - n = write(1, &reply, sizeof(reply)); - if (n != sizeof(reply)) { - fprintf(debug_f, "reply failed %d\n", n); - return; - } - } -} - -int main(void) -{ - debug_f = fopen("/dev/kmsg", "w"); - setvbuf(debug_f, 0, _IOLBF, 0); - fprintf(debug_f, "<5>Started bpfilter\n"); - loop(); - fclose(debug_f); - return 0; -} diff --git a/net/bpfilter/msgfmt.h b/net/bpfilter/msgfmt.h deleted file mode 100644 index 98d121c62945..000000000000 --- a/net/bpfilter/msgfmt.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _NET_BPFILTER_MSGFMT_H -#define _NET_BPFILTER_MSGFMT_H - -struct mbox_request { - __u64 addr; - __u32 len; - __u32 is_set; - __u32 cmd; - __u32 pid; -}; - -struct mbox_reply { - __u32 status; -}; - -#endif diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index e144a02a6a61..ec36d2ec059e 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -16,8 +16,6 @@ obj-y := route.o inetpeer.o protocol.o \ inet_fragment.o ping.o ip_tunnel_core.o gre_offload.o \ metrics.o netlink.o nexthop.o udp_tunnel_stub.o -obj-$(CONFIG_BPFILTER) += bpfilter/ - obj-$(CONFIG_NET_IP_TUNNEL) += ip_tunnel.o obj-$(CONFIG_SYSCTL) += sysctl_net_ipv4.o obj-$(CONFIG_PROC_FS) += proc.o diff --git a/net/ipv4/bpfilter/Makefile b/net/ipv4/bpfilter/Makefile deleted file mode 100644 index 00af5305e05a..000000000000 --- a/net/ipv4/bpfilter/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_BPFILTER) += sockopt.o diff --git a/net/ipv4/bpfilter/sockopt.c b/net/ipv4/bpfilter/sockopt.c deleted file mode 100644 index 193bcc2acccc..000000000000 --- a/net/ipv4/bpfilter/sockopt.c +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct bpfilter_umh_ops bpfilter_ops; -EXPORT_SYMBOL_GPL(bpfilter_ops); - -static int bpfilter_mbox_request(struct sock *sk, int optname, sockptr_t optval, - unsigned int optlen, bool is_set) -{ - int err; - mutex_lock(&bpfilter_ops.lock); - if (!bpfilter_ops.sockopt) { - mutex_unlock(&bpfilter_ops.lock); - request_module("bpfilter"); - mutex_lock(&bpfilter_ops.lock); - - if (!bpfilter_ops.sockopt) { - err = -ENOPROTOOPT; - goto out; - } - } - if (bpfilter_ops.info.tgid && - thread_group_exited(bpfilter_ops.info.tgid)) - umd_cleanup_helper(&bpfilter_ops.info); - - if (!bpfilter_ops.info.tgid) { - err = bpfilter_ops.start(); - if (err) - goto out; - } - err = bpfilter_ops.sockopt(sk, optname, optval, optlen, is_set); -out: - mutex_unlock(&bpfilter_ops.lock); - return err; -} - -int bpfilter_ip_set_sockopt(struct sock *sk, int optname, sockptr_t optval, - unsigned int optlen) -{ - return bpfilter_mbox_request(sk, optname, optval, optlen, true); -} - -int bpfilter_ip_get_sockopt(struct sock *sk, int optname, char __user *optval, - int __user *optlen) -{ - int len; - - if (get_user(len, optlen)) - return -EFAULT; - - return bpfilter_mbox_request(sk, optname, USER_SOCKPTR(optval), len, - false); -} - -static int __init bpfilter_sockopt_init(void) -{ - mutex_init(&bpfilter_ops.lock); - bpfilter_ops.info.tgid = NULL; - bpfilter_ops.info.driver_name = "bpfilter_umh"; - - return 0; -} -device_initcall(bpfilter_sockopt_init); diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 66247e8b429e..7aa9dc0e6760 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -47,8 +47,6 @@ #include #include -#include - /* * SOL_IP control messages. */ @@ -1411,11 +1409,6 @@ int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, return -ENOPROTOOPT; err = do_ip_setsockopt(sk, level, optname, optval, optlen); -#if IS_ENABLED(CONFIG_BPFILTER_UMH) - if (optname >= BPFILTER_IPT_SO_SET_REPLACE && - optname < BPFILTER_IPT_SET_MAX) - err = bpfilter_ip_set_sockopt(sk, optname, optval, optlen); -#endif #ifdef CONFIG_NETFILTER /* we need to exclude all possible ENOPROTOOPTs except default case */ if (err == -ENOPROTOOPT && optname != IP_HDRINCL && @@ -1763,11 +1756,6 @@ int ip_getsockopt(struct sock *sk, int level, err = do_ip_getsockopt(sk, level, optname, USER_SOCKPTR(optval), USER_SOCKPTR(optlen)); -#if IS_ENABLED(CONFIG_BPFILTER_UMH) - if (optname >= BPFILTER_IPT_SO_GET_INFO && - optname < BPFILTER_IPT_GET_MAX) - err = bpfilter_ip_get_sockopt(sk, optname, optval, optlen); -#endif #ifdef CONFIG_NETFILTER /* we need to exclude all possible ENOPROTOOPTs except default case */ if (err == -ENOPROTOOPT && optname != IP_PKTOPTIONS && diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index edda4fc2c4d0..708733b0ea06 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -426,10 +426,6 @@ static void probe_kernel_image_config(const char *define_prefix) { "CONFIG_BPF_STREAM_PARSER", }, /* xt_bpf module for passing BPF programs to netfilter */ { "CONFIG_NETFILTER_XT_MATCH_BPF", }, - /* bpfilter back-end for iptables */ - { "CONFIG_BPFILTER", }, - /* bpftilter module with "user mode helper" */ - { "CONFIG_BPFILTER_UMH", }, /* test_bpf module for BPF tests */ { "CONFIG_TEST_BPF", }, diff --git a/tools/testing/selftests/bpf/config.aarch64 b/tools/testing/selftests/bpf/config.aarch64 index 29c8635c5722..3720b7611523 100644 --- a/tools/testing/selftests/bpf/config.aarch64 +++ b/tools/testing/selftests/bpf/config.aarch64 @@ -11,7 +11,6 @@ CONFIG_BLK_DEV_IO_TRACE=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_SD=y CONFIG_BONDING=y -CONFIG_BPFILTER=y CONFIG_BPF_JIT_ALWAYS_ON=y CONFIG_BPF_JIT_DEFAULT_ON=y CONFIG_BPF_PRELOAD_UMD=y diff --git a/tools/testing/selftests/bpf/config.s390x b/tools/testing/selftests/bpf/config.s390x index e93330382849..706931a8c2c6 100644 --- a/tools/testing/selftests/bpf/config.s390x +++ b/tools/testing/selftests/bpf/config.s390x @@ -9,7 +9,6 @@ CONFIG_BPF_JIT_ALWAYS_ON=y CONFIG_BPF_JIT_DEFAULT_ON=y CONFIG_BPF_PRELOAD=y CONFIG_BPF_PRELOAD_UMD=y -CONFIG_BPFILTER=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_FREEZER=y diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64 index b946088017f1..5680befae8c6 100644 --- a/tools/testing/selftests/bpf/config.x86_64 +++ b/tools/testing/selftests/bpf/config.x86_64 @@ -19,7 +19,6 @@ CONFIG_BOOTTIME_TRACING=y CONFIG_BPF_JIT_ALWAYS_ON=y CONFIG_BPF_PRELOAD=y CONFIG_BPF_PRELOAD_UMD=y -CONFIG_BPFILTER=y CONFIG_BSD_DISKLABEL=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_CFS_BANDWIDTH=y diff --git a/tools/testing/selftests/hid/config b/tools/testing/selftests/hid/config index 4f425178b56f..1758b055f295 100644 --- a/tools/testing/selftests/hid/config +++ b/tools/testing/selftests/hid/config @@ -1,5 +1,4 @@ CONFIG_BPF_EVENTS=y -CONFIG_BPFILTER=y CONFIG_BPF_JIT_ALWAYS_ON=y CONFIG_BPF_JIT=y CONFIG_BPF_KPROBE_OVERRIDE=y -- cgit v1.2.3 From fe1eb24bd5ade085914248c527044e942f75e06a Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 4 Jan 2024 16:04:35 -0800 Subject: Revert "Introduce PHY listing and link_topology tracking" This reverts commit 32bb4515e34469975abc936deb0a116c4a445817. This reverts commit d078d480639a4f3b5fc2d56247afa38e0956483a. This reverts commit fcc4b105caa4b844bf043375bf799c20a9c99db1. This reverts commit 345237dbc1bdbb274c9fb9ec38976261ff4a40b8. This reverts commit 7db69ec9cfb8b4ab50420262631fb2d1908b25bf. This reverts commit 95132a018f00f5dad38bdcfd4180d1af955d46f6. This reverts commit 63d5eaf35ac36cad00cfb3809d794ef0078c822b. This reverts commit c29451aefcb42359905d18678de38e52eccb3bb5. This reverts commit 2ab0edb505faa9ac90dee1732571390f074e8113. This reverts commit dedd702a35793ab462fce4c737eeba0badf9718e. This reverts commit 034fcc210349b873ece7356905be5c6ca11eef2a. This reverts commit 9c5625f559ad6fe9f6f733c11475bf470e637d34. This reverts commit 02018c544ef113e980a2349eba89003d6f399d22. Looks like we need more time for reviews, and incremental changes will be hard to make sense of. So revert. Link: https://lore.kernel.org/all/ZZP6FV5sXEf+xd58@shell.armlinux.org.uk/ Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/ethtool.yaml | 68 ------ Documentation/networking/ethtool-netlink.rst | 51 ----- Documentation/networking/index.rst | 1 - Documentation/networking/phy-link-topology.rst | 121 ---------- MAINTAINERS | 2 - drivers/net/phy/Makefile | 2 +- drivers/net/phy/at803x.c | 2 - drivers/net/phy/marvell-88x2222.c | 2 - drivers/net/phy/marvell.c | 2 - drivers/net/phy/marvell10g.c | 2 - drivers/net/phy/phy_device.c | 55 ----- drivers/net/phy/phy_link_topology.c | 66 ------ drivers/net/phy/phylink.c | 3 +- drivers/net/phy/sfp-bus.c | 15 +- include/linux/netdevice.h | 4 +- include/linux/phy.h | 6 - include/linux/phy_link_topology.h | 67 ------ include/linux/phy_link_topology_core.h | 19 -- include/linux/sfp.h | 8 +- include/uapi/linux/ethtool.h | 16 -- include/uapi/linux/ethtool_netlink.h | 30 --- net/core/dev.c | 3 - net/ethtool/Makefile | 2 +- net/ethtool/cabletest.c | 12 +- net/ethtool/netlink.c | 33 --- net/ethtool/netlink.h | 12 +- net/ethtool/phy.c | 306 ------------------------- net/ethtool/plca.c | 13 +- net/ethtool/pse-pd.c | 9 +- net/ethtool/strset.c | 15 +- 30 files changed, 35 insertions(+), 912 deletions(-) delete mode 100644 Documentation/networking/phy-link-topology.rst delete mode 100644 drivers/net/phy/phy_link_topology.c delete mode 100644 include/linux/phy_link_topology.h delete mode 100644 include/linux/phy_link_topology_core.h delete mode 100644 net/ethtool/phy.c (limited to 'include/linux') diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 7f6fb1f61dd4..197208f419dc 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -16,11 +16,6 @@ definitions: name: stringset type: enum entries: [] - - - name: phy-upstream-type - enum-name: - type: enum - entries: [ mac, phy ] attribute-sets: - @@ -35,9 +30,6 @@ attribute-sets: - name: flags type: u32 - - - name: phy-index - type: u32 - name: bitset-bit @@ -950,45 +942,6 @@ attribute-sets: - name: burst-tmr type: u32 - - - name: phy-upstream - attributes: - - - name: index - type: u32 - - - name: sfp-name - type: string - - - name: phy - attributes: - - - name: header - type: nest - nested-attributes: header - - - name: index - type: u32 - - - name: drvname - type: string - - - name: name - type: string - - - name: upstream-type - type: u8 - enum: phy-upstream-type - - - name: upstream - type: nest - nested-attributes: phy-upstream - - - name: downstream-sfp-name - type: string - - - name: id - type: u32 operations: enum-model: directional @@ -1740,24 +1693,3 @@ operations: name: mm-ntf doc: Notification for change in MAC Merge configuration. notify: mm-get - - - name: phy-get - doc: Get PHY devices attached to an interface - - attribute-set: phy - - do: &phy-get-op - request: - attributes: - - header - reply: - attributes: - - header - - index - - drvname - - name - - upstream-type - - upstream - - downstream-sfp-name - - id - dump: *phy-get-op diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 97ff787a7dd8..d583d9abf2f8 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -57,7 +57,6 @@ Structure of this header is ``ETHTOOL_A_HEADER_DEV_INDEX`` u32 device ifindex ``ETHTOOL_A_HEADER_DEV_NAME`` string device name ``ETHTOOL_A_HEADER_FLAGS`` u32 flags common for all requests - ``ETHTOOL_A_HEADER_PHY_INDEX`` u32 phy device index ============================== ====== ============================= ``ETHTOOL_A_HEADER_DEV_INDEX`` and ``ETHTOOL_A_HEADER_DEV_NAME`` identify the @@ -82,12 +81,6 @@ the behaviour is backward compatible, i.e. requests from old clients not aware of the flag should be interpreted the way the client expects. A client must not set flags it does not understand. -``ETHTOOL_A_HEADER_PHY_INDEX`` identify the ethernet PHY the message relates to. -As there are numerous commands that are related to PHY configuration, and because -we can have more than one PHY on the link, the PHY index can be passed in the -request for the commands that needs it. It is however not mandatory, and if it -is not passed for commands that target a PHY, the net_device.phydev pointer -is used, as a fallback that keeps the legacy behaviour. Bit sets ======== @@ -2011,49 +2004,6 @@ The attributes are propagated to the driver through the following structure: .. kernel-doc:: include/linux/ethtool.h :identifiers: ethtool_mm_cfg -PHY_GET -======= - -Retrieve information about a given Ethernet PHY sitting on the link. As there -can be more than one PHY, the DUMP operation can be used to list the PHYs -present on a given interface, by passing an interface index or name in -the dump request - -Request contents: - - ==================================== ====== ========================== - ``ETHTOOL_A_PHY_HEADER`` nested request header - ==================================== ====== ========================== - -Kernel response contents: - - ===================================== ====== ========================== - ``ETHTOOL_A_PHY_HEADER`` nested request header - ``ETHTOOL_A_PHY_INDEX`` u32 the phy's unique index, that can - be used for phy-specific requests - ``ETHTOOL_A_PHY_DRVNAME`` string the phy driver name - ``ETHTOOL_A_PHY_NAME`` string the phy device name - ``ETHTOOL_A_PHY_UPSTREAM_TYPE`` u32 the type of device this phy is - connected to - ``ETHTOOL_A_PHY_UPSTREAM_PHY`` nested if the phy is connected to another - phy, this nest contains info on - that connection - ``ETHTOOL_A_PHY_DOWNSTREAM_SFP_NAME`` string if the phy controls an sfp bus, - the name of the sfp bus - ``ETHTOOL_A_PHY_ID`` u32 the phy id if the phy is C22 - ===================================== ====== ========================== - -When ``ETHTOOL_A_PHY_UPSTREAM_TYPE`` is PHY_UPSTREAM_PHY, the PHY's parent is -another PHY. Information on the parent PHY will be set in the -``ETHTOOL_A_PHY_UPSTREAM_PHY`` nest, which has the following structure : - - =================================== ====== ========================== - ``ETHTOOL_A_PHY_UPSTREAM_INDEX`` u32 the PHY index of the upstream PHY - ``ETHTOOL_A_PHY_UPSTREAM_SFP_NAME`` string if this PHY is connected to it's - parent PHY through an SFP bus, the - name of this sfp bus - =================================== ====== ========================== - Request translation =================== @@ -2160,5 +2110,4 @@ are netlink only. n/a ``ETHTOOL_MSG_PLCA_GET_STATUS`` n/a ``ETHTOOL_MSG_MM_GET`` n/a ``ETHTOOL_MSG_MM_SET`` - n/a ``ETHTOOL_MSG_PHY_GET`` =================================== ===================================== diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst index a2c45a75a4a6..69f3d6dcd9fd 100644 --- a/Documentation/networking/index.rst +++ b/Documentation/networking/index.rst @@ -88,7 +88,6 @@ Contents: operstates packet_mmap phonet - phy-link-topology pktgen plip ppp_generic diff --git a/Documentation/networking/phy-link-topology.rst b/Documentation/networking/phy-link-topology.rst deleted file mode 100644 index 1fd8e904ef4b..000000000000 --- a/Documentation/networking/phy-link-topology.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -================= -PHY link topology -================= - -Overview -======== - -The PHY link topology representation in the networking stack aims at representing -the hardware layout for any given Ethernet link. - -An Ethernet Interface from userspace's point of view is nothing but a -:c:type:`struct net_device `, which exposes configuration options -through the legacy ioctls and the ethool netlink commands. The base assumption -when designing these configuration channels were that the link looked -something like this :: - - +-----------------------+ +----------+ +--------------+ - | Ethernet Controller / | | Ethernet | | Connector / | - | MAC | ------ | PHY | ---- | Port | ---... to LP - +-----------------------+ +----------+ +--------------+ - struct net_device struct phy_device - -Commands that needs to configure the PHY will go through the net_device.phydev -field to reach the PHY and perform the relevant configuration. - -This assumption falls apart in more complex topologies that can arise when, -for example, using SFP transceivers (although that's not the only specific case). - -Here, we have 2 basic scenarios. Either the MAC is able to output a serialized -interface, that can directly be fed to an SFP cage, such as SGMII, 1000BaseX, -10GBaseR, etc. - -The link topology then looks like this (when an SFP module is inserted) :: - - +-----+ SGMII +------------+ - | MAC | ------- | SFP Module | - +-----+ +------------+ - -Knowing that some modules embed a PHY, the actual link is more like :: - - +-----+ SGMII +--------------+ - | MAC | -------- | PHY (on SFP) | - +-----+ +--------------+ - -In this case, the SFP PHY is handled by phylib, and registered by phylink through -its SFP upstream ops. - -Now some Ethernet controllers aren't able to output a serialized interface, so -we can't directly connect them to an SFP cage. However, some PHYs can be used -as media-converters, to translate the non-serialized MAC MII interface to a -serialized MII interface fed to the SFP :: - - +-----+ RGMII +-----------------------+ SGMII +--------------+ - | MAC | ------- | PHY (media converter) | ------- | PHY (on SFP) | - +-----+ +-----------------------+ +--------------+ - -This is where the model of having a single net_device.phydev pointer shows its -limitations, as we now have 2 PHYs on the link. - -The phy_link topology framework aims at providing a way to keep track of every -PHY on the link, for use by both kernel drivers and subsystems, but also to -report the topology to userspace, allowing to target individual PHYs in configuration -commands. - -API -=== - -The :c:type:`struct phy_link_topology ` is a per-netdevice -resource, that gets initialized at netdevice creation. Once it's initialized, -it is then possible to register PHYs to the topology through : - -:c:func:`phy_link_topo_add_phy` - -Besides registering the PHY to the topology, this call will also assign a unique -index to the PHY, which can then be reported to userspace to refer to this PHY -(akin to the ifindex). This index is a u32, ranging from 1 to U32_MAX. The value -0 is reserved to indicate the PHY doesn't belong to any topology yet. - -The PHY can then be removed from the topology through - -:c:func:`phy_link_topo_del_phy` - -These function are already hooked into the phylib subsystem, so all PHYs that -are linked to a net_device through :c:func:`phy_attach_direct` will automatically -join the netdev's topology. - -PHYs that are on a SFP module will also be automatically registered IF the SFP -upstream is phylink (so, no media-converter). - -PHY drivers that can be used as SFP upstream need to call :c:func:`phy_sfp_attach_phy` -and :c:func:`phy_sfp_detach_phy`, which can be used as a -.attach_phy / .detach_phy implementation for the -:c:type:`struct sfp_upstream_ops `. - -UAPI -==== - -There exist a set of netlink commands to query the link topology from userspace, -see ``Documentation/networking/ethtool-netlink.rst``. - -The whole point of having a topology representation is to assign the phyindex -field in :c:type:`struct phy_device `. This index is reported to -userspace using the ``ETHTOOL_MSG_PHY_GET`` ethtnl command. Performing a DUMP operation -will result in all PHYs from all net_device being listed. The DUMP command -accepts either a ``ETHTOOL_A_HEADER_DEV_INDEX`` or ``ETHTOOL_A_HEADER_DEV_NAME`` -to be passed in the request to filter the DUMP to a single net_device. - -The retrieved index can then be passed as a request parameter using the -``ETHTOOL_A_HEADER_PHY_INDEX`` field in the following ethnl commands : - -* ``ETHTOOL_MSG_STRSET_GET`` to get the stats string set from a given PHY -* ``ETHTOOL_MSG_CABLE_TEST_ACT`` and ``ETHTOOL_MSG_CABLE_TEST_ACT``, to perform - cable testing on a given PHY on the link (most likely the outermost PHY) -* ``ETHTOOL_MSG_PSE_SET`` and ``ETHTOOL_MSG_PSE_GET`` for PHY-controlled PoE and PSE settings -* ``ETHTOOL_MSG_PLCA_GET_CFG``, ``ETHTOOL_MSG_PLCA_SET_CFG`` and ``ETHTOOL_MSG_PLCA_GET_STATUS`` - to set the PLCA (Physical Layer Collision Avoidance) parameters - -Note that the PHY index can be passed to other requests, which will silently -ignore it if present and irrelevant. diff --git a/MAINTAINERS b/MAINTAINERS index 79ac49b113dc..2b916990d7f0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7871,8 +7871,6 @@ F: include/linux/mii.h F: include/linux/of_net.h F: include/linux/phy.h F: include/linux/phy_fixed.h -F: include/linux/phy_link_topology.h -F: include/linux/phy_link_topology_core.h F: include/linux/phylib_stubs.h F: include/linux/platform_data/mdio-bcm-unimac.h F: include/linux/platform_data/mdio-gpio.h diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile index f218954fd7a8..6097afd44392 100644 --- a/drivers/net/phy/Makefile +++ b/drivers/net/phy/Makefile @@ -2,7 +2,7 @@ # Makefile for Linux PHY drivers libphy-y := phy.o phy-c45.o phy-core.o phy_device.o \ - linkmode.o phy_link_topology.o + linkmode.o mdio-bus-y += mdio_bus.o mdio_device.o ifdef CONFIG_MDIO_DEVICE diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index aaf6c654aaed..19cfbf36fe80 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -1452,8 +1452,6 @@ static const struct sfp_upstream_ops at8031_sfp_ops = { .attach = phy_sfp_attach, .detach = phy_sfp_detach, .module_insert = at8031_sfp_insert, - .connect_phy = phy_sfp_connect_phy, - .disconnect_phy = phy_sfp_disconnect_phy, }; static int at8031_parse_dt(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c index 3f77bbc7e04f..e3aa30dad2e6 100644 --- a/drivers/net/phy/marvell-88x2222.c +++ b/drivers/net/phy/marvell-88x2222.c @@ -555,8 +555,6 @@ static const struct sfp_upstream_ops sfp_phy_ops = { .link_down = mv2222_sfp_link_down, .attach = phy_sfp_attach, .detach = phy_sfp_detach, - .connect_phy = phy_sfp_connect_phy, - .disconnect_phy = phy_sfp_disconnect_phy, }; static int mv2222_probe(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index 674e29bce2cc..eba652a4c1d8 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -3254,8 +3254,6 @@ static const struct sfp_upstream_ops m88e1510_sfp_ops = { .module_remove = m88e1510_sfp_remove, .attach = phy_sfp_attach, .detach = phy_sfp_detach, - .connect_phy = phy_sfp_connect_phy, - .disconnect_phy = phy_sfp_disconnect_phy, }; static int m88e1510_probe(struct phy_device *phydev) diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c index 6642eb642d4b..ad43e280930c 100644 --- a/drivers/net/phy/marvell10g.c +++ b/drivers/net/phy/marvell10g.c @@ -503,8 +503,6 @@ static int mv3310_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) static const struct sfp_upstream_ops mv3310_sfp_ops = { .attach = phy_sfp_attach, .detach = phy_sfp_detach, - .connect_phy = phy_sfp_connect_phy, - .disconnect_phy = phy_sfp_disconnect_phy, .module_insert = mv3310_sfp_insert, }; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 1e595762afea..3611ea64875e 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -266,14 +265,6 @@ static void phy_mdio_device_remove(struct mdio_device *mdiodev) static struct phy_driver genphy_driver; -static struct phy_link_topology *phy_get_link_topology(struct phy_device *phydev) -{ - if (phydev->attached_dev) - return &phydev->attached_dev->link_topo; - - return NULL; -} - static LIST_HEAD(phy_fixup_list); static DEFINE_MUTEX(phy_fixup_lock); @@ -1363,46 +1354,6 @@ phy_standalone_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(phy_standalone); -/** - * phy_sfp_connect_phy - Connect the SFP module's PHY to the upstream PHY - * @upstream: pointer to the upstream phy device - * @phy: pointer to the SFP module's phy device - * - * This helper allows keeping track of PHY devices on the link. It adds the - * SFP module's phy to the phy namespace of the upstream phy - */ -int phy_sfp_connect_phy(void *upstream, struct phy_device *phy) -{ - struct phy_device *phydev = upstream; - struct phy_link_topology *topo = phy_get_link_topology(phydev); - - if (topo) - return phy_link_topo_add_phy(topo, phy, PHY_UPSTREAM_PHY, phydev); - - return 0; -} -EXPORT_SYMBOL(phy_sfp_connect_phy); - -/** - * phy_sfp_disconnect_phy - Disconnect the SFP module's PHY from the upstream PHY - * @upstream: pointer to the upstream phy device - * @phy: pointer to the SFP module's phy device - * - * This helper allows keeping track of PHY devices on the link. It removes the - * SFP module's phy to the phy namespace of the upstream phy. As the module phy - * will be destroyed, re-inserting the same module will add a new phy with a - * new index. - */ -void phy_sfp_disconnect_phy(void *upstream, struct phy_device *phy) -{ - struct phy_device *phydev = upstream; - struct phy_link_topology *topo = phy_get_link_topology(phydev); - - if (topo) - phy_link_topo_del_phy(topo, phy); -} -EXPORT_SYMBOL(phy_sfp_disconnect_phy); - /** * phy_sfp_attach - attach the SFP bus to the PHY upstream network device * @upstream: pointer to the phy device @@ -1540,11 +1491,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, if (phydev->sfp_bus_attached) dev->sfp_bus = phydev->sfp_bus; - - err = phy_link_topo_add_phy(&dev->link_topo, phydev, - PHY_UPSTREAM_MAC, dev); - if (err) - goto error; } /* Some Ethernet drivers try to connect to a PHY device before @@ -1874,7 +1820,6 @@ void phy_detach(struct phy_device *phydev) if (dev) { phydev->attached_dev->phydev = NULL; phydev->attached_dev = NULL; - phy_link_topo_del_phy(&dev->link_topo, phydev); } phydev->phylink = NULL; diff --git a/drivers/net/phy/phy_link_topology.c b/drivers/net/phy/phy_link_topology.c deleted file mode 100644 index 34e7e08fbfc3..000000000000 --- a/drivers/net/phy/phy_link_topology.c +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Infrastructure to handle all PHY devices connected to a given netdev, - * either directly or indirectly attached. - * - * Copyright (c) 2023 Maxime Chevallier - */ - -#include -#include -#include -#include -#include - -int phy_link_topo_add_phy(struct phy_link_topology *topo, - struct phy_device *phy, - enum phy_upstream upt, void *upstream) -{ - struct phy_device_node *pdn; - int ret; - - pdn = kzalloc(sizeof(*pdn), GFP_KERNEL); - if (!pdn) - return -ENOMEM; - - pdn->phy = phy; - switch (upt) { - case PHY_UPSTREAM_MAC: - pdn->upstream.netdev = (struct net_device *)upstream; - if (phy_on_sfp(phy)) - pdn->parent_sfp_bus = pdn->upstream.netdev->sfp_bus; - break; - case PHY_UPSTREAM_PHY: - pdn->upstream.phydev = (struct phy_device *)upstream; - if (phy_on_sfp(phy)) - pdn->parent_sfp_bus = pdn->upstream.phydev->sfp_bus; - break; - default: - ret = -EINVAL; - goto err; - } - pdn->upstream_type = upt; - - ret = xa_alloc_cyclic(&topo->phys, &phy->phyindex, pdn, xa_limit_32b, - &topo->next_phy_index, GFP_KERNEL); - if (ret) - goto err; - - return 0; - -err: - kfree(pdn); - return ret; -} -EXPORT_SYMBOL_GPL(phy_link_topo_add_phy); - -void phy_link_topo_del_phy(struct phy_link_topology *topo, - struct phy_device *phy) -{ - struct phy_device_node *pdn = xa_erase(&topo->phys, phy->phyindex); - - phy->phyindex = 0; - - kfree(pdn); -} -EXPORT_SYMBOL_GPL(phy_link_topo_del_phy); diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index a816391add12..ed0b4ccaa6a6 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -3385,8 +3385,7 @@ static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy) return ret; } -static void phylink_sfp_disconnect_phy(void *upstream, - struct phy_device *phydev) +static void phylink_sfp_disconnect_phy(void *upstream) { phylink_disconnect_phy(upstream); } diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c index fb1c102714b5..6fa679b36290 100644 --- a/drivers/net/phy/sfp-bus.c +++ b/drivers/net/phy/sfp-bus.c @@ -486,7 +486,7 @@ static void sfp_unregister_bus(struct sfp_bus *bus) bus->socket_ops->stop(bus->sfp); bus->socket_ops->detach(bus->sfp); if (bus->phydev && ops && ops->disconnect_phy) - ops->disconnect_phy(bus->upstream, bus->phydev); + ops->disconnect_phy(bus->upstream); } bus->registered = false; } @@ -742,7 +742,7 @@ void sfp_remove_phy(struct sfp_bus *bus) const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus); if (ops && ops->disconnect_phy) - ops->disconnect_phy(bus->upstream, bus->phydev); + ops->disconnect_phy(bus->upstream); bus->phydev = NULL; } EXPORT_SYMBOL_GPL(sfp_remove_phy); @@ -859,14 +859,3 @@ void sfp_unregister_socket(struct sfp_bus *bus) sfp_bus_put(bus); } EXPORT_SYMBOL_GPL(sfp_unregister_socket); - -const char *sfp_get_name(struct sfp_bus *bus) -{ - ASSERT_RTNL(); - - if (bus->sfp_dev) - return dev_name(bus->sfp_dev); - - return NULL; -} -EXPORT_SYMBOL_GPL(sfp_get_name); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e265aa1f2169..118c40258d07 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -40,6 +40,7 @@ #include #endif #include + #include #include #include @@ -51,7 +52,6 @@ #include #include #include -#include struct netpoll_info; struct device; @@ -2047,7 +2047,6 @@ enum netdev_stat_type { * @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp * * @priomap: XXX: need comments on this one - * @link_topo: Physical link topology tracking attached PHYs * @phydev: Physical device may attach itself * for hardware timestamping * @sfp_bus: attached &struct sfp_bus structure. @@ -2442,7 +2441,6 @@ struct net_device { #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) struct netprio_map __rcu *priomap; #endif - struct phy_link_topology link_topo; struct phy_device *phydev; struct sfp_bus *sfp_bus; struct lock_class_key *qdisc_tx_busylock; diff --git a/include/linux/phy.h b/include/linux/phy.h index 6cb9d843aee9..e9e85d347587 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -544,9 +544,6 @@ struct macsec_ops; * @drv: Pointer to the driver for this PHY instance * @devlink: Create a link between phy dev and mac dev, if the external phy * used by current mac interface is managed by another mac interface. - * @phyindex: Unique id across the phy's parent tree of phys to address the PHY - * from userspace, similar to ifindex. A zero index means the PHY - * wasn't assigned an id yet. * @phy_id: UID for this device found during discovery * @c45_ids: 802.3-c45 Device Identifiers if is_c45. * @is_c45: Set to true if this PHY uses clause 45 addressing. @@ -646,7 +643,6 @@ struct phy_device { struct device_link *devlink; - u32 phyindex; u32 phy_id; struct phy_c45_device_ids c45_ids; @@ -1726,8 +1722,6 @@ int phy_suspend(struct phy_device *phydev); int phy_resume(struct phy_device *phydev); int __phy_resume(struct phy_device *phydev); int phy_loopback(struct phy_device *phydev, bool enable); -int phy_sfp_connect_phy(void *upstream, struct phy_device *phy); -void phy_sfp_disconnect_phy(void *upstream, struct phy_device *phy); void phy_sfp_attach(void *upstream, struct sfp_bus *bus); void phy_sfp_detach(void *upstream, struct sfp_bus *bus); int phy_sfp_probe(struct phy_device *phydev, diff --git a/include/linux/phy_link_topology.h b/include/linux/phy_link_topology.h deleted file mode 100644 index 91902263ec0e..000000000000 --- a/include/linux/phy_link_topology.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * PHY device list allow maintaining a list of PHY devices that are - * part of a netdevice's link topology. PHYs can for example be chained, - * as is the case when using a PHY that exposes an SFP module, on which an - * SFP transceiver that embeds a PHY is connected. - * - * This list can then be used by userspace to leverage individual PHY - * capabilities. - */ -#ifndef __PHY_LINK_TOPOLOGY_H -#define __PHY_LINK_TOPOLOGY_H - -#include -#include - -struct xarray; -struct phy_device; -struct net_device; -struct sfp_bus; - -struct phy_device_node { - enum phy_upstream upstream_type; - - union { - struct net_device *netdev; - struct phy_device *phydev; - } upstream; - - struct sfp_bus *parent_sfp_bus; - - struct phy_device *phy; -}; - -static inline struct phy_device * -phy_link_topo_get_phy(struct phy_link_topology *topo, u32 phyindex) -{ - struct phy_device_node *pdn = xa_load(&topo->phys, phyindex); - - if (pdn) - return pdn->phy; - - return NULL; -} - -#if IS_ENABLED(CONFIG_PHYLIB) -int phy_link_topo_add_phy(struct phy_link_topology *topo, - struct phy_device *phy, - enum phy_upstream upt, void *upstream); - -void phy_link_topo_del_phy(struct phy_link_topology *lt, struct phy_device *phy); - -#else -static inline int phy_link_topo_add_phy(struct phy_link_topology *topo, - struct phy_device *phy, - enum phy_upstream upt, void *upstream) -{ - return 0; -} - -static inline void phy_link_topo_del_phy(struct phy_link_topology *topo, - struct phy_device *phy) -{ -} -#endif - -#endif /* __PHY_LINK_TOPOLOGY_H */ diff --git a/include/linux/phy_link_topology_core.h b/include/linux/phy_link_topology_core.h deleted file mode 100644 index 78c75f909489..000000000000 --- a/include/linux/phy_link_topology_core.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __PHY_LINK_TOPOLOGY_CORE_H -#define __PHY_LINK_TOPOLOGY_CORE_H - -struct xarray; - -struct phy_link_topology { - struct xarray phys; - - u32 next_phy_index; -}; - -static inline void phy_link_topo_init(struct phy_link_topology *topo) -{ - xa_init_flags(&topo->phys, XA_FLAGS_ALLOC1); - topo->next_phy_index = 1; -} - -#endif /* __PHY_LINK_TOPOLOGY_CORE_H */ diff --git a/include/linux/sfp.h b/include/linux/sfp.h index 55c0ab17c9e2..9346cd44814d 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -544,7 +544,7 @@ struct sfp_upstream_ops { void (*link_down)(void *priv); void (*link_up)(void *priv); int (*connect_phy)(void *priv, struct phy_device *); - void (*disconnect_phy)(void *priv, struct phy_device *); + void (*disconnect_phy)(void *priv); }; #if IS_ENABLED(CONFIG_SFP) @@ -570,7 +570,6 @@ struct sfp_bus *sfp_bus_find_fwnode(const struct fwnode_handle *fwnode); int sfp_bus_add_upstream(struct sfp_bus *bus, void *upstream, const struct sfp_upstream_ops *ops); void sfp_bus_del_upstream(struct sfp_bus *bus); -const char *sfp_get_name(struct sfp_bus *bus); #else static inline int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id, @@ -649,11 +648,6 @@ static inline int sfp_bus_add_upstream(struct sfp_bus *bus, void *upstream, static inline void sfp_bus_del_upstream(struct sfp_bus *bus) { } - -static inline const char *sfp_get_name(struct sfp_bus *bus) -{ - return NULL; -} #endif #endif diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index 01ba529dbb6d..06ef6b78b7de 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -2220,20 +2220,4 @@ struct ethtool_link_settings { * __u32 map_lp_advertising[link_mode_masks_nwords]; */ }; - -/** - * enum phy_upstream - Represents the upstream component a given PHY device - * is connected to, as in what is on the other end of the MII bus. Most PHYs - * will be attached to an Ethernet MAC controller, but in some cases, there's - * an intermediate PHY used as a media-converter, which will driver another - * MII interface as its output. - * @PHY_UPSTREAM_MAC: Upstream component is a MAC (a switch port, - * or ethernet controller) - * @PHY_UPSTREAM_PHY: Upstream component is a PHY (likely a media converter) - */ -enum phy_upstream { - PHY_UPSTREAM_MAC, - PHY_UPSTREAM_PHY, -}; - #endif /* _UAPI_LINUX_ETHTOOL_H */ diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index 00cd7ad16709..3f89074aa06c 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -57,7 +57,6 @@ enum { ETHTOOL_MSG_PLCA_GET_STATUS, ETHTOOL_MSG_MM_GET, ETHTOOL_MSG_MM_SET, - ETHTOOL_MSG_PHY_GET, /* add new constants above here */ __ETHTOOL_MSG_USER_CNT, @@ -110,8 +109,6 @@ enum { ETHTOOL_MSG_PLCA_NTF, ETHTOOL_MSG_MM_GET_REPLY, ETHTOOL_MSG_MM_NTF, - ETHTOOL_MSG_PHY_GET_REPLY, - ETHTOOL_MSG_PHY_NTF, /* add new constants above here */ __ETHTOOL_MSG_KERNEL_CNT, @@ -136,7 +133,6 @@ enum { ETHTOOL_A_HEADER_DEV_INDEX, /* u32 */ ETHTOOL_A_HEADER_DEV_NAME, /* string */ ETHTOOL_A_HEADER_FLAGS, /* u32 - ETHTOOL_FLAG_* */ - ETHTOOL_A_HEADER_PHY_INDEX, /* u32 */ /* add new constants above here */ __ETHTOOL_A_HEADER_CNT, @@ -980,32 +976,6 @@ enum { ETHTOOL_A_MM_MAX = (__ETHTOOL_A_MM_CNT - 1) }; -enum { - ETHTOOL_A_PHY_UPSTREAM_UNSPEC, - ETHTOOL_A_PHY_UPSTREAM_INDEX, /* u32 */ - ETHTOOL_A_PHY_UPSTREAM_SFP_NAME, /* string */ - - /* add new constants above here */ - __ETHTOOL_A_PHY_UPSTREAM_CNT, - ETHTOOL_A_PHY_UPSTREAM_MAX = (__ETHTOOL_A_PHY_UPSTREAM_CNT - 1) -}; - -enum { - ETHTOOL_A_PHY_UNSPEC, - ETHTOOL_A_PHY_HEADER, /* nest - _A_HEADER_* */ - ETHTOOL_A_PHY_INDEX, /* u32 */ - ETHTOOL_A_PHY_DRVNAME, /* string */ - ETHTOOL_A_PHY_NAME, /* string */ - ETHTOOL_A_PHY_UPSTREAM_TYPE, /* u8 */ - ETHTOOL_A_PHY_UPSTREAM, /* nest - _A_PHY_UPSTREAM_* */ - ETHTOOL_A_PHY_DOWNSTREAM_SFP_NAME, /* string */ - ETHTOOL_A_PHY_ID, /* u32 */ - - /* add new constants above here */ - __ETHTOOL_A_PHY_CNT, - ETHTOOL_A_PHY_MAX = (__ETHTOOL_A_PHY_CNT - 1) -}; - /* generic netlink info */ #define ETHTOOL_GENL_NAME "ethtool" #define ETHTOOL_GENL_VERSION 1 diff --git a/net/core/dev.c b/net/core/dev.c index bc4ac49d4643..f01a9b858347 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -153,7 +153,6 @@ #include #include #include -#include #include "dev.h" #include "net-sysfs.h" @@ -10876,8 +10875,6 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, #ifdef CONFIG_NET_SCHED hash_init(dev->qdisc_hash); #endif - phy_link_topo_init(&dev->link_topo); - dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; setup(dev); diff --git a/net/ethtool/Makefile b/net/ethtool/Makefile index 0ccd0e9afd3f..504f954a1b28 100644 --- a/net/ethtool/Makefile +++ b/net/ethtool/Makefile @@ -8,4 +8,4 @@ ethtool_nl-y := netlink.o bitset.o strset.o linkinfo.o linkmodes.o rss.o \ linkstate.o debug.o wol.o features.o privflags.o rings.o \ channels.o coalesce.o pause.o eee.o tsinfo.o cabletest.o \ tunnels.o fec.o eeprom.o stats.o phc_vclocks.o mm.o \ - module.o pse-pd.o plca.o mm.o phy.o + module.o pse-pd.o plca.o mm.o diff --git a/net/ethtool/cabletest.c b/net/ethtool/cabletest.c index 6b00d0800f23..06a151165c31 100644 --- a/net/ethtool/cabletest.c +++ b/net/ethtool/cabletest.c @@ -69,7 +69,7 @@ int ethnl_act_cable_test(struct sk_buff *skb, struct genl_info *info) return ret; dev = req_info.dev; - if (!req_info.phydev) { + if (!dev->phydev) { ret = -EOPNOTSUPP; goto out_dev_put; } @@ -85,12 +85,12 @@ int ethnl_act_cable_test(struct sk_buff *skb, struct genl_info *info) if (ret < 0) goto out_rtnl; - ret = ops->start_cable_test(req_info.phydev, info->extack); + ret = ops->start_cable_test(dev->phydev, info->extack); ethnl_ops_complete(dev); if (!ret) - ethnl_cable_test_started(req_info.phydev, + ethnl_cable_test_started(dev->phydev, ETHTOOL_MSG_CABLE_TEST_NTF); out_rtnl: @@ -321,7 +321,7 @@ int ethnl_act_cable_test_tdr(struct sk_buff *skb, struct genl_info *info) return ret; dev = req_info.dev; - if (!req_info.phydev) { + if (!dev->phydev) { ret = -EOPNOTSUPP; goto out_dev_put; } @@ -342,12 +342,12 @@ int ethnl_act_cable_test_tdr(struct sk_buff *skb, struct genl_info *info) if (ret < 0) goto out_rtnl; - ret = ops->start_cable_test_tdr(req_info.phydev, info->extack, &cfg); + ret = ops->start_cable_test_tdr(dev->phydev, info->extack, &cfg); ethnl_ops_complete(dev); if (!ret) - ethnl_cable_test_started(req_info.phydev, + ethnl_cable_test_started(dev->phydev, ETHTOOL_MSG_CABLE_TEST_TDR_NTF); out_rtnl: diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 92b0dd8ca046..fe3553f60bf3 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -4,7 +4,6 @@ #include #include #include "netlink.h" -#include static struct genl_family ethtool_genl_family; @@ -21,7 +20,6 @@ const struct nla_policy ethnl_header_policy[] = { .len = ALTIFNAMSIZ - 1 }, [ETHTOOL_A_HEADER_FLAGS] = NLA_POLICY_MASK(NLA_U32, ETHTOOL_FLAGS_BASIC), - [ETHTOOL_A_HEADER_PHY_INDEX] = NLA_POLICY_MIN(NLA_U32, 1), }; const struct nla_policy ethnl_header_policy_stats[] = { @@ -30,7 +28,6 @@ const struct nla_policy ethnl_header_policy_stats[] = { .len = ALTIFNAMSIZ - 1 }, [ETHTOOL_A_HEADER_FLAGS] = NLA_POLICY_MASK(NLA_U32, ETHTOOL_FLAGS_STATS), - [ETHTOOL_A_HEADER_PHY_INDEX] = NLA_POLICY_MIN(NLA_U32, 1), }; int ethnl_ops_begin(struct net_device *dev) @@ -94,7 +91,6 @@ int ethnl_parse_header_dev_get(struct ethnl_req_info *req_info, { struct nlattr *tb[ARRAY_SIZE(ethnl_header_policy)]; const struct nlattr *devname_attr; - struct phy_device *phydev = NULL; struct net_device *dev = NULL; u32 flags = 0; int ret; @@ -149,26 +145,6 @@ int ethnl_parse_header_dev_get(struct ethnl_req_info *req_info, return -EINVAL; } - if (dev) { - if (tb[ETHTOOL_A_HEADER_PHY_INDEX]) { - u32 phy_index = nla_get_u32(tb[ETHTOOL_A_HEADER_PHY_INDEX]); - - phydev = phy_link_topo_get_phy(&dev->link_topo, - phy_index); - if (!phydev) { - NL_SET_ERR_MSG_ATTR(extack, header, - "no phy matches phy index"); - return -EINVAL; - } - } else { - /* If we need a PHY but no phy index is specified, fallback - * to dev->phydev - */ - phydev = dev->phydev; - } - } - - req_info->phydev = phydev; req_info->dev = dev; req_info->flags = flags; return 0; @@ -1153,15 +1129,6 @@ static const struct genl_ops ethtool_genl_ops[] = { .policy = ethnl_mm_set_policy, .maxattr = ARRAY_SIZE(ethnl_mm_set_policy) - 1, }, - { - .cmd = ETHTOOL_MSG_PHY_GET, - .doit = ethnl_phy_doit, - .start = ethnl_phy_start, - .dumpit = ethnl_phy_dumpit, - .done = ethnl_phy_done, - .policy = ethnl_phy_get_policy, - .maxattr = ARRAY_SIZE(ethnl_phy_get_policy) - 1, - }, }; static const struct genl_multicast_group ethtool_nl_mcgrps[] = { diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 5e6a43e35a09..9a333a8d04c1 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -250,7 +250,6 @@ static inline unsigned int ethnl_reply_header_size(void) * @dev: network device the request is for (may be null) * @dev_tracker: refcount tracker for @dev reference * @flags: request flags common for all request types - * @phydev: phy_device connected to @dev this request is for (may be null) * * This is a common base for request specific structures holding data from * parsed userspace request. These always embed struct ethnl_req_info at @@ -260,7 +259,6 @@ struct ethnl_req_info { struct net_device *dev; netdevice_tracker dev_tracker; u32 flags; - struct phy_device *phydev; }; static inline void ethnl_parse_header_dev_put(struct ethnl_req_info *req_info) @@ -397,10 +395,9 @@ extern const struct ethnl_request_ops ethnl_rss_request_ops; extern const struct ethnl_request_ops ethnl_plca_cfg_request_ops; extern const struct ethnl_request_ops ethnl_plca_status_request_ops; extern const struct ethnl_request_ops ethnl_mm_request_ops; -extern const struct ethnl_request_ops ethnl_phy_request_ops; -extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_PHY_INDEX + 1]; -extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_PHY_INDEX + 1]; +extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_FLAGS + 1]; +extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_FLAGS + 1]; extern const struct nla_policy ethnl_strset_get_policy[ETHTOOL_A_STRSET_COUNTS_ONLY + 1]; extern const struct nla_policy ethnl_linkinfo_get_policy[ETHTOOL_A_LINKINFO_HEADER + 1]; extern const struct nla_policy ethnl_linkinfo_set_policy[ETHTOOL_A_LINKINFO_TP_MDIX_CTRL + 1]; @@ -444,7 +441,6 @@ extern const struct nla_policy ethnl_plca_set_cfg_policy[ETHTOOL_A_PLCA_MAX + 1] extern const struct nla_policy ethnl_plca_get_status_policy[ETHTOOL_A_PLCA_HEADER + 1]; extern const struct nla_policy ethnl_mm_get_policy[ETHTOOL_A_MM_HEADER + 1]; extern const struct nla_policy ethnl_mm_set_policy[ETHTOOL_A_MM_MAX + 1]; -extern const struct nla_policy ethnl_phy_get_policy[ETHTOOL_A_PHY_HEADER + 1]; int ethnl_set_features(struct sk_buff *skb, struct genl_info *info); int ethnl_act_cable_test(struct sk_buff *skb, struct genl_info *info); @@ -452,10 +448,6 @@ int ethnl_act_cable_test_tdr(struct sk_buff *skb, struct genl_info *info); int ethnl_tunnel_info_doit(struct sk_buff *skb, struct genl_info *info); int ethnl_tunnel_info_start(struct netlink_callback *cb); int ethnl_tunnel_info_dumpit(struct sk_buff *skb, struct netlink_callback *cb); -int ethnl_phy_start(struct netlink_callback *cb); -int ethnl_phy_doit(struct sk_buff *skb, struct genl_info *info); -int ethnl_phy_dumpit(struct sk_buff *skb, struct netlink_callback *cb); -int ethnl_phy_done(struct netlink_callback *cb); extern const char stats_std_names[__ETHTOOL_STATS_CNT][ETH_GSTRING_LEN]; extern const char stats_eth_phy_names[__ETHTOOL_A_STATS_ETH_PHY_CNT][ETH_GSTRING_LEN]; diff --git a/net/ethtool/phy.c b/net/ethtool/phy.c deleted file mode 100644 index 5add2840aaeb..000000000000 --- a/net/ethtool/phy.c +++ /dev/null @@ -1,306 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2023 Bootlin - * - */ -#include "common.h" -#include "netlink.h" - -#include -#include -#include - -struct phy_req_info { - struct ethnl_req_info base; - struct phy_device_node pdn; -}; - -#define PHY_REQINFO(__req_base) \ - container_of(__req_base, struct phy_req_info, base) - -const struct nla_policy ethnl_phy_get_policy[ETHTOOL_A_PHY_HEADER + 1] = { - [ETHTOOL_A_PHY_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), -}; - -/* Caller holds rtnl */ -static ssize_t -ethnl_phy_reply_size(const struct ethnl_req_info *req_base, - struct netlink_ext_ack *extack) -{ - struct phy_link_topology *topo; - struct phy_device_node *pdn; - struct phy_device *phydev; - unsigned long index; - size_t size; - - ASSERT_RTNL(); - - topo = &req_base->dev->link_topo; - - size = nla_total_size(0); - - xa_for_each(&topo->phys, index, pdn) { - phydev = pdn->phy; - - /* ETHTOOL_A_PHY_INDEX */ - size += nla_total_size(sizeof(u32)); - - /* ETHTOOL_A_DRVNAME */ - size += nla_total_size(strlen(phydev->drv->name) + 1); - - /* ETHTOOL_A_NAME */ - size += nla_total_size(strlen(dev_name(&phydev->mdio.dev)) + 1); - - /* ETHTOOL_A_PHY_UPSTREAM_TYPE */ - size += nla_total_size(sizeof(u8)); - - /* ETHTOOL_A_PHY_ID */ - size += nla_total_size(sizeof(u32)); - - if (phy_on_sfp(phydev)) { - const char *upstream_sfp_name = sfp_get_name(pdn->parent_sfp_bus); - - /* ETHTOOL_A_PHY_UPSTREAM_SFP_NAME */ - if (upstream_sfp_name) - size += nla_total_size(strlen(upstream_sfp_name) + 1); - - /* ETHTOOL_A_PHY_UPSTREAM_INDEX */ - size += nla_total_size(sizeof(u32)); - } - - /* ETHTOOL_A_PHY_DOWNSTREAM_SFP_NAME */ - if (phydev->sfp_bus) { - const char *sfp_name = sfp_get_name(phydev->sfp_bus); - - if (sfp_name) - size += nla_total_size(strlen(sfp_name) + 1); - } - } - - return size; -} - -static int -ethnl_phy_fill_reply(const struct ethnl_req_info *req_base, struct sk_buff *skb) -{ - struct phy_req_info *req_info = PHY_REQINFO(req_base); - struct phy_device_node *pdn = &req_info->pdn; - struct phy_device *phydev = pdn->phy; - enum phy_upstream ptype; - struct nlattr *nest; - - ptype = pdn->upstream_type; - - if (nla_put_u32(skb, ETHTOOL_A_PHY_INDEX, phydev->phyindex) || - nla_put_string(skb, ETHTOOL_A_PHY_DRVNAME, phydev->drv->name) || - nla_put_string(skb, ETHTOOL_A_PHY_NAME, dev_name(&phydev->mdio.dev)) || - nla_put_u8(skb, ETHTOOL_A_PHY_UPSTREAM_TYPE, ptype) || - nla_put_u32(skb, ETHTOOL_A_PHY_ID, phydev->phy_id)) - return -EMSGSIZE; - - if (ptype == PHY_UPSTREAM_PHY) { - struct phy_device *upstream = pdn->upstream.phydev; - const char *sfp_upstream_name; - - nest = nla_nest_start(skb, ETHTOOL_A_PHY_UPSTREAM); - if (!nest) - return -EMSGSIZE; - - /* Parent index */ - if (nla_put_u32(skb, ETHTOOL_A_PHY_UPSTREAM_INDEX, upstream->phyindex)) - return -EMSGSIZE; - - if (pdn->parent_sfp_bus) { - sfp_upstream_name = sfp_get_name(pdn->parent_sfp_bus); - if (sfp_upstream_name && nla_put_string(skb, - ETHTOOL_A_PHY_UPSTREAM_SFP_NAME, - sfp_upstream_name)) - return -EMSGSIZE; - } - - nla_nest_end(skb, nest); - } - - if (phydev->sfp_bus) { - const char *sfp_name = sfp_get_name(phydev->sfp_bus); - - if (sfp_name && - nla_put_string(skb, ETHTOOL_A_PHY_DOWNSTREAM_SFP_NAME, - sfp_name)) - return -EMSGSIZE; - } - - return 0; -} - -static int ethnl_phy_parse_request(struct ethnl_req_info *req_base, - struct nlattr **tb) -{ - struct phy_link_topology *topo = &req_base->dev->link_topo; - struct phy_req_info *req_info = PHY_REQINFO(req_base); - struct phy_device_node *pdn; - - if (!req_base->phydev) - return 0; - - pdn = xa_load(&topo->phys, req_base->phydev->phyindex); - memcpy(&req_info->pdn, pdn, sizeof(*pdn)); - - return 0; -} - -int ethnl_phy_doit(struct sk_buff *skb, struct genl_info *info) -{ - struct phy_req_info req_info = {}; - struct nlattr **tb = info->attrs; - struct sk_buff *rskb; - void *reply_payload; - int reply_len; - int ret; - - ret = ethnl_parse_header_dev_get(&req_info.base, - tb[ETHTOOL_A_PHY_HEADER], - genl_info_net(info), info->extack, - true); - if (ret < 0) - return ret; - - rtnl_lock(); - - ret = ethnl_phy_parse_request(&req_info.base, tb); - if (ret < 0) - goto err_unlock_rtnl; - - /* No PHY, return early */ - if (!req_info.pdn.phy) - goto err_unlock_rtnl; - - ret = ethnl_phy_reply_size(&req_info.base, info->extack); - if (ret < 0) - goto err_unlock_rtnl; - reply_len = ret + ethnl_reply_header_size(); - - rskb = ethnl_reply_init(reply_len, req_info.base.dev, - ETHTOOL_MSG_PHY_GET_REPLY, - ETHTOOL_A_PHY_HEADER, - info, &reply_payload); - if (!rskb) { - ret = -ENOMEM; - goto err_unlock_rtnl; - } - - ret = ethnl_phy_fill_reply(&req_info.base, rskb); - if (ret) - goto err_free_msg; - - rtnl_unlock(); - ethnl_parse_header_dev_put(&req_info.base); - genlmsg_end(rskb, reply_payload); - - return genlmsg_reply(rskb, info); - -err_free_msg: - nlmsg_free(rskb); -err_unlock_rtnl: - rtnl_unlock(); - ethnl_parse_header_dev_put(&req_info.base); - return ret; -} - -struct ethnl_phy_dump_ctx { - struct phy_req_info *phy_req_info; -}; - -int ethnl_phy_start(struct netlink_callback *cb) -{ - const struct genl_dumpit_info *info = genl_dumpit_info(cb); - struct ethnl_phy_dump_ctx *ctx = (void *)cb->ctx; - struct nlattr **tb = info->info.attrs; - int ret; - - BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx)); - - ctx->phy_req_info = kzalloc(sizeof(*ctx->phy_req_info), GFP_KERNEL); - if (!ctx->phy_req_info) - return -ENOMEM; - - ret = ethnl_parse_header_dev_get(&ctx->phy_req_info->base, - tb[ETHTOOL_A_PHY_HEADER], - sock_net(cb->skb->sk), cb->extack, - false); - return ret; -} - -int ethnl_phy_done(struct netlink_callback *cb) -{ - struct ethnl_phy_dump_ctx *ctx = (void *)cb->ctx; - - kfree(ctx->phy_req_info); - - return 0; -} - -static int ethnl_phy_dump_one_dev(struct sk_buff *skb, struct net_device *dev, - struct netlink_callback *cb) -{ - struct ethnl_phy_dump_ctx *ctx = (void *)cb->ctx; - struct phy_req_info *pri = ctx->phy_req_info; - struct phy_device_node *pdn; - unsigned long index = 1; - int ret = 0; - void *ehdr; - - pri->base.dev = dev; - - xa_for_each(&dev->link_topo.phys, index, pdn) { - ehdr = ethnl_dump_put(skb, cb, - ETHTOOL_MSG_PHY_GET_REPLY); - if (!ehdr) { - ret = -EMSGSIZE; - break; - } - - ret = ethnl_fill_reply_header(skb, dev, - ETHTOOL_A_PHY_HEADER); - if (ret < 0) { - genlmsg_cancel(skb, ehdr); - break; - } - - memcpy(&pri->pdn, pdn, sizeof(*pdn)); - ret = ethnl_phy_fill_reply(&pri->base, skb); - - genlmsg_end(skb, ehdr); - } - - return ret; -} - -int ethnl_phy_dumpit(struct sk_buff *skb, struct netlink_callback *cb) -{ - struct ethnl_phy_dump_ctx *ctx = (void *)cb->ctx; - struct net *net = sock_net(skb->sk); - unsigned long ifindex = 1; - struct net_device *dev; - int ret = 0; - - rtnl_lock(); - - if (ctx->phy_req_info->base.dev) { - ret = ethnl_phy_dump_one_dev(skb, ctx->phy_req_info->base.dev, cb); - ethnl_parse_header_dev_put(&ctx->phy_req_info->base); - ctx->phy_req_info->base.dev = NULL; - } else { - for_each_netdev_dump(net, dev, ifindex) { - ret = ethnl_phy_dump_one_dev(skb, dev, cb); - if (ret) - break; - } - } - rtnl_unlock(); - - if (ret == -EMSGSIZE && skb->len) - return skb->len; - return ret; -} - diff --git a/net/ethtool/plca.c b/net/ethtool/plca.c index 2b3e419f4dc2..b1e2e3b5027f 100644 --- a/net/ethtool/plca.c +++ b/net/ethtool/plca.c @@ -61,7 +61,7 @@ static int plca_get_cfg_prepare_data(const struct ethnl_req_info *req_base, int ret; // check that the PHY device is available and connected - if (!req_base->phydev) { + if (!dev->phydev) { ret = -EOPNOTSUPP; goto out; } @@ -80,7 +80,7 @@ static int plca_get_cfg_prepare_data(const struct ethnl_req_info *req_base, memset(&data->plca_cfg, 0xff, sizeof_field(struct plca_reply_data, plca_cfg)); - ret = ops->get_plca_cfg(req_base->phydev, &data->plca_cfg); + ret = ops->get_plca_cfg(dev->phydev, &data->plca_cfg); ethnl_ops_complete(dev); out: @@ -141,6 +141,7 @@ const struct nla_policy ethnl_plca_set_cfg_policy[] = { static int ethnl_set_plca(struct ethnl_req_info *req_info, struct genl_info *info) { + struct net_device *dev = req_info->dev; const struct ethtool_phy_ops *ops; struct nlattr **tb = info->attrs; struct phy_plca_cfg plca_cfg; @@ -148,7 +149,7 @@ ethnl_set_plca(struct ethnl_req_info *req_info, struct genl_info *info) int ret; // check that the PHY device is available and connected - if (!req_info->phydev) + if (!dev->phydev) return -EOPNOTSUPP; ops = ethtool_phy_ops; @@ -167,7 +168,7 @@ ethnl_set_plca(struct ethnl_req_info *req_info, struct genl_info *info) if (!mod) return 0; - ret = ops->set_plca_cfg(req_info->phydev, &plca_cfg, info->extack); + ret = ops->set_plca_cfg(dev->phydev, &plca_cfg, info->extack); return ret < 0 ? ret : 1; } @@ -203,7 +204,7 @@ static int plca_get_status_prepare_data(const struct ethnl_req_info *req_base, int ret; // check that the PHY device is available and connected - if (!req_base->phydev) { + if (!dev->phydev) { ret = -EOPNOTSUPP; goto out; } @@ -222,7 +223,7 @@ static int plca_get_status_prepare_data(const struct ethnl_req_info *req_base, memset(&data->plca_st, 0xff, sizeof_field(struct plca_reply_data, plca_st)); - ret = ops->get_plca_status(req_base->phydev, &data->plca_st); + ret = ops->get_plca_status(dev->phydev, &data->plca_st); ethnl_ops_complete(dev); out: return ret; diff --git a/net/ethtool/pse-pd.c b/net/ethtool/pse-pd.c index 4a1c8d37bd3d..cc478af77111 100644 --- a/net/ethtool/pse-pd.c +++ b/net/ethtool/pse-pd.c @@ -31,10 +31,12 @@ const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1] = { [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), }; -static int pse_get_pse_attributes(struct phy_device *phydev, +static int pse_get_pse_attributes(struct net_device *dev, struct netlink_ext_ack *extack, struct pse_reply_data *data) { + struct phy_device *phydev = dev->phydev; + if (!phydev) { NL_SET_ERR_MSG(extack, "No PHY is attached"); return -EOPNOTSUPP; @@ -62,7 +64,7 @@ static int pse_prepare_data(const struct ethnl_req_info *req_base, if (ret < 0) return ret; - ret = pse_get_pse_attributes(req_base->phydev, info->extack, data); + ret = pse_get_pse_attributes(dev, info->extack, data); ethnl_ops_complete(dev); @@ -122,6 +124,7 @@ ethnl_set_pse_validate(struct ethnl_req_info *req_info, struct genl_info *info) static int ethnl_set_pse(struct ethnl_req_info *req_info, struct genl_info *info) { + struct net_device *dev = req_info->dev; struct pse_control_config config = {}; struct nlattr **tb = info->attrs; struct phy_device *phydev; @@ -129,7 +132,7 @@ ethnl_set_pse(struct ethnl_req_info *req_info, struct genl_info *info) /* this values are already validated by the ethnl_pse_set_policy */ config.admin_cotrol = nla_get_u32(tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]); - phydev = req_info->phydev; + phydev = dev->phydev; if (!phydev) { NL_SET_ERR_MSG(info->extack, "No PHY is attached"); return -EOPNOTSUPP; diff --git a/net/ethtool/strset.c b/net/ethtool/strset.c index 70c00631c51f..c678b484a079 100644 --- a/net/ethtool/strset.c +++ b/net/ethtool/strset.c @@ -233,18 +233,17 @@ static void strset_cleanup_data(struct ethnl_reply_data *reply_base) } static int strset_prepare_set(struct strset_info *info, struct net_device *dev, - struct phy_device *phydev, unsigned int id, - bool counts_only) + unsigned int id, bool counts_only) { const struct ethtool_phy_ops *phy_ops = ethtool_phy_ops; const struct ethtool_ops *ops = dev->ethtool_ops; void *strings; int count, ret; - if (id == ETH_SS_PHY_STATS && phydev && + if (id == ETH_SS_PHY_STATS && dev->phydev && !ops->get_ethtool_phy_stats && phy_ops && phy_ops->get_sset_count) - ret = phy_ops->get_sset_count(phydev); + ret = phy_ops->get_sset_count(dev->phydev); else if (ops->get_sset_count && ops->get_strings) ret = ops->get_sset_count(dev, id); else @@ -259,10 +258,10 @@ static int strset_prepare_set(struct strset_info *info, struct net_device *dev, strings = kcalloc(count, ETH_GSTRING_LEN, GFP_KERNEL); if (!strings) return -ENOMEM; - if (id == ETH_SS_PHY_STATS && phydev && + if (id == ETH_SS_PHY_STATS && dev->phydev && !ops->get_ethtool_phy_stats && phy_ops && phy_ops->get_strings) - phy_ops->get_strings(phydev, strings); + phy_ops->get_strings(dev->phydev, strings); else ops->get_strings(dev, id, strings); info->strings = strings; @@ -306,8 +305,8 @@ static int strset_prepare_data(const struct ethnl_req_info *req_base, !data->sets[i].per_dev) continue; - ret = strset_prepare_set(&data->sets[i], dev, req_base->phydev, - i, req_info->counts_only); + ret = strset_prepare_set(&data->sets[i], dev, i, + req_info->counts_only); if (ret < 0) goto err_ops; } -- cgit v1.2.3 From 19bfcdf9498aa968ea293417fbbc39e523527ca8 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 3 Jan 2024 20:05:44 +0100 Subject: bpf: Relax tracing prog recursive attach rules Currently, it's not allowed to attach an fentry/fexit prog to another one fentry/fexit. At the same time it's not uncommon to see a tracing program with lots of logic in use, and the attachment limitation prevents usage of fentry/fexit for performance analysis (e.g. with "bpftool prog profile" command) in this case. An example could be falcosecurity libs project that uses tp_btf tracing programs. Following the corresponding discussion [1], the reason for that is to avoid tracing progs call cycles without introducing more complex solutions. But currently it seems impossible to load and attach tracing programs in a way that will form such a cycle. The limitation is coming from the fact that attach_prog_fd is specified at the prog load (thus making it impossible to attach to a program loaded after it in this way), as well as tracing progs not implementing link_detach. Replace "no same type" requirement with verification that no more than one level of attachment nesting is allowed. In this way only one fentry/fexit program could be attached to another fentry/fexit to cover profiling use case, and still no cycle could be formed. To implement, add a new field into bpf_prog_aux to track nested attachment for tracing programs. [1]: https://lore.kernel.org/bpf/20191108064039.2041889-16-ast@kernel.org/ Acked-by: Jiri Olsa Acked-by: Song Liu Signed-off-by: Dmitrii Dolgov <9erthalion6@gmail.com> Link: https://lore.kernel.org/r/20240103190559.14750-2-9erthalion6@gmail.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf.h | 1 + kernel/bpf/syscall.c | 23 ++++++++++++++++++++++- kernel/bpf/verifier.c | 39 +++++++++++++++++++++++++-------------- 3 files changed, 48 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7671530d6e4e..e30100597d0a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1449,6 +1449,7 @@ struct bpf_prog_aux { bool dev_bound; /* Program is bound to the netdev. */ bool offload_requested; /* Program is bound and offloaded to the netdev. */ bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ + bool attach_tracing_prog; /* true if tracing another tracing program */ bool func_proto_unreliable; bool sleepable; bool tail_call_reachable; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 1bf9805ee185..d15f9998bc20 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2738,6 +2738,22 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) goto free_prog_sec; } + /* + * Bookkeeping for managing the program attachment chain. + * + * It might be tempting to set attach_tracing_prog flag at the attachment + * time, but this will not prevent from loading bunch of tracing prog + * first, then attach them one to another. + * + * The flag attach_tracing_prog is set for the whole program lifecycle, and + * doesn't have to be cleared in bpf_tracing_link_release, since tracing + * programs cannot change attachment target. + */ + if (type == BPF_PROG_TYPE_TRACING && dst_prog && + dst_prog->type == BPF_PROG_TYPE_TRACING) { + prog->aux->attach_tracing_prog = true; + } + /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) @@ -3171,7 +3187,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, } if (tgt_prog_fd) { - /* For now we only allow new targets for BPF_PROG_TYPE_EXT */ + /* + * For now we only allow new targets for BPF_PROG_TYPE_EXT. If this + * part would be changed to implement the same for + * BPF_PROG_TYPE_TRACING, do not forget to update the way how + * attach_tracing_prog flag is set. + */ if (prog->type != BPF_PROG_TYPE_EXT) { err = -EINVAL; goto out_put_prog; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d5f4ff1eb235..adbf330d364b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20317,6 +20317,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_attach_target_info *tgt_info) { bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; + bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING; const char prefix[] = "btf_trace_"; int ret = 0, subprog = -1, i; const struct btf_type *t; @@ -20387,10 +20388,21 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, bpf_log(log, "Can attach to only JITed progs\n"); return -EINVAL; } - if (tgt_prog->type == prog->type) { - /* Cannot fentry/fexit another fentry/fexit program. - * Cannot attach program extension to another extension. - * It's ok to attach fentry/fexit to extension program. + if (prog_tracing) { + if (aux->attach_tracing_prog) { + /* + * Target program is an fentry/fexit which is already attached + * to another tracing program. More levels of nesting + * attachment are not allowed. + */ + bpf_log(log, "Cannot nest tracing program attach more than once\n"); + return -EINVAL; + } + } else if (tgt_prog->type == prog->type) { + /* + * To avoid potential call chain cycles, prevent attaching of a + * program extension to another extension. It's ok to attach + * fentry/fexit to extension program. */ bpf_log(log, "Cannot recursively attach\n"); return -EINVAL; @@ -20403,16 +20415,15 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, * except fentry/fexit. The reason is the following. * The fentry/fexit programs are used for performance * analysis, stats and can be attached to any program - * type except themselves. When extension program is - * replacing XDP function it is necessary to allow - * performance analysis of all functions. Both original - * XDP program and its program extension. Hence - * attaching fentry/fexit to BPF_PROG_TYPE_EXT is - * allowed. If extending of fentry/fexit was allowed it - * would be possible to create long call chain - * fentry->extension->fentry->extension beyond - * reasonable stack size. Hence extending fentry is not - * allowed. + * type. When extension program is replacing XDP function + * it is necessary to allow performance analysis of all + * functions. Both original XDP program and its program + * extension. Hence attaching fentry/fexit to + * BPF_PROG_TYPE_EXT is allowed. If extending of + * fentry/fexit was allowed it would be possible to create + * long call chain fentry->extension->fentry->extension + * beyond reasonable stack size. Hence extending fentry + * is not allowed. */ bpf_log(log, "Cannot extend fentry/fexit\n"); return -EINVAL; -- cgit v1.2.3 From 8a6286c1804e2c7144aef3154a0357c4b496e10b Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 3 Jan 2024 14:28:36 +0100 Subject: dpll: expose fractional frequency offset value to user Add a new netlink attribute to expose fractional frequency offset value for a pin. Add an op to get the value from the driver. Signed-off-by: Jiri Pirko Acked-by: Vadim Fedorenko Acked-by: Arkadiusz Kubalewski Link: https://lore.kernel.org/r/20240103132838.1501801-2-jiri@resnulli.us Signed-off-by: Jakub Kicinski --- Documentation/netlink/specs/dpll.yaml | 11 +++++++++++ drivers/dpll/dpll_netlink.c | 24 ++++++++++++++++++++++++ include/linux/dpll.h | 3 +++ include/uapi/linux/dpll.h | 1 + 4 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml index cf8abe1c0550..b14aed18065f 100644 --- a/Documentation/netlink/specs/dpll.yaml +++ b/Documentation/netlink/specs/dpll.yaml @@ -296,6 +296,16 @@ attribute-sets: - name: phase-offset type: s64 + - + name: fractional-frequency-offset + type: sint + doc: | + The FFO (Fractional Frequency Offset) between the RX and TX + symbol rate on the media associated with the pin: + (rx_frequency-tx_frequency)/rx_frequency + Value is in PPM (parts per million). + This may be implemented for example for pin of type + PIN_TYPE_SYNCE_ETH_PORT. - name: pin-parent-device subset-of: pin @@ -460,6 +470,7 @@ operations: - phase-adjust-min - phase-adjust-max - phase-adjust + - fractional-frequency-offset dump: pre: dpll-lock-dumpit diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index 21c627e9401a..3370dbddb86b 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -263,6 +263,27 @@ dpll_msg_add_phase_offset(struct sk_buff *msg, struct dpll_pin *pin, return 0; } +static int dpll_msg_add_ffo(struct sk_buff *msg, struct dpll_pin *pin, + struct dpll_pin_ref *ref, + struct netlink_ext_ack *extack) +{ + const struct dpll_pin_ops *ops = dpll_pin_ops(ref); + struct dpll_device *dpll = ref->dpll; + s64 ffo; + int ret; + + if (!ops->ffo_get) + return 0; + ret = ops->ffo_get(pin, dpll_pin_on_dpll_priv(dpll, pin), + dpll, dpll_priv(dpll), &ffo, extack); + if (ret) { + if (ret == -ENODATA) + return 0; + return ret; + } + return nla_put_sint(msg, DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET, ffo); +} + static int dpll_msg_add_pin_freq(struct sk_buff *msg, struct dpll_pin *pin, struct dpll_pin_ref *ref, struct netlink_ext_ack *extack) @@ -440,6 +461,9 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin, prop->phase_range.max)) return -EMSGSIZE; ret = dpll_msg_add_pin_phase_adjust(msg, pin, ref, extack); + if (ret) + return ret; + ret = dpll_msg_add_ffo(msg, pin, ref, extack); if (ret) return ret; if (xa_empty(&pin->parent_refs)) diff --git a/include/linux/dpll.h b/include/linux/dpll.h index b1a5f9ca8ee5..9cf896ea1d41 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -77,6 +77,9 @@ struct dpll_pin_ops { const struct dpll_device *dpll, void *dpll_priv, const s32 phase_adjust, struct netlink_ext_ack *extack); + int (*ffo_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + s64 *ffo, struct netlink_ext_ack *extack); }; struct dpll_pin_frequency { diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h index 715a491d2727..b4e947f9bfbc 100644 --- a/include/uapi/linux/dpll.h +++ b/include/uapi/linux/dpll.h @@ -179,6 +179,7 @@ enum dpll_a_pin { DPLL_A_PIN_PHASE_ADJUST_MAX, DPLL_A_PIN_PHASE_ADJUST, DPLL_A_PIN_PHASE_OFFSET, + DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET, __DPLL_A_PIN_MAX, DPLL_A_PIN_MAX = (__DPLL_A_PIN_MAX - 1) -- cgit v1.2.3 From 477bd4beb93bf9ace9bda71f1437b191befa9cf4 Mon Sep 17 00:00:00 2001 From: Swee Leong Ching Date: Fri, 5 Jan 2024 15:09:23 +0800 Subject: net: stmmac: Make MSI interrupt routine generic There is no support for per DMA channel interrupt for non-MSI platform, where the MAC's per channel interrupt hooks up to interrupt controller(GIC) through shared peripheral interrupt(SPI) to handle interrupt from TX/RX transmit channel. This patch generalize the existing MSI ISR to also support non-MSI platform. Signed-off-by: Teoh Ji Sheng Signed-off-by: Swee Leong Ching Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c | 4 +-- .../net/ethernet/stmicro/stmmac/dwmac-socfpga.c | 3 +++ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c | 2 +- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 30 +++++++++++----------- include/linux/stmmac.h | 4 +-- 5 files changed, 23 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 60283543ffc8..f0ec69af96c9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -952,7 +952,7 @@ static int stmmac_config_single_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, 0); res->wol_irq = res->irq; - plat->flags &= ~STMMAC_FLAG_MULTI_MSI_EN; + plat->flags &= ~STMMAC_FLAG_MULTI_IRQ_EN; dev_info(&pdev->dev, "%s: Single IRQ enablement successful\n", __func__); @@ -1004,7 +1004,7 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) res->sfty_ue_irq = pci_irq_vector(pdev, plat->msi_sfty_ue_vec); - plat->flags |= STMMAC_FLAG_MULTI_MSI_EN; + plat->flags |= STMMAC_FLAG_MULTI_IRQ_EN; dev_info(&pdev->dev, "%s: multi MSI enablement successful\n", __func__); return 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index ba2ce776bd4d..cf43fb3c6cc5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -427,6 +427,9 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) plat_dat->bsp_priv = dwmac; plat_dat->fix_mac_speed = socfpga_dwmac_fix_mac_speed; + if (stmmac_res.rx_irq[0] > 0 && stmmac_res.tx_irq[0] > 0) + plat_dat->flags |= STMMAC_FLAG_MULTI_IRQ_EN; + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (ret) return ret; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index 84d3a8551b03..5f649106ffcd 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -175,7 +175,7 @@ static void dwmac4_dma_init(void __iomem *ioaddr, value = readl(ioaddr + DMA_BUS_MODE); - if (dma_cfg->multi_msi_en) { + if (dma_cfg->multi_irq_en) { value &= ~DMA_BUS_MODE_INTM_MASK; value |= (DMA_BUS_MODE_INTM_MODE1 << DMA_BUS_MODE_INTM_SHIFT); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 47de466e432c..57873b879b33 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -129,8 +129,8 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id); /* For MSI interrupts handling */ static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); -static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); -static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); +static irqreturn_t stmmac_dma_tx_interrupt(int irq, void *data); +static irqreturn_t stmmac_dma_rx_interrupt(int irq, void *data); static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_queues_param(struct stmmac_priv *priv); @@ -3602,7 +3602,7 @@ static void stmmac_free_irq(struct net_device *dev, } } -static int stmmac_request_irq_multi_msi(struct net_device *dev) +static int stmmac_request_irq_multi(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); enum request_irq_err irq_err; @@ -3697,7 +3697,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) } } - /* Request Rx MSI irq */ + /* Request Rx irq */ for (i = 0; i < priv->plat->rx_queues_to_use; i++) { if (i >= MTL_MAX_RX_QUEUES) break; @@ -3707,11 +3707,11 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) int_name = priv->int_name_rx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); ret = request_irq(priv->rx_irq[i], - stmmac_msi_intr_rx, + stmmac_dma_rx_interrupt, 0, int_name, &priv->dma_conf.rx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, - "%s: alloc rx-%d MSI %d (error: %d)\n", + "%s: alloc rx-%d dma rx_irq %d (error: %d)\n", __func__, i, priv->rx_irq[i], ret); irq_err = REQ_IRQ_ERR_RX; irq_idx = i; @@ -3722,7 +3722,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); } - /* Request Tx MSI irq */ + /* Request Tx irq */ for (i = 0; i < priv->plat->tx_queues_to_use; i++) { if (i >= MTL_MAX_TX_QUEUES) break; @@ -3732,11 +3732,11 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) int_name = priv->int_name_tx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); ret = request_irq(priv->tx_irq[i], - stmmac_msi_intr_tx, + stmmac_dma_tx_interrupt, 0, int_name, &priv->dma_conf.tx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, - "%s: alloc tx-%d MSI %d (error: %d)\n", + "%s: alloc tx-%d dma tx_irq %d (error: %d)\n", __func__, i, priv->tx_irq[i], ret); irq_err = REQ_IRQ_ERR_TX; irq_idx = i; @@ -3811,8 +3811,8 @@ static int stmmac_request_irq(struct net_device *dev) int ret; /* Request the IRQ lines */ - if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) - ret = stmmac_request_irq_multi_msi(dev); + if (priv->plat->flags & STMMAC_FLAG_MULTI_IRQ_EN) + ret = stmmac_request_irq_multi(dev); else ret = stmmac_request_irq_single(dev); @@ -6075,7 +6075,7 @@ static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) +static irqreturn_t stmmac_dma_tx_interrupt(int irq, void *data) { struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; struct stmmac_dma_conf *dma_conf; @@ -6107,7 +6107,7 @@ static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) return IRQ_HANDLED; } -static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) +static irqreturn_t stmmac_dma_rx_interrupt(int irq, void *data) { struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; struct stmmac_dma_conf *dma_conf; @@ -7456,8 +7456,8 @@ int stmmac_dvr_probe(struct device *device, priv->plat = plat_dat; priv->ioaddr = res->addr; priv->dev->base_addr = (unsigned long)res->addr; - priv->plat->dma_cfg->multi_msi_en = - (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); + priv->plat->dma_cfg->multi_irq_en = + (priv->plat->flags & STMMAC_FLAG_MULTI_IRQ_EN); priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index dee5ad6e48c5..b950e6f9761d 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -98,7 +98,7 @@ struct stmmac_dma_cfg { int mixed_burst; bool aal; bool eame; - bool multi_msi_en; + bool multi_irq_en; bool dche; }; @@ -215,7 +215,7 @@ struct dwmac4_addrs { #define STMMAC_FLAG_TSO_EN BIT(4) #define STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP BIT(5) #define STMMAC_FLAG_VLAN_FAIL_Q_EN BIT(6) -#define STMMAC_FLAG_MULTI_MSI_EN BIT(7) +#define STMMAC_FLAG_MULTI_IRQ_EN BIT(7) #define STMMAC_FLAG_EXT_SNAPSHOT_EN BIT(8) #define STMMAC_FLAG_INT_SNAPSHOT_EN BIT(9) #define STMMAC_FLAG_RX_CLK_RUNS_IN_LPI BIT(10) -- cgit v1.2.3 From e9ee910218ffd420454b52a052d6f1087354905b Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Sun, 7 Jan 2024 17:11:38 -0800 Subject: Revert "net: stmmac: Enable Per DMA Channel interrupt" Revert "net: stmmac: Use interrupt mode INTM=1 for per channel irq" This reverts commit 36af9f25ddfd311da82628f194c794786467cb12. Revert "net: stmmac: Add support for TX/RX channel interrupt" This reverts commit 9072e03d32088137a435ddf3aa95fd6e038d69d8. Revert "net: stmmac: Make MSI interrupt routine generic" This reverts commit 477bd4beb93bf9ace9bda71f1437b191befa9cf4. Revert "dt-bindings: net: snps,dwmac: per channel irq" This reverts commit 67d47c8ada0f8795bfcdb85cc8f2ad3ce556674b. Device tree bindings need to be reviewed. Link: https://lore.kernel.org/all/2df9fe3e-7971-4aa2-89a9-0e085b3b00d7@linaro.org/ Signed-off-by: Jakub Kicinski --- .../devicetree/bindings/net/snps,dwmac.yaml | 24 +++++----------- drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c | 4 +-- .../net/ethernet/stmicro/stmmac/dwmac-socfpga.c | 3 -- drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c | 2 +- drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h | 3 -- drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c | 32 +++++++++------------- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 30 ++++++++++---------- .../net/ethernet/stmicro/stmmac/stmmac_platform.c | 28 ------------------- include/linux/stmmac.h | 4 +-- 9 files changed, 40 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml index e72dded824f4..5c2769dc689a 100644 --- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml +++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml @@ -103,27 +103,17 @@ properties: interrupts: minItems: 1 - maxItems: 19 + items: + - description: Combined signal for various interrupt events + - description: The interrupt to manage the remote wake-up packet detection + - description: The interrupt that occurs when Rx exits the LPI state interrupt-names: minItems: 1 - maxItems: 19 items: - oneOf: - - description: Combined signal for various interrupt events - const: macirq - - description: The interrupt to manage the remote wake-up packet detection - const: eth_wake_irq - - description: The interrupt that occurs when Rx exits the LPI state - const: eth_lpi - - description: DMA Tx per-channel interrupt - pattern: '^dma_tx[0-7]?$' - - description: DMA Rx per-channel interrupt - pattern: '^dma_rx[0-7]?$' - - allOf: - - contains: - const: macirq + - const: macirq + - enum: [eth_wake_irq, eth_lpi] + - const: eth_lpi clocks: minItems: 1 diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index f0ec69af96c9..60283543ffc8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -952,7 +952,7 @@ static int stmmac_config_single_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, 0); res->wol_irq = res->irq; - plat->flags &= ~STMMAC_FLAG_MULTI_IRQ_EN; + plat->flags &= ~STMMAC_FLAG_MULTI_MSI_EN; dev_info(&pdev->dev, "%s: Single IRQ enablement successful\n", __func__); @@ -1004,7 +1004,7 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) res->sfty_ue_irq = pci_irq_vector(pdev, plat->msi_sfty_ue_vec); - plat->flags |= STMMAC_FLAG_MULTI_IRQ_EN; + plat->flags |= STMMAC_FLAG_MULTI_MSI_EN; dev_info(&pdev->dev, "%s: multi MSI enablement successful\n", __func__); return 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index cf43fb3c6cc5..ba2ce776bd4d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -427,9 +427,6 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) plat_dat->bsp_priv = dwmac; plat_dat->fix_mac_speed = socfpga_dwmac_fix_mac_speed; - if (stmmac_res.rx_irq[0] > 0 && stmmac_res.tx_irq[0] > 0) - plat_dat->flags |= STMMAC_FLAG_MULTI_IRQ_EN; - ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); if (ret) return ret; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index 5f649106ffcd..84d3a8551b03 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -175,7 +175,7 @@ static void dwmac4_dma_init(void __iomem *ioaddr, value = readl(ioaddr + DMA_BUS_MODE); - if (dma_cfg->multi_irq_en) { + if (dma_cfg->multi_msi_en) { value &= ~DMA_BUS_MODE_INTM_MASK; value |= (DMA_BUS_MODE_INTM_MODE1 << DMA_BUS_MODE_INTM_SHIFT); } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h index 04bf731cb7ea..207ff1799f2c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h @@ -346,9 +346,6 @@ /* DMA Registers */ #define XGMAC_DMA_MODE 0x00003000 #define XGMAC_SWR BIT(0) -#define XGMAC_DMA_MODE_INTM_MASK GENMASK(13, 12) -#define XGMAC_DMA_MODE_INTM_SHIFT 12 -#define XGMAC_DMA_MODE_INTM_MODE1 0x1 #define XGMAC_DMA_SYSBUS_MODE 0x00003004 #define XGMAC_WR_OSR_LMT GENMASK(29, 24) #define XGMAC_WR_OSR_LMT_SHIFT 24 diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c index dcb9f094415d..3cde695fec91 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c @@ -31,13 +31,6 @@ static void dwxgmac2_dma_init(void __iomem *ioaddr, value |= XGMAC_EAME; writel(value, ioaddr + XGMAC_DMA_SYSBUS_MODE); - - if (dma_cfg->multi_irq_en) { - value = readl(ioaddr + XGMAC_DMA_MODE); - value &= ~XGMAC_DMA_MODE_INTM_MASK; - value |= (XGMAC_DMA_MODE_INTM_MODE1 << XGMAC_DMA_MODE_INTM_SHIFT); - writel(value, ioaddr + XGMAC_DMA_MODE); - } } static void dwxgmac2_dma_init_chan(struct stmmac_priv *priv, @@ -372,18 +365,19 @@ static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv, } /* TX/RX NORMAL interrupts */ - if (likely(intr_status & XGMAC_RI)) { - u64_stats_update_begin(&rxq_stats->syncp); - rxq_stats->rx_normal_irq_n++; - u64_stats_update_end(&rxq_stats->syncp); - ret |= handle_rx; - } - - if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) { - u64_stats_update_begin(&txq_stats->syncp); - txq_stats->tx_normal_irq_n++; - u64_stats_update_end(&txq_stats->syncp); - ret |= handle_tx; + if (likely(intr_status & XGMAC_NIS)) { + if (likely(intr_status & XGMAC_RI)) { + u64_stats_update_begin(&rxq_stats->syncp); + rxq_stats->rx_normal_irq_n++; + u64_stats_update_end(&rxq_stats->syncp); + ret |= handle_rx; + } + if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) { + u64_stats_update_begin(&txq_stats->syncp); + txq_stats->tx_normal_irq_n++; + u64_stats_update_end(&txq_stats->syncp); + ret |= handle_tx; + } } /* Clear interrupts */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 57873b879b33..47de466e432c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -129,8 +129,8 @@ static irqreturn_t stmmac_interrupt(int irq, void *dev_id); /* For MSI interrupts handling */ static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); -static irqreturn_t stmmac_dma_tx_interrupt(int irq, void *data); -static irqreturn_t stmmac_dma_rx_interrupt(int irq, void *data); +static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); +static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); static void stmmac_reset_queues_param(struct stmmac_priv *priv); @@ -3602,7 +3602,7 @@ static void stmmac_free_irq(struct net_device *dev, } } -static int stmmac_request_irq_multi(struct net_device *dev) +static int stmmac_request_irq_multi_msi(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); enum request_irq_err irq_err; @@ -3697,7 +3697,7 @@ static int stmmac_request_irq_multi(struct net_device *dev) } } - /* Request Rx irq */ + /* Request Rx MSI irq */ for (i = 0; i < priv->plat->rx_queues_to_use; i++) { if (i >= MTL_MAX_RX_QUEUES) break; @@ -3707,11 +3707,11 @@ static int stmmac_request_irq_multi(struct net_device *dev) int_name = priv->int_name_rx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); ret = request_irq(priv->rx_irq[i], - stmmac_dma_rx_interrupt, + stmmac_msi_intr_rx, 0, int_name, &priv->dma_conf.rx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, - "%s: alloc rx-%d dma rx_irq %d (error: %d)\n", + "%s: alloc rx-%d MSI %d (error: %d)\n", __func__, i, priv->rx_irq[i], ret); irq_err = REQ_IRQ_ERR_RX; irq_idx = i; @@ -3722,7 +3722,7 @@ static int stmmac_request_irq_multi(struct net_device *dev) irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); } - /* Request Tx irq */ + /* Request Tx MSI irq */ for (i = 0; i < priv->plat->tx_queues_to_use; i++) { if (i >= MTL_MAX_TX_QUEUES) break; @@ -3732,11 +3732,11 @@ static int stmmac_request_irq_multi(struct net_device *dev) int_name = priv->int_name_tx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); ret = request_irq(priv->tx_irq[i], - stmmac_dma_tx_interrupt, + stmmac_msi_intr_tx, 0, int_name, &priv->dma_conf.tx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, - "%s: alloc tx-%d dma tx_irq %d (error: %d)\n", + "%s: alloc tx-%d MSI %d (error: %d)\n", __func__, i, priv->tx_irq[i], ret); irq_err = REQ_IRQ_ERR_TX; irq_idx = i; @@ -3811,8 +3811,8 @@ static int stmmac_request_irq(struct net_device *dev) int ret; /* Request the IRQ lines */ - if (priv->plat->flags & STMMAC_FLAG_MULTI_IRQ_EN) - ret = stmmac_request_irq_multi(dev); + if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) + ret = stmmac_request_irq_multi_msi(dev); else ret = stmmac_request_irq_single(dev); @@ -6075,7 +6075,7 @@ static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static irqreturn_t stmmac_dma_tx_interrupt(int irq, void *data) +static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) { struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; struct stmmac_dma_conf *dma_conf; @@ -6107,7 +6107,7 @@ static irqreturn_t stmmac_dma_tx_interrupt(int irq, void *data) return IRQ_HANDLED; } -static irqreturn_t stmmac_dma_rx_interrupt(int irq, void *data) +static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) { struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; struct stmmac_dma_conf *dma_conf; @@ -7456,8 +7456,8 @@ int stmmac_dvr_probe(struct device *device, priv->plat = plat_dat; priv->ioaddr = res->addr; priv->dev->base_addr = (unsigned long)res->addr; - priv->plat->dma_cfg->multi_irq_en = - (priv->plat->flags & STMMAC_FLAG_MULTI_IRQ_EN); + priv->plat->dma_cfg->multi_msi_en = + (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index ae6859153e98..70eadc83ca68 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -710,10 +710,6 @@ EXPORT_SYMBOL_GPL(devm_stmmac_probe_config_dt); int stmmac_get_platform_resources(struct platform_device *pdev, struct stmmac_resources *stmmac_res) { - char irq_name[9]; - int i; - int irq; - memset(stmmac_res, 0, sizeof(*stmmac_res)); /* Get IRQ information early to have an ability to ask for deferred @@ -747,30 +743,6 @@ int stmmac_get_platform_resources(struct platform_device *pdev, dev_info(&pdev->dev, "IRQ eth_lpi not found\n"); } - /* For RX Channel */ - for (i = 0; i < MTL_MAX_RX_QUEUES; i++) { - snprintf(irq_name, sizeof(irq_name), "dma_rx%i", i); - irq = platform_get_irq_byname_optional(pdev, irq_name); - if (irq == -EPROBE_DEFER) - return irq; - else if (irq < 0) - break; - - stmmac_res->rx_irq[i] = irq; - } - - /* For TX Channel */ - for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { - snprintf(irq_name, sizeof(irq_name), "dma_tx%i", i); - irq = platform_get_irq_byname_optional(pdev, irq_name); - if (irq == -EPROBE_DEFER) - return irq; - else if (irq < 0) - break; - - stmmac_res->tx_irq[i] = irq; - } - stmmac_res->addr = devm_platform_ioremap_resource(pdev, 0); return PTR_ERR_OR_ZERO(stmmac_res->addr); diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index b950e6f9761d..dee5ad6e48c5 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -98,7 +98,7 @@ struct stmmac_dma_cfg { int mixed_burst; bool aal; bool eame; - bool multi_irq_en; + bool multi_msi_en; bool dche; }; @@ -215,7 +215,7 @@ struct dwmac4_addrs { #define STMMAC_FLAG_TSO_EN BIT(4) #define STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP BIT(5) #define STMMAC_FLAG_VLAN_FAIL_Q_EN BIT(6) -#define STMMAC_FLAG_MULTI_IRQ_EN BIT(7) +#define STMMAC_FLAG_MULTI_MSI_EN BIT(7) #define STMMAC_FLAG_EXT_SNAPSHOT_EN BIT(8) #define STMMAC_FLAG_INT_SNAPSHOT_EN BIT(9) #define STMMAC_FLAG_RX_CLK_RUNS_IN_LPI BIT(10) -- cgit v1.2.3 From 3fbf61207c66ff7ac9b60ab76d4bfd239f97e973 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Sun, 7 Jan 2024 17:14:51 -0800 Subject: Revert "mlx5 updates 2023-12-20" Revert "net/mlx5: Implement management PF Ethernet profile" This reverts commit 22c4640698a1d47606b5a4264a584e8046641784. Revert "net/mlx5: Enable SD feature" This reverts commit c88c49ac9c18fb7c3fa431126de1d8f8f555e912. Revert "net/mlx5e: Block TLS device offload on combined SD netdev" This reverts commit 83a59ce0057b7753d7fbece194b89622c663b2a6. Revert "net/mlx5e: Support per-mdev queue counter" This reverts commit d72baceb92539a178d2610b0e9ceb75706a75b55. Revert "net/mlx5e: Support cross-vhca RSS" This reverts commit c73a3ab8fa6e93a783bd563938d7cf00d62d5d34. Revert "net/mlx5e: Let channels be SD-aware" This reverts commit e4f9686bdee7b4dd89e0ed63cd03606e4bda4ced. Revert "net/mlx5e: Create EN core HW resources for all secondary devices" This reverts commit c4fb94aa822d6c9d05fc3c5aee35c7e339061dc1. Revert "net/mlx5e: Create single netdev per SD group" This reverts commit e2578b4f983cfcd47837bbe3bcdbf5920e50b2ad. Revert "net/mlx5: SD, Add informative prints in kernel log" This reverts commit c82d360325112ccc512fc11a3b68cdcdf04a1478. Revert "net/mlx5: SD, Implement steering for primary and secondaries" This reverts commit 605fcce33b2d1beb0139b6e5913fa0b2062116b2. Revert "net/mlx5: SD, Implement devcom communication and primary election" This reverts commit a45af9a96740873db9a4b5bb493ce2ad81ccb4d5. Revert "net/mlx5: SD, Implement basic query and instantiation" This reverts commit 63b9ce944c0e26c44c42cdd5095c2e9851c1a8ff. Revert "net/mlx5: SD, Introduce SD lib" This reverts commit 4a04a31f49320d078b8078e1da4b0e2faca5dfa3. Revert "net/mlx5: Fix query of sd_group field" This reverts commit e04984a37398b3f4f5a79c993b94c6b1224184cc. Revert "net/mlx5e: Use the correct lag ports number when creating TISes" This reverts commit a7e7b40c4bc115dbf2a2bb453d7bbb2e0ea99703. There are some unanswered questions on the list, and we don't have any docs. Given the lack of replies so far and the fact that v6.8 merge window has started - let's revert this and revisit for v6.9. Link: https://lore.kernel.org/all/20231221005721.186607-1-saeed@kernel.org/ Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/dev.c | 3 - drivers/net/ethernet/mellanox/mlx5/core/ecpf.c | 6 - drivers/net/ethernet/mellanox/mlx5/core/en.h | 15 +- .../net/ethernet/mellanox/mlx5/core/en/channels.c | 10 +- .../net/ethernet/mellanox/mlx5/core/en/channels.h | 6 +- .../net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c | 268 ------------ .../ethernet/mellanox/mlx5/core/en/monitor_stats.c | 48 +- .../net/ethernet/mellanox/mlx5/core/en/params.c | 9 +- .../net/ethernet/mellanox/mlx5/core/en/params.h | 3 + drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c | 12 +- drivers/net/ethernet/mellanox/mlx5/core/en/qos.c | 8 +- .../ethernet/mellanox/mlx5/core/en/reporter_rx.c | 4 +- .../ethernet/mellanox/mlx5/core/en/reporter_tx.c | 3 +- drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c | 123 +----- drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h | 9 +- drivers/net/ethernet/mellanox/mlx5/core/en/rss.c | 17 +- drivers/net/ethernet/mellanox/mlx5/core/en/rss.h | 4 +- .../net/ethernet/mellanox/mlx5/core/en/rx_res.c | 62 +-- .../net/ethernet/mellanox/mlx5/core/en/rx_res.h | 1 - drivers/net/ethernet/mellanox/mlx5/core/en/trap.c | 11 +- .../net/ethernet/mellanox/mlx5/core/en/xsk/pool.c | 6 +- .../net/ethernet/mellanox/mlx5/core/en/xsk/setup.c | 8 +- .../ethernet/mellanox/mlx5/core/en_accel/ktls.c | 2 +- .../ethernet/mellanox/mlx5/core/en_accel/ktls.h | 4 +- .../ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c | 6 +- .../net/ethernet/mellanox/mlx5/core/en_common.c | 21 +- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 200 ++------- drivers/net/ethernet/mellanox/mlx5/core/en_stats.c | 39 +- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 4 +- drivers/net/ethernet/mellanox/mlx5/core/eswitch.c | 2 +- .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c | 2 +- .../net/ethernet/mellanox/mlx5/core/lib/devcom.h | 1 - drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h | 12 - drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c | 487 --------------------- drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h | 38 -- drivers/net/ethernet/mellanox/mlx5/core/vport.c | 21 - include/linux/mlx5/driver.h | 10 - include/linux/mlx5/mlx5_ifc.h | 24 +- include/linux/mlx5/vport.h | 1 - 40 files changed, 192 insertions(+), 1320 deletions(-) delete mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c delete mode 100644 drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c delete mode 100644 drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index f36232dead1a..c44870b175f9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -29,7 +29,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ - en/mgmt_pf.o lib/crypto.o lib/sd.o + lib/crypto.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index aa1b471e13fa..cf0477f53dc4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -190,9 +190,6 @@ bool mlx5_rdma_supported(struct mlx5_core_dev *dev) if (is_mp_supported(dev)) return false; - if (mlx5_core_is_mgmt_pf(dev)) - return false; - return true; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c index aa397e3ebe6d..d000236ddbac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c @@ -75,9 +75,6 @@ int mlx5_ec_init(struct mlx5_core_dev *dev) if (!mlx5_core_is_ecpf(dev)) return 0; - if (mlx5_core_is_mgmt_pf(dev)) - return 0; - return mlx5_host_pf_init(dev); } @@ -88,9 +85,6 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev) if (!mlx5_core_is_ecpf(dev)) return; - if (mlx5_core_is_mgmt_pf(dev)) - return; - mlx5_host_pf_cleanup(dev); err = mlx5_wait_for_pages(dev, &dev->priv.page_counters[MLX5_HOST_PF]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 922b63c25154..0bfe1ca8a364 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -60,10 +60,8 @@ #include "lib/clock.h" #include "en/rx_res.h" #include "en/selq.h" -#include "lib/sd.h" extern const struct net_device_ops mlx5e_netdev_ops; -extern const struct net_device_ops mlx5e_mgmt_netdev_ops; struct page_pool; #define MLX5E_METADATA_ETHER_TYPE (0x8CE4) @@ -793,8 +791,6 @@ struct mlx5e_channel { struct hwtstamp_config *tstamp; DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES); int ix; - int vec_ix; - int sd_ix; int cpu; /* Sync between icosq recovery and XSK enable/disable. */ struct mutex icosq_recovery_lock; @@ -918,7 +914,7 @@ struct mlx5e_priv { bool tx_ptp_opened; bool rx_ptp_opened; struct hwtstamp_config tstamp; - u16 q_counter[MLX5_SD_MAX_GROUP_SZ]; + u16 q_counter; u16 drop_rq_q_counter; struct notifier_block events_nb; struct notifier_block blocking_events_nb; @@ -1033,12 +1029,12 @@ struct mlx5e_xsk_param; struct mlx5e_rq_param; int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, + struct mlx5e_xsk_param *xsk, int node, struct mlx5e_rq *rq); #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_close_rq(struct mlx5e_rq *rq); -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter); +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param); void mlx5e_destroy_rq(struct mlx5e_rq *rq); struct mlx5e_sq_param; @@ -1126,10 +1122,9 @@ static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) } extern const struct ethtool_ops mlx5e_ethtool_ops; -extern const struct mlx5e_profile mlx5e_mgmt_pf_nic_profile; int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); -int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises); +int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev); void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev); int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb, bool enable_mc_lb); @@ -1232,8 +1227,6 @@ netdev_features_t mlx5e_features_check(struct sk_buff *skb, struct net_device *netdev, netdev_features_t features); int mlx5e_set_features(struct net_device *netdev, netdev_features_t features); -void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv); - #ifdef CONFIG_MLX5_ESWITCH int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac); int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c index 874a1016623c..48581ea3adcb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c @@ -23,26 +23,20 @@ bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix) return test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); } -void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, - u32 *vhca_id) +void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) { struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); *rqn = c->rq.rqn; - if (vhca_id) - *vhca_id = MLX5_CAP_GEN(c->mdev, vhca_id); } -void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, - u32 *vhca_id) +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) { struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); WARN_ON_ONCE(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)); *rqn = c->xskrq.rqn; - if (vhca_id) - *vhca_id = MLX5_CAP_GEN(c->mdev, vhca_id); } bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h index 6715aa9383b9..637ca90daaa8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h @@ -10,10 +10,8 @@ struct mlx5e_channels; unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs); bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix); -void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, - u32 *vhca_id); -void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn, - u32 *vhca_id); +void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn); #endif /* __MLX5_EN_CHANNELS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c b/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c deleted file mode 100644 index 77b5805895b9..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/mgmt_pf.c +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB -// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -#include -#include "en/params.h" -#include "en/health.h" -#include "lib/eq.h" -#include "en/dcbnl.h" -#include "en_accel/ipsec.h" -#include "en_accel/en_accel.h" -#include "en/trap.h" -#include "en/monitor_stats.h" -#include "en/hv_vhca_stats.h" -#include "en_rep.h" -#include "en.h" - -static int mgmt_pf_async_event(struct notifier_block *nb, unsigned long event, void *data) -{ - struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); - struct mlx5_eqe *eqe = data; - - if (event != MLX5_EVENT_TYPE_PORT_CHANGE) - return NOTIFY_DONE; - - switch (eqe->sub_type) { - case MLX5_PORT_CHANGE_SUBTYPE_DOWN: - case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: - queue_work(priv->wq, &priv->update_carrier_work); - break; - default: - return NOTIFY_DONE; - } - - return NOTIFY_OK; -} - -static void mlx5e_mgmt_pf_enable_async_events(struct mlx5e_priv *priv) -{ - priv->events_nb.notifier_call = mgmt_pf_async_event; - mlx5_notifier_register(priv->mdev, &priv->events_nb); -} - -static void mlx5e_disable_mgmt_pf_async_events(struct mlx5e_priv *priv) -{ - mlx5_notifier_unregister(priv->mdev, &priv->events_nb); -} - -static void mlx5e_modify_mgmt_pf_admin_state(struct mlx5_core_dev *mdev, - enum mlx5_port_status state) -{ - struct mlx5_eswitch *esw = mdev->priv.eswitch; - int vport_admin_state; - - mlx5_set_port_admin_status(mdev, state); - - if (state == MLX5_PORT_UP) - vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; - else - vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; - - mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); -} - -static void mlx5e_build_mgmt_pf_nic_params(struct mlx5e_priv *priv, u16 mtu) -{ - struct mlx5e_params *params = &priv->channels.params; - struct mlx5_core_dev *mdev = priv->mdev; - u8 rx_cq_period_mode; - - params->sw_mtu = mtu; - params->hard_mtu = MLX5E_ETH_HARD_MTU; - params->num_channels = 1; - - /* SQ */ - params->log_sq_size = is_kdump_kernel() ? - MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : - MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; - MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); - - MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); - - /* RQ */ - mlx5e_build_rq_params(mdev, params); - - /* CQ moderation params */ - rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? - MLX5_CQ_PERIOD_MODE_START_FROM_CQE : - MLX5_CQ_PERIOD_MODE_START_FROM_EQE; - params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); - params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); - mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); - mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); - - /* TX inline */ - mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); -} - -static int mlx5e_mgmt_pf_init(struct mlx5_core_dev *mdev, - struct net_device *netdev) -{ - struct mlx5e_priv *priv = netdev_priv(netdev); - struct mlx5e_flow_steering *fs; - int err; - - mlx5e_build_mgmt_pf_nic_params(priv, netdev->mtu); - - mlx5e_timestamp_init(priv); - - fs = mlx5e_fs_init(priv->profile, mdev, - !test_bit(MLX5E_STATE_DESTROYING, &priv->state), - priv->dfs_root); - if (!fs) { - err = -ENOMEM; - mlx5_core_err(mdev, "FS initialization failed, %d\n", err); - return err; - } - priv->fs = fs; - - mlx5e_health_create_reporters(priv); - - return 0; -} - -static void mlx5e_mgmt_pf_cleanup(struct mlx5e_priv *priv) -{ - mlx5e_health_destroy_reporters(priv); - mlx5e_fs_cleanup(priv->fs); - priv->fs = NULL; -} - -static int mlx5e_mgmt_pf_init_rx(struct mlx5e_priv *priv) -{ - struct mlx5_core_dev *mdev = priv->mdev; - int err; - - priv->rx_res = mlx5e_rx_res_create(mdev, 0, priv->max_nch, priv->drop_rq.rqn, - &priv->channels.params.packet_merge, - priv->channels.params.num_channels); - if (!priv->rx_res) - return -ENOMEM; - - mlx5e_create_q_counters(priv); - - err = mlx5e_open_drop_rq(priv, &priv->drop_rq); - if (err) { - mlx5_core_err(mdev, "open drop rq failed, %d\n", err); - goto err_destroy_q_counters; - } - - err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, - priv->netdev); - if (err) { - mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); - goto err_destroy_rx_res; - } - - return 0; - -err_destroy_rx_res: - mlx5e_rx_res_destroy(priv->rx_res); - priv->rx_res = NULL; - mlx5e_close_drop_rq(&priv->drop_rq); -err_destroy_q_counters: - mlx5e_destroy_q_counters(priv); - return err; -} - -static void mlx5e_mgmt_pf_cleanup_rx(struct mlx5e_priv *priv) -{ - mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), - priv->profile); - mlx5e_rx_res_destroy(priv->rx_res); - priv->rx_res = NULL; - mlx5e_close_drop_rq(&priv->drop_rq); - mlx5e_destroy_q_counters(priv); -} - -static int mlx5e_mgmt_pf_init_tx(struct mlx5e_priv *priv) -{ - return 0; -} - -static void mlx5e_mgmt_pf_cleanup_tx(struct mlx5e_priv *priv) -{ -} - -static void mlx5e_mgmt_pf_enable(struct mlx5e_priv *priv) -{ - struct net_device *netdev = priv->netdev; - struct mlx5_core_dev *mdev = priv->mdev; - - mlx5e_fs_init_l2_addr(priv->fs, netdev); - - /* Marking the link as currently not needed by the Driver */ - if (!netif_running(netdev)) - mlx5e_modify_mgmt_pf_admin_state(mdev, MLX5_PORT_DOWN); - - mlx5e_set_netdev_mtu_boundaries(priv); - mlx5e_set_dev_port_mtu(priv); - - mlx5e_mgmt_pf_enable_async_events(priv); - if (mlx5e_monitor_counter_supported(priv)) - mlx5e_monitor_counter_init(priv); - - mlx5e_hv_vhca_stats_create(priv); - if (netdev->reg_state != NETREG_REGISTERED) - return; - mlx5e_dcbnl_init_app(priv); - - mlx5e_nic_set_rx_mode(priv); - - rtnl_lock(); - if (netif_running(netdev)) - mlx5e_open(netdev); - udp_tunnel_nic_reset_ntf(priv->netdev); - netif_device_attach(netdev); - rtnl_unlock(); -} - -static void mlx5e_mgmt_pf_disable(struct mlx5e_priv *priv) -{ - if (priv->netdev->reg_state == NETREG_REGISTERED) - mlx5e_dcbnl_delete_app(priv); - - rtnl_lock(); - if (netif_running(priv->netdev)) - mlx5e_close(priv->netdev); - netif_device_detach(priv->netdev); - rtnl_unlock(); - - mlx5e_nic_set_rx_mode(priv); - - mlx5e_hv_vhca_stats_destroy(priv); - if (mlx5e_monitor_counter_supported(priv)) - mlx5e_monitor_counter_cleanup(priv); - - mlx5e_disable_mgmt_pf_async_events(priv); - mlx5e_ipsec_cleanup(priv); -} - -static int mlx5e_mgmt_pf_update_rx(struct mlx5e_priv *priv) -{ - return mlx5e_refresh_tirs(priv, false, false); -} - -static int mlx5e_mgmt_pf_max_nch_limit(struct mlx5_core_dev *mdev) -{ - return 1; -} - -const struct mlx5e_profile mlx5e_mgmt_pf_nic_profile = { - .init = mlx5e_mgmt_pf_init, - .cleanup = mlx5e_mgmt_pf_cleanup, - .init_rx = mlx5e_mgmt_pf_init_rx, - .cleanup_rx = mlx5e_mgmt_pf_cleanup_rx, - .init_tx = mlx5e_mgmt_pf_init_tx, - .cleanup_tx = mlx5e_mgmt_pf_cleanup_tx, - .enable = mlx5e_mgmt_pf_enable, - .disable = mlx5e_mgmt_pf_disable, - .update_rx = mlx5e_mgmt_pf_update_rx, - .update_stats = mlx5e_stats_update_ndo_stats, - .update_carrier = mlx5e_update_carrier, - .rx_handlers = &mlx5e_rx_handlers_nic, - .max_tc = 1, - .max_nch_limit = mlx5e_mgmt_pf_max_nch_limit, - .stats_grps = mlx5e_nic_stats_grps, - .stats_grps_num = mlx5e_nic_stats_grps_num -}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c index e2d8d2754be0..40c8df111754 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/monitor_stats.c @@ -20,8 +20,10 @@ #define NUM_REQ_PPCNT_COUNTER_S1 MLX5_CMD_SET_MONITOR_NUM_PPCNT_COUNTER_SET1 #define NUM_REQ_Q_COUNTERS_S1 MLX5_CMD_SET_MONITOR_NUM_Q_COUNTERS_SET1 -static int mlx5e_monitor_counter_cap(struct mlx5_core_dev *mdev) +int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) { + struct mlx5_core_dev *mdev = priv->mdev; + if (!MLX5_CAP_GEN(mdev, max_num_of_monitor_counters)) return false; if (MLX5_CAP_PCAM_REG(mdev, ppcnt) && @@ -34,38 +36,24 @@ static int mlx5e_monitor_counter_cap(struct mlx5_core_dev *mdev) return true; } -int mlx5e_monitor_counter_supported(struct mlx5e_priv *priv) -{ - struct mlx5_core_dev *pos; - int i; - - mlx5_sd_for_each_dev(i, priv->mdev, pos) - if (!mlx5e_monitor_counter_cap(pos)) - return false; - return true; -} - -static void mlx5e_monitor_counter_arm(struct mlx5_core_dev *mdev) +static void mlx5e_monitor_counter_arm(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(arm_monitor_counter_in)] = {}; MLX5_SET(arm_monitor_counter_in, in, opcode, MLX5_CMD_OP_ARM_MONITOR_COUNTER); - mlx5_cmd_exec_in(mdev, arm_monitor_counter, in); + mlx5_cmd_exec_in(priv->mdev, arm_monitor_counter, in); } static void mlx5e_monitor_counters_work(struct work_struct *work) { struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, monitor_counters_work); - struct mlx5_core_dev *pos; - int i; mutex_lock(&priv->state_lock); mlx5e_stats_update_ndo_stats(priv); mutex_unlock(&priv->state_lock); - mlx5_sd_for_each_dev(i, priv->mdev, pos) - mlx5e_monitor_counter_arm(pos); + mlx5e_monitor_counter_arm(priv); } static int mlx5e_monitor_event_handler(struct notifier_block *nb, @@ -109,13 +97,15 @@ static int fill_monitor_counter_q_counter_set1(int cnt, int q_counter, u32 *in) } /* check if mlx5e_monitor_counter_supported before calling this function*/ -static void mlx5e_set_monitor_counter(struct mlx5_core_dev *mdev, int q_counter) +static void mlx5e_set_monitor_counter(struct mlx5e_priv *priv) { + struct mlx5_core_dev *mdev = priv->mdev; int max_num_of_counters = MLX5_CAP_GEN(mdev, max_num_of_monitor_counters); int num_q_counters = MLX5_CAP_GEN(mdev, num_q_monitor_counters); int num_ppcnt_counters = !MLX5_CAP_PCAM_REG(mdev, ppcnt) ? 0 : MLX5_CAP_GEN(mdev, num_ppcnt_monitor_counters); u32 in[MLX5_ST_SZ_DW(set_monitor_counter_in)] = {}; + int q_counter = priv->q_counter; int cnt = 0; if (num_ppcnt_counters >= NUM_REQ_PPCNT_COUNTER_S1 && @@ -137,17 +127,13 @@ static void mlx5e_set_monitor_counter(struct mlx5_core_dev *mdev, int q_counter) /* check if mlx5e_monitor_counter_supported before calling this function*/ void mlx5e_monitor_counter_init(struct mlx5e_priv *priv) { - struct mlx5_core_dev *pos; - int i; - INIT_WORK(&priv->monitor_counters_work, mlx5e_monitor_counters_work); MLX5_NB_INIT(&priv->monitor_counters_nb, mlx5e_monitor_event_handler, MONITOR_COUNTER); - mlx5_sd_for_each_dev(i, priv->mdev, pos) { - mlx5_eq_notifier_register(pos, &priv->monitor_counters_nb); - mlx5e_set_monitor_counter(pos, priv->q_counter[i]); - mlx5e_monitor_counter_arm(pos); - } + mlx5_eq_notifier_register(priv->mdev, &priv->monitor_counters_nb); + + mlx5e_set_monitor_counter(priv); + mlx5e_monitor_counter_arm(priv); queue_work(priv->wq, &priv->update_stats_work); } @@ -155,15 +141,11 @@ void mlx5e_monitor_counter_init(struct mlx5e_priv *priv) void mlx5e_monitor_counter_cleanup(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(set_monitor_counter_in)] = {}; - struct mlx5_core_dev *pos; - int i; MLX5_SET(set_monitor_counter_in, in, opcode, MLX5_CMD_OP_SET_MONITOR_COUNTER); - mlx5_sd_for_each_dev(i, priv->mdev, pos) { - mlx5_cmd_exec_in(pos, set_monitor_counter, in); - mlx5_eq_notifier_unregister(pos, &priv->monitor_counters_nb); - } + mlx5_cmd_exec_in(priv->mdev, set_monitor_counter, in); + mlx5_eq_notifier_unregister(priv->mdev, &priv->monitor_counters_nb); cancel_work_sync(&priv->monitor_counters_work); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index fb10bb166fbb..284253b79266 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -674,7 +674,7 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e .napi = &c->napi, .ch_stats = c->stats, .node = cpu_to_node(c->cpu), - .ix = c->vec_ix, + .ix = c->ix, }; } @@ -945,6 +945,7 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, + u16 q_counter, struct mlx5e_rq_param *param) { void *rqc = param->rqc; @@ -1006,6 +1007,7 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); MLX5_SET(wq, wq, pd, mdev->mlx5e_res.hw_objs.pdn); + MLX5_SET(rqc, rqc, counter_set_id, q_counter); MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); @@ -1016,6 +1018,7 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, } void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, + u16 q_counter, struct mlx5e_rq_param *param) { void *rqc = param->rqc; @@ -1024,6 +1027,7 @@ void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); + MLX5_SET(rqc, rqc, counter_set_id, q_counter); param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); } @@ -1288,12 +1292,13 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + u16 q_counter, struct mlx5e_channel_param *cparam) { u8 icosq_log_wq_sz, async_icosq_log_wq_sz; int err; - err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq); + err = mlx5e_build_rq_param(mdev, params, NULL, q_counter, &cparam->rq); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 9a781f18b57f..6800949dafbc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -130,8 +130,10 @@ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, + u16 q_counter, struct mlx5e_rq_param *param); void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, + u16 q_counter, struct mlx5e_rq_param *param); void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); @@ -147,6 +149,7 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + u16 q_counter, struct mlx5e_channel_param *cparam); u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index cafb41895f94..c206cc0a8483 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -646,6 +646,7 @@ static void mlx5e_ptp_build_sq_param(struct mlx5_core_dev *mdev, static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, struct net_device *netdev, + u16 q_counter, struct mlx5e_ptp_params *ptp_params) { struct mlx5e_rq_param *rq_params = &ptp_params->rq_param; @@ -654,7 +655,7 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = netdev->max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, rq_params); + mlx5e_build_rq_param(mdev, params, NULL, q_counter, rq_params); } static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, @@ -680,7 +681,7 @@ static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, /* RQ */ if (test_bit(MLX5E_PTP_STATE_RX, c->state)) { params->vlan_strip_disable = orig->vlan_strip_disable; - mlx5e_ptp_build_rq_param(c->mdev, c->netdev, cparams); + mlx5e_ptp_build_rq_param(c->mdev, c->netdev, c->priv->q_counter, cparams); } } @@ -713,16 +714,13 @@ static int mlx5e_ptp_open_rq(struct mlx5e_ptp *c, struct mlx5e_params *params, struct mlx5e_rq_param *rq_param) { int node = dev_to_node(c->mdev->device); - int err, sd_ix; - u16 q_counter; + int err; err = mlx5e_init_ptp_rq(c, params, &c->rq); if (err) return err; - sd_ix = mlx5_sd_ch_ix_get_dev_ix(c->mdev, MLX5E_PTP_CHANNEL_IX); - q_counter = c->priv->q_counter[sd_ix]; - return mlx5e_open_rq(params, rq_param, NULL, node, q_counter, &c->rq); + return mlx5e_open_rq(params, rq_param, NULL, node, &c->rq); } static int mlx5e_ptp_open_queues(struct mlx5e_ptp *c, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index e87e26f2c669..34adf8c3f81a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -122,8 +122,8 @@ int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, memset(¶m_sq, 0, sizeof(param_sq)); memset(¶m_cq, 0, sizeof(param_cq)); - mlx5e_build_sq_param(c->mdev, params, ¶m_sq); - mlx5e_build_tx_cq_param(c->mdev, params, ¶m_cq); + mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); + mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); if (err) goto err_free_sq; @@ -176,7 +176,7 @@ int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) */ smp_wmb(); - qos_dbg(sq->mdev, "Activate QoS SQ qid %u\n", node_qid); + qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); mlx5e_activate_txqsq(sq); return 0; @@ -190,7 +190,7 @@ void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) if (!sq) /* Handle the case when the SQ failed to open. */ return; - qos_dbg(sq->mdev, "Deactivate QoS SQ qid %u\n", qid); + qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); mlx5e_deactivate_txqsq(sq); priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index 25d751eba99b..4358798d6ce1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -294,8 +294,8 @@ static void mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, params = &priv->channels.params; rq_sz = mlx5e_rqwq_get_size(rq); - real_time = mlx5_is_real_time_rq(rq->mdev); - rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(rq->mdev, params, NULL)); + real_time = mlx5_is_real_time_rq(priv->mdev); + rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 0ab9db319530..6b44ddce14e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -219,6 +219,7 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, struct mlx5e_txqsq *sq, int tc) { bool stopped = netif_xmit_stopped(sq->txq); + struct mlx5e_priv *priv = sq->priv; u8 state; int err; @@ -226,7 +227,7 @@ mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix); devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn); - err = mlx5_core_query_sq_state(sq->mdev, sq->sqn, &state); + err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state); if (!err) devlink_fmsg_u8_pair_put(fmsg, "HW state", state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c index bcafb4bf9415..7b8ff7a71003 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c @@ -4,33 +4,6 @@ #include "rqt.h" #include -static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, - unsigned int size) -{ - unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); - int i; - - /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */ - for (i = 0; i < size; i++) - if (vhca_ids[i] >= max_num_vhca_id) - return false; - return true; -} - -static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, - unsigned int size) -{ - if (!vhca_ids) - return true; - - if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt)) - return false; - if (!verify_num_vhca_ids(mdev, vhca_ids, size)) - return false; - - return true; -} - void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, unsigned int num_channels) { @@ -40,38 +13,19 @@ void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, indir->table[i] = i % num_channels; } -static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size) -{ - unsigned int i; - - if (vhca_ids) { - MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1); - for (i = 0; i < size; i++) { - MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]); - MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]); - } - } else { - for (i = 0; i < size; i++) - MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); - } -} static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size) + u16 max_size, u32 *init_rqns, u16 init_size) { - int entry_sz; void *rqtc; int inlen; int err; u32 *in; - - if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size)) - return -EOPNOTSUPP; + int i; rqt->mdev = mdev; rqt->size = max_size; - entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); - inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size; + inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * init_size; in = kvzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -79,9 +33,10 @@ static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size); - MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); - fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size); + MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); + for (i = 0; i < init_size; i++) + MLX5_SET(rqtc, rqtc, rq_num[i], init_rqns[i]); err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn); @@ -94,7 +49,7 @@ int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, { u16 max_size = indir_enabled ? indir_table_size : 1; - return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1); + return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, 1); } static int mlx5e_bits_invert(unsigned long a, int size) @@ -108,8 +63,7 @@ static int mlx5e_bits_invert(unsigned long a, int size) return inv; } -static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids, - unsigned int num_rqns, +static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { unsigned int i; @@ -128,42 +82,30 @@ static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u3 */ return -EINVAL; rss_rqns[i] = rqns[ix]; - if (vhca_ids) - rss_vhca_ids[i] = vhca_ids[ix]; } return 0; } int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, + u32 *rqns, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { - u32 *rss_rqns, *rss_vhca_ids = NULL; + u32 *rss_rqns; int err; rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); if (!rss_rqns) return -ENOMEM; - if (vhca_ids) { - rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), - GFP_KERNEL); - if (!rss_vhca_ids) { - kvfree(rss_rqns); - return -ENOMEM; - } - } - - err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); + err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir); if (err) goto out; - err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids, + err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, indir->actual_table_size); out: - kvfree(rss_vhca_ids); kvfree(rss_rqns); return err; } @@ -184,20 +126,15 @@ void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn); } -static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, - unsigned int size) +static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int size) { - int entry_sz; + unsigned int i; void *rqtc; int inlen; u32 *in; int err; - if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size)) - return -EINVAL; - - entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); - inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size; + inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * size; in = kvzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -206,8 +143,8 @@ static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); MLX5_SET(rqtc, rqtc, rqt_actual_size, size); - - fill_rqn_list(rqtc, rqns, vhca_ids, size); + for (i = 0; i < size; i++) + MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen); @@ -215,21 +152,17 @@ static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, return err; } -int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id) +int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn) { - return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1); + return mlx5e_rqt_redirect(rqt, &rqn, 1); } -int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, - unsigned int num_rqns, +int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir) { - u32 *rss_rqns, *rss_vhca_ids = NULL; + u32 *rss_rqns; int err; - if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns)) - return -EINVAL; - if (WARN_ON(rqt->size != indir->max_table_size)) return -EINVAL; @@ -237,23 +170,13 @@ int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, if (!rss_rqns) return -ENOMEM; - if (vhca_ids) { - rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), - GFP_KERNEL); - if (!rss_vhca_ids) { - kvfree(rss_rqns); - return -ENOMEM; - } - } - - err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); + err = mlx5e_calc_indir_rqns(rss_rqns, rqns, num_rqns, hfunc, indir); if (err) goto out; - err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size); + err = mlx5e_rqt_redirect(rqt, rss_rqns, indir->actual_table_size); out: - kvfree(rss_vhca_ids); kvfree(rss_rqns); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h index e0bc30308c77..77fba3ebd18d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h @@ -20,7 +20,7 @@ void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, unsigned int num_channels); struct mlx5e_rqt { - struct mlx5_core_dev *mdev; /* primary */ + struct mlx5_core_dev *mdev; u32 rqtn; u16 size; }; @@ -28,7 +28,7 @@ struct mlx5e_rqt { int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, bool indir_enabled, u32 init_rqn, u32 indir_table_size); int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, - u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, + u32 *rqns, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir); void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt); @@ -38,9 +38,8 @@ static inline u32 mlx5e_rqt_get_rqtn(struct mlx5e_rqt *rqt) } u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels); -int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id); -int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, - unsigned int num_rqns, +int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn); +int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, unsigned int num_rqns, u8 hfunc, struct mlx5e_rss_params_indir *indir); #endif /* __MLX5_EN_RQT_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c index 5f742f896600..c1545a2e8d6d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c @@ -74,7 +74,7 @@ struct mlx5e_rss { struct mlx5e_tir *tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS]; struct mlx5e_rqt rqt; - struct mlx5_core_dev *mdev; /* primary */ + struct mlx5_core_dev *mdev; u32 drop_rqn; bool inner_ft_support; bool enabled; @@ -473,22 +473,21 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, return 0; } -static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) +static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) { int err; - err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, vhca_ids, num_rqns, rss->hash.hfunc, - &rss->indir); + err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, num_rqns, rss->hash.hfunc, &rss->indir); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n", mlx5e_rqt_get_rqtn(&rss->rqt), err); return err; } -void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) +void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) { rss->enabled = true; - mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); + mlx5e_rss_apply(rss, rqns, num_rqns); } void mlx5e_rss_disable(struct mlx5e_rss *rss) @@ -496,7 +495,7 @@ void mlx5e_rss_disable(struct mlx5e_rss *rss) int err; rss->enabled = false; - err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn, NULL); + err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn); if (err) mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n", mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); @@ -569,7 +568,7 @@ int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc) int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, const u8 *key, const u8 *hfunc, - u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) + u32 *rqns, unsigned int num_rqns) { bool changed_indir = false; bool changed_hash = false; @@ -609,7 +608,7 @@ int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, } if (changed_indir && rss->enabled) { - err = mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); + err = mlx5e_rss_apply(rss, rqns, num_rqns); if (err) { mlx5e_rss_copy(rss, old_rss); goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h index d0df98963c8d..d1d0bc350e92 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h @@ -39,7 +39,7 @@ int mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, const struct mlx5e_packet_merge_param *init_pkt_merge_param, bool inner, u32 *tirn); -void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); +void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns); void mlx5e_rss_disable(struct mlx5e_rss *rss); int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, @@ -47,7 +47,7 @@ int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss, int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc); int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, const u8 *key, const u8 *hfunc, - u32 *rqns, u32 *vhca_ids, unsigned int num_rqns); + u32 *rqns, unsigned int num_rqns); struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss); u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt); int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index a86eade9a9e0..b23e224e3763 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -8,7 +8,7 @@ #define MLX5E_MAX_NUM_RSS 16 struct mlx5e_rx_res { - struct mlx5_core_dev *mdev; /* primary */ + struct mlx5_core_dev *mdev; enum mlx5e_rx_res_features features; unsigned int max_nch; u32 drop_rqn; @@ -19,7 +19,6 @@ struct mlx5e_rx_res { struct mlx5e_rss *rss[MLX5E_MAX_NUM_RSS]; bool rss_active; u32 *rss_rqns; - u32 *rss_vhca_ids; unsigned int rss_nch; struct { @@ -35,13 +34,6 @@ struct mlx5e_rx_res { /* API for rx_res_rss_* */ -static u32 *get_vhca_ids(struct mlx5e_rx_res *res, int offset) -{ - bool multi_vhca = res->features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; - - return multi_vhca ? res->rss_vhca_ids + offset : NULL; -} - void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch) { int i; @@ -93,11 +85,8 @@ int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 *rss_idx, unsigned int i return PTR_ERR(rss); mlx5e_rss_set_indir_uniform(rss, init_nch); - if (res->rss_active) { - u32 *vhca_ids = get_vhca_ids(res, 0); - - mlx5e_rss_enable(rss, res->rss_rqns, vhca_ids, res->rss_nch); - } + if (res->rss_active) + mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); res->rss[i] = rss; *rss_idx = i; @@ -164,12 +153,10 @@ static void mlx5e_rx_res_rss_enable(struct mlx5e_rx_res *res) for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { struct mlx5e_rss *rss = res->rss[i]; - u32 *vhca_ids; if (!rss) continue; - vhca_ids = get_vhca_ids(res, 0); - mlx5e_rss_enable(rss, res->rss_rqns, vhca_ids, res->rss_nch); + mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); } } @@ -213,7 +200,6 @@ int mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, const u32 *indir, const u8 *key, const u8 *hfunc) { - u32 *vhca_ids = get_vhca_ids(res, 0); struct mlx5e_rss *rss; if (rss_idx >= MLX5E_MAX_NUM_RSS) @@ -223,8 +209,7 @@ int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, if (!rss) return -ENOENT; - return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, vhca_ids, - res->rss_nch); + return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, res->rss_nch); } int mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, @@ -295,13 +280,11 @@ struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx) static void mlx5e_rx_res_free(struct mlx5e_rx_res *res) { - kvfree(res->rss_vhca_ids); kvfree(res->rss_rqns); kvfree(res); } -static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsigned int max_nch, - bool multi_vhca) +static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsigned int max_nch) { struct mlx5e_rx_res *rx_res; @@ -315,15 +298,6 @@ static struct mlx5e_rx_res *mlx5e_rx_res_alloc(struct mlx5_core_dev *mdev, unsig return NULL; } - if (multi_vhca) { - rx_res->rss_vhca_ids = kvcalloc(max_nch, sizeof(*rx_res->rss_vhca_ids), GFP_KERNEL); - if (!rx_res->rss_vhca_ids) { - kvfree(rx_res->rss_rqns); - kvfree(rx_res); - return NULL; - } - } - return rx_res; } @@ -450,11 +424,10 @@ mlx5e_rx_res_create(struct mlx5_core_dev *mdev, enum mlx5e_rx_res_features featu const struct mlx5e_packet_merge_param *init_pkt_merge_param, unsigned int init_nch) { - bool multi_vhca = features & MLX5E_RX_RES_FEATURE_MULTI_VHCA; struct mlx5e_rx_res *res; int err; - res = mlx5e_rx_res_alloc(mdev, max_nch, multi_vhca); + res = mlx5e_rx_res_alloc(mdev, max_nch); if (!res) return ERR_PTR(-ENOMEM); @@ -531,11 +504,10 @@ static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, unsigned int ix) { - u32 *vhca_id = get_vhca_ids(res, ix); u32 rqn = res->rss_rqns[ix]; int err; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn, vhca_id); + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), @@ -547,7 +519,7 @@ static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, { int err; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn, NULL); + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), @@ -562,12 +534,10 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann nch = mlx5e_channels_get_num(chs); for (ix = 0; ix < chs->num; ix++) { - u32 *vhca_id = get_vhca_ids(res, ix); - if (mlx5e_channels_is_xsk(chs, ix)) - mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); else - mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); } res->rss_nch = chs->num; @@ -584,7 +554,7 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann if (!mlx5e_channels_get_ptp_rqn(chs, &rqn)) rqn = res->drop_rqn; - err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn, NULL); + err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (PTP): err = %d\n", mlx5e_rqt_get_rqtn(&res->ptp.rqt), @@ -603,7 +573,7 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { - err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn, NULL); + err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); if (err) mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (PTP): err = %d\n", mlx5e_rqt_get_rqtn(&res->ptp.rqt), @@ -614,12 +584,10 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, unsigned int ix, bool xsk) { - u32 *vhca_id = get_vhca_ids(res, ix); - if (xsk) - mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); else - mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix], vhca_id); + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); mlx5e_rx_res_rss_enable(res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 7b1a9f0f1874..82aaba8a82b3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -18,7 +18,6 @@ struct mlx5e_rss_params_hash; enum mlx5e_rx_res_features { MLX5E_RX_RES_FEATURE_INNER_FT = BIT(0), MLX5E_RX_RES_FEATURE_PTP = BIT(1), - MLX5E_RX_RES_FEATURE_MULTI_VHCA = BIT(2), }; /* Setup */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c index 53ca16cb9c41..ac458a8d10e0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -63,12 +63,10 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) struct mlx5e_create_cq_param ccp = {}; struct dim_cq_moder trap_moder = {}; struct mlx5e_rq *rq = &t->rq; - u16 q_counter; int node; int err; node = dev_to_node(mdev->device); - q_counter = priv->q_counter[0]; ccp.netdev = priv->netdev; ccp.wq = priv->wq; @@ -81,7 +79,7 @@ static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct mlx5e_trap *t) return err; mlx5e_init_trap_rq(t, &t->params, rq); - err = mlx5e_open_rq(&t->params, rq_param, NULL, node, q_counter, rq); + err = mlx5e_open_rq(&t->params, rq_param, NULL, node, rq); if (err) goto err_destroy_cq; @@ -118,14 +116,15 @@ static int mlx5e_create_trap_direct_rq_tir(struct mlx5_core_dev *mdev, struct ml } static void mlx5e_build_trap_params(struct mlx5_core_dev *mdev, - int max_mtu, struct mlx5e_trap *t) + int max_mtu, u16 q_counter, + struct mlx5e_trap *t) { struct mlx5e_params *params = &t->params; params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, &t->rq_param); + mlx5e_build_rq_param(mdev, params, NULL, q_counter, &t->rq_param); } static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) @@ -139,7 +138,7 @@ static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) if (!t) return ERR_PTR(-ENOMEM); - mlx5e_build_trap_params(priv->mdev, netdev->max_mtu, t); + mlx5e_build_trap_params(priv->mdev, netdev->max_mtu, priv->q_counter, t); t->priv = priv; t->mdev = priv->mdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index db776e515b6a..ebada0c5af3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -6,10 +6,10 @@ #include "setup.h" #include "en/params.h" -static int mlx5e_xsk_map_pool(struct mlx5_core_dev *mdev, +static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool) { - struct device *dev = mlx5_core_dma_dev(mdev); + struct device *dev = mlx5_core_dma_dev(priv->mdev); return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC); } @@ -89,7 +89,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, if (unlikely(!mlx5e_xsk_is_pool_sane(pool))) return -EINVAL; - err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool); + err = mlx5e_xsk_map_pool(priv, pool); if (unlikely(err)) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 06592b9f0424..82e6abbc1734 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -49,9 +49,10 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, + u16 q_counter, struct mlx5e_channel_param *cparam) { - mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); + mlx5e_build_rq_param(mdev, params, xsk, q_counter, &cparam->rq); mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq); } @@ -92,7 +93,6 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk) { - u16 q_counter = c->priv->q_counter[c->sd_ix]; struct mlx5e_rq *xskrq = &c->xskrq; int err; @@ -100,7 +100,7 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param if (err) return err; - err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq); + err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), xskrq); if (err) return err; @@ -125,7 +125,7 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (!cparam) return -ENOMEM; - mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam); + mlx5e_build_xsk_cparam(priv->mdev, params, xsk, priv->q_counter, cparam); err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->xskrq.cq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index e3e57c849436..984fa04bd331 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -96,7 +96,7 @@ bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) { u8 max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); - if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx) || mlx5_get_sd(mdev)) + if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx)) return false; /* Check the possibility to post the required ICOSQ WQEs. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index adc6d8ea0960..f11075e67658 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -11,7 +11,6 @@ #ifdef CONFIG_MLX5_EN_TLS #include "lib/crypto.h" -#include "lib/mlx5.h" struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, struct tls_crypto_info *crypto_info); @@ -62,8 +61,7 @@ void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_ static inline bool mlx5e_is_ktls_tx(struct mlx5_core_dev *mdev) { - return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx) && - !mlx5_get_sd(mdev); + return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx); } bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 65ccb33edafb..9b597cb24598 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -267,7 +267,7 @@ resync_post_get_progress_params(struct mlx5e_icosq *sq, goto err_out; } - pdev = mlx5_core_dma_dev(sq->channel->mdev); + pdev = mlx5_core_dma_dev(sq->channel->priv->mdev); buf->dma_addr = dma_map_single(pdev, &buf->progress, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { @@ -425,12 +425,14 @@ void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, { struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; struct mlx5e_ktls_offload_context_rx *priv_rx; + struct mlx5e_ktls_rx_resync_ctx *resync; u8 tracker_state, auth_state, *ctx; struct device *dev; u32 hw_seq; priv_rx = buf->priv_rx; - dev = mlx5_core_dma_dev(sq->channel->mdev); + resync = &priv_rx->resync; + dev = mlx5_core_dma_dev(resync->priv->mdev); if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) goto out; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 6ed3a32b7e22..67f546683e85 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -95,7 +95,7 @@ static void mlx5e_destroy_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PO { int tc, i; - for (i = 0; i < mlx5e_get_num_lag_ports(mdev); i++) + for (i = 0; i < MLX5_MAX_PORTS; i++) for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) mlx5e_destroy_tis(mdev, tisn[i][tc]); } @@ -110,7 +110,7 @@ static int mlx5e_create_tises(struct mlx5_core_dev *mdev, u32 tisn[MLX5_MAX_PORT int tc, i; int err; - for (i = 0; i < mlx5e_get_num_lag_ports(mdev); i++) { + for (i = 0; i < MLX5_MAX_PORTS; i++) { for (tc = 0; tc < MLX5_MAX_NUM_TC; tc++) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; void *tisc; @@ -140,7 +140,7 @@ err_close_tises: return err; } -int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) +int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; int err; @@ -169,15 +169,11 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises) goto err_destroy_mkey; } - if (create_tises) { - err = mlx5e_create_tises(mdev, res->tisn); - if (err) { - mlx5_core_err(mdev, "alloc tises failed, %d\n", err); - goto err_destroy_bfreg; - } - res->tisn_valid = true; + err = mlx5e_create_tises(mdev, res->tisn); + if (err) { + mlx5_core_err(mdev, "alloc tises failed, %d\n", err); + goto err_destroy_bfreg; } - INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); @@ -207,8 +203,7 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); mdev->mlx5e_res.dek_priv = NULL; - if (res->tisn_valid) - mlx5e_destroy_tises(mdev, res->tisn); + mlx5e_destroy_tises(mdev, res->tisn); mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 40626b6108fb..b5f1c4ca38ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -70,7 +70,6 @@ #include "qos.h" #include "en/trap.h" #include "lib/devcom.h" -#include "lib/sd.h" bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, enum mlx5e_mpwrq_umr_mode umr_mode) @@ -1025,7 +1024,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) mlx5_wq_destroy(&rq->wq_ctrl); } -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) { struct mlx5_core_dev *mdev = rq->mdev; u8 ts_format; @@ -1052,7 +1051,6 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); MLX5_SET(rqc, rqc, ts_format, ts_format); - MLX5_SET(rqc, rqc, counter_set_id, q_counter); MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); @@ -1276,7 +1274,7 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) } int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, + struct mlx5e_xsk_param *xsk, int node, struct mlx5e_rq *rq) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1289,7 +1287,7 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, if (err) return err; - err = mlx5e_create_rq(rq, param, q_counter); + err = mlx5e_create_rq(rq, param); if (err) goto err_free_rq; @@ -2335,14 +2333,13 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, struct mlx5e_rq_param *rq_params) { - u16 q_counter = c->priv->q_counter[c->sd_ix]; int err; err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); if (err) return err; - return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); + return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), &c->rq); } static int mlx5e_open_queues(struct mlx5e_channel *c, @@ -2529,20 +2526,14 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct xsk_buff_pool *xsk_pool, struct mlx5e_channel **cp) { + int cpu = mlx5_comp_vector_get_cpu(priv->mdev, ix); struct net_device *netdev = priv->netdev; - struct mlx5_core_dev *mdev; struct mlx5e_xsk_param xsk; struct mlx5e_channel *c; unsigned int irq; - int vec_ix; - int cpu; int err; - mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); - vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); - cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); - - err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); + err = mlx5_comp_irqn_get(priv->mdev, ix, &irq); if (err) return err; @@ -2555,20 +2546,18 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, return -ENOMEM; c->priv = priv; - c->mdev = mdev; + c->mdev = priv->mdev; c->tstamp = &priv->tstamp; c->ix = ix; - c->vec_ix = vec_ix; - c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); c->cpu = cpu; - c->pdev = mlx5_core_dma_dev(mdev); + c->pdev = mlx5_core_dma_dev(priv->mdev); c->netdev = priv->netdev; - c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); + c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey); c->num_tc = mlx5e_get_dcb_num_tc(params); c->xdp = !!params->xdp_prog; c->stats = &priv->channel_stats[ix]->ch; c->aff_mask = irq_get_effective_affinity_mask(irq); - c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); + c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); netif_napi_add(netdev, &c->napi, mlx5e_napi_poll); @@ -2658,7 +2647,7 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, if (!chs->c || !cparam) goto err_free; - err = mlx5e_build_channel_param(priv->mdev, &chs->params, cparam); + err = mlx5e_build_channel_param(priv->mdev, &chs->params, priv->q_counter, cparam); if (err) goto err_free; @@ -2946,18 +2935,15 @@ static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_netdev_queues); static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, struct mlx5e_params *params) { - int ix; + struct mlx5_core_dev *mdev = priv->mdev; + int num_comp_vectors, ix, irq; - for (ix = 0; ix < params->num_channels; ix++) { - int num_comp_vectors, irq, vec_ix; - struct mlx5_core_dev *mdev; + num_comp_vectors = mlx5_comp_vectors_max(mdev); - mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); - num_comp_vectors = mlx5_comp_vectors_max(mdev); + for (ix = 0; ix < params->num_channels; ix++) { cpumask_clear(priv->scratchpad.cpumask); - vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); - for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { + for (irq = ix; irq < num_comp_vectors; irq += params->num_channels) { int cpu = mlx5_comp_vector_get_cpu(mdev, irq); cpumask_set_cpu(cpu, priv->scratchpad.cpumask); @@ -3349,7 +3335,7 @@ int mlx5e_open_drop_rq(struct mlx5e_priv *priv, struct mlx5e_cq *cq = &drop_rq->cq; int err; - mlx5e_build_drop_rq_param(mdev, &rq_param); + mlx5e_build_drop_rq_param(mdev, priv->drop_rq_q_counter, &rq_param); err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); if (err) @@ -3363,7 +3349,7 @@ int mlx5e_open_drop_rq(struct mlx5e_priv *priv, if (err) goto err_destroy_cq; - err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); + err = mlx5e_create_rq(drop_rq, &rq_param); if (err) goto err_free_rq; @@ -3799,7 +3785,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; } -void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) +static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) { if (mlx5e_is_uplink_rep(priv)) return; /* no rx mode for uplink rep */ @@ -5004,15 +4990,6 @@ const struct net_device_ops mlx5e_netdev_ops = { #endif }; -const struct net_device_ops mlx5e_mgmt_netdev_ops = { - .ndo_open = mlx5e_open, - .ndo_stop = mlx5e_close, - .ndo_start_xmit = mlx5e_xmit, - .ndo_get_stats64 = mlx5e_get_stats, - .ndo_change_mtu = mlx5e_change_nic_mtu, - .ndo_set_rx_mode = mlx5e_set_rx_mode, -}; - static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) { int i; @@ -5152,11 +5129,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) SET_NETDEV_DEV(netdev, mdev->device); - if (mlx5_core_is_mgmt_pf(mdev)) - netdev->netdev_ops = &mlx5e_mgmt_netdev_ops; - else - netdev->netdev_ops = &mlx5e_netdev_ops; - + netdev->netdev_ops = &mlx5e_netdev_ops; netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; @@ -5291,17 +5264,13 @@ void mlx5e_create_q_counters(struct mlx5e_priv *priv) u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; struct mlx5_core_dev *mdev = priv->mdev; - struct mlx5_core_dev *pos; - int err, i; + int err; MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); - - mlx5_sd_for_each_dev(i, mdev, pos) { - err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); - if (!err) - priv->q_counter[i] = - MLX5_GET(alloc_q_counter_out, out, counter_set_id); - } + err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); + if (!err) + priv->q_counter = + MLX5_GET(alloc_q_counter_out, out, counter_set_id); err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); if (!err) @@ -5312,17 +5281,13 @@ void mlx5e_create_q_counters(struct mlx5e_priv *priv) void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) { u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; - struct mlx5_core_dev *pos; - int i; MLX5_SET(dealloc_q_counter_in, in, opcode, MLX5_CMD_OP_DEALLOC_Q_COUNTER); - mlx5_sd_for_each_dev(i, priv->mdev, pos) { - if (priv->q_counter[i]) { - MLX5_SET(dealloc_q_counter_in, in, counter_set_id, - priv->q_counter[i]); - mlx5_cmd_exec_in(pos, dealloc_q_counter, in); - } + if (priv->q_counter) { + MLX5_SET(dealloc_q_counter_in, in, counter_set_id, + priv->q_counter); + mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); } if (priv->drop_rq_q_counter) { @@ -5406,8 +5371,6 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) features = MLX5E_RX_RES_FEATURE_PTP; if (mlx5_tunnel_inner_ft_supported(mdev)) features |= MLX5E_RX_RES_FEATURE_INNER_FT; - if (mlx5_get_sd(priv->mdev)) - features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, &priv->channels.params.packet_merge, @@ -6017,52 +5980,28 @@ void mlx5e_destroy_netdev(struct mlx5e_priv *priv) free_netdev(netdev); } -static int _mlx5e_resume(struct auxiliary_device *adev) +static int mlx5e_resume(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = edev->mdev; - struct mlx5_core_dev *pos, *to; - int err, i; + int err; if (netif_device_present(netdev)) return 0; - mlx5_sd_for_each_dev(i, mdev, pos) { - err = mlx5e_create_mdev_resources(pos, true); - if (err) - goto err_destroy_mdev_res; - } - - err = mlx5e_attach_netdev(priv); + err = mlx5e_create_mdev_resources(mdev); if (err) - goto err_destroy_mdev_res; - - return 0; - -err_destroy_mdev_res: - to = pos; - mlx5_sd_for_each_dev_to(i, mdev, to, pos) - mlx5e_destroy_mdev_resources(pos); - return err; -} - -static int mlx5e_resume(struct auxiliary_device *adev) -{ - struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); - struct mlx5_core_dev *mdev = edev->mdev; - struct auxiliary_device *actual_adev; - int err; + return err; - err = mlx5_sd_init(mdev); - if (err) + err = mlx5e_attach_netdev(priv); + if (err) { + mlx5e_destroy_mdev_resources(mdev); return err; + } - actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); - if (actual_adev) - return _mlx5e_resume(actual_adev); return 0; } @@ -6072,53 +6011,33 @@ static int _mlx5e_suspend(struct auxiliary_device *adev) struct mlx5e_priv *priv = mlx5e_dev->priv; struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - struct mlx5_core_dev *pos; - int i; if (!netif_device_present(netdev)) { if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) - mlx5_sd_for_each_dev(i, mdev, pos) - mlx5e_destroy_mdev_resources(pos); + mlx5e_destroy_mdev_resources(mdev); return -ENODEV; } mlx5e_detach_netdev(priv); - mlx5_sd_for_each_dev(i, mdev, pos) - mlx5e_destroy_mdev_resources(pos); - + mlx5e_destroy_mdev_resources(mdev); return 0; } static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) { - struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); - struct mlx5_core_dev *mdev = edev->mdev; - struct auxiliary_device *actual_adev; - int err = 0; - - actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); - if (actual_adev) - err = _mlx5e_suspend(actual_adev); - - mlx5_sd_cleanup(mdev); - return err; + return _mlx5e_suspend(adev); } static int _mlx5e_probe(struct auxiliary_device *adev) { struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); + const struct mlx5e_profile *profile = &mlx5e_nic_profile; struct mlx5_core_dev *mdev = edev->mdev; - const struct mlx5e_profile *profile; struct mlx5e_dev *mlx5e_dev; struct net_device *netdev; struct mlx5e_priv *priv; int err; - if (mlx5_core_is_mgmt_pf(mdev)) - profile = &mlx5e_mgmt_pf_nic_profile; - else - profile = &mlx5e_nic_profile; - mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); if (IS_ERR(mlx5e_dev)) return PTR_ERR(mlx5e_dev); @@ -6152,9 +6071,9 @@ static int _mlx5e_probe(struct auxiliary_device *adev) goto err_destroy_netdev; } - err = _mlx5e_resume(adev); + err = mlx5e_resume(adev); if (err) { - mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); + mlx5_core_err(mdev, "mlx5e_resume failed, %d\n", err); goto err_profile_cleanup; } @@ -6185,29 +6104,15 @@ err_devlink_unregister: static int mlx5e_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { - struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); - struct mlx5_core_dev *mdev = edev->mdev; - struct auxiliary_device *actual_adev; - int err; - - err = mlx5_sd_init(mdev); - if (err) - return err; - - actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); - if (actual_adev) - return _mlx5e_probe(actual_adev); - return 0; + return _mlx5e_probe(adev); } -static void _mlx5e_remove(struct auxiliary_device *adev) +static void mlx5e_remove(struct auxiliary_device *adev) { - struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); struct mlx5e_priv *priv = mlx5e_dev->priv; - struct mlx5_core_dev *mdev = edev->mdev; - mlx5_core_uplink_netdev_set(mdev, NULL); + mlx5_core_uplink_netdev_set(priv->mdev, NULL); mlx5e_dcbnl_delete_app(priv); unregister_netdev(priv->netdev); _mlx5e_suspend(adev); @@ -6217,19 +6122,6 @@ static void _mlx5e_remove(struct auxiliary_device *adev) mlx5e_destroy_devlink(mlx5e_dev); } -static void mlx5e_remove(struct auxiliary_device *adev) -{ - struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); - struct mlx5_core_dev *mdev = edev->mdev; - struct auxiliary_device *actual_adev; - - actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); - if (actual_adev) - _mlx5e_remove(actual_adev); - - mlx5_sd_cleanup(mdev); -} - static const struct auxiliary_device_id mlx5e_id_table[] = { { .name = MLX5_ADEV_NAME ".eth", }, {}, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index f3d0898bdbc6..4b96ad657145 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -561,23 +561,11 @@ static const struct counter_desc drop_rq_stats_desc[] = { #define NUM_Q_COUNTERS ARRAY_SIZE(q_stats_desc) #define NUM_DROP_RQ_COUNTERS ARRAY_SIZE(drop_rq_stats_desc) -static bool q_counter_any(struct mlx5e_priv *priv) -{ - struct mlx5_core_dev *pos; - int i; - - mlx5_sd_for_each_dev(i, priv->mdev, pos) - if (priv->q_counter[i++]) - return true; - - return false; -} - static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qcnt) { int num_stats = 0; - if (q_counter_any(priv)) + if (priv->q_counter) num_stats += NUM_Q_COUNTERS; if (priv->drop_rq_q_counter) @@ -590,7 +578,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qcnt) { int i; - for (i = 0; i < NUM_Q_COUNTERS && q_counter_any(priv); i++) + for (i = 0; i < NUM_Q_COUNTERS && priv->q_counter; i++) strcpy(data + (idx++) * ETH_GSTRING_LEN, q_stats_desc[i].format); @@ -605,7 +593,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qcnt) { int i; - for (i = 0; i < NUM_Q_COUNTERS && q_counter_any(priv); i++) + for (i = 0; i < NUM_Q_COUNTERS && priv->q_counter; i++) data[idx++] = MLX5E_READ_CTR32_CPU(&priv->stats.qcnt, q_stats_desc, i); for (i = 0; i < NUM_DROP_RQ_COUNTERS && priv->drop_rq_q_counter; i++) @@ -619,23 +607,18 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(qcnt) struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt; u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; - struct mlx5_core_dev *pos; - u32 rx_out_of_buffer = 0; - int ret, i; + int ret; MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); - mlx5_sd_for_each_dev(i, priv->mdev, pos) { - if (priv->q_counter[i]) { - MLX5_SET(query_q_counter_in, in, counter_set_id, - priv->q_counter[i]); - ret = mlx5_cmd_exec_inout(pos, query_q_counter, in, out); - if (!ret) - rx_out_of_buffer += MLX5_GET(query_q_counter_out, - out, out_of_buffer); - } + if (priv->q_counter) { + MLX5_SET(query_q_counter_in, in, counter_set_id, + priv->q_counter); + ret = mlx5_cmd_exec_inout(priv->mdev, query_q_counter, in, out); + if (!ret) + qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, + out, out_of_buffer); } - qcnt->rx_out_of_buffer = rx_out_of_buffer; if (priv->drop_rq_q_counter) { MLX5_SET(query_q_counter_in, in, counter_set_id, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index b8ceb972df9e..30932c9c9a8f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -766,7 +766,7 @@ static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp) return err; mlx5e_rss_params_indir_init_uniform(&indir, hp->num_channels); - err = mlx5e_rqt_init_indir(&hp->indir_rqt, mdev, hp->pair->rqn, NULL, hp->num_channels, + err = mlx5e_rqt_init_indir(&hp->indir_rqt, mdev, hp->pair->rqn, hp->num_channels, mlx5e_rx_res_get_current_hash(priv->rx_res).hfunc, &indir); @@ -1169,7 +1169,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz), MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); - params.q_counter = priv->q_counter[0]; + params.q_counter = priv->q_counter; err = devl_param_driverinit_value_get( devlink, MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, &val); if (err) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 3bf419d06d53..3047d7015c52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1665,7 +1665,7 @@ int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 * void *hca_caps; int err; - if (!mlx5_core_is_ecpf(dev) || mlx5_core_is_mgmt_pf(dev)) { + if (!mlx5_core_is_ecpf(dev)) { *max_sfs = 0; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index d77be1b4dd9c..58845121954c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -783,7 +783,7 @@ static int mlx5_rdma_setup_rn(struct ib_device *ibdev, u32 port_num, } /* This should only be called once per mdev */ - err = mlx5e_create_mdev_resources(mdev, false); + err = mlx5e_create_mdev_resources(mdev); if (err) goto destroy_ht; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index d58032dd0df7..ec32b686f586 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -10,7 +10,6 @@ enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, MLX5_DEVCOM_MPV, MLX5_DEVCOM_HCA_PORTS, - MLX5_DEVCOM_SD_GROUP, MLX5_DEVCOM_NUM_COMPONENTS, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 37d5f445598c..2b5826a785c4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -54,16 +54,4 @@ static inline struct net_device *mlx5_uplink_netdev_get(struct mlx5_core_dev *md { return mdev->mlx5e_res.uplink_netdev; } - -struct mlx5_sd; - -static inline struct mlx5_sd *mlx5_get_sd(struct mlx5_core_dev *dev) -{ - return dev->sd; -} - -static inline void mlx5_set_sd(struct mlx5_core_dev *dev, struct mlx5_sd *sd) -{ - dev->sd = sd; -} #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c deleted file mode 100644 index f68942277c62..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ /dev/null @@ -1,487 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB -/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ - -#include "lib/sd.h" -#include "mlx5_core.h" -#include "lib/mlx5.h" -#include "fs_cmd.h" -#include - -#define sd_info(__dev, format, ...) \ - dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) -#define sd_warn(__dev, format, ...) \ - dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__) - -struct mlx5_sd { - u32 group_id; - u8 host_buses; - struct mlx5_devcom_comp_dev *devcom; - bool primary; - union { - struct { /* primary */ - struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; - struct mlx5_flow_table *tx_ft; - }; - struct { /* secondary */ - struct mlx5_core_dev *primary_dev; - u32 alias_obj_id; - }; - }; -}; - -static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - - if (!sd) - return 1; - - return sd->host_buses; -} - -static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - - if (!sd) - return dev; - - return sd->primary ? dev : sd->primary_dev; -} - -struct mlx5_core_dev * -mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) -{ - struct mlx5_sd *sd; - - if (idx == 0) - return primary; - - if (idx >= mlx5_sd_get_host_buses(primary)) - return NULL; - - sd = mlx5_get_sd(primary); - return sd->secondaries[idx - 1]; -} - -int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) -{ - return ch_ix % mlx5_sd_get_host_buses(dev); -} - -int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix) -{ - return ch_ix / mlx5_sd_get_host_buses(dev); -} - -struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix) -{ - int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix); - - return mlx5_sd_primary_get_peer(primary, mdev_idx); -} - -static bool ft_create_alias_supported(struct mlx5_core_dev *dev) -{ - u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access); - u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported); - - if (!(obj_supp & - MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE)) - return false; - - if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE)) - return false; - - return true; -} - -static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses) -{ - /* Feature is currently implemented for PFs only */ - if (!mlx5_core_is_pf(dev)) - return false; - - /* Honor the SW implementation limit */ - if (host_buses > MLX5_SD_MAX_GROUP_SZ) - return false; - - /* Disconnect secondaries from the network */ - if (!MLX5_CAP_GEN(dev, eswitch_manager)) - return false; - if (!MLX5_CAP_GEN(dev, silent_mode)) - return false; - - /* RX steering from primary to secondaries */ - if (!MLX5_CAP_GEN(dev, cross_vhca_rqt)) - return false; - if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id)) - return false; - - /* TX steering from secondaries to primary */ - if (!ft_create_alias_supported(dev)) - return false; - if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default)) - return false; - - return true; -} - -static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm, - u8 *host_buses, u8 *sd_group) -{ - u32 out[MLX5_ST_SZ_DW(mpir_reg)]; - int err; - - err = mlx5_query_mpir_reg(dev, out); - if (err) - return err; - - err = mlx5_query_nic_vport_sd_group(dev, sd_group); - if (err) - return err; - - *sdm = MLX5_GET(mpir_reg, out, sdm); - *host_buses = MLX5_GET(mpir_reg, out, host_buses); - - return 0; -} - -static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group) -{ - return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group); -} - -static int sd_init(struct mlx5_core_dev *dev) -{ - u8 host_buses, sd_group; - struct mlx5_sd *sd; - u32 group_id; - bool sdm; - int err; - - err = mlx5_query_sd(dev, &sdm, &host_buses, &sd_group); - if (err) - return err; - - if (!sdm) - return 0; - - if (!sd_group) - return 0; - - group_id = mlx5_sd_group_id(dev, sd_group); - - if (!mlx5_sd_is_supported(dev, host_buses)) { - sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n", - group_id); - return 0; - } - - sd = kzalloc(sizeof(*sd), GFP_KERNEL); - if (!sd) - return -ENOMEM; - - sd->host_buses = host_buses; - sd->group_id = group_id; - - mlx5_set_sd(dev, sd); - - return 0; -} - -static void sd_cleanup(struct mlx5_core_dev *dev) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - - mlx5_set_sd(dev, NULL); - kfree(sd); -} - -static int sd_register(struct mlx5_core_dev *dev) -{ - struct mlx5_devcom_comp_dev *devcom, *pos; - struct mlx5_core_dev *peer, *primary; - struct mlx5_sd *sd, *primary_sd; - int err, i; - - sd = mlx5_get_sd(dev); - devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, - sd->group_id, NULL, dev); - if (!devcom) - return -ENOMEM; - - sd->devcom = devcom; - - if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) - return 0; - - mlx5_devcom_comp_lock(devcom); - mlx5_devcom_comp_set_ready(devcom, true); - mlx5_devcom_comp_unlock(devcom); - - if (!mlx5_devcom_for_each_peer_begin(devcom)) { - err = -ENODEV; - goto err_devcom_unreg; - } - - primary = dev; - mlx5_devcom_for_each_peer_entry(devcom, peer, pos) - if (peer->pdev->bus->number < primary->pdev->bus->number) - primary = peer; - - primary_sd = mlx5_get_sd(primary); - primary_sd->primary = true; - i = 0; - /* loop the secondaries */ - mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { - struct mlx5_sd *peer_sd = mlx5_get_sd(peer); - - primary_sd->secondaries[i++] = peer; - peer_sd->primary = false; - peer_sd->primary_dev = primary; - } - - mlx5_devcom_for_each_peer_end(devcom); - return 0; - -err_devcom_unreg: - mlx5_devcom_comp_lock(sd->devcom); - mlx5_devcom_comp_set_ready(sd->devcom, false); - mlx5_devcom_comp_unlock(sd->devcom); - mlx5_devcom_unregister_component(sd->devcom); - return err; -} - -static void sd_unregister(struct mlx5_core_dev *dev) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - - mlx5_devcom_comp_lock(sd->devcom); - mlx5_devcom_comp_set_ready(sd->devcom, false); - mlx5_devcom_comp_unlock(sd->devcom); - mlx5_devcom_unregister_component(sd->devcom); -} - -static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key) -{ - struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {}; - struct mlx5_sd *sd = mlx5_get_sd(primary); - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5_flow_namespace *nic_ns; - struct mlx5_flow_table *ft; - int err; - - nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS); - if (!nic_ns) - return -EOPNOTSUPP; - - ft = mlx5_create_flow_table(nic_ns, &ft_attr); - if (IS_ERR(ft)) { - err = PTR_ERR(ft); - return err; - } - sd->tx_ft = ft; - memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN); - allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; - allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; - - err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr); - if (err) { - mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n", - err); - mlx5_destroy_flow_table(ft); - return err; - } - - return 0; -} - -static void sd_cmd_unset_primary(struct mlx5_core_dev *primary) -{ - struct mlx5_sd *sd = mlx5_get_sd(primary); - - mlx5_destroy_flow_table(sd->tx_ft); -} - -static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary, - struct mlx5_core_dev *primary, - struct mlx5_flow_table *ft, - u32 *obj_id, u8 *alias_key) -{ - u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id; - u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id); - struct mlx5_cmd_alias_obj_create_attr alias_attr = {}; - int ret; - - memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN); - alias_attr.obj_id = aliased_object_id; - alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS; - alias_attr.vhca_id = vhca_id_to_be_accessed; - ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id); - if (ret) { - mlx5_core_err(secondary, "Failed to create alias object err=%d\n", - ret); - return ret; - } - - return 0; -} - -static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary) -{ - struct mlx5_sd *sd = mlx5_get_sd(secondary); - - mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id, - MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS); -} - -static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary, - struct mlx5_core_dev *primary, - u8 *alias_key) -{ - struct mlx5_sd *primary_sd = mlx5_get_sd(primary); - struct mlx5_sd *sd = mlx5_get_sd(secondary); - int err; - - err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1); - if (err) - return err; - - err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft, - &sd->alias_obj_id, alias_key); - if (err) - goto err_unset_silent; - - err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false); - if (err) - goto err_destroy_alias_ft; - - return 0; - -err_destroy_alias_ft: - sd_secondary_destroy_alias_ft(secondary); -err_unset_silent: - mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); - return err; -} - -static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary) -{ - mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true); - sd_secondary_destroy_alias_ft(secondary); - mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0); -} - -static void sd_print_group(struct mlx5_core_dev *primary) -{ - struct mlx5_sd *sd = mlx5_get_sd(primary); - struct mlx5_core_dev *pos; - int i; - - sd_info(primary, "group id %#x, primary %s, vhca %u\n", - sd->group_id, pci_name(primary->pdev), - MLX5_CAP_GEN(primary, vhca_id)); - mlx5_sd_for_each_secondary(i, primary, pos) - sd_info(primary, "group id %#x, secondary#%d %s, vhca %u\n", - sd->group_id, i - 1, pci_name(pos->pdev), - MLX5_CAP_GEN(pos, vhca_id)); -} - -int mlx5_sd_init(struct mlx5_core_dev *dev) -{ - struct mlx5_core_dev *primary, *pos, *to; - struct mlx5_sd *sd = mlx5_get_sd(dev); - u8 alias_key[ACCESS_KEY_LEN]; - int err, i; - - err = sd_init(dev); - if (err) - return err; - - sd = mlx5_get_sd(dev); - if (!sd) - return 0; - - err = sd_register(dev); - if (err) - goto err_sd_cleanup; - - if (!mlx5_devcom_comp_is_ready(sd->devcom)) - return 0; - - primary = mlx5_sd_get_primary(dev); - - for (i = 0; i < ACCESS_KEY_LEN; i++) - alias_key[i] = get_random_u8(); - - err = sd_cmd_set_primary(primary, alias_key); - if (err) - goto err_sd_unregister; - - mlx5_sd_for_each_secondary(i, primary, pos) { - err = sd_cmd_set_secondary(pos, primary, alias_key); - if (err) - goto err_unset_secondaries; - } - - sd_info(primary, "group id %#x, size %d, combined\n", - sd->group_id, mlx5_devcom_comp_get_size(sd->devcom)); - sd_print_group(primary); - - return 0; - -err_unset_secondaries: - to = pos; - mlx5_sd_for_each_secondary_to(i, primary, to, pos) - sd_cmd_unset_secondary(pos); - sd_cmd_unset_primary(primary); -err_sd_unregister: - sd_unregister(dev); -err_sd_cleanup: - sd_cleanup(dev); - return err; -} - -void mlx5_sd_cleanup(struct mlx5_core_dev *dev) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - struct mlx5_core_dev *primary, *pos; - int i; - - if (!sd) - return; - - if (!mlx5_devcom_comp_is_ready(sd->devcom)) - goto out; - - primary = mlx5_sd_get_primary(dev); - mlx5_sd_for_each_secondary(i, primary, pos) - sd_cmd_unset_secondary(pos); - sd_cmd_unset_primary(primary); - - sd_info(primary, "group id %#x, uncombined\n", sd->group_id); -out: - sd_unregister(dev); - sd_cleanup(dev); -} - -struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, - struct auxiliary_device *adev, - int idx) -{ - struct mlx5_sd *sd = mlx5_get_sd(dev); - struct mlx5_core_dev *primary; - - if (!sd) - return adev; - - if (!mlx5_devcom_comp_is_ready(sd->devcom)) - return NULL; - - primary = mlx5_sd_get_primary(dev); - if (dev == primary) - return adev; - - return &primary->priv.adev[idx]->adev; -} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h deleted file mode 100644 index 137efaf9aabc..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ - -#ifndef __MLX5_LIB_SD_H__ -#define __MLX5_LIB_SD_H__ - -#define MLX5_SD_MAX_GROUP_SZ 2 - -struct mlx5_sd; - -struct mlx5_core_dev *mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx); -int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix); -int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix); -struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix); -struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, - struct auxiliary_device *adev, - int idx); - -int mlx5_sd_init(struct mlx5_core_dev *dev); -void mlx5_sd_cleanup(struct mlx5_core_dev *dev); - -#define mlx5_sd_for_each_dev_from_to(i, primary, ix_from, to, pos) \ - for (i = ix_from; \ - (pos = mlx5_sd_primary_get_peer(primary, i)) && pos != (to); i++) - -#define mlx5_sd_for_each_dev(i, primary, pos) \ - mlx5_sd_for_each_dev_from_to(i, primary, 0, NULL, pos) - -#define mlx5_sd_for_each_dev_to(i, primary, to, pos) \ - mlx5_sd_for_each_dev_from_to(i, primary, 0, to, pos) - -#define mlx5_sd_for_each_secondary(i, primary, pos) \ - mlx5_sd_for_each_dev_from_to(i, primary, 1, NULL, pos) - -#define mlx5_sd_for_each_secondary_to(i, primary, to, pos) \ - mlx5_sd_for_each_dev_from_to(i, primary, 1, to, pos) - -#endif /* __MLX5_LIB_SD_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 1005bb6935b6..21753f327868 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -440,27 +440,6 @@ out: } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_system_image_guid); -int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group) -{ - int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out); - u32 *out; - int err; - - out = kvzalloc(outlen, GFP_KERNEL); - if (!out) - return -ENOMEM; - - err = mlx5_query_nic_vport_context(mdev, 0, out); - if (err) - goto out; - - *sd_group = MLX5_GET(query_nic_vport_context_out, out, - nic_vport_context.sd_group); -out: - kvfree(out); - return err; -} - int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid) { u32 *out; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 2bba88c67f58..7ee5b79ff3d6 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -681,7 +681,6 @@ struct mlx5e_resources { struct mlx5_sq_bfreg bfreg; #define MLX5_MAX_NUM_TC 8 u32 tisn[MLX5_MAX_PORTS][MLX5_MAX_NUM_TC]; - bool tisn_valid; } hw_objs; struct net_device *uplink_netdev; struct mutex uplink_netdev_lock; @@ -822,7 +821,6 @@ struct mlx5_core_dev { struct blocking_notifier_head macsec_nh; #endif u64 num_ipsec_offloads; - struct mlx5_sd *sd; }; struct mlx5_db { @@ -1224,14 +1222,6 @@ static inline bool mlx5_core_is_ecpf(const struct mlx5_core_dev *dev) return dev->caps.embedded_cpu; } -static inline bool mlx5_core_is_mgmt_pf(const struct mlx5_core_dev *dev) -{ - if (!MLX5_CAP_GEN_2(dev, local_mng_port_valid)) - return false; - - return MLX5_CAP_GEN_2(dev, local_mng_port); -} - static inline bool mlx5_core_is_ecpf_esw_manager(const struct mlx5_core_dev *dev) { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 586569209254..fee20fc010c2 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1954,10 +1954,8 @@ enum { struct mlx5_ifc_cmd_hca_cap_2_bits { u8 reserved_at_0[0x80]; - u8 migratable[0x1]; - u8 reserved_at_81[0x19]; - u8 local_mng_port[0x1]; - u8 reserved_at_9b[0x5]; + u8 migratable[0x1]; + u8 reserved_at_81[0x1f]; u8 max_reformat_insert_size[0x8]; u8 max_reformat_insert_offset[0x8]; @@ -1975,13 +1973,7 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 allowed_object_for_other_vhca_access[0x40]; - u8 reserved_at_140[0x20]; - - u8 reserved_at_160[0xa]; - u8 local_mng_port_valid[0x1]; - u8 reserved_at_16b[0x15]; - - u8 reserved_at_180[0x20]; + u8 reserved_at_140[0x60]; u8 flow_table_type_2_type[0x8]; u8 reserved_at_1a8[0x3]; @@ -4038,13 +4030,8 @@ struct mlx5_ifc_nic_vport_context_bits { u8 affiliation_criteria[0x4]; u8 affiliated_vhca_id[0x10]; - u8 reserved_at_60[0xa0]; + u8 reserved_at_60[0xd0]; - u8 reserved_at_100[0x1]; - u8 sd_group[0x3]; - u8 reserved_at_104[0x1c]; - - u8 reserved_at_120[0x10]; u8 mtu[0x10]; u8 system_image_guid[0x40]; @@ -10129,7 +10116,8 @@ struct mlx5_ifc_mpir_reg_bits { u8 reserved_at_20[0x20]; u8 local_port[0x8]; - u8 reserved_at_28[0x18]; + u8 reserved_at_28[0x15]; + u8 sd_group[0x3]; u8 reserved_at_60[0x20]; }; diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index c36cc6d82926..fbb9bf447889 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -72,7 +72,6 @@ int mlx5_query_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 *mtu); int mlx5_modify_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 mtu); int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev, u64 *system_image_guid); -int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group); int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid); int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, u16 vport, u64 node_guid); -- cgit v1.2.3