From 6c92bd5cd4650c39dd929565ee172984c680fead Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Wed, 1 Jul 2020 23:44:07 +0200 Subject: selftests/bpf: Test_progs indicate to shell on non-actions When a user selects a non-existing test the summary is printed with indication 0 for all info types, and shell "success" (EXIT_SUCCESS) is indicated. This can be understood by a human end-user, but for shell scripting is it useful to indicate a shell failure (EXIT_FAILURE). Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/159363984736.930467.17956007131403952343.stgit@firesoul --- tools/testing/selftests/bpf/test_progs.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 54fa5fa688ce..da70a4f72f54 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -687,5 +687,8 @@ int main(int argc, char **argv) free_str_set(&env.subtest_selector.whitelist); free(env.subtest_selector.num_set); + if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0) + return EXIT_FAILURE; + return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS; } -- cgit v1.2.3 From 643e7233aa948901dce81d4573c91ed99fdd272e Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Wed, 1 Jul 2020 23:44:12 +0200 Subject: selftests/bpf: Test_progs option for getting number of tests It can be practial to get the number of tests that test_progs contain. This could for example be used to create a shell for-loop construct that runs the individual tests. Like: for N in $(seq 1 $(./test_progs -c)); do ./test_progs -n $N 2>&1 > result_test_${N}.log & done ; wait V2: Add the ability to return the count for the selected tests. This is useful for getting a count e.g. after excluding some tests with option -b. The current beakers test script like to report the max test count upfront. Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/159363985244.930467.12617117873058936829.stgit@firesoul --- tools/testing/selftests/bpf/test_progs.c | 18 ++++++++++++++++++ tools/testing/selftests/bpf/test_progs.h | 1 + 2 files changed, 19 insertions(+) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index da70a4f72f54..a5dba14b2025 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -366,6 +366,7 @@ enum ARG_KEYS { ARG_TEST_NAME_BLACKLIST = 'b', ARG_VERIFIER_STATS = 's', ARG_VERBOSE = 'v', + ARG_GET_TEST_CNT = 'c', }; static const struct argp_option opts[] = { @@ -379,6 +380,8 @@ static const struct argp_option opts[] = { "Output verifier statistics", }, { "verbose", ARG_VERBOSE, "LEVEL", OPTION_ARG_OPTIONAL, "Verbose output (use -vv or -vvv for progressively verbose output)" }, + { "count", ARG_GET_TEST_CNT, NULL, 0, + "Get number of selected top-level tests " }, {}, }; @@ -511,6 +514,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) } } break; + case ARG_GET_TEST_CNT: + env->get_test_cnt = true; + break; case ARGP_KEY_ARG: argp_usage(state); break; @@ -654,6 +660,11 @@ int main(int argc, char **argv) test->test_num, test->test_name)) continue; + if (env.get_test_cnt) { + env.succ_cnt++; + continue; + } + test->run_test(); /* ensure last sub-test is finalized properly */ if (test->subtest_name) @@ -677,9 +688,16 @@ int main(int argc, char **argv) cleanup_cgroup_environment(); } stdio_restore(); + + if (env.get_test_cnt) { + printf("%d\n", env.succ_cnt); + goto out; + } + fprintf(stdout, "Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", env.succ_cnt, env.sub_succ_cnt, env.skip_cnt, env.fail_cnt); +out: free_str_set(&env.test_selector.blacklist); free_str_set(&env.test_selector.whitelist); free(env.test_selector.num_set); diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index f4503c926aca..0030584619c3 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -66,6 +66,7 @@ struct test_env { enum verbosity verbosity; bool jit_enabled; + bool get_test_cnt; struct prog_test_def *test; FILE *stdout; -- cgit v1.2.3 From c1f1f3656eee3a59a40e1805699041ec1c14ab83 Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Wed, 1 Jul 2020 23:44:17 +0200 Subject: selftests/bpf: Test_progs option for listing test names The program test_progs have some very useful ability to specify a list of test name substrings for selecting which tests to run. This patch add the ability to list the selected test names without running them. This is practical for seeing which tests gets selected with given select arguments (which can also contain a exclude list via --name-blacklist). This output can also be used by shell-scripts in a for-loop: for N in $(./test_progs --list -t xdp); do \ ./test_progs -t $N 2>&1 > result_test_${N}.log & \ done ; wait This features can also be used for looking up a test number and returning a testname. If the selection was empty then a shell EXIT_FAILURE is returned. This is useful for scripting. e.g. like this: n=1; while [ $(./test_progs --list -n $n) ] ; do \ ./test_progs -n $n ; n=$(( n+1 )); \ done Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/159363985751.930467.9610992940793316982.stgit@firesoul --- tools/testing/selftests/bpf/test_progs.c | 15 +++++++++++++++ tools/testing/selftests/bpf/test_progs.h | 1 + 2 files changed, 16 insertions(+) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index a5dba14b2025..ef05d2f0e7bb 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -367,6 +367,7 @@ enum ARG_KEYS { ARG_VERIFIER_STATS = 's', ARG_VERBOSE = 'v', ARG_GET_TEST_CNT = 'c', + ARG_LIST_TEST_NAMES = 'l', }; static const struct argp_option opts[] = { @@ -382,6 +383,8 @@ static const struct argp_option opts[] = { "Verbose output (use -vv or -vvv for progressively verbose output)" }, { "count", ARG_GET_TEST_CNT, NULL, 0, "Get number of selected top-level tests " }, + { "list", ARG_LIST_TEST_NAMES, NULL, 0, + "List test names that would run (without running them) " }, {}, }; @@ -517,6 +520,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case ARG_GET_TEST_CNT: env->get_test_cnt = true; break; + case ARG_LIST_TEST_NAMES: + env->list_test_names = true; + break; case ARGP_KEY_ARG: argp_usage(state); break; @@ -665,6 +671,12 @@ int main(int argc, char **argv) continue; } + if (env.list_test_names) { + fprintf(env.stdout, "%s\n", test->test_name); + env.succ_cnt++; + continue; + } + test->run_test(); /* ensure last sub-test is finalized properly */ if (test->subtest_name) @@ -694,6 +706,9 @@ int main(int argc, char **argv) goto out; } + if (env.list_test_names) + goto out; + fprintf(stdout, "Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", env.succ_cnt, env.sub_succ_cnt, env.skip_cnt, env.fail_cnt); diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index 0030584619c3..ec31f382e7fd 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -67,6 +67,7 @@ struct test_env { bool jit_enabled; bool get_test_cnt; + bool list_test_names; struct prog_test_def *test; FILE *stdout; -- cgit v1.2.3 From 811d7e375d08312dba23f3b6bf7e58ec14aa5dcb Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Wed, 1 Jul 2020 17:48:58 -0700 Subject: bpf: selftests: Restore netns after each test It is common for networking tests creating its netns and making its own setting under this new netns (e.g. changing tcp sysctl). If the test forgot to restore to the original netns, it would affect the result of other tests. This patch saves the original netns at the beginning and then restores it after every test. Since the restore "setns()" is not expensive, it does it on all tests without tracking if a test has created a new netns or not. The new restore_netns() could also be done in test__end_subtest() such that each subtest will get an automatic netns reset. However, the individual test would lose flexibility to have total control on netns for its own subtests. In some cases, forcing a test to do unnecessary netns re-configure for each subtest is time consuming. e.g. In my vm, forcing netns re-configure on each subtest in sk_assign.c increased the runtime from 1s to 8s. On top of that, test_progs.c is also doing per-test (instead of per-subtest) cleanup for cgroup. Thus, this patch also does per-test restore_netns(). The only existing per-subtest cleanup is reset_affinity() and no test is depending on this. Thus, it is removed from test__end_subtest() to give a consistent expectation to the individual tests. test_progs.c only ensures any affinity/netns/cgroup change made by an earlier test does not affect the following tests. Signed-off-by: Martin KaFai Lau Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20200702004858.2103728-1-kafai@fb.com --- tools/testing/selftests/bpf/test_progs.c | 23 +++++++++++++++++++++-- tools/testing/selftests/bpf/test_progs.h | 2 ++ 2 files changed, 23 insertions(+), 2 deletions(-) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index ef05d2f0e7bb..104e833d0087 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -121,6 +121,24 @@ static void reset_affinity() { } } +static void save_netns(void) +{ + env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY); + if (env.saved_netns_fd == -1) { + perror("open(/proc/self/ns/net)"); + exit(-1); + } +} + +static void restore_netns(void) +{ + if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) { + stdio_restore(); + perror("setns(CLONE_NEWNS)"); + exit(-1); + } +} + void test__end_subtest() { struct prog_test_def *test = env.test; @@ -138,8 +156,6 @@ void test__end_subtest() test->test_num, test->subtest_num, test->subtest_name, sub_error_cnt ? "FAIL" : "OK"); - reset_affinity(); - free(test->subtest_name); test->subtest_name = NULL; } @@ -655,6 +671,7 @@ int main(int argc, char **argv) return -1; } + save_netns(); stdio_hijack(); for (i = 0; i < prog_test_cnt; i++) { struct prog_test_def *test = &prog_test_defs[i]; @@ -696,6 +713,7 @@ int main(int argc, char **argv) test->error_cnt ? "FAIL" : "OK"); reset_affinity(); + restore_netns(); if (test->need_cgroup_cleanup) cleanup_cgroup_environment(); } @@ -719,6 +737,7 @@ out: free_str_set(&env.subtest_selector.blacklist); free_str_set(&env.subtest_selector.whitelist); free(env.subtest_selector.num_set); + close(env.saved_netns_fd); if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0) return EXIT_FAILURE; diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index ec31f382e7fd..6e09bf738473 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -80,6 +80,8 @@ struct test_env { int sub_succ_cnt; /* successful sub-tests */ int fail_cnt; /* total failed tests + sub-tests */ int skip_cnt; /* skipped tests */ + + int saved_netns_fd; }; extern struct test_env env; -- cgit v1.2.3 From 3220fb667842a9725cbb71656f406eadb03c094b Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Tue, 7 Jul 2020 09:12:19 +0200 Subject: selftests/bpf: test_progs use another shell exit on non-actions This is a follow up adjustment to commit 6c92bd5cd465 ("selftests/bpf: Test_progs indicate to shell on non-actions"), that returns shell exit indication EXIT_FAILURE (value 1) when user selects a non-existing test. The problem with using EXIT_FAILURE is that a shell script cannot tell the difference between a non-existing test and the test failing. This patch uses value 2 as shell exit indication. (Aside note unrecognized option parameters use value 64). Fixes: 6c92bd5cd465 ("selftests/bpf: Test_progs indicate to shell on non-actions") Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/159410593992.1093222.90072558386094370.stgit@firesoul --- tools/testing/selftests/bpf/test_progs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 104e833d0087..65d3f8686e29 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -12,6 +12,8 @@ #include #include /* backtrace */ +#define EXIT_NO_TEST 2 + /* defined in test_progs.h */ struct test_env env = {}; @@ -740,7 +742,7 @@ out: close(env.saved_netns_fd); if (env.succ_cnt + env.fail_cnt + env.skip_cnt == 0) - return EXIT_FAILURE; + return EXIT_NO_TEST; return env.fail_cnt ? EXIT_FAILURE : EXIT_SUCCESS; } -- cgit v1.2.3 From b8c50df0cb3eb9008f8372e4ff0317eee993b8d1 Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Tue, 7 Jul 2020 09:12:25 +0200 Subject: selftests/bpf: test_progs avoid minus shell exit codes There are a number of places in test_progs that use minus-1 as the argument to exit(). This is confusing as a process exit status is masked to be a number between 0 and 255 as defined in man exit(3). Thus, users will see status 255 instead of minus-1. This patch use positive exit code 3 instead of minus-1. These cases are put in the same group of infrastructure setup errors. Fixes: fd27b1835e70 ("selftests/bpf: Reset process and thread affinity after each test/sub-test") Fixes: 811d7e375d08 ("bpf: selftests: Restore netns after each test") Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/159410594499.1093222.11080787853132708654.stgit@firesoul --- tools/testing/selftests/bpf/test_progs.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'tools/testing/selftests/bpf/test_progs.c') diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 65d3f8686e29..b1e4dadacd9b 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -13,6 +13,7 @@ #include /* backtrace */ #define EXIT_NO_TEST 2 +#define EXIT_ERR_SETUP_INFRA 3 /* defined in test_progs.h */ struct test_env env = {}; @@ -113,13 +114,13 @@ static void reset_affinity() { if (err < 0) { stdio_restore(); fprintf(stderr, "Failed to reset process affinity: %d!\n", err); - exit(-1); + exit(EXIT_ERR_SETUP_INFRA); } err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset); if (err < 0) { stdio_restore(); fprintf(stderr, "Failed to reset thread affinity: %d!\n", err); - exit(-1); + exit(EXIT_ERR_SETUP_INFRA); } } @@ -128,7 +129,7 @@ static void save_netns(void) env.saved_netns_fd = open("/proc/self/ns/net", O_RDONLY); if (env.saved_netns_fd == -1) { perror("open(/proc/self/ns/net)"); - exit(-1); + exit(EXIT_ERR_SETUP_INFRA); } } @@ -137,7 +138,7 @@ static void restore_netns(void) if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) { stdio_restore(); perror("setns(CLONE_NEWNS)"); - exit(-1); + exit(EXIT_ERR_SETUP_INFRA); } } -- cgit v1.2.3