From 9f57cc76eccc1e0a369bb051c4b0d596e7d15e30 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Fri, 5 Nov 2021 18:30:58 -0700 Subject: kunit: tool: add --kconfig_add to allow easily tweaking kunitconfigs E.g. run tests but with KASAN $ ./tools/testing/kunit/kunit.py run --arch=x86_64 --kconfig_add=CONFIG_KASAN=y This also works with --kunitconfig $ ./tools/testing/kunit/kunit.py run --arch=x86_64 --kunitconfig=fs/ext4 --kconfig_add=CONFIG_KASAN=y This flag is inspired by TuxMake's --kconfig-add, see https://gitlab.com/Linaro/tuxmake#examples. Our version just uses "_" as the delimiter for consistency with pre-existing flags like --build_dir, --make_options, --kernel_args, etc. Note: this does make it easier to run into a pre-existing edge case: $ ./tools/testing/kunit/kunit.py run --arch=x86_64 --kconfig_add=CONFIG_KASAN=y $ ./tools/testing/kunit/kunit.py run --arch=x86_64 This second invocation ^ still has KASAN enabled! kunit.py won't call olddefconfig if our current .config is already a superset of the provided kunitconfig. Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 68e6f461c758..be58f4c93806 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -280,6 +280,10 @@ def add_common_opts(parser) -> None: ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 'will get automatically appended.', metavar='kunitconfig') + parser.add_argument('--kconfig_add', + help='Additional Kconfig options to append to the ' + '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', + action='append') parser.add_argument('--arch', help=('Specifies the architecture to run tests under. ' @@ -398,6 +402,7 @@ def main(argv, linux=None): if not linux: linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, kunitconfig_path=cli_args.kunitconfig, + kconfig_add=cli_args.kconfig_add, arch=cli_args.arch, cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) @@ -423,6 +428,7 @@ def main(argv, linux=None): if not linux: linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, kunitconfig_path=cli_args.kunitconfig, + kconfig_add=cli_args.kconfig_add, arch=cli_args.arch, cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) @@ -439,6 +445,7 @@ def main(argv, linux=None): if not linux: linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, kunitconfig_path=cli_args.kunitconfig, + kconfig_add=cli_args.kconfig_add, arch=cli_args.arch, cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) @@ -457,6 +464,7 @@ def main(argv, linux=None): if not linux: linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, kunitconfig_path=cli_args.kunitconfig, + kconfig_add=cli_args.kconfig_add, arch=cli_args.arch, cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) -- cgit v1.2.3 From 7fa7ffcf9babaea2f0a81681b4ef460ee4b93278 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Fri, 19 Nov 2021 19:24:01 -0800 Subject: kunit: tool: suggest using decode_stacktrace.sh on kernel crash kunit.py isn't very clear that 1) it stashes a copy of the unparsed output in $BUILD_DIR/test.log 2) it sets $BUILD_DIR=.kunit by default So it's trickier than it should be for a user to come up with the right command to do so. Make kunit.py print out a command for this if a) we saw a test case crash b) we only ran one kernel (test.log only contains output from the last) Example suggested command: $ scripts/decode_stacktrace.sh .kunit/vmlinux .kunit < .kunit/test.log | tee .kunit/decoded.log | ./tools/testing/kunit/kunit.py parse Without debug info a user might see something like [14:11:25] Call Trace: [14:11:25] ? kunit_binary_assert_format (:?) [14:11:25] kunit_try_run_case (test.c:?) [14:11:25] ? __kthread_parkme (kthread.c:?) [14:11:25] kunit_generic_run_threadfn_adapter (try-catch.c:?) [14:11:25] ? kunit_generic_run_threadfn_adapter (try-catch.c:?) [14:11:25] kthread (kthread.c:?) [14:11:25] new_thread_handler (:?) [14:11:25] [CRASHED] `tee` is in GNU coreutils, so it seems fine to add that into the pipeline by default, that way users can inspect the otuput in more detail. Note: to turn on debug info, users would need to do something like $ echo -e 'CONFIG_DEBUG_KERNEL=y\nCONFIG_DEBUG_INFO=y' >> .kunit/.kunitconfig $ ./tools/testing/kunit/kunit.py config $ ./tools/testing/kunit/kunit.py build $ This feels too clunky to include in the instructions. With --kconfig_add [1], it would become a bit less painful. [1] https://lore.kernel.org/linux-kselftest/20211106013058.2621799-2-dlatypov@google.com/ Signed-off-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index be58f4c93806..8c52faf5787c 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -156,6 +156,12 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, test_counts.add_subtest_counts(result.result.test.counts) + if len(filter_globs) == 1 and test_counts.crashed > 0: + bd = request.build_dir + print('The kernel seems to have crashed; you can decode the stack traces with:') + print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format( + bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) + kunit_status = _map_to_overall_status(test_counts.get_status()) return KunitResult(status=kunit_status, result=result.result, elapsed_time=exec_time) -- cgit v1.2.3 From db1679813f9f86b05bbbc6f05f4cdbe481d59352 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 14 Dec 2021 11:26:11 -0800 Subject: kunit: tool: use dataclass instead of collections.namedtuple namedtuple is a terse way of defining a collection of fields. However, it does not allow us to annotate the type of these fields. It also doesn't let us have any sort of inheritance between types. Since commit df4b0807ca1a ("kunit: tool: Assert the version requirement"), kunit.py has asserted that it's running on python >=3.7. So in that case use a 3.7 feature, dataclasses, to replace these. Changes in detail: * Make KunitExecRequest contain all the fields needed for exec_tests * Use inheritance to dedupe fields * also allows us to e.g. pass a KUnitRequest in as a KUnitParseRequest * this has changed around the order of some fields * Use named arguments when constructing all request objects in kunit.py * This is to prevent accidentally mixing up fields, etc. Signed-off-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 139 +++++++++++++++++---------------- tools/testing/kunit/kunit_tool_test.py | 6 +- 2 files changed, 75 insertions(+), 70 deletions(-) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 8c52faf5787c..4f5c7984cf3f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -15,38 +15,57 @@ import time assert sys.version_info >= (3, 7), "Python version is too old" -from collections import namedtuple +from dataclasses import dataclass from enum import Enum, auto -from typing import Iterable, Sequence, List +from typing import Any, Iterable, Sequence, List, Optional import kunit_json import kunit_kernel import kunit_parser -KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) - -KunitConfigRequest = namedtuple('KunitConfigRequest', - ['build_dir', 'make_options']) -KunitBuildRequest = namedtuple('KunitBuildRequest', - ['jobs', 'build_dir', 'alltests', - 'make_options']) -KunitExecRequest = namedtuple('KunitExecRequest', - ['timeout', 'build_dir', 'alltests', - 'filter_glob', 'kernel_args', 'run_isolated']) -KunitParseRequest = namedtuple('KunitParseRequest', - ['raw_output', 'build_dir', 'json']) -KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', - 'build_dir', 'alltests', 'filter_glob', - 'kernel_args', 'run_isolated', 'json', 'make_options']) - -KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] - class KunitStatus(Enum): SUCCESS = auto() CONFIG_FAILURE = auto() BUILD_FAILURE = auto() TEST_FAILURE = auto() +@dataclass +class KunitResult: + status: KunitStatus + result: Any + elapsed_time: float + +@dataclass +class KunitConfigRequest: + build_dir: str + make_options: Optional[List[str]] + +@dataclass +class KunitBuildRequest(KunitConfigRequest): + jobs: int + alltests: bool + +@dataclass +class KunitParseRequest: + raw_output: Optional[str] + build_dir: str + json: Optional[str] + +@dataclass +class KunitExecRequest(KunitParseRequest): + timeout: int + alltests: bool + filter_glob: str + kernel_args: Optional[List[str]] + run_isolated: Optional[str] + +@dataclass +class KunitRequest(KunitExecRequest, KunitBuildRequest): + pass + + +KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] + def get_kernel_root_path() -> str: path = sys.argv[0] if not __file__ else __file__ parts = os.path.realpath(path).split('tools/testing/kunit') @@ -121,8 +140,7 @@ def _suites_from_test_list(tests: List[str]) -> List[str]: -def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, - parse_request: KunitParseRequest) -> KunitResult: +def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: filter_globs = [request.filter_glob] if request.run_isolated: tests = _list_tests(linux, request) @@ -147,7 +165,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, filter_glob=filter_glob, build_dir=request.build_dir) - result = parse_tests(parse_request, run_result) + result = parse_tests(request, run_result) # run_kernel() doesn't block on the kernel exiting. # That only happens after we get the last line of output from `run_result`. # So exec_time here actually contains parsing + execution time, which is fine. @@ -217,27 +235,15 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitRequest) -> KunitResult: run_start = time.time() - config_request = KunitConfigRequest(request.build_dir, - request.make_options) - config_result = config_tests(linux, config_request) + config_result = config_tests(linux, request) if config_result.status != KunitStatus.SUCCESS: return config_result - build_request = KunitBuildRequest(request.jobs, request.build_dir, - request.alltests, - request.make_options) - build_result = build_tests(linux, build_request) + build_result = build_tests(linux, request) if build_result.status != KunitStatus.SUCCESS: return build_result - exec_request = KunitExecRequest(request.timeout, request.build_dir, - request.alltests, request.filter_glob, - request.kernel_args, request.run_isolated) - parse_request = KunitParseRequest(request.raw_output, - request.build_dir, - request.json) - - exec_result = exec_tests(linux, exec_request, parse_request) + exec_result = exec_tests(linux, request) run_end = time.time() @@ -413,16 +419,16 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitRequest(cli_args.raw_output, - cli_args.timeout, - cli_args.jobs, - cli_args.build_dir, - cli_args.alltests, - cli_args.filter_glob, - cli_args.kernel_args, - cli_args.run_isolated, - cli_args.json, - cli_args.make_options) + request = KunitRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options, + jobs=cli_args.jobs, + alltests=cli_args.alltests, + raw_output=cli_args.raw_output, + json=cli_args.json, + timeout=cli_args.timeout, + filter_glob=cli_args.filter_glob, + kernel_args=cli_args.kernel_args, + run_isolated=cli_args.run_isolated) result = run_tests(linux, request) if result.status != KunitStatus.SUCCESS: sys.exit(1) @@ -439,8 +445,8 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitConfigRequest(cli_args.build_dir, - cli_args.make_options) + request = KunitConfigRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options) result = config_tests(linux, request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -456,10 +462,10 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - request = KunitBuildRequest(cli_args.jobs, - cli_args.build_dir, - cli_args.alltests, - cli_args.make_options) + request = KunitBuildRequest(build_dir=cli_args.build_dir, + make_options=cli_args.make_options, + jobs=cli_args.jobs, + alltests=cli_args.alltests) result = build_tests(linux, request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -475,16 +481,15 @@ def main(argv, linux=None): cross_compile=cli_args.cross_compile, qemu_config_path=cli_args.qemu_config) - exec_request = KunitExecRequest(cli_args.timeout, - cli_args.build_dir, - cli_args.alltests, - cli_args.filter_glob, - cli_args.kernel_args, - cli_args.run_isolated) - parse_request = KunitParseRequest(cli_args.raw_output, - cli_args.build_dir, - cli_args.json) - result = exec_tests(linux, exec_request, parse_request) + exec_request = KunitExecRequest(raw_output=cli_args.raw_output, + build_dir=cli_args.build_dir, + json=cli_args.json, + timeout=cli_args.timeout, + alltests=cli_args.alltests, + filter_glob=cli_args.filter_glob, + kernel_args=cli_args.kernel_args, + run_isolated=cli_args.run_isolated) + result = exec_tests(linux, exec_request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % (result.elapsed_time)) if result.status != KunitStatus.SUCCESS: @@ -496,9 +501,9 @@ def main(argv, linux=None): else: with open(cli_args.file, 'r', errors='backslashreplace') as f: kunit_output = f.read().splitlines() - request = KunitParseRequest(cli_args.raw_output, - None, - cli_args.json) + request = KunitParseRequest(raw_output=cli_args.raw_output, + build_dir='', + json=cli_args.json) result = parse_tests(request, kunit_output) if result.status != KunitStatus.SUCCESS: sys.exit(1) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 97a4c72af2b8..e30e162bd084 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -691,7 +691,7 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want got = kunit._list_tests(self.linux_source_mock, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'suite')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'suite')) self.assertEqual(got, want) # Should respect the user's filter glob when listing tests. @@ -706,7 +706,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*.test*', None, 'suite')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*.test*', None, 'suite')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300), @@ -719,7 +719,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'test')) + kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'test')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300), -- cgit v1.2.3 From e0cc8c052a3992b01f51df1d51eaae49e5b2710f Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 14 Dec 2021 11:26:12 -0800 Subject: kunit: tool: delete kunit_parser.TestResult type The `log` field is unused, and the `status` field is accessible via `test.status`. So it's simpler to just return the main `Test` object directly. And since we're no longer returning a namedtuple, which has no type annotations, this hopefully means typecheckers are better equipped to find any errors. Signed-off-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 14 ++++++-------- tools/testing/kunit/kunit_json.py | 6 +++--- tools/testing/kunit/kunit_parser.py | 10 +++------- tools/testing/kunit/kunit_tool_test.py | 34 +++++++++++++++++----------------- 4 files changed, 29 insertions(+), 35 deletions(-) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 4f5c7984cf3f..417dc2d11f4f 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -172,7 +172,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - test_end = time.time() exec_time += test_end - test_start - test_counts.add_subtest_counts(result.result.test.counts) + test_counts.add_subtest_counts(result.result.counts) if len(filter_globs) == 1 and test_counts.crashed > 0: bd = request.build_dir @@ -181,7 +181,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) kunit_status = _map_to_overall_status(test_counts.get_status()) - return KunitResult(status=kunit_status, result=result.result, elapsed_time=exec_time) + return KunitResult(status=kunit_status, result=result, elapsed_time=exec_time) def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): @@ -192,14 +192,12 @@ def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitResult: parse_start = time.time() - test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, - kunit_parser.Test(), - 'Tests not Parsed.') + test_result = kunit_parser.Test() if request.raw_output: # Treat unparsed results as one passing test. - test_result.test.status = kunit_parser.TestStatus.SUCCESS - test_result.test.counts.passed = 1 + test_result.status = kunit_parser.TestStatus.SUCCESS + test_result.counts.passed = 1 output: Iterable[str] = input_data if request.raw_output == 'all': @@ -217,7 +215,7 @@ def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitR if request.json: json_obj = kunit_json.get_json_result( - test_result=test_result, + test=test_result, def_config='kunit_defconfig', build_dir=request.build_dir, json_path=request.json) diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py index b6e66c5d64d1..6862671709bc 100644 --- a/tools/testing/kunit/kunit_json.py +++ b/tools/testing/kunit/kunit_json.py @@ -11,7 +11,7 @@ import os import kunit_parser -from kunit_parser import Test, TestResult, TestStatus +from kunit_parser import Test, TestStatus from typing import Any, Dict, Optional JsonObj = Dict[str, Any] @@ -50,9 +50,9 @@ def _get_group_json(test: Test, def_config: str, } return test_group -def get_json_result(test_result: TestResult, def_config: str, +def get_json_result(test: Test, def_config: str, build_dir: Optional[str], json_path: str) -> str: - test_group = _get_group_json(test_result.test, def_config, build_dir) + test_group = _get_group_json(test, def_config, build_dir) test_group["name"] = "KUnit Test Group" json_obj = json.dumps(test_group, indent=4) if json_path != 'stdout': diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 8e42b6ef3fe3..66a7f2fb314a 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -12,14 +12,11 @@ from __future__ import annotations import re -from collections import namedtuple from datetime import datetime from enum import Enum, auto from functools import reduce from typing import Iterable, Iterator, List, Optional, Tuple -TestResult = namedtuple('TestResult', ['status','test','log']) - class Test(object): """ A class to represent a test parsed from KTAP results. All KTAP @@ -805,7 +802,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: print_test_result(test) return test -def parse_run_tests(kernel_output: Iterable[str]) -> TestResult: +def parse_run_tests(kernel_output: Iterable[str]) -> Test: """ Using kernel output, extract KTAP lines, parse the lines for test results and print condensed test results and summary line . @@ -814,8 +811,7 @@ def parse_run_tests(kernel_output: Iterable[str]) -> TestResult: kernel_output - Iterable object contains lines of kernel output Return: - TestResult - Tuple containg status of main test object, main test - object with all subtests, and log of all KTAP lines. + Test - the main test object with all subtests. """ print_with_timestamp(DIVIDER) lines = extract_tap_lines(kernel_output) @@ -829,4 +825,4 @@ def parse_run_tests(kernel_output: Iterable[str]) -> TestResult: test.status = test.counts.get_status() print_with_timestamp(DIVIDER) print_summary_line(test) - return TestResult(test.status, test, lines) + return test diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index e30e162bd084..1f6b177ca5c2 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -178,7 +178,7 @@ class KUnitParserTest(unittest.TestCase): with open(empty_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) - self.assertEqual(0, len(result.test.subtests)) + self.assertEqual(0, len(result.subtests)) self.assertEqual( kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS, result.status) @@ -191,9 +191,9 @@ class KUnitParserTest(unittest.TestCase): kunit_parser.extract_tap_lines( file.readlines())) # A missing test plan is not an error. - self.assertEqual(0, result.test.counts.errors) + self.assertEqual(0, result.counts.errors) # All tests should be accounted for. - self.assertEqual(10, result.test.counts.total()) + self.assertEqual(10, result.counts.total()) self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) @@ -203,7 +203,7 @@ class KUnitParserTest(unittest.TestCase): with open(header_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) - self.assertEqual(0, len(result.test.subtests)) + self.assertEqual(0, len(result.subtests)) self.assertEqual( kunit_parser.TestStatus.NO_TESTS, result.status) @@ -213,11 +213,11 @@ class KUnitParserTest(unittest.TestCase): with open(no_plan_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) - self.assertEqual(0, len(result.test.subtests[0].subtests[0].subtests)) + self.assertEqual(0, len(result.subtests[0].subtests[0].subtests)) self.assertEqual( kunit_parser.TestStatus.NO_TESTS, - result.test.subtests[0].subtests[0].status) - self.assertEqual(1, result.test.counts.errors) + result.subtests[0].subtests[0].status) + self.assertEqual(1, result.counts.errors) def test_no_kunit_output(self): @@ -228,7 +228,7 @@ class KUnitParserTest(unittest.TestCase): kunit_parser.extract_tap_lines(file.readlines())) print_mock.assert_any_call(StrContains('invalid KTAP input!')) print_mock.stop() - self.assertEqual(0, len(result.test.subtests)) + self.assertEqual(0, len(result.subtests)) def test_crashed_test(self): crashed_log = test_data_path('test_is_test_passed-crash.log') @@ -269,10 +269,10 @@ class KUnitParserTest(unittest.TestCase): result.status) self.assertEqual( "sysctl_test", - result.test.subtests[0].name) + result.subtests[0].name) self.assertEqual( "example", - result.test.subtests[1].name) + result.subtests[1].name) file.close() @@ -283,7 +283,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def test_ignores_multiple_prefixes(self): prefix_log = test_data_path('test_multiple_prefixes.log') @@ -292,7 +292,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def test_prefix_mixed_kernel_output(self): mixed_prefix_log = test_data_path('test_interrupted_tap_output.log') @@ -301,7 +301,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def test_prefix_poundsign(self): pound_log = test_data_path('test_pound_sign.log') @@ -310,7 +310,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def test_kernel_panic_end(self): panic_log = test_data_path('test_kernel_panic_interrupt.log') @@ -319,7 +319,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.TEST_CRASHED, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def test_pound_no_prefix(self): pound_log = test_data_path('test_pound_no_prefix.log') @@ -328,7 +328,7 @@ class KUnitParserTest(unittest.TestCase): self.assertEqual( kunit_parser.TestStatus.SUCCESS, result.status) - self.assertEqual('kunit-resource-test', result.test.subtests[0].name) + self.assertEqual('kunit-resource-test', result.subtests[0].name) def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream: return kunit_parser.LineStream(enumerate(strs, start=1)) @@ -467,7 +467,7 @@ class KUnitJsonTest(unittest.TestCase): with open(test_data_path(log_file)) as file: test_result = kunit_parser.parse_run_tests(file) json_obj = kunit_json.get_json_result( - test_result=test_result, + test=test_result, def_config='kunit_defconfig', build_dir=None, json_path='stdout') -- cgit v1.2.3 From 1ee2ba89bea86d6389509e426583b49ac19b86f2 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Tue, 14 Dec 2021 11:30:10 -0800 Subject: kunit: tool: make `build` subcommand also reconfigure if needed If I created a kunitconfig file that was incomplete, then $ ./tools/testing/kunit/kunit.py build --kunitconfig=my_kunitconfig would silently drop all the options with unmet dependencies! This is because it doesn't do the config check that `kunit.py config` does. So if I want to safely build a kernel for testing, I have to do $ ./tools/testing/kunit/kunit.py config $ ./tools/testing/kunit/kunit.py build It seems unlikely that any user of kunit.py would want the current `build` semantics. So make it effectively do `kunit.py config` + `kunit.py build`. Signed-off-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 10 +++++++++- tools/testing/kunit/kunit_tool_test.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index 417dc2d11f4f..f1be71811369 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -110,6 +110,14 @@ def build_tests(linux: kunit_kernel.LinuxSourceTree, 'built kernel successfully', build_end - build_start) +def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree, + request: KunitBuildRequest) -> KunitResult: + config_result = config_tests(linux, request) + if config_result.status != KunitStatus.SUCCESS: + return config_result + + return build_tests(linux, request) + def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: args = ['kunit.action=list'] if request.kernel_args: @@ -464,7 +472,7 @@ def main(argv, linux=None): make_options=cli_args.make_options, jobs=cli_args.jobs, alltests=cli_args.alltests) - result = build_tests(linux, request) + result = config_and_build_tests(linux, request) kunit_parser.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( result.elapsed_time)) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 1f6b177ca5c2..b80e333a20cb 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -526,7 +526,7 @@ class KUnitMainTest(unittest.TestCase): def test_build_passes_args_pass(self): kunit.main(['build'], self.linux_source_mock) - self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 0) + self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1) self.linux_source_mock.build_kernel.assert_called_once_with(False, 8, '.kunit', None) self.assertEqual(self.linux_source_mock.run_kernel.call_count, 0) -- cgit v1.2.3 From ad659ccb5412874c6a89d3588cb18857c00e9d0f Mon Sep 17 00:00:00 2001 From: David Gow Date: Wed, 15 Dec 2021 12:50:14 -0800 Subject: kunit: tool: Default --jobs to number of CPUs The --jobs parameter for kunit_tool currently defaults to 8 CPUs, regardless of the number available. For systems with significantly more (or less), this is not as efficient. Instead, default --jobs to the number of CPUs available to the process: while there are as many superstitions as to exactly what the ideal jobs:CPU ratio is, this seems sufficiently sensible to me. A new helper function to get the default number of jobs is added: get_default_jobs() -- this is used in kunit_tool_test instead of a hardcoded value, or an explicit call to len(os.sched_getaffinity()), so should be more flexible if this needs to change in the future. Signed-off-by: David Gow Signed-off-by: Daniel Latypov Reviewed-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 5 ++++- tools/testing/kunit/kunit_tool_test.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'tools/testing/kunit/kunit.py') diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index f1be71811369..7a706f96f68d 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -282,6 +282,9 @@ def massage_argv(argv: Sequence[str]) -> Sequence[str]: return f'{arg}={pseudo_bool_flag_defaults[arg]}' return list(map(massage_arg, argv)) +def get_default_jobs() -> int: + return len(os.sched_getaffinity(0)) + def add_common_opts(parser) -> None: parser.add_argument('--build_dir', help='As in the make command, it specifies the build ' @@ -332,7 +335,7 @@ def add_build_opts(parser) -> None: parser.add_argument('--jobs', help='As in the make command, "Specifies the number of ' 'jobs (commands) to run simultaneously."', - type=int, default=8, metavar='jobs') + type=int, default=get_default_jobs(), metavar='jobs') def add_exec_opts(parser) -> None: parser.add_argument('--timeout', diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index b80e333a20cb..352369dffbd9 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -527,7 +527,7 @@ class KUnitMainTest(unittest.TestCase): def test_build_passes_args_pass(self): kunit.main(['build'], self.linux_source_mock) self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1) - self.linux_source_mock.build_kernel.assert_called_once_with(False, 8, '.kunit', None) + self.linux_source_mock.build_kernel.assert_called_once_with(False, kunit.get_default_jobs(), '.kunit', None) self.assertEqual(self.linux_source_mock.run_kernel.call_count, 0) def test_exec_passes_args_pass(self): @@ -633,8 +633,9 @@ class KUnitMainTest(unittest.TestCase): def test_build_builddir(self): build_dir = '.kunit' + jobs = kunit.get_default_jobs() kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock) - self.linux_source_mock.build_kernel.assert_called_once_with(False, 8, build_dir, None) + self.linux_source_mock.build_kernel.assert_called_once_with(False, jobs, build_dir, None) def test_exec_builddir(self): build_dir = '.kunit' -- cgit v1.2.3