diff options
| -rwxr-xr-x | tests/run-multitests.py | 22 |
1 files changed, 21 insertions, 1 deletions
diff --git a/tests/run-multitests.py b/tests/run-multitests.py index f27af3b0e..edf2238a5 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -303,6 +303,10 @@ def run_test_on_instances(test_file, num_instances, instances): def run_tests(test_files, instances_truth, instances_test): + skipped_tests = [] + passed_tests = [] + failed_tests = [] + for test_file, num_instances in test_files: instances_str = "|".join(str(instances_test[i]) for i in range(num_instances)) print("{} on {}: ".format(test_file, instances_str), end="") @@ -335,10 +339,13 @@ def run_tests(test_files, instances_truth, instances_test): # Print result of test if skip: print("skip") + skipped_tests.append(test_file) elif output_test == output_truth: print("pass") + passed_tests.append(test_file) else: print("FAIL") + failed_tests.append(test_file) if not cmd_args.show_output: print("### TEST ###") print(output_test, end="") @@ -348,6 +355,16 @@ def run_tests(test_files, instances_truth, instances_test): if cmd_args.show_output: print() + print("{} tests performed".format(len(skipped_tests) + len(passed_tests) + len(failed_tests))) + print("{} tests passed".format(len(passed_tests))) + + if skipped_tests: + print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests))) + if failed_tests: + print("{} tests failed: {}".format(len(failed_tests), " ".join(failed_tests))) + + return not failed_tests + def main(): global cmd_args @@ -384,13 +401,16 @@ def main(): instances_test.append(PyInstanceSubProcess([MICROPYTHON])) try: - run_tests(test_files, instances_truth, instances_test) + all_pass = run_tests(test_files, instances_truth, instances_test) finally: for i in instances_truth: i.close() for i in instances_test: i.close() + if not all_pass: + sys.exit(1) + if __name__ == "__main__": main() |