From 5a6b9c81554a905b9798e4df1b2ab2e5a79a0c1f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 28 Apr 2025 09:30:46 +0200 Subject: t/perf: fix benchmarks with alternate repo formats Many of our benchmarks operate on a user-defined repository that we copy over before running the benchmarked logic. To keep unintentional side effects caused by on-disk state at bay we skip copying some files. This includes for example hooks, but also the repo's configuration. It is quite sensible to not copy over the configuration, as it is quite easy to inadvertently carry over configuration that may significantly impact the performance measurements. But we cannot fully ignore the configuration either, as it may contain information about the repository format. This will cause failures when for example using a repository with SHA256 object format or the reftable ref format. Fix the issue by parsing the reference and object formats from the source repository and passing them to git-init(1). Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/perf/perf-lib.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index 8ab6d9c469..1a9a51ca3c 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -98,6 +98,8 @@ test_perf_create_repo_from () { source_git="$("$MODERN_GIT" -C "$source" rev-parse --git-dir)" objects_dir="$("$MODERN_GIT" -C "$source" rev-parse --git-path objects)" common_dir="$("$MODERN_GIT" -C "$source" rev-parse --git-common-dir)" + refformat="$("$MODERN_GIT" -C "$source" rev-parse --show-ref-format)" + objectformat="$("$MODERN_GIT" -C "$source" rev-parse --show-object-format)" mkdir -p "$repo/.git" ( cd "$source" && @@ -114,7 +116,7 @@ test_perf_create_repo_from () { ) && ( cd "$repo" && - "$MODERN_GIT" init -q && + "$MODERN_GIT" init -q --ref-format="$refformat" --object-format="$objectformat" && test_perf_do_repo_symlink_config_ && mv .git/hooks .git/hooks-disabled 2>/dev/null && if test -f .git/index.lock -- cgit v1.2.3 From d84b990883284920234f5bc951ea285161fc7bca Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 28 Apr 2025 09:30:47 +0200 Subject: t/perf: use configured PERL_PATH Our benchmarks use a couple of Perl scripts to compute results. These Perl scripts get executed directly, and as the shebang is hardcoded to "/usr/bin/perl" this will fail on any system where the Perl interpreter is located in a different path. Our build infrastructure already lets users configure the location of Perl, which ultimately gets written into the GIT-BUILD-OPTIONS file. This file is being sourced by "test-lib.sh", and consequently we already have the "PERL_PATH" variable available that contains its configured location. Use "PERL_PATH" to execute Perl scripts, which makes them work on more esoteric systems like NixOS. Furthermore, adapt the shebang to use env(1) to execute Perl so that users who have Perl in PATH, but in a non-standard location can execute the script directly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/perf/aggregate.perl | 2 +- t/perf/perf-lib.sh | 4 ++-- t/perf/run | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/t/perf/aggregate.perl b/t/perf/aggregate.perl index 575d2000cc..1791c7528a 100755 --- a/t/perf/aggregate.perl +++ b/t/perf/aggregate.perl @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl use lib '../../perl/build/lib'; use strict; diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index 1a9a51ca3c..4173eee4de 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -276,7 +276,7 @@ test_perf_ () { else test_ok_ "$1" fi - "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".result + "$PERL_PATH" "$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".result rm test_time.* } @@ -324,7 +324,7 @@ test_at_end_hook_ () { if test -z "$GIT_PERF_AGGREGATING_LATER"; then ( cd "$TEST_DIRECTORY"/perf && - ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $(basename "$0") + "$PERL_PATH" ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $(basename "$0") ) fi } diff --git a/t/perf/run b/t/perf/run index 486ead2198..073bcb2aff 100755 --- a/t/perf/run +++ b/t/perf/run @@ -192,10 +192,10 @@ run_subsection () { if test -z "$GIT_PERF_SEND_TO_CODESPEED" then - ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $codespeed_opt "$@" + "$PERL_PATH" ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $codespeed_opt "$@" else json_res_file=""$TEST_RESULTS_DIR"/$GIT_PERF_SUBSECTION/aggregate.json" - ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" --codespeed "$@" | tee "$json_res_file" + "$PERL_PATH" ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" --codespeed "$@" | tee "$json_res_file" send_data_url="$GIT_PERF_SEND_TO_CODESPEED/result/add/json/" curl -v --request POST --data-urlencode "json=$(cat "$json_res_file")" "$send_data_url" fi -- cgit v1.2.3 From 5756ccd181d4cc938dfcd0fe96083ff6c1455afd Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 28 Apr 2025 09:30:48 +0200 Subject: t/perf: fix benchmarks with out-of-tree builds The "perf-lib.sh" script is sourced by all of our benchmarking suites to make available common infrastructure. The script assumes that build and source directory are the same, which works for our Makefile. But the assumption breaks with both CMake and Meson, where the build directory can be located in an arbitrary place. Adapt the script so that it works with out-of-tree builds. Most importantly, this requires us to figure out the location of the build directory: - When running benchmarks via our Makefile the build directory is the same as the source directory. We already know to derive the test directory ("t/") via `$(pwd)/..`, which works because we chdir into "t/perf" before executing benchmarks. We can thus derive the build directory by appending another "/.." to that path. - When running benchmarks via Meson the build directory is located at an arbitrary location. The build system thus has to make the path known by exporting the `GIT_BUILD_DIR` environment variable. This change prepares us for wiring up benchmarks in Meson. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/perf/perf-lib.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index 4173eee4de..5406557b7c 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -25,7 +25,29 @@ TEST_OUTPUT_DIRECTORY=$(pwd) TEST_NO_CREATE_REPO=t TEST_NO_MALLOC_CHECK=t -. ../test-lib.sh +# While test-lib.sh computes the build directory for us, we also have to do the +# same thing in order to locate the script via GIT-BUILD-OPTIONS in the first +# place. +GIT_BUILD_DIR="${GIT_BUILD_DIR:-$TEST_DIRECTORY/..}" +if test -f "$GIT_BUILD_DIR/GIT-BUILD-DIR" +then + GIT_BUILD_DIR="$(cat "$GIT_BUILD_DIR/GIT-BUILD-DIR")" || exit 1 + # On Windows, we must convert Windows paths lest they contain a colon + case "$(uname -s)" in + *MINGW*) + GIT_BUILD_DIR="$(cygpath -au "$GIT_BUILD_DIR")" + ;; + esac +fi + +if test ! -f "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS +then + echo >&2 'error: GIT-BUILD-OPTIONS missing (has Git been built?).' + exit 1 +fi + +. "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS +. "$GIT_SOURCE_DIR"/t/test-lib.sh unset GIT_CONFIG_NOSYSTEM GIT_CONFIG_SYSTEM="$TEST_DIRECTORY/perf/config" @@ -324,7 +346,7 @@ test_at_end_hook_ () { if test -z "$GIT_PERF_AGGREGATING_LATER"; then ( cd "$TEST_DIRECTORY"/perf && - "$PERL_PATH" ./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $(basename "$0") + "$PERL_PATH" "$GIT_SOURCE_DIR"/t/perf/aggregate.perl --results-dir="$TEST_RESULTS_DIR" $(basename "$0") ) fi } -- cgit v1.2.3 From d84eefaeea10cba53fda063f2b68e6c47f250029 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 28 Apr 2025 09:30:49 +0200 Subject: meson: wire up benchmarks Wire up benchmarks in Meson. The setup is mostly the same as how we wire up our tests. The only difference is that benchmarks get wired up via the `benchmark()` option instead of via `test()`, which gives them a bit of special treatment: - Benchmarks never run in parallel. - Benchmarks aren't run by default when tests are executed. - Meson does not inject the `MALLOC_PERTURB` environment variable. Using benchmarks is quite simple: ``` $ meson setup build # Run all benchmarks. $ meson test -C build --benchmark # Run a specific benchmark. $ meson test -C build --benchmark p0000-* ``` Other than that the usual command line arguments accepted when running tests are also accepted when running benchmarks. Note that the benchmarking target is somewhat limited because it will only run benchmarks for the current build. Other use cases, like running benchmarks against multiple different versions of Git, are not currently supported. Users should continue to use "t/perf/run" for those use cases. The script should get extended at one point in time to support Meson, but this is outside of the scope of this series. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 16 +++++++++--- meson_options.txt | 2 ++ t/meson.build | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/meson.build b/meson.build index 410bbf93da..939a2eae32 100644 --- a/meson.build +++ b/meson.build @@ -70,9 +70,15 @@ # # Execute single test interactively such that features like `debug ()` work. # $ meson test -i --test-args='-ix' t1400-update-ref # -# Test execution is parallelized by default and scales with the number of -# processor cores available. You can change the number of processes by passing -# the `-jN` flag to `meson test`. +# # Execute all benchmarks. +# $ meson test -i --benchmark +# +# # Execute single benchmark. +# $ meson test -i --benchmark p0000-* +# +# Test execution (but not benchmark execution) is parallelized by default and +# scales with the number of processor cores available. You can change the +# number of processes by passing the `-jN` flag to `meson test`. # # 4. Install the Git distribution. Again, this can be done via Meson, Ninja or # Samurai: @@ -235,6 +241,7 @@ git = find_program('git', dirs: program_path, native: true, required: false) sed = find_program('sed', dirs: program_path, native: true) shell = find_program('sh', dirs: program_path, native: true) tar = find_program('tar', dirs: program_path, native: true) +time = find_program('time', dirs: program_path, required: get_option('benchmarks')) target_shell = find_program('sh', dirs: program_path, native: false) @@ -836,7 +843,7 @@ endif # features. It is optional if you want to neither execute tests nor use any of # these optional features. perl_required = get_option('perl') -if get_option('gitweb').enabled() or 'netrc' in get_option('credential_helpers') or get_option('docs') != [] +if get_option('benchmarks').enabled() or get_option('gitweb').enabled() or 'netrc' in get_option('credential_helpers') or get_option('docs') != [] perl_required = true endif @@ -2082,6 +2089,7 @@ meson.add_dist_script( ) summary({ + 'benchmarks': get_option('tests') and perl.found() and time.found(), 'curl': curl.found(), 'expat': expat.found(), 'gettext': intl.found(), diff --git a/meson_options.txt b/meson_options.txt index 8ac30a5223..7f5bca5c02 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -101,6 +101,8 @@ option('docs_backend', type: 'combo', choices: ['asciidoc', 'asciidoctor', 'auto description: 'Which backend to use to generate documentation.') # Testing. +option('benchmarks', type: 'feature', value: 'auto', + description: 'Enable benchmarks. This requires Perl and GNU time.') option('coccinelle', type: 'feature', value: 'auto', description: 'Provide a coccicheck target that generates a Coccinelle patch.') option('tests', type: 'boolean', value: true, diff --git a/t/meson.build b/t/meson.build index bfb744e886..59438b0626 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1097,11 +1097,71 @@ integration_tests = [ 't9903-bash-prompt.sh', ] +benchmarks = [ + 'perf/p0000-perf-lib-sanity.sh', + 'perf/p0001-rev-list.sh', + 'perf/p0002-read-cache.sh', + 'perf/p0003-delta-base-cache.sh', + 'perf/p0004-lazy-init-name-hash.sh', + 'perf/p0005-status.sh', + 'perf/p0006-read-tree-checkout.sh', + 'perf/p0007-write-cache.sh', + 'perf/p0008-odb-fsync.sh', + 'perf/p0071-sort.sh', + 'perf/p0090-cache-tree.sh', + 'perf/p0100-globbing.sh', + 'perf/p1006-cat-file.sh', + 'perf/p1400-update-ref.sh', + 'perf/p1450-fsck.sh', + 'perf/p1451-fsck-skip-list.sh', + 'perf/p1500-graph-walks.sh', + 'perf/p2000-sparse-operations.sh', + 'perf/p3400-rebase.sh', + 'perf/p3404-rebase-interactive.sh', + 'perf/p4000-diff-algorithms.sh', + 'perf/p4001-diff-no-index.sh', + 'perf/p4002-diff-color-moved.sh', + 'perf/p4205-log-pretty-formats.sh', + 'perf/p4209-pickaxe.sh', + 'perf/p4211-line-log.sh', + 'perf/p4220-log-grep-engines.sh', + 'perf/p4221-log-grep-engines-fixed.sh', + 'perf/p5302-pack-index.sh', + 'perf/p5303-many-packs.sh', + 'perf/p5304-prune.sh', + 'perf/p5310-pack-bitmaps.sh', + 'perf/p5311-pack-bitmaps-fetch.sh', + 'perf/p5312-pack-bitmaps-revs.sh', + 'perf/p5313-pack-objects.sh', + 'perf/p5314-name-hash.sh', + 'perf/p5326-multi-pack-bitmaps.sh', + 'perf/p5332-multi-pack-reuse.sh', + 'perf/p5333-pseudo-merge-bitmaps.sh', + 'perf/p5550-fetch-tags.sh', + 'perf/p5551-fetch-rescan.sh', + 'perf/p5600-partial-clone.sh', + 'perf/p5601-clone-reference.sh', + 'perf/p6100-describe.sh', + 'perf/p6300-for-each-ref.sh', + 'perf/p7000-filter-branch.sh', + 'perf/p7102-reset.sh', + 'perf/p7300-clean.sh', + 'perf/p7519-fsmonitor.sh', + 'perf/p7527-builtin-fsmonitor.sh', + 'perf/p7810-grep.sh', + 'perf/p7820-grep-engines.sh', + 'perf/p7821-grep-engines-fixed.sh', + 'perf/p7822-grep-perl-character.sh', + 'perf/p9210-scalar.sh', + 'perf/p9300-fast-import-export.sh', +] + # Sanity check that we are not missing any tests present in 't/'. This check # only runs once at configure time and is thus best-effort, only. It is # sufficient to catch missing test suites in our CI though. foreach glob, tests : { 't[0-9][0-9][0-9][0-9]-*.sh': integration_tests, + 'perf/p[0-9][0-9][0-9][0-9]-*.sh': benchmarks, 'unit-tests/t-*.c': unit_test_programs, 'unit-tests/u-*.c': clar_test_suites, } @@ -1153,3 +1213,20 @@ foreach integration_test : integration_tests timeout: 0, ) endforeach + +if perl.found() and time.found() + benchmark_environment = test_environment + benchmark_environment.set('GTIME', time.full_path()) + + foreach benchmark : benchmarks + benchmark(fs.stem(benchmark), shell, + args: [ + fs.name(benchmark), + ], + workdir: meson.current_source_dir() / 'perf', + env: benchmark_environment, + depends: test_dependencies + bin_wrappers, + timeout: 0, + ) + endforeach +endif -- cgit v1.2.3 From c3fc5c68f65fdd50d94672267cf2be77d96cfcf7 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 28 Apr 2025 09:30:50 +0200 Subject: meson: wire up benchmarking options Wire up a couple of benchmarking options that we end up writing into our "GIT-BUILD-OPTIONS" file. These options allow users to control how exactly benchmarks are executed. Note that neither `GIT_PERF_MAKE_COMMAND` nor `GIT_PERF_MAKE_OPTS` are exposed as a build option. Those options are used by "t/perf/run", which is not used by Meson. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 6 +++--- meson_options.txt | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 939a2eae32..d06f82ff9f 100644 --- a/meson.build +++ b/meson.build @@ -706,11 +706,11 @@ builtin_sources += custom_target( # build options to our tests. build_options_config = configuration_data() build_options_config.set('GIT_INTEROP_MAKE_OPTS', '') -build_options_config.set('GIT_PERF_LARGE_REPO', '') +build_options_config.set_quoted('GIT_PERF_LARGE_REPO', get_option('benchmark_large_repo')) build_options_config.set('GIT_PERF_MAKE_COMMAND', '') build_options_config.set('GIT_PERF_MAKE_OPTS', '') -build_options_config.set('GIT_PERF_REPEAT_COUNT', '') -build_options_config.set('GIT_PERF_REPO', '') +build_options_config.set_quoted('GIT_PERF_REPEAT_COUNT', get_option('benchmark_repeat_count').to_string()) +build_options_config.set_quoted('GIT_PERF_REPO', get_option('benchmark_repo')) build_options_config.set('GIT_TEST_CMP_USE_COPIED_CONTEXT', '') build_options_config.set('GIT_TEST_INDEX_VERSION', '') build_options_config.set('GIT_TEST_OPTS', '') diff --git a/meson_options.txt b/meson_options.txt index 7f5bca5c02..8547c0eb47 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -103,6 +103,12 @@ option('docs_backend', type: 'combo', choices: ['asciidoc', 'asciidoctor', 'auto # Testing. option('benchmarks', type: 'feature', value: 'auto', description: 'Enable benchmarks. This requires Perl and GNU time.') +option('benchmark_repo', type: 'string', value: '', + description: 'Repository to copy for the performance tests. Should be at least the size of the Git repository.') +option('benchmark_large_repo', type: 'string', value: '', + description: 'Large repository to copy for the performance tests. Should be at least the size of the Linux repository.') +option('benchmark_repeat_count', type: 'integer', value: 3, + description: 'Number of times a test should be repeated for best-of-N measurements.') option('coccinelle', type: 'feature', value: 'auto', description: 'Provide a coccicheck target that generates a Coccinelle patch.') option('tests', type: 'boolean', value: true, -- cgit v1.2.3